{"id":78564,"date":"2026-02-27T22:29:25","date_gmt":"2026-02-27T14:29:25","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/78564.html"},"modified":"2026-02-27T22:29:25","modified_gmt":"2026-02-27T14:29:25","slug":"hadoop-3-x%e5%ae%9e%e6%88%98%ef%bc%9a%e5%9f%ba%e4%ba%8ehdfssparkflink%e7%9a%84%e5%ae%9e%e6%97%b6%e7%94%a8%e6%88%b7%e8%a1%8c%e4%b8%ba%e5%88%86%e6%9e%90%e5%b9%b3%e5%8f%b0%ef%bc%88%e5%90%abkerberos","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/78564.html","title":{"rendered":"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09"},"content":{"rendered":"<h2>Hadoop 3.x\u5b9e\u6218&#xff1a;\u57fa\u4e8eHDFS&#043;Spark&#043;Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff08;\u542bKerberos\u5b89\u5168\u914d\u7f6e&#043;\u51b7\u70ed\u6570\u636e\u5206\u5c42&#xff09;<\/h2>\n<p>\u5728\u5927\u6570\u636e\u5b9e\u65f6\u5316\u3001\u5b89\u5168\u5408\u89c4\u8981\u6c42\u5347\u7ea7\u7684\u5f53\u4e0b&#xff0c;\u4f20\u7edfHadoop\u96c6\u7fa4\u9762\u4e34\u201c\u5b9e\u65f6\u5904\u7406\u5f31\u3001\u5b89\u5168\u9632\u62a4\u4e0d\u8db3\u3001\u5b58\u50a8\u6210\u672c\u9ad8\u201d\u4e09\u5927\u75db\u70b9\u3002\u672c\u6587\u57fa\u4e8e Hadoop 3.3.6&#xff08;\u6700\u65b0\u7a33\u5b9a\u7248&#xff09;&#xff0c;\u8bbe\u8ba1\u4e00\u5957\u201c\u79bb\u7ebf\u6279\u5904\u7406&#043;\u5b9e\u65f6\u6d41\u8ba1\u7b97\u201d\u4e00\u4f53\u5316\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff0c;\u6574\u5408HDFS 3.x\u3001Spark 3.4.1\u3001Flink 1.17.0\u6838\u5fc3\u7ec4\u4ef6&#xff0c;\u878d\u5165Kerberos\u8eab\u4efd\u8ba4\u8bc1\u3001HDFS\u51b7\u70ed\u6570\u636e\u5206\u5c42\u3001\u5217\u5f0f\u5b58\u50a8\u4f18\u5316\u7b49\u4f01\u4e1a\u7ea7\u7279\u6027&#xff0c;\u5168\u7a0b\u62c6\u89e3\u4ece\u96c6\u7fa4\u90e8\u7f72\u5230\u4e1a\u52a1\u843d\u5730\u7684\u5b8c\u6574\u6d41\u7a0b&#xff0c;\u9644\u5173\u952e\u914d\u7f6e\u4e0e\u4ee3\u7801\u5b9e\u73b0&#xff0c;\u53ef\u76f4\u63a5\u9002\u914d\u7535\u5546\u3001\u77ed\u89c6\u9891\u7b49\u573a\u666f\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u9700\u6c42\u3002<\/p>\n<h3>\u4e00\u3001\u4e1a\u52a1\u80cc\u666f\u4e0e\u6280\u672f\u9009\u578b<\/h3>\n<h4>1. \u4e1a\u52a1\u9700\u6c42&#xff08;\u67d0\u77ed\u89c6\u9891\u5e73\u53f0&#xff09;<\/h4>\n<ul>\n<li>\u6570\u636e\u89c4\u6a21&#xff1a;\u65e5\u5747\u7528\u6237\u884c\u4e3a\u6570\u636e10TB&#xff08;\u70b9\u51fb\/\u6ed1\u52a8\/\u505c\u7559\/\u70b9\u8d5e\u7b4912\u7c7b\u4e8b\u4ef6&#xff09;&#xff0c;\u7528\u6237\u89c4\u6a211\u4ebf&#043;&#xff0c;\u5546\u54c1\/\u89c6\u9891\u5e935000\u4e07&#043;&#xff1b;<\/li>\n<li>\u6838\u5fc3\u9700\u6c42&#xff1a;\n<li>\u79bb\u7ebf\u5206\u6790&#xff1a;T&#043;1\u7edf\u8ba1\u7528\u6237\u753b\u50cf&#xff08;\u5e74\u9f84\/\u5730\u57df\/\u5174\u8da3\u6807\u7b7e&#xff09;\u3001\u89c6\u9891\u8f6c\u5316\u7387\u3001\u6e20\u9053\u6548\u679c\u5f52\u56e0&#xff1b;<\/li>\n<li>\u5b9e\u65f6\u5206\u6790&#xff1a;\u79d2\u7ea7\u76d1\u63a7\u70ed\u95e8\u89c6\u9891\u66dd\u5149\u91cf\u3001\u5b9e\u65f6\u63a8\u8350\u5019\u9009\u96c6\u66f4\u65b0\u3001\u5f02\u5e38\u884c\u4e3a&#xff08;\u5237\u91cf\/\u6076\u610f\u70b9\u51fb&#xff09;\u68c0\u6d4b&#xff1b;<\/li>\n<li>\u5b89\u5168\u5408\u89c4&#xff1a;\u654f\u611f\u6570\u636e&#xff08;\u7528\u6237ID\/\u624b\u673a\u53f7&#xff09;\u52a0\u5bc6\u5b58\u50a8&#xff0c;\u96c6\u7fa4\u8bbf\u95ee\u6743\u9650\u7cbe\u7ec6\u5316\u63a7\u5236&#xff1b;<\/li>\n<li>\u6210\u672c\u4f18\u5316&#xff1a;\u51b7\u70ed\u6570\u636e\u5206\u79bb\u5b58\u50a8&#xff0c;\u964d\u4f4e80%\u5386\u53f2\u6570\u636e\u7684\u5b58\u50a8\u6210\u672c\u3002<\/li>\n<\/li>\n<\/ul>\n<h4>2. \u6280\u672f\u9009\u578b&#xff08;Hadoop 3.x\u751f\u6001\u6838\u5fc3\u7ec4\u4ef6&#xff09;<\/h4>\n<table>\n<tr>\u6a21\u5757\u6280\u672f\u9009\u578b\u6838\u5fc3\u4f18\u52bf&#xff08;Hadoop 3.x\u7279\u6027&#xff09;<\/tr>\n<tbody>\n<tr>\n<td>\u5206\u5e03\u5f0f\u5b58\u50a8<\/td>\n<td>HDFS 3.3.6&#xff08;Erasure Coding&#043;\u51b7\u70ed\u6570\u636e\u5206\u5c42&#xff09;<\/td>\n<td>\u7ea0\u5220\u7801\u5b58\u50a8&#xff08;EC&#xff09;\u66ff\u4ee3\u526f\u672c&#xff0c;\u51b7\u6570\u636e\u5b58\u50a8\u6210\u672c\u964d\u4f4e50%&#xff1b;\u652f\u6301\u5b58\u50a8\u7b56\u7565\u63d2\u4ef6&#xff0c;\u81ea\u52a8\u8fc1\u79fb\u51b7\u70ed\u6570\u636e<\/td>\n<\/tr>\n<tr>\n<td>\u79bb\u7ebf\u8ba1\u7b97<\/td>\n<td>Spark 3.4.1&#xff08;On YARN\u6a21\u5f0f&#xff09;<\/td>\n<td>\u517c\u5bb9Hadoop 3.x\u7684YARN Federation&#xff0c;\u652f\u6301\u52a8\u6001\u8d44\u6e90\u8c03\u6574&#xff0c;\u6279\u5904\u7406\u6027\u80fd\u8f83Hadoop 2.x\u63d0\u534730%<\/td>\n<\/tr>\n<tr>\n<td>\u5b9e\u65f6\u8ba1\u7b97<\/td>\n<td>Flink 1.17.0&#xff08;Kafka Connector&#043;YARN Per-Job\u6a21\u5f0f&#xff09;<\/td>\n<td>\u4e0eHadoop 3.x\u7684HDFS\/HBase\u517c\u5bb9\u6027\u4f18\u5316&#xff0c;\u652f\u6301Checkpoint\u5b58\u50a8\u5230HDFS&#xff0c;Exactly-Once\u8bed\u4e49\u4fdd\u969c<\/td>\n<\/tr>\n<tr>\n<td>\u5b89\u5168\u8ba4\u8bc1<\/td>\n<td>Kerberos&#043;Apache Ranger<\/td>\n<td>Hadoop 3.x\u589e\u5f3aKerberos\u4e92\u4fe1\u673a\u5236&#xff0c;Ranger\u5b9e\u73b0\u7ec6\u7c92\u5ea6\u6743\u9650\u63a7\u5236&#xff08;\u5e93\u8868\/\u6587\u4ef6\u7ea7\u6388\u6743&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5143\u6570\u636e\u7ba1\u7406<\/td>\n<td>Hive 3.1.3&#xff08;ACID\u4e8b\u52a1&#043;\u5217\u5f0f\u5b58\u50a8ORC&#xff09;<\/td>\n<td>\u652f\u6301\u4e8b\u52a1\u64cd\u4f5c&#xff0c;ORC\u683c\u5f0f\u538b\u7f29\u6bd4\u8fbe1:10&#xff0c;\u67e5\u8be2\u6548\u7387\u8f83Text\u683c\u5f0f\u63d0\u53475\u500d\u4ee5\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>\u6570\u636e\u540c\u6b65<\/td>\n<td>Sqoop 1.4.7&#xff08;\u9002\u914dHadoop 3.x&#xff09;&#043;DataX<\/td>\n<td>\u652f\u6301\u4eceMySQL\/Oracle\u589e\u91cf\u540c\u6b65\u6570\u636e\u5230HDFS&#xff0c;\u9002\u914dHadoop 3.x\u7684\u9ad8\u53ef\u7528&#xff08;HA&#xff09;\u96c6\u7fa4<\/td>\n<\/tr>\n<tr>\n<td>\u76d1\u63a7\u8fd0\u7ef4<\/td>\n<td>Prometheus&#043;Grafana&#043;Hadoop 3.x Metrics2<\/td>\n<td>Hadoop 3.x\u539f\u751f\u652f\u6301Metrics2\u63a5\u53e3&#xff0c;\u53ef\u76f4\u63a5\u5bf9\u63a5\u76d1\u63a7\u7cfb\u7edf&#xff0c;\u5b9e\u65f6\u91c7\u96c6\u96c6\u7fa4\u8d44\u6e90\/\u4efb\u52a1\u6307\u6807<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>3. \u6574\u4f53\u67b6\u6784&#xff08;\u4e09\u5c42\u67b6\u6784&#043;\u6570\u636e\u6d41\u95ed\u73af&#xff09;<\/h4>\n<p>\u6570\u636e\u91c7\u96c6\u5c42&#xff1a;\u7528\u6237\u884c\u4e3a\u65e5\u5fd7&#xff08;Flume&#043;Kafka&#xff09;&#043; \u4e1a\u52a1\u6570\u636e\u5e93&#xff08;Sqoop\/DataX&#xff09;<br \/>\n\u6570\u636e\u5b58\u50a8\u5c42&#xff1a;HDFS 3.x&#xff08;\u70ed\u6570\u636e&#xff1a;\u526f\u672c\u5b58\u50a8&#xff1b;\u51b7\u6570\u636e&#xff1a;EC\u7ea0\u5220\u7801&#xff09;&#043; HBase&#xff08;\u5b9e\u65f6\u67e5\u8be2&#xff09;<br \/>\n\u8ba1\u7b97\u5c42&#xff1a;\u79bb\u7ebf\u8ba1\u7b97&#xff08;Spark SQL\/MLlib&#xff09;&#043; \u5b9e\u65f6\u8ba1\u7b97&#xff08;Flink StreamProcessing&#xff09;<br \/>\n\u5e94\u7528\u5c42&#xff1a;\u7528\u6237\u753b\u50cf\u7cfb\u7edf\u3001\u5b9e\u65f6\u63a8\u8350\u3001\u8fd0\u8425\u62a5\u8868\u3001\u5f02\u5e38\u68c0\u6d4b<\/p>\n<h3>\u4e8c\u3001\u6838\u5fc3\u6280\u672f\u62c6\u89e3&#xff1a;Hadoop 3.x\u5173\u952e\u7279\u6027\u843d\u5730<\/h3>\n<h4>1. HDFS 3.x\u90e8\u7f72\u4e0e\u4f18\u5316&#xff08;HA&#043;EC\u7ea0\u5220\u7801&#043;\u51b7\u70ed\u5206\u5c42&#xff09;<\/h4>\n<h5>&#xff08;1&#xff09;\u96c6\u7fa4\u90e8\u7f72&#xff08;3\u4e3b3\u4ece\u9ad8\u53ef\u7528\u67b6\u6784&#xff09;<\/h5>\n<ul>\n<li>\u4e3b\u8282\u70b9&#xff1a;NameNode&#xff08;2\u4e2a&#xff0c;Active\/Standby&#xff09;&#043; JournalNode&#xff08;3\u4e2a&#xff0c;\u5143\u6570\u636e\u540c\u6b65&#xff09;&#043; ZKFC&#xff08;ZooKeeper Failover Controller&#xff09;<\/li>\n<li>\u4ece\u8282\u70b9&#xff1a;DataNode&#xff08;3\u4e2a&#xff0c;\u5b58\u50a8\u6570\u636e\u5757&#xff09;&#043; NodeManager&#xff08;3\u4e2a&#xff0c;YARN\u8ba1\u7b97\u8282\u70b9&#xff09;<\/li>\n<li>\u6838\u5fc3\u914d\u7f6e&#xff08;hdfs-site.xml&#xff09;&#xff1a;<\/li>\n<\/ul>\n<p><span class=\"token comment\">&lt;!&#8211; \u542f\u7528HA\u9ad8\u53ef\u7528 &#8211;&gt;<\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.nameservices<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>hadoop-cluster-3x<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.ha.namenodes.hadoop-cluster-3x<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>nn1,nn2<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token comment\">&lt;!&#8211; JournalNode\u5730\u5740&#xff08;\u5143\u6570\u636e\u540c\u6b65&#xff09; &#8211;&gt;<\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.namenode.shared.edits.dir<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>qjournal:\/\/node1:8485;node2:8485;node3:8485\/hadoop-cluster-3x<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token comment\">&lt;!&#8211; \u542f\u7528\u7ea0\u5220\u7801&#xff08;EC&#xff09;\u5b58\u50a8 &#8211;&gt;<\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.erasurecoding.enabled<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>true<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.erasurecoding.codec<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>rs-6-3-1024k<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span> <span class=\"token comment\">&lt;!&#8211; 6\u6570\u636e\u5757&#043;3\u6821\u9a8c\u5757&#xff0c;\u9002\u5408\u51b7\u6570\u636e &#8211;&gt;<\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token comment\">&lt;!&#8211; \u51b7\u70ed\u6570\u636e\u5206\u5c42\u5b58\u50a8\u7b56\u7565 &#8211;&gt;<\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>dfs.storage.policy.enabled<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>true<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><\/p>\n<h5>&#xff08;2&#xff09;\u51b7\u70ed\u6570\u636e\u5206\u5c42\u5b9e\u8df5<\/h5>\n<ul>\n<li>\u70ed\u6570\u636e&#xff08;\u8fd17\u5929\u7528\u6237\u884c\u4e3a\u65e5\u5fd7&#xff09;&#xff1a;\u91c7\u75283\u526f\u672c\u5b58\u50a8&#xff08;\u526f\u672c\u6570&#061;3&#xff09;&#xff0c;\u5b58\u50a8\u8def\u5f84\/user\/hive\/warehouse\/behavior_hot.db&#xff0c;\u4fdd\u969c\u9ad8\u5e76\u53d1\u8bfb\u5199&#xff1b;<\/li>\n<li>\u51b7\u6570\u636e&#xff08;7\u5929\u524d\u5386\u53f2\u6570\u636e&#xff09;&#xff1a;\u91c7\u7528EC\u7ea0\u5220\u7801\u5b58\u50a8&#xff08;rs-6-3&#xff09;&#xff0c;\u5b58\u50a8\u8def\u5f84\/user\/hive\/warehouse\/behavior_cold.db&#xff0c;\u901a\u8fc7HDFS\u5b58\u50a8\u7b56\u7565\u81ea\u52a8\u8fc1\u79fb&#xff1a;\n<li>\u521b\u5efa\u5b58\u50a8\u7b56\u7565&#xff1a;hdfs storagepolicies -setStoragePolicy -path \/user\/hive\/warehouse\/behavior_cold.db -policy ErasureCoding\n<\/li>\n<li>\u5b9a\u65f6\u8fc1\u79fb\u811a\u672c&#xff08;Shell&#043;Crontab&#xff09;&#xff1a;<span class=\"token comment\"># \u6bcf\u5929\u51cc\u66682\u70b9\u5c067\u5929\u524d\u7684\u6570\u636e\u8fc1\u79fb\u5230\u51b7\u6570\u636e\u76ee\u5f55\u5e76\u542f\u7528EC<\/span><br \/>\nhive -e <span class=\"token string\">&#034;INSERT OVERWRITE TABLE behavior_cold.user_behavior SELECT * FROM behavior_hot.user_behavior WHERE dt &lt; date_sub(current_date(),7)&#034;<\/span><br \/>\nhdfs storagepolicies -setStoragePolicy -path \/user\/hive\/warehouse\/behavior_cold.db\/user_behavior\/dt<span class=\"token operator\">&#061;<\/span>xxx -policy ErasureCoding<\/p>\n<\/li>\n<\/li>\n<li>\u6548\u679c&#xff1a;\u51b7\u6570\u636e\u5b58\u50a8\u6210\u672c\u964d\u4f4e50%&#xff0c;\u96c6\u7fa4\u6574\u4f53\u5b58\u50a8\u5229\u7528\u7387\u4ece65%\u63d0\u5347\u81f385%\u3002<\/li>\n<\/ul>\n<h5>&#xff08;3&#xff09;HDFS\u6027\u80fd\u4f18\u5316&#xff08;Hadoop 3.x\u4e13\u5c5e&#xff09;<\/h5>\n<ul>\n<li>\u542f\u7528Short-Circuit Local Reads&#xff1a;\u7ed5\u8fc7DataNode\u7f51\u7edcIO&#xff0c;\u672c\u5730\u8bfb\u53d6\u6027\u80fd\u63d0\u534740%&#xff08;\u914d\u7f6edfs.client.read.shortcircuit&#061;true&#xff09;&#xff1b;<\/li>\n<li>\u4f18\u5316DataNode\u5757\u6c47\u62a5&#xff1a;dfs.datanode.blockreport.intervalMsec&#061;3600000&#xff08;1\u5c0f\u65f6\u4e00\u6b21\u5757\u6c47\u62a5&#xff0c;\u51cf\u5c11\u96c6\u7fa4\u901a\u4fe1\u5f00\u9500&#xff09;&#xff1b;<\/li>\n<li>\u5143\u6570\u636e\u7f13\u5b58\u4f18\u5316&#xff1a;dfs.namenode.fslimits.max-xattrs-per-inode&#061;100&#xff08;\u63d0\u5347\u5143\u6570\u636e\u67e5\u8be2\u6548\u7387&#xff09;\u3002<\/li>\n<\/ul>\n<h4>2. Kerberos&#043;Ranger\u5b89\u5168\u914d\u7f6e&#xff08;\u4f01\u4e1a\u7ea7\u5408\u89c4\u4fdd\u969c&#xff09;<\/h4>\n<p>Hadoop 3.x\u5bf9Kerberos\u7684\u652f\u6301\u66f4\u5b8c\u5584&#xff0c;\u7ed3\u5408Ranger\u5b9e\u73b0\u7ec6\u7c92\u5ea6\u6743\u9650\u63a7\u5236&#xff0c;\u89e3\u51b3\u201c\u533f\u540d\u8bbf\u95ee\u3001\u6743\u9650\u6df7\u4e71\u201d\u95ee\u9898\u3002<\/p>\n<h5>&#xff08;1&#xff09;Kerberos\u8ba4\u8bc1\u914d\u7f6e<\/h5>\n<li>\u5b89\u88c5Kerberos\u670d\u52a1&#xff08;KDC&#xff09;&#xff1a;yum <span class=\"token function\">install<\/span> -y krb5-server krb5-libs krb5-workstation\n<\/li>\n<li>\u914d\u7f6ekrb5.conf&#xff08;\u6838\u5fc3\u53c2\u6570&#xff09;&#xff1a;[realms]<br \/>\nHADOOP.COM &#061; {<br \/>\n  kdc &#061; node1.hadoop.com  # KDC\u670d\u52a1\u5668\u5730\u5740<br \/>\n  admin_server &#061; node1.hadoop.com<br \/>\n  default_domain &#061; hadoop.com<br \/>\n}<\/p>\n<\/li>\n<li>\u521b\u5efaHadoop\u96c6\u7fa4\u670d\u52a1\u4e3b\u4f53&#xff1a;kadmin.local -q <span class=\"token string\">&#034;addprinc -randkey hdfs\/node1.hadoop.com&#064;HADOOP.COM&#034;<\/span><br \/>\nkadmin.local -q <span class=\"token string\">&#034;addprinc -randkey yarn\/node1.hadoop.com&#064;HADOOP.COM&#034;<\/span><br \/>\nkadmin.local -q <span class=\"token string\">&#034;ktadd -k \/etc\/hadoop\/hdfs.keytab hdfs\/node1.hadoop.com&#064;HADOOP.COM&#034;<\/span><\/p>\n<\/li>\n<li>\u542f\u7528Hadoop\u670d\u52a1Kerberos\u8ba4\u8bc1&#xff08;core-site.xml&#xff09;&#xff1a;<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>hadoop.security.authentication<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>kerberos<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>hadoop.security.authorization<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>name<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n  <span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span>true<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>value<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><br \/>\n<span class=\"token tag\"><span class=\"token tag\"><span class=\"token punctuation\">&lt;\/<\/span>property<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><\/p>\n<\/li>\n<h5>&#xff08;2&#xff09;Ranger\u7ec6\u7c92\u5ea6\u6743\u9650\u63a7\u5236<\/h5>\n<li>\u96c6\u6210Ranger\u4e0eHadoop 3.x&#xff1a;\u914d\u7f6eRanger\u8bbf\u95eeHDFS\/Hive\u7684Kerberos\u51ed\u8bc1&#xff1b;<\/li>\n<li>\u521b\u5efaHDFS\u6743\u9650\u7b56\u7565&#xff1a;\n<ul>\n<li>\u5141\u8bb8data_analyst\u89d2\u8272\u8bfb\u5199\u70ed\u6570\u636e\u76ee\u5f55\/user\/hive\/warehouse\/behavior_hot.db&#xff1b;<\/li>\n<li>\u4ec5\u5141\u8bb8admin\u89d2\u8272\u4fee\u6539\u51b7\u6570\u636e\u76ee\u5f55\/user\/hive\/warehouse\/behavior_cold.db&#xff1b;<\/li>\n<\/ul>\n<\/li>\n<li>\u521b\u5efaHive\u6743\u9650\u7b56\u7565&#xff1a;\n<ul>\n<li>\u9650\u5236\u8fd0\u8425\u89d2\u8272\u4ec5\u67e5\u8be2user_profile\u8868\u7684\u975e\u654f\u611f\u5b57\u6bb5&#xff08;\u5c4f\u853d\u624b\u673a\u53f7\u3001\u8eab\u4efd\u8bc1\u53f7&#xff09;&#xff1b;<\/li>\n<li>\u7981\u6b62\u666e\u901a\u7528\u6237\u6267\u884cDROP\/ALTER\u64cd\u4f5c\u3002<\/li>\n<\/ul>\n<\/li>\n<h4>3. \u79bb\u7ebf\u8ba1\u7b97&#xff1a;Spark on YARN\u6279\u5904\u7406&#xff08;\u7528\u6237\u753b\u50cf\u8ba1\u7b97&#xff09;<\/h4>\n<p>\u57fa\u4e8eSpark 3.4.1\u5904\u7406T&#043;1\u7528\u6237\u884c\u4e3a\u6570\u636e&#xff0c;\u751f\u6210\u7528\u6237\u5174\u8da3\u6807\u7b7e&#xff0c;\u5b58\u50a8\u5230Hive ORC\u8868\u3002<\/p>\n<h5>&#xff08;1&#xff09;Spark\u4e0eHadoop 3.x\u517c\u5bb9\u914d\u7f6e&#xff08;spark-defaults.conf&#xff09;<\/h5>\n<p>spark.yarn.jars&#061;hdfs:\/\/hadoop-cluster-3x\/spark\/jars\/*<br \/>\nspark.hadoop.hadoop.security.authentication&#061;kerberos<br \/>\nspark.hadoop.hadoop.security.authorization&#061;true<br \/>\nspark.yarn.appMasterEnv.JAVA_HOME&#061;\/usr\/local\/jdk1.8.0_381<br \/>\nspark.executorEnv.JAVA_HOME&#061;\/usr\/local\/jdk1.8.0_381<\/p>\n<h5>&#xff08;2&#xff09;\u7528\u6237\u753b\u50cf\u8ba1\u7b97\u6838\u5fc3\u4ee3\u7801&#xff08;Scala&#xff09;<\/h5>\n<p><span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>spark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span><\/span>SparkSession<br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>spark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span>functions<span class=\"token punctuation\">.<\/span><\/span>_<\/p>\n<p><span class=\"token keyword\">object<\/span> UserProfileBuild <span class=\"token punctuation\">{<\/span><br \/>\n  <span class=\"token keyword\">def<\/span> main<span class=\"token punctuation\">(<\/span>args<span class=\"token operator\">:<\/span> Array<span class=\"token punctuation\">[<\/span><span class=\"token builtin\">String<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token operator\">:<\/span> <span class=\"token builtin\">Unit<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token comment\">\/\/ \u521d\u59cb\u5316SparkSession&#xff08;\u9002\u914dHadoop 3.x Kerberos\u8ba4\u8bc1&#xff09;<\/span><br \/>\n    <span class=\"token keyword\">val<\/span> spark <span class=\"token operator\">&#061;<\/span> SparkSession<span class=\"token punctuation\">.<\/span>builder<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>appName<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;UserProfileBuild-Hadoop3.x&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>enableHiveSupport<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;spark.security.credentials.hive.enabled&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;true&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>getOrCreate<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\">\/\/ \u8bfb\u53d6Hive\u4e2d\u7684\u7528\u6237\u884c\u4e3a\u70ed\u6570\u636e&#xff08;ORC\u683c\u5f0f&#xff09;<\/span><br \/>\n    <span class=\"token keyword\">val<\/span> behaviorDF <span class=\"token operator\">&#061;<\/span> spark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">(<\/span><br \/>\n      <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        |SELECT user_id, video_id, action_type, city, age, view_duration, dt<br \/>\n        |FROM behavior_hot.user_behavior<br \/>\n        |WHERE dt &#061; date_sub(current_date(), 1)<br \/>\n      &#034;&#034;&#034;<\/span><span class=\"token punctuation\">.<\/span>stripMargin<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\">\/\/ 1. \u8ba1\u7b97\u7528\u6237\u57fa\u7840\u6807\u7b7e&#xff08;\u5730\u57df\/\u5e74\u9f84\/\u6027\u522b&#xff09;<\/span><br \/>\n    <span class=\"token keyword\">val<\/span> baseTagDF <span class=\"token operator\">&#061;<\/span> behaviorDF<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span><br \/>\n        first<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;city&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;city&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        first<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token comment\">\/\/ \u5e74\u9f84\u5206\u6bb5\u6807\u7b7e<\/span><br \/>\n        when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">18<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;teenager&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n          <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>between<span class=\"token punctuation\">(<\/span><span class=\"token number\">18<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">35<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;young_adult&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n          <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">35<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;middle_aged&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;age_tag&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\">\/\/ 2. \u8ba1\u7b97\u7528\u6237\u5174\u8da3\u6807\u7b7e&#xff08;Top3\u89c6\u9891\u7c7b\u76ee&#xff09;<\/span><br \/>\n    <span class=\"token keyword\">val<\/span> interestTagDF <span class=\"token operator\">&#061;<\/span> behaviorDF<br \/>\n      <span class=\"token punctuation\">.<\/span>join<span class=\"token punctuation\">(<\/span>spark<span class=\"token punctuation\">.<\/span>table<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;dim_video&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;video_id&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;left&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;category&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;action_type&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;action_cnt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>window<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>desc<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;action_cnt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>where<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;row_number() &lt;&#061; 3&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span>collect_list<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;category&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;interest_tags&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\">\/\/ 3. \u5408\u5e76\u6807\u7b7e&#xff0c;\u5199\u5165Hive ORC\u8868&#xff08;\u542f\u7528\u538b\u7f29&#xff09;<\/span><br \/>\n    baseTagDF<span class=\"token punctuation\">.<\/span>join<span class=\"token punctuation\">(<\/span>interestTagDF<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;left&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;dt&#034;<\/span><span class=\"token punctuation\">,<\/span> current_date<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>write<br \/>\n      <span class=\"token punctuation\">.<\/span>mode<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;append&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>option<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;compression&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;snappy&#034;<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\">\/\/ ORC&#043;Snappy\u538b\u7f29<\/span><br \/>\n      <span class=\"token punctuation\">.<\/span>orc<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;hdfs:\/\/hadoop-cluster-3x\/user\/hive\/warehouse\/user_profile.db\/user_tag&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    spark<span class=\"token punctuation\">.<\/span>stop<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<h5>&#xff08;3&#xff09;\u63d0\u4ea4Spark\u4efb\u52a1\u5230YARN&#xff08;Kerberos\u73af\u5883&#xff09;<\/h5>\n<p><span class=\"token comment\"># \u5148\u83b7\u53d6Kerberos\u51ed\u8bc1<\/span><br \/>\nkinit -kt \/etc\/hadoop\/spark.keytab spark\/node1.hadoop.com&#064;HADOOP.COM<\/p>\n<p><span class=\"token comment\"># \u63d0\u4ea4\u4efb\u52a1&#xff08;\u6307\u5b9aYARN\u961f\u5217&#xff0c;\u52a8\u6001\u8d44\u6e90\u5206\u914d&#xff09;<\/span><br \/>\nspark-submit <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;class UserProfileBuild <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;master <span class=\"token function\">yarn<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;deploy-mode cluster <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;queue offline <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;executor-memory 4G <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;executor-cores <span class=\"token number\">2<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;num-executors <span class=\"token number\">10<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;conf spark.dynamicAllocation.enabled<span class=\"token operator\">&#061;<\/span>true <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;conf spark.dynamicAllocation.minExecutors<span class=\"token operator\">&#061;<\/span><span class=\"token number\">5<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;conf spark.dynamicAllocation.maxExecutors<span class=\"token operator\">&#061;<\/span><span class=\"token number\">20<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  user-profile-1.0.jar<\/p>\n<h4>4. \u5b9e\u65f6\u8ba1\u7b97&#xff1a;Flink on YARN&#xff08;\u70ed\u95e8\u89c6\u9891\u5b9e\u65f6\u7edf\u8ba1&#xff09;<\/h4>\n<p>\u57fa\u4e8eFlink 1.17.0\u6d88\u8d39Kafka\u4e2d\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u6570\u636e&#xff0c;\u79d2\u7ea7\u7edf\u8ba1\u70ed\u95e8\u89c6\u9891TOP10&#xff0c;\u7ed3\u679c\u5199\u5165HBase\u4f9b\u63a8\u8350\u7cfb\u7edf\u8c03\u7528\u3002<\/p>\n<h5>&#xff08;1&#xff09;Flink\u4e0eHadoop 3.x\u96c6\u6210\u914d\u7f6e&#xff08;flink-conf.yaml&#xff09;<\/h5>\n<p><span class=\"token comment\"># \u542f\u7528Kerberos\u8ba4\u8bc1<\/span><br \/>\n<span class=\"token key atrule\">security.kerberos.login.enable<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean important\">true<\/span><br \/>\n<span class=\"token key atrule\">security.kerberos.login.keytab<\/span><span class=\"token punctuation\">:<\/span> \/etc\/hadoop\/flink.keytab<br \/>\n<span class=\"token key atrule\">security.kerberos.login.principal<\/span><span class=\"token punctuation\">:<\/span> flink\/node1.hadoop.com&#064;HADOOP.COM<br \/>\n<span class=\"token comment\"># \u914d\u7f6eHDFS\u4f5c\u4e3aCheckpoint\u5b58\u50a8<\/span><br \/>\n<span class=\"token key atrule\">state.backend<\/span><span class=\"token punctuation\">:<\/span> filesystem<br \/>\n<span class=\"token key atrule\">state.checkpoints.dir<\/span><span class=\"token punctuation\">:<\/span> hdfs<span class=\"token punctuation\">:<\/span>\/\/hadoop<span class=\"token punctuation\">&#8211;<\/span>cluster<span class=\"token punctuation\">&#8211;<\/span>3x\/flink\/checkpoints<br \/>\n<span class=\"token comment\"># \u9002\u914dYARN Per-Job\u6a21\u5f0f<\/span><br \/>\n<span class=\"token key atrule\">execution.target<\/span><span class=\"token punctuation\">:<\/span> yarn<span class=\"token punctuation\">&#8211;<\/span>per<span class=\"token punctuation\">&#8211;<\/span>job<\/p>\n<h5>&#xff08;2&#xff09;\u70ed\u95e8\u89c6\u9891\u5b9e\u65f6\u7edf\u8ba1\u6838\u5fc3\u4ee3\u7801&#xff08;Java&#xff09;<\/h5>\n<p><span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>common<span class=\"token punctuation\">.<\/span>eventtime<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">WatermarkStrategy<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>common<span class=\"token punctuation\">.<\/span>functions<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">AggregateFunction<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>common<span class=\"token punctuation\">.<\/span>serialization<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">SimpleStringSchema<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>java<span class=\"token punctuation\">.<\/span>tuple<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>datastream<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">DataStream<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>datastream<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">SingleOutputStreamOperator<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>environment<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">StreamExecutionEnvironment<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>windowing<span class=\"token punctuation\">.<\/span>assigners<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">SlidingEventTimeWindows<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>api<span class=\"token punctuation\">.<\/span>windowing<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Time<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>connectors<span class=\"token punctuation\">.<\/span>kafka<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">FlinkKafkaConsumer<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>hadoop<span class=\"token punctuation\">.<\/span>hbase<span class=\"token punctuation\">.<\/span>client<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Put<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>hadoop<span class=\"token punctuation\">.<\/span>hbase<span class=\"token punctuation\">.<\/span>io<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">ImmutableBytesWritable<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>hadoop<span class=\"token punctuation\">.<\/span>hbase<span class=\"token punctuation\">.<\/span>mapreduce<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">TableOutputFormat<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>hadoop<span class=\"token punctuation\">.<\/span>hbase<span class=\"token punctuation\">.<\/span>util<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Bytes<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> <span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>hadoop<span class=\"token punctuation\">.<\/span>mapreduce<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Job<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> <span class=\"token namespace\">java<span class=\"token punctuation\">.<\/span>util<span class=\"token punctuation\">.<\/span><\/span><span class=\"token class-name\">Properties<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token keyword\">public<\/span> <span class=\"token keyword\">class<\/span> <span class=\"token class-name\">HotVideoRealTime<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token keyword\">public<\/span> <span class=\"token keyword\">static<\/span> <span class=\"token keyword\">void<\/span> <span class=\"token function\">main<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span> args<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">throws<\/span> <span class=\"token class-name\">Exception<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n        <span class=\"token comment\">\/\/ \u521d\u59cb\u5316Flink\u6267\u884c\u73af\u5883&#xff08;\u9002\u914dHadoop 3.x&#xff09;<\/span><br \/>\n        <span class=\"token class-name\">StreamExecutionEnvironment<\/span> env <span class=\"token operator\">&#061;<\/span> <span class=\"token class-name\">StreamExecutionEnvironment<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">getExecutionEnvironment<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        env<span class=\"token punctuation\">.<\/span><span class=\"token function\">enableCheckpointing<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">5000<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span>  <span class=\"token comment\">\/\/ 5\u79d2\u4e00\u6b21Checkpoint&#xff0c;\u5b58\u50a8\u5230HDFS<\/span><\/p>\n<p>        <span class=\"token comment\">\/\/ 1. \u8bfb\u53d6Kafka\u5b9e\u65f6\u884c\u4e3a\u6570\u636e<\/span><br \/>\n        <span class=\"token class-name\">Properties<\/span> kafkaProps <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">Properties<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        kafkaProps<span class=\"token punctuation\">.<\/span><span class=\"token function\">setProperty<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bootstrap.servers&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;node1:9092,node2:9092,node3:9092&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        kafkaProps<span class=\"token punctuation\">.<\/span><span class=\"token function\">setProperty<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;group.id&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;hot_video_group&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        kafkaProps<span class=\"token punctuation\">.<\/span><span class=\"token function\">setProperty<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;key.deserializer&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;org.apache.kafka.common.serialization.StringDeserializer&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        kafkaProps<span class=\"token punctuation\">.<\/span><span class=\"token function\">setProperty<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;value.deserializer&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;org.apache.kafka.common.serialization.StringDeserializer&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>        <span class=\"token class-name\">DataStream<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">&gt;<\/span><\/span> kafkaStream <span class=\"token operator\">&#061;<\/span> env<span class=\"token punctuation\">.<\/span><span class=\"token function\">addSource<\/span><span class=\"token punctuation\">(<\/span><br \/>\n                <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">FlinkKafkaConsumer<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_behavior_real-time&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">SimpleStringSchema<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> kafkaProps<span class=\"token punctuation\">)<\/span><br \/>\n                        <span class=\"token punctuation\">.<\/span><span class=\"token function\">assignTimestampsAndWatermarks<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">WatermarkStrategy<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">forMonotonousTimestamps<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>        <span class=\"token comment\">\/\/ 2. \u6570\u636e\u89e3\u6790\u4e0e\u8f6c\u6362&#xff08;JSON\u2192Tuple2&lt;video_id, 1&gt;&#xff09;<\/span><br \/>\n        <span class=\"token class-name\">SingleOutputStreamOperator<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Integer<\/span><span class=\"token punctuation\">&gt;<\/span><span class=\"token punctuation\">&gt;<\/span><\/span> videoStream <span class=\"token operator\">&#061;<\/span> kafkaStream<br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">map<\/span><span class=\"token punctuation\">(<\/span>json <span class=\"token operator\">-&gt;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                    <span class=\"token comment\">\/\/ \u89e3\u6790JSON\u683c\u5f0f\u7684\u884c\u4e3a\u6570\u636e&#xff08;\u7b80\u5316\u5904\u7406&#xff09;<\/span><br \/>\n                    <span class=\"token class-name\">String<\/span> videoId <span class=\"token operator\">&#061;<\/span> json<span class=\"token punctuation\">.<\/span><span class=\"token function\">split<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;,&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">;<\/span><br \/>\n                    <span class=\"token keyword\">return<\/span> <span class=\"token class-name\">Tuple2<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">of<\/span><span class=\"token punctuation\">(<\/span>videoId<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n                <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>        <span class=\"token comment\">\/\/ 3. \u6ed1\u52a8\u7a97\u53e3\u805a\u5408&#xff08;1\u5206\u949f\u7a97\u53e3&#xff0c;10\u79d2\u6ed1\u52a8\u4e00\u6b21&#xff09;<\/span><br \/>\n        <span class=\"token class-name\">SingleOutputStreamOperator<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Long<\/span><span class=\"token punctuation\">&gt;<\/span><span class=\"token punctuation\">&gt;<\/span><\/span> hotVideoStream <span class=\"token operator\">&#061;<\/span> videoStream<br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">keyBy<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token operator\">::<\/span><span class=\"token function\">f0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">window<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">SlidingEventTimeWindows<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">of<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Time<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">minutes<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Time<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">seconds<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">aggregate<\/span><span class=\"token punctuation\">(<\/span><span class=\"token keyword\">new<\/span> <span class=\"token class-name\">AggregateFunction<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Integer<\/span><span class=\"token punctuation\">&gt;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Long<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Long<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                    <span class=\"token annotation punctuation\">&#064;Override<\/span><br \/>\n                    <span class=\"token keyword\">public<\/span> <span class=\"token class-name\">Long<\/span> <span class=\"token function\">createAccumulator<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                        <span class=\"token keyword\">return<\/span> <span class=\"token number\">0L<\/span><span class=\"token punctuation\">;<\/span><br \/>\n                    <span class=\"token punctuation\">}<\/span><\/p>\n<p>                    <span class=\"token annotation punctuation\">&#064;Override<\/span><br \/>\n                    <span class=\"token keyword\">public<\/span> <span class=\"token class-name\">Long<\/span> <span class=\"token function\">add<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Tuple2<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token class-name\">String<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Integer<\/span><span class=\"token punctuation\">&gt;<\/span><\/span> value<span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Long<\/span> accumulator<span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                        <span class=\"token keyword\">return<\/span> accumulator <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">;<\/span><br \/>\n                    <span class=\"token punctuation\">}<\/span><\/p>\n<p>                    <span class=\"token annotation punctuation\">&#064;Override<\/span><br \/>\n                    <span class=\"token keyword\">public<\/span> <span class=\"token class-name\">Long<\/span> <span class=\"token function\">getResult<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Long<\/span> accumulator<span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                        <span class=\"token keyword\">return<\/span> accumulator<span class=\"token punctuation\">;<\/span><br \/>\n                    <span class=\"token punctuation\">}<\/span><\/p>\n<p>                    <span class=\"token annotation punctuation\">&#064;Override<\/span><br \/>\n                    <span class=\"token keyword\">public<\/span> <span class=\"token class-name\">Long<\/span> <span class=\"token function\">merge<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Long<\/span> a<span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Long<\/span> b<span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                        <span class=\"token keyword\">return<\/span> a <span class=\"token operator\">&#043;<\/span> b<span class=\"token punctuation\">;<\/span><br \/>\n                    <span class=\"token punctuation\">}<\/span><br \/>\n                <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">keyBy<\/span><span class=\"token punctuation\">(<\/span>t <span class=\"token operator\">-&gt;<\/span> <span class=\"token string\">&#034;top10&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">window<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">SlidingEventTimeWindows<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">of<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Time<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">minutes<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token class-name\">Time<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">seconds<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">.<\/span><span class=\"token function\">process<\/span><span class=\"token punctuation\">(<\/span><span class=\"token keyword\">new<\/span> <span class=\"token class-name\">TopNProcessFunction<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span>  <span class=\"token comment\">\/\/ \u81ea\u5b9a\u4e49ProcessFunction\u53d6TOP10<\/span><\/p>\n<p>        <span class=\"token comment\">\/\/ 4. \u7ed3\u679c\u5199\u5165HBase&#xff08;Hadoop 3.x\u517c\u5bb9&#xff09;<\/span><br \/>\n        hotVideoStream<span class=\"token punctuation\">.<\/span><span class=\"token function\">map<\/span><span class=\"token punctuation\">(<\/span>t <span class=\"token operator\">-&gt;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token class-name\">String<\/span> videoId <span class=\"token operator\">&#061;<\/span> t<span class=\"token punctuation\">.<\/span>f0<span class=\"token punctuation\">;<\/span><br \/>\n            <span class=\"token class-name\">Long<\/span> viewCount <span class=\"token operator\">&#061;<\/span> t<span class=\"token punctuation\">.<\/span>f1<span class=\"token punctuation\">;<\/span><br \/>\n            <span class=\"token class-name\">Put<\/span> put <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">Put<\/span><span class=\"token punctuation\">(<\/span><span class=\"token class-name\">Bytes<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">toBytes<\/span><span class=\"token punctuation\">(<\/span>videoId<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n            put<span class=\"token punctuation\">.<\/span><span class=\"token function\">addColumn<\/span><span class=\"token punctuation\">(<\/span><br \/>\n                    <span class=\"token class-name\">Bytes<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">toBytes<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;info&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                    <span class=\"token class-name\">Bytes<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">toBytes<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;view_count&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                    <span class=\"token class-name\">Bytes<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">toBytes<\/span><span class=\"token punctuation\">(<\/span>viewCount<span class=\"token punctuation\">.<\/span><span class=\"token function\">toString<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">Tuple2<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><span class=\"token punctuation\">(<\/span><span class=\"token keyword\">new<\/span> <span class=\"token class-name\">ImmutableBytesWritable<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> put<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token function\">addSink<\/span><span class=\"token punctuation\">(<\/span><span class=\"token keyword\">new<\/span> <span class=\"token class-name\"><span class=\"token namespace\">org<span class=\"token punctuation\">.<\/span>apache<span class=\"token punctuation\">.<\/span>flink<span class=\"token punctuation\">.<\/span>streaming<span class=\"token punctuation\">.<\/span>connectors<span class=\"token punctuation\">.<\/span>hbase<span class=\"token punctuation\">.<\/span><\/span>HBaseSink<\/span><span class=\"token generics\"><span class=\"token punctuation\">&lt;<\/span><span class=\"token punctuation\">&gt;<\/span><\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;hot_video&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token keyword\">new<\/span> <span class=\"token class-name\">HBaseConfiguration<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>        env<span class=\"token punctuation\">.<\/span><span class=\"token function\">execute<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;HotVideoRealTime-Hadoop3.x&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n    <span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<h3>\u4e09\u3001\u5de5\u7a0b\u5316\u843d\u5730\u4e0e\u76d1\u63a7\u8fd0\u7ef4<\/h3>\n<h4>1. \u96c6\u7fa4\u76d1\u63a7&#xff08;Prometheus&#043;Grafana&#xff09;<\/h4>\n<ul>\n<li>\u90e8\u7f72Hadoop 3.x Exporter&#xff1a;\u91c7\u96c6NameNode\/DataNode\/YARN\u7684Metrics\u6307\u6807&#xff1b;<\/li>\n<li>\u914d\u7f6eGrafana\u4eea\u8868\u76d8&#xff1a;\u76d1\u63a7HDFS\u5b58\u50a8\u7a7a\u95f4&#xff08;\u70ed\/\u51b7\u6570\u636e\u5360\u6bd4&#xff09;\u3001Spark\/Flink\u4efb\u52a1\u5ef6\u8fdf\u3001Kerberos\u51ed\u8bc1\u6709\u6548\u671f&#xff1b;<\/li>\n<li>\u544a\u8b66\u914d\u7f6e&#xff1a;\u5f53HDFS\u53ef\u7528\u7a7a\u95f4\u4f4e\u4e8e20%\u3001Flink Checkpoint\u5931\u8d25\u65f6\u89e6\u53d1\u90ae\u4ef6\/\u77ed\u4fe1\u544a\u8b66\u3002<\/li>\n<\/ul>\n<h4>2. \u6570\u636e\u8d28\u91cf\u4fdd\u969c<\/h4>\n<ul>\n<li>\u79bb\u7ebf\u6570\u636e&#xff1a;Spark\u4efb\u52a1\u4e2d\u52a0\u5165\u6570\u636e\u6821\u9a8c&#xff08;\u975e\u7a7a\u6821\u9a8c\u3001\u683c\u5f0f\u6821\u9a8c&#xff09;&#xff0c;\u8f93\u51fa\u6570\u636e\u8d28\u91cf\u62a5\u544a\u5230HDFS&#xff1b;<\/li>\n<li>\u5b9e\u65f6\u6570\u636e&#xff1a;Flink\u4e2d\u542f\u7528Side Output\u673a\u5236&#xff0c;\u5c06\u5f02\u5e38\u6570\u636e&#xff08;\u683c\u5f0f\u9519\u8bef\u3001\u7f3a\u5931\u5173\u952e\u5b57\u6bb5&#xff09;\u5206\u6d41\u5230HDFS\u5f02\u5e38\u76ee\u5f55&#xff0c;\u540e\u7eed\u4eba\u5de5\u5904\u7406&#xff1b;<\/li>\n<li>\u6570\u636e\u4e00\u81f4\u6027&#xff1a;\u79bb\u7ebfT&#043;1\u7ed3\u679c\u4e0e\u5b9e\u65f6\u7edf\u8ba1\u7ed3\u679c\u8fdb\u884c\u6bd4\u5bf9&#xff0c;\u8bef\u5dee\u8d85\u8fc75%\u65f6\u89e6\u53d1\u544a\u8b66\u3002<\/li>\n<\/ul>\n<h4>3. \u8fd0\u7ef4\u81ea\u52a8\u5316<\/h4>\n<ul>\n<li>\u96c6\u7fa4\u542f\u505c\u811a\u672c&#xff1a;\u57fa\u4e8eShell\u811a\u672c\u5c01\u88c5Hadoop 3.x\u96c6\u7fa4\u542f\u505c\u3001Kerberos\u51ed\u8bc1\u81ea\u52a8\u5237\u65b0\u903b\u8f91&#xff1b;<\/li>\n<li>\u4efb\u52a1\u8c03\u5ea6&#xff1a;\u4f7f\u7528Airflow\u8c03\u5ea6Spark\u79bb\u7ebf\u4efb\u52a1&#xff0c;\u5b9a\u65f6\u6267\u884c\u51b7\u70ed\u6570\u636e\u8fc1\u79fb\u3001\u7528\u6237\u753b\u50cf\u8ba1\u7b97&#xff1b;<\/li>\n<li>\u65e5\u5fd7\u6536\u96c6&#xff1a;Flink\/Spark\u4efb\u52a1\u65e5\u5fd7\u5199\u5165HDFS&#xff0c;\u901a\u8fc7Elasticsearch&#043;Kibana\u5b9e\u73b0\u65e5\u5fd7\u68c0\u7d22\u3002<\/li>\n<\/ul>\n<h3>\u56db\u3001\u6027\u80fd\u4e0e\u6210\u672c\u4f18\u5316\u6548\u679c<\/h3>\n<table>\n<tr>\u6307\u6807\u4f18\u5316\u524d&#xff08;Hadoop 2.x&#xff09;\u4f18\u5316\u540e&#xff08;Hadoop 3.x&#xff09;\u63d0\u5347\u6548\u679c<\/tr>\n<tbody>\n<tr>\n<td>HDFS\u51b7\u6570\u636e\u5b58\u50a8\u6210\u672c<\/td>\n<td>10TB\u6570\u636e\u7ea6\u970030TB\u5b58\u50a8<\/td>\n<td>10TB\u6570\u636e\u7ea6\u970015TB\u5b58\u50a8<\/td>\n<td>\u6210\u672c\u964d\u4f4e50%&#xff08;EC\u7ea0\u5220\u7801&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>Spark\u79bb\u7ebf\u4efb\u52a1\u5ef6\u8fdf<\/td>\n<td>10TB\u6570\u636e\u5904\u7406\u7ea64\u5c0f\u65f6<\/td>\n<td>10TB\u6570\u636e\u5904\u7406\u7ea62.8\u5c0f\u65f6<\/td>\n<td>\u6548\u7387\u63d0\u534730%&#xff08;YARN\u52a8\u6001\u8d44\u6e90&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>Flink\u5b9e\u65f6\u4efb\u52a1\u5ef6\u8fdf<\/td>\n<td>\u79d2\u7ea7\u7a97\u53e3\u5ef6\u8fdf\u7ea6500ms<\/td>\n<td>\u79d2\u7ea7\u7a97\u53e3\u5ef6\u8fdf\u7ea6150ms<\/td>\n<td>\u5ef6\u8fdf\u964d\u4f4e70%&#xff08;HDFS\u4f18\u5316&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u96c6\u7fa4\u5e76\u53d1\u4efb\u52a1\u6570<\/td>\n<td>\u652f\u630110\u4e2a\u5e76\u53d1Spark\u4efb\u52a1<\/td>\n<td>\u652f\u630125\u4e2a\u5e76\u53d1Spark\u4efb\u52a1<\/td>\n<td>\u5e76\u53d1\u80fd\u529b\u63d0\u5347150%&#xff08;YARN Federation&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h3>\u4e94\u3001\u603b\u7ed3\u4e0e\u672a\u6765\u65b9\u5411<\/h3>\n<h4>1. \u9879\u76ee\u6210\u679c<\/h4>\n<ul>\n<li>\u6280\u672f\u5c42\u9762&#xff1a;\u57fa\u4e8eHadoop 3.x\u6784\u5efa\u4e86\u201c\u5b89\u5168&#043;\u5b9e\u65f6&#043;\u4f4e\u6210\u672c\u201d\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff0c;\u843d\u5730Kerberos\u5b89\u5168\u8ba4\u8bc1\u3001HDFS\u51b7\u70ed\u5206\u5c42\u3001EC\u7ea0\u5220\u7801\u7b49\u6838\u5fc3\u7279\u6027&#xff1b;<\/li>\n<li>\u4e1a\u52a1\u5c42\u9762&#xff1a;\u652f\u6491\u65e5\u574710TB\u6570\u636e\u7684\u79bb\u7ebf&#043;\u5b9e\u65f6\u5904\u7406&#xff0c;\u7528\u6237\u753b\u50cf\u66f4\u65b0\u5ef6\u8fdf\u4eceT&#043;3\u964d\u81f3T&#043;1&#xff0c;\u70ed\u95e8\u89c6\u9891\u63a8\u8350\u5b9e\u65f6\u6027\u63d0\u5347\u81f3\u79d2\u7ea7&#xff0c;\u5b58\u50a8\u6210\u672c\u964d\u4f4e50%\u3002<\/li>\n<\/ul>\n<h4>2. \u672a\u6765\u4f18\u5316\u65b9\u5411<\/h4>\n<ul>\n<li>\u5f15\u5165Hadoop 3.x\u7684GPU\u8c03\u5ea6\u652f\u6301&#xff1a;\u52a0\u901fSpark MLlib\u7684\u7528\u6237\u5174\u8da3\u6a21\u578b\u8bad\u7ec3&#xff1b;<\/li>\n<li>\u878d\u5408Lakehouse\u67b6\u6784&#xff1a;\u4f7f\u7528Delta Lake\u66ff\u4ee3Hive ORC&#xff0c;\u652f\u6301\u79bb\u7ebf\u4e0e\u5b9e\u65f6\u6570\u636e\u7684\u7edf\u4e00\u5b58\u50a8\u4e0e\u67e5\u8be2&#xff1b;<\/li>\n<li>\u81ea\u52a8\u5316\u8fd0\u7ef4&#xff1a;\u57fa\u4e8eAIops\u5de5\u5177&#xff08;\u5982Apache DolphinScheduler&#xff09;\u5b9e\u73b0\u96c6\u7fa4\u6545\u969c\u81ea\u6108\u3001\u4efb\u52a1\u667a\u80fd\u8c03\u4f18\u3002<\/li>\n<\/ul>\n<h3>\u9644\u5f55&#xff1a;\u6838\u5fc3\u914d\u7f6e\u6587\u4ef6\u6c47\u603b<\/h3>\n<li>HDFS 3.x\u6838\u5fc3\u914d\u7f6e&#xff08;hdfs-site.xml&#xff09;&#xff1b;<\/li>\n<li>Kerberos\u8ba4\u8bc1\u914d\u7f6e&#xff08;krb5.conf\u3001core-site.xml&#xff09;&#xff1b;<\/li>\n<li>Spark\/Flink\u4e0eHadoop 3.x\u517c\u5bb9\u914d\u7f6e&#xff1b;<\/li>\n<li>\u51b7\u70ed\u6570\u636e\u8fc1\u79fb\u811a\u672c\u3002<\/li>\n","protected":false},"excerpt":{"rendered":"<p>Hadoop 3.x\u5b9e\u6218&#xff1a;\u57fa\u4e8eHDFSSparkFlink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff08;\u542bKerberos\u5b89\u5168\u914d\u7f6e\u51b7\u70ed\u6570\u636e\u5206\u5c42&#xff09;<br \/>\n\u5728\u5927\u6570\u636e\u5b9e\u65f6\u5316\u3001\u5b89\u5168\u5408\u89c4\u8981\u6c42\u5347\u7ea7\u7684\u5f53\u4e0b&#xff0c;\u4f20\u7edfHadoop\u96c6\u7fa4\u9762\u4e34\u201c\u5b9e\u65f6\u5904\u7406\u5f31\u3001\u5b89\u5168\u9632\u62a4\u4e0d\u8db3\u3001\u5b58\u50a8\u6210\u672c\u9ad8\u201d\u4e09\u5927\u75db\u70b9\u3002\u672c\u6587\u57fa\u4e8e Hadoop 3.3.6&#xff08;\u6700\u65b0\u7a33\u5b9a\u7248&#xff09;&#xff0c;\u8bbe\u8ba1\u4e00\u5957\u201c\u79bb\u7ebf\u6279\u5904\u7406\u5b9e\u65f6\u6d41\u8ba1\u7b97\u201d\u4e00\u4f53\u5316\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff0c;\u6574\u5408HDFS 3.x\u3001<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[779,2828,365],"topic":[],"class_list":["post-78564","post","type-post","status-publish","format-standard","hentry","category-server","tag-hadoop","tag-hdfs","tag-spark"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/78564.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"Hadoop 3.x\u5b9e\u6218&#xff1a;\u57fa\u4e8eHDFSSparkFlink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff08;\u542bKerberos\u5b89\u5168\u914d\u7f6e\u51b7\u70ed\u6570\u636e\u5206\u5c42&#xff09; \u5728\u5927\u6570\u636e\u5b9e\u65f6\u5316\u3001\u5b89\u5168\u5408\u89c4\u8981\u6c42\u5347\u7ea7\u7684\u5f53\u4e0b&#xff0c;\u4f20\u7edfHadoop\u96c6\u7fa4\u9762\u4e34\u201c\u5b9e\u65f6\u5904\u7406\u5f31\u3001\u5b89\u5168\u9632\u62a4\u4e0d\u8db3\u3001\u5b58\u50a8\u6210\u672c\u9ad8\u201d\u4e09\u5927\u75db\u70b9\u3002\u672c\u6587\u57fa\u4e8e Hadoop 3.3.6&#xff08;\u6700\u65b0\u7a33\u5b9a\u7248&#xff09;&#xff0c;\u8bbe\u8ba1\u4e00\u5957\u201c\u79bb\u7ebf\u6279\u5904\u7406\u5b9e\u65f6\u6d41\u8ba1\u7b97\u201d\u4e00\u4f53\u5316\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff0c;\u6574\u5408HDFS 3.x\u3001\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/78564.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-02-27T14:29:25+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"9 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/78564.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/78564.html\",\"name\":\"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-02-27T14:29:25+00:00\",\"dateModified\":\"2026-02-27T14:29:25+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/78564.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/78564.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/78564.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/78564.html","og_locale":"zh_CN","og_type":"article","og_title":"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"Hadoop 3.x\u5b9e\u6218&#xff1a;\u57fa\u4e8eHDFSSparkFlink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff08;\u542bKerberos\u5b89\u5168\u914d\u7f6e\u51b7\u70ed\u6570\u636e\u5206\u5c42&#xff09; \u5728\u5927\u6570\u636e\u5b9e\u65f6\u5316\u3001\u5b89\u5168\u5408\u89c4\u8981\u6c42\u5347\u7ea7\u7684\u5f53\u4e0b&#xff0c;\u4f20\u7edfHadoop\u96c6\u7fa4\u9762\u4e34\u201c\u5b9e\u65f6\u5904\u7406\u5f31\u3001\u5b89\u5168\u9632\u62a4\u4e0d\u8db3\u3001\u5b58\u50a8\u6210\u672c\u9ad8\u201d\u4e09\u5927\u75db\u70b9\u3002\u672c\u6587\u57fa\u4e8e Hadoop 3.3.6&#xff08;\u6700\u65b0\u7a33\u5b9a\u7248&#xff09;&#xff0c;\u8bbe\u8ba1\u4e00\u5957\u201c\u79bb\u7ebf\u6279\u5904\u7406\u5b9e\u65f6\u6d41\u8ba1\u7b97\u201d\u4e00\u4f53\u5316\u7684\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0&#xff0c;\u6574\u5408HDFS 3.x\u3001","og_url":"https:\/\/www.wsisp.com\/helps\/78564.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-02-27T14:29:25+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"9 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/78564.html","url":"https:\/\/www.wsisp.com\/helps\/78564.html","name":"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-02-27T14:29:25+00:00","dateModified":"2026-02-27T14:29:25+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/78564.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/78564.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/78564.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"Hadoop 3.x\u5b9e\u6218\uff1a\u57fa\u4e8eHDFS+Spark+Flink\u7684\u5b9e\u65f6\u7528\u6237\u884c\u4e3a\u5206\u6790\u5e73\u53f0\uff08\u542bKerberos\u5b89\u5168\u914d\u7f6e+\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff09"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/78564","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=78564"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/78564\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=78564"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=78564"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=78564"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=78564"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}