{"id":121717,"date":"2026-11-03T13:16:40","date_gmt":"2026-11-03T05:16:40","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/121717.html"},"modified":"2026-11-03T13:16:40","modified_gmt":"2026-11-03T05:16:40","slug":"20000-%e4%b8%aa%e6%96%87%e4%bb%b6%e8%b7%91%e4%ba%86-11500-%e7%a7%92%ef%bc%9a%e4%bd%a0%e5%8e%8b%e7%9a%84%e6%98%af-yarn-%e8%b0%83%e5%ba%a6%e5%99%a8%ef%bc%8c%e4%b8%8d%e6%98%af-hdfs","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/121717.html","title":{"rendered":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS"},"content":{"rendered":"<h2>20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2&#xff1a;\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668&#xff0c;\u4e0d\u662f HDFS<\/h2>\n<p>\u5173\u952e\u8bcd&#xff1a;TestDFSIO\u3001HDFS \u57fa\u51c6\u6d4b\u8bd5\u3001YARN \u8c03\u5ea6\u5668\u3001maximum-offswitch-assignments\u3001\u5fc3\u8df3\u95f4\u9694\u3001\u5bb9\u5668\u5206\u914d\u901f\u7387\u3001Hadoop \u6027\u80fd\u6d4b\u8bd5\u3001TeraSort\u3001\u5c0f\u6587\u4ef6\u3001\u57fa\u51c6\u6d4b\u8bd5\u8bbe\u8ba1<\/p>\n<h3>\u76ee\u5f55<\/h3>\n<ul>\n<li>\u4e00\u3001\u5148\u8bf4\u7ed3\u8bba<\/li>\n<li>\u4e8c\u3001\u6d4b\u8bd5\u73b0\u573a<\/li>\n<li>\u4e09\u3001\u7ed3\u679c&#xff1a;\u4e94\u8f6e\u5199\u5165&#xff0c;\u7a33\u5b9a\u5f97\u4e0d\u50cf I\/O<\/li>\n<li>\u56db\u3001\u7b2c\u4e00\u53cd\u5e94&#xff1a;\u662f\u4e0d\u662f\u78c1\u76d8\u4e0d\u884c<\/li>\n<li>\u4e94\u3001\u8f6c\u6298&#xff1a;\u6709\u4eba\u95ee\u4e86\u4e00\u53e5&#034;\u5206\u914d container \u8981\u591a\u4e45&#034;<\/li>\n<li>\u516d\u3001\u7b97\u8d26&#xff1a;\u96c6\u7fa4\u6bcf\u79d2\u80fd\u53d1\u591a\u5c11\u4e2a\u4efb\u52a1<\/li>\n<li>\u4e03\u3001\u4e09\u4e2a\u53c2\u6570\u9501\u6b7b\u4e86\u8c03\u5ea6\u901f\u7387<\/li>\n<li>\u516b\u30011333 \u79d2\u53ea\u662f\u4e0b\u9650<\/li>\n<li>\u4e5d\u300190% \u7684\u65f6\u95f4&#xff0c;\u78c1\u76d8\u5728\u7a7a\u8f6c<\/li>\n<li>\u5341\u3001\u8bfb\u53d6\u4e3a\u4ec0\u4e48\u5feb 2.1 \u500d<\/li>\n<li>\u5341\u4e00\u3001\u540c\u4e00\u6279\u6d4b\u8bd5\u91cc\u7684 TeraSort<\/li>\n<li>\u5341\u4e8c\u3001\u600e\u4e48\u6539&#xff1a;\u5c11\u4efb\u52a1&#xff0c;\u5927\u6587\u4ef6<\/li>\n<li>\u5341\u4e09\u3001\u5982\u679c\u771f\u7684\u8981\u538b\u5c0f\u6587\u4ef6<\/li>\n<li>\u5341\u56db\u3001\u590d\u76d8<\/li>\n<\/ul>\n<hr \/>\n<h3>\u4e00\u3001\u5148\u8bf4\u7ed3\u8bba<\/h3>\n<p>5 \u8282\u70b9\u7684 HDFS \u96c6\u7fa4&#xff0c;TestDFSIO \u5199 20000 \u4e2a 100MB \u6587\u4ef6&#xff0c;\u4e94\u8f6e\u5e73\u5747\u8017\u65f6 11572 \u79d2&#xff08;\u7ea6 3.2 \u5c0f\u65f6&#xff09;&#xff0c;\u541e\u5410\u53ea\u6709 177 MB\/s\u3002<\/p>\n<p>\u7b2c\u4e00\u53cd\u5e94\u4e00\u5b9a\u662f&#034;\u78c1\u76d8\u4e0d\u884c&#034;\u6216\u8005&#034;\u7f51\u7edc\u4e0d\u884c&#034;\u3002<\/p>\n<p>\u4f46\u7b97\u5b8c\u4e00\u7b14\u8d26\u4e4b\u540e&#xff0c;\u7ed3\u8bba\u662f\u53e6\u4e00\u4e2a\u65b9\u5411&#xff1a;<\/p>\n<p>\u8fd9\u6279\u6570\u636e\u4e0d\u80fd\u4ee3\u8868 HDFS \u7684\u6027\u80fd\u3002\u5b83\u6d4b\u7684\u662f YARN \u63a7\u5236\u5e73\u9762\u7684\u4efb\u52a1\u5206\u53d1\u80fd\u529b\u3002<\/p>\n<p>\u652f\u6491\u8fd9\u4e2a\u7ed3\u8bba\u7684\u6709\u56db\u5c42\u8bc1\u636e&#xff1a;<\/p>\n<li>\u8c03\u5ea6\u901f\u7387\u88ab\u4e09\u4e2a\u53c2\u6570\u9501\u6b7b&#xff1a;\u5168\u96c6\u7fa4\u6bcf\u79d2\u6700\u591a\u5206\u53d1 15 \u4e2a\u5bb9\u5668&#xff0c;20000 \u4e2a task \u5149&#034;\u53d1\u53f7&#034;\u5c31\u8981 1333 \u79d2<\/li>\n<li>\u52a8\u6001\u5fc3\u8df3\u662f\u5173\u95ed\u7684&#xff1a;\u8282\u70b9\u6ca1\u6709\u4efb\u4f55\u52a0\u901f\u6c47\u62a5\u7684\u53ef\u80fd&#xff0c;\u5373\u4f7f\u4efb\u52a1\u961f\u5217\u5806\u79ef\u5982\u5c71<\/li>\n<li>\u4e94\u8f6e\u8017\u65f6\u6781\u5dee\u53ea\u6709 4.2%&#xff0c;\u8bfb\u53d6 4 \u8f6e\u53ea\u6709 1%&#xff1a;\u8fd9\u79cd\u4e00\u81f4\u6027\u662f\u786e\u5b9a\u6027\u74f6\u9888\u7684\u7279\u5f81&#xff0c;\u4e0d\u662f I\/O \u6296\u52a8<\/li>\n<li>\u5355\u4efb\u52a1\u7c92\u5ea6\u592a\u5c0f&#xff1a;\u6bcf\u4e2a task \u53ea\u5199 100MB&#xff08;\u7ea6 1 \u79d2\u7684 I\/O&#xff09;&#xff0c;\u5374\u8981\u8d70\u5b8c\u6574\u7684\u5bb9\u5668\u7533\u8bf7\u4e0e JVM \u542f\u52a8\u6d41\u7a0b<\/li>\n<p>\u4e00\u53e5\u8bdd&#xff1a;<\/p>\n<p>\u4f60\u6d4b\u5230\u7684\u4e0d\u662f&#034;\u5b57\u8282\u541e\u5410&#034;&#xff0c;\u662f&#034;\u4efb\u52a1\u541e\u5410&#034;\u3002<\/p>\n<hr \/>\n<h3>\u4e8c\u3001\u6d4b\u8bd5\u73b0\u573a<\/h3>\n<h4>2.1 \u547d\u4ee4<\/h4>\n<p>hadoop jar hadoop-mapreduce-client-jobclient-3.3.3-tests.jar TestDFSIO <span class=\"token parameter variable\">-write<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">20000<\/span> <span class=\"token parameter variable\">-fileSize<\/span> <span class=\"token number\">100<\/span><\/p>\n<table>\n<tr>\u9879\u503c<\/tr>\n<tbody>\n<tr>\n<td>\u6587\u4ef6\u6570<\/td>\n<td>20000<\/td>\n<\/tr>\n<tr>\n<td>\u5355\u6587\u4ef6\u5927\u5c0f<\/td>\n<td>100 MB<\/td>\n<\/tr>\n<tr>\n<td>\u603b\u6570\u636e\u91cf<\/td>\n<td>20000 \u00d7 100MB &#061; 2 TB<\/td>\n<\/tr>\n<tr>\n<td>\u96c6\u7fa4\u89c4\u6a21<\/td>\n<td>5 \u8282\u70b9&#xff08;2 NN &#043; 3~5 DN\/NM&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5199\u5165\u8f6e\u6b21<\/td>\n<td>5<\/td>\n<\/tr>\n<tr>\n<td>\u8bfb\u53d6\u8f6e\u6b21<\/td>\n<td>4<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>-nrFiles 20000 \u8fd9\u4e2a\u53c2\u6570&#xff0c;\u76f4\u63a5\u51b3\u5b9a\u4e86\u8fd9\u4e2a\u6d4b\u8bd5\u4f1a\u521b\u5efa 20000 \u4e2a map task\u3002<\/p>\n<p>\u800c 20000 \u4e2a task&#xff0c;\u5c31\u662f\u8fd9\u4e2a\u95ee\u9898\u7684\u8d77\u70b9\u3002<\/p>\n<h4>2.2 \u57fa\u51c6\u811a\u672c<\/h4>\n<p><span class=\"token shebang important\">#!\/bin\/bash<\/span><br \/>\n<span class=\"token comment\"># hadoop_benchmark.sh<\/span><br \/>\n<span class=\"token assign-left variable\">HADOOP_BIN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/bin\/hadoop&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">TEST_JAR<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-client-jobclient-3.3.3-tests.jar&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">EXAMPLE_JAR<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-3.3.3.jar&#034;<\/span><\/p>\n<p><span class=\"token assign-left variable\">LOG_FILE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;.\/benchmark_<span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%Y%m%d-%H%M%S<span class=\"token variable\">)<\/span><\/span>.log&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">CSV_FILE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;.\/benchmark_result_<span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%Y%m%d-%H%M%S<span class=\"token variable\">)<\/span><\/span>.csv&#034;<\/span><\/p>\n<p><span class=\"token assign-left variable\">RUN_TIMES<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">${1<span class=\"token operator\">:-<\/span>1}<\/span><br \/>\n<span class=\"token assign-left variable\">MODE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">${2<span class=\"token operator\">:-<\/span>&#034;dfsio-write&#034;}<\/span><\/p>\n<p><span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;round,test_type,start_time,end_time,cost_seconds&#034;<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token variable\">$CSV_FILE<\/span><\/p>\n<p><span class=\"token function-name function\">clean_hdfs<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/test\/terasort-input   <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/test\/terasort-output  <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/benchmarks\/TestDFSIO* <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p><span class=\"token function-name function\">run_test<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token builtin class-name\">local<\/span> <span class=\"token assign-left variable\">i<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">$1<\/span><br \/>\n    clean_hdfs<\/p>\n<p>    <span class=\"token assign-left variable\">START_TIME<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%s<span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span>  <span class=\"token assign-left variable\">START_HUMAN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> <span class=\"token string\">&#034;&#043;%Y-%m-%d %H:%M:%S&#034;<\/span><span class=\"token variable\">)<\/span><\/span><\/p>\n<p>    <span class=\"token keyword\">case<\/span> <span class=\"token variable\">$MODE<\/span> <span class=\"token keyword\">in<\/span><br \/>\n        dfsio-write<span class=\"token punctuation\">)<\/span> <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-write<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">20000<\/span> <span class=\"token parameter variable\">-fileSize<\/span> <span class=\"token number\">100<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token variable\">$LOG_FILE<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        dfsio-read<span class=\"token punctuation\">)<\/span>  <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-read<\/span>  <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">20000<\/span> <span class=\"token parameter variable\">-fileSize<\/span> <span class=\"token number\">100<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token variable\">$LOG_FILE<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        teragen<span class=\"token punctuation\">)<\/span>     <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$EXAMPLE_JAR<\/span> teragen <span class=\"token number\">10000000000<\/span> \/test\/terasort-input <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token variable\">$LOG_FILE<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        terasort<span class=\"token punctuation\">)<\/span>    <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$EXAMPLE_JAR<\/span> terasort \/test\/terasort-input \/test\/terasort-output <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token variable\">$LOG_FILE<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        *<span class=\"token punctuation\">)<\/span> <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;[ERROR] Unknown mode: <span class=\"token variable\">$MODE<\/span>&#034;<\/span><span class=\"token punctuation\">;<\/span> <span class=\"token builtin class-name\">exit<\/span> <span class=\"token number\">1<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n    <span class=\"token keyword\">esac<\/span><\/p>\n<p>    <span class=\"token assign-left variable\">END_TIME<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%s<span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span>  <span class=\"token assign-left variable\">END_HUMAN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> <span class=\"token string\">&#034;&#043;%Y-%m-%d %H:%M:%S&#034;<\/span><span class=\"token variable\">)<\/span><\/span><br \/>\n    <span class=\"token assign-left variable\">COST<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$((<\/span>END_TIME <span class=\"token operator\">&#8211;<\/span> START_TIME<span class=\"token variable\">))<\/span><\/span><br \/>\n    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$i<\/span>,<span class=\"token variable\">$MODE<\/span>,<span class=\"token variable\">$START_HUMAN<\/span>,<span class=\"token variable\">$END_HUMAN<\/span>,<span class=\"token variable\">$COST<\/span>&#034;<\/span> <span class=\"token operator\">&gt;&gt;<\/span> <span class=\"token variable\">$CSV_FILE<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p><span class=\"token keyword\">for<\/span> <span class=\"token variable\"><span class=\"token punctuation\">((<\/span>i<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">;<\/span> i<span class=\"token operator\">&lt;&#061;<\/span>RUN_TIMES<span class=\"token punctuation\">;<\/span> i<span class=\"token operator\">&#043;&#043;<\/span><span class=\"token punctuation\">))<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span> run_test <span class=\"token variable\">$i<\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">done<\/span><\/p>\n<p>\u811a\u672c\u672c\u8eab\u6ca1\u95ee\u9898\u2014\u2014\u6e05\u7406\u3001\u8ba1\u65f6\u3001\u843d CSV\u3001\u591a\u8f6e\u5faa\u73af\u90fd\u6709\u3002<\/p>\n<p>\u95ee\u9898\u4e0d\u5728\u811a\u672c&#xff0c;\u5728\u53c2\u6570\u3002<\/p>\n<hr \/>\n<h3>\u4e09\u3001\u7ed3\u679c&#xff1a;\u4e94\u8f6e\u5199\u5165&#xff0c;\u7a33\u5b9a\u5f97\u4e0d\u50cf I\/O<\/h3>\n<h4>3.1 \u5199\u5165 5 \u8f6e<\/h4>\n<table>\n<tr>\u8f6e\u6b21\u8017\u65f6&#xff08;\u79d2&#xff09;<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>11855<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>11538<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>11701<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>11399<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>11370<\/td>\n<\/tr>\n<tr>\n<td>\u5e73\u5747<\/td>\n<td>11572<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6781\u5dee 485 \u79d2&#xff0c;\u6ce2\u52a8\u7387 4.2%\u3002<\/p>\n<h4>3.2 \u8bfb\u53d6 4 \u8f6e<\/h4>\n<table>\n<tr>\u8f6e\u6b21\u8017\u65f6&#xff08;\u79d2&#xff09;<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>5324<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>5360<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>5304<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>5341<\/td>\n<\/tr>\n<tr>\n<td>\u5e73\u5747<\/td>\n<td>5332<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6781\u5dee 56 \u79d2&#xff0c;\u6ce2\u52a8\u7387 1.05%\u3002<\/p>\n<h4>3.3 \u6362\u7b97\u541e\u5410<\/h4>\n<p>\u5199\u5165&#xff1a;2 TB \u00f7 11572 s \u2248 177 MB\/s<br \/>\n\u8bfb\u53d6&#xff1a;2 TB \u00f7 5332 s \u2248 384 MB\/s<\/p>\n<h4>3.4 \u8fd9\u91cc\u6709\u4e2a\u7ec6\u8282\u503c\u5f97\u76ef\u4e00\u4e0b<\/h4>\n<p>\u8bfb\u53d6 4 \u8f6e\u7684\u6ce2\u52a8\u53ea\u6709 1%\u3002<\/p>\n<p>\u771f\u5b9e\u7684\u78c1\u76d8 I\/O \u5f88\u96be\u8fd9\u4e48\u7a33\u2014\u2014\u9875\u7f13\u5b58\u547d\u4e2d\u7387\u3001\u540e\u53f0 compaction\u3001GC\u3001\u7f51\u7edc\u6296\u52a8&#xff0c;\u90fd\u4f1a\u8ba9\u7ed3\u679c\u4e0a\u4e0b\u6d6e\u52a8\u4e2a\u51e0\u79d2\u5230\u51e0\u5341\u79d2\u3002<\/p>\n<p>1% \u7684\u6ce2\u52a8\u7387&#xff0c;\u8bf4\u660e\u74f6\u9888\u662f\u786e\u5b9a\u6027\u7684\u3001\u6bcf\u6b21\u90fd\u4f1a\u53d1\u751f\u3001\u4e14\u6bcf\u6b21\u90fd\u6d88\u8017\u51e0\u4e4e\u76f8\u540c\u7684\u65f6\u95f4\u3002<\/p>\n<p>\u8fd9\u4e0d\u662f&#034;\u78c1\u76d8\u72b6\u6001\u6709\u6ce2\u52a8&#034;\u7684\u7279\u5f81&#xff0c;\u662f&#034;\u6bcf\u4e2a\u4efb\u52a1\u90fd\u8981\u8d70\u4e00\u904d\u56fa\u5b9a\u6d41\u7a0b&#034;\u7684\u7279\u5f81\u3002<\/p>\n<hr \/>\n<h3>\u56db\u3001\u7b2c\u4e00\u53cd\u5e94&#xff1a;\u662f\u4e0d\u662f\u78c1\u76d8\u4e0d\u884c<\/h3>\n<p>\u770b\u5230 177 MB\/s&#xff0c;\u7b2c\u4e00\u53cd\u5e94\u662f\u6392\u67e5\u786c\u4ef6&#xff1a;<\/p>\n<table>\n<tr>\u68c0\u67e5\u9879\u7ed3\u679c<\/tr>\n<tbody>\n<tr>\n<td>\u78c1\u76d8\u5065\u5eb7&#xff08;SMART \/ iostat&#xff09;<\/td>\n<td>\u6b63\u5e38<\/td>\n<\/tr>\n<tr>\n<td>\u7f51\u7edc\u5e26\u5bbd<\/td>\n<td>\u6b63\u5e38<\/td>\n<\/tr>\n<tr>\n<td>DataNode \u5b58\u6d3b<\/td>\n<td>5 \u53f0\u90fd\u5728<\/td>\n<\/tr>\n<tr>\n<td>\u526f\u672c\u6570 \/ \u5757\u5927\u5c0f<\/td>\n<td>\u9ed8\u8ba4&#xff0c;\u6b63\u5e38<\/td>\n<\/tr>\n<tr>\n<td>NameNode RPC \u961f\u5217<\/td>\n<td>\u4e0d\u79ef\u538b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u786c\u4ef6\u5c42\u9762\u6ca1\u6709\u660e\u663e\u77ed\u677f\u3002<\/p>\n<p>\u7167\u8fd9\u4e2a\u65b9\u5411\u7ee7\u7eed\u67e5\u4e0b\u53bb&#xff0c;\u4f1a\u9677\u5165&#034;\u6bcf\u5757\u76d8\u770b\u8d77\u6765\u90fd\u6b63\u5e38&#xff0c;\u4f46\u6574\u4f53\u5c31\u662f\u6162&#034;\u7684\u56f0\u5883\u3002<\/p>\n<hr \/>\n<h3>\u4e94\u3001\u8f6c\u6298&#xff1a;\u6709\u4eba\u95ee\u4e86\u4e00\u53e5&#034;\u5206\u914d container \u8981\u591a\u4e45&#034;<\/h3>\n<p>\u5f53\u65f6\u5df2\u7ecf\u51c6\u5907\u53bb\u6d4b\u5355\u76d8 IOPS \u4e86\u3002<\/p>\n<p>\u7136\u540e\u88ab\u95ee\u4e86\u4e00\u4e2a\u5f88\u7b80\u5355\u7684\u95ee\u9898&#xff1a;<\/p>\n<p>\u201c20000 \u4e2a map&#xff0c;\u5149\u5206\u914d container \u8981\u591a\u4e45&#xff1f;\u8ddf HDFS \u6027\u80fd\u6709\u5173\u7cfb\u5417&#xff1f;\u201d<\/p>\n<p>\u8fd9\u53e5\u8bdd\u628a\u65b9\u5411\u6574\u4e2a\u63b0\u4e86\u8fc7\u6765\u3002<\/p>\n<p>\u56e0\u4e3a TestDFSIO \u7684 20000 \u4e2a\u6587\u4ef6 &#061; 20000 \u4e2a map task&#xff0c;\u800c\u6bcf\u4e00\u4e2a task \u90fd\u5fc5\u987b&#xff1a;<\/p>\n<p>\u6392\u961f \u2192 \u5411 RM \u7533\u8bf7\u5bb9\u5668 \u2192 \u62ff\u5230\u5bb9\u5668 \u2192 \u542f\u52a8 JVM \u2192 \u5e72\u6d3b \u2192 \u56de\u6536<\/p>\n<p>&#034;\u5e72\u6d3b&#034;\u53ea\u662f\u8fd9\u6761\u94fe\u4e0a\u7684\u4e00\u73af\u3002<\/p>\n<p>\u5982\u679c\u8fd9\u6761\u94fe\u7684\u5165\u53e3\u6bcf\u79d2\u53ea\u80fd\u653e\u8fdb 15 \u4e2a&#xff0c;\u90a3 20000 \u4e2a\u5c31\u8981 1300 \u591a\u79d2\u2014\u2014\u8fd9\u8ddf\u78c1\u76d8\u5feb\u6162\u4e00\u70b9\u5173\u7cfb\u90fd\u6ca1\u6709\u3002<\/p>\n<hr \/>\n<h3>\u516d\u3001\u7b97\u8d26&#xff1a;\u96c6\u7fa4\u6bcf\u79d2\u80fd\u53d1\u591a\u5c11\u4e2a\u4efb\u52a1<\/h3>\n<p>YARN \u662f\u62c9\u53d6\u5f0f\u8c03\u5ea6&#xff1a;ResourceManager \u4e0d\u4f1a\u4e3b\u52a8\u628a\u4efb\u52a1\u63a8\u7ed9\u8282\u70b9&#xff0c;\u5fc5\u987b\u7b49 NodeManager \u53d1\u5fc3\u8df3\u6765\u9886\u3002<\/p>\n<p>\u6240\u4ee5&#034;\u6bcf\u79d2\u80fd\u6d3e\u53d1\u591a\u5c11\u4efb\u52a1&#034;\u7531\u4e09\u4e2a\u4e1c\u897f\u51b3\u5b9a&#xff1a;<\/p>\n<p>\u5206\u914d\u901f\u7387 &#061; \u8ba1\u7b97\u8282\u70b9\u6570 \u00d7 \u5fc3\u8df3\u9891\u7387 \u00d7 \u5355\u6b21\u5fc3\u8df3\u53ef\u5206\u914d\u5bb9\u5668\u6570<\/p>\n<p>\u4ee3\u5165\u5b9e\u6d4b\u53c2\u6570&#xff1a;<\/p>\n<p>\u8ba1\u7b97\u8282\u70b9\u6570        : 3&#xff08;5 \u8282\u70b9\u96c6\u7fa4\u4e2d 2 \u53f0\u662f NN&#xff09;<br \/>\n\u5fc3\u8df3\u9891\u7387          : 1 \u6b21\/\u79d2&#xff08;1000ms&#xff09;<br \/>\n\u5355\u6b21\u53ef\u5206\u914d\u5bb9\u5668\u6570   : 5<\/p>\n<p>\u5206\u914d\u901f\u7387 &#061; 3 \u00d7 1 \u00d7 5 &#061; 15 \u5bb9\u5668\/\u79d2<\/p>\n<h4>6.1 20000 \u4e2a\u4efb\u52a1\u9700\u8981\u591a\u4e45\u624d\u80fd&#034;\u53d1\u5b8c&#034;<\/h4>\n<p>20000 \u00f7 15 \u2248 1333 \u79d2   &#xff08;\u7ea6 22 \u5206\u949f&#xff09;<\/p>\n<p>\u5982\u679c 5 \u4e2a\u8ba1\u7b97\u8282\u70b9\u5168\u5728&#xff08;\u800c\u4e0d\u662f 3 \u4e2a&#xff09;&#xff1a;<\/p>\n<p>5 \u00d7 1 \u00d7 5 &#061; 25 \u5bb9\u5668\/\u79d2<br \/>\n20000 \u00f7 25 &#061; 800 \u79d2    &#xff08;\u7ea6 13 \u5206\u949f&#xff09;<\/p>\n<p>\u5149\u628a\u4efb\u52a1\u6d3e\u53d1\u4e0b\u53bb&#xff0c;\u6700\u5c11\u4e5f\u8981 13\u201322 \u5206\u949f\u3002<\/p>\n<h4>6.2 \u8fd9\u6bb5\u65f6\u95f4\u91cc\u96c6\u7fa4\u5728\u5e72\u4ec0\u4e48<\/h4>\n<p>\u4efb\u52a1\u4e0d\u662f&#034;\u4e00\u6b21\u6027\u5168\u53d1\u4e0b\u53bb\u7136\u540e\u540c\u65f6\u8dd1&#034;&#xff0c;\u800c\u662f&#xff1a;<\/p>\n<p>t&#061;0s    \u53d1\u51fa\u7b2c 1\u201315 \u4e2a&#xff0c;\u5f00\u59cb\u8dd1<br \/>\nt&#061;1s    \u53d1\u51fa\u7b2c 16\u201330 \u4e2a<br \/>\nt&#061;2s    &#8230;<br \/>\n        \u2193<br \/>\n\u524d\u9762\u53d1\u4e0b\u53bb\u7684\u4efb\u52a1\u65e9\u5c31\u8dd1\u5b8c\u4e86&#xff08;\u6bcf\u4e2a\u53ea\u5199 100MB&#xff09;<br \/>\n\u540e\u9762\u7684\u4efb\u52a1\u8fd8\u5728\u6392\u961f\u7b49\u53f7<br \/>\n        \u2193<br \/>\n\u96c6\u7fa4\u7684\u771f\u5b9e\u72b6\u6001&#xff1a;\u8dd1\u4e00\u9635&#xff0c;\u7b49\u4e00\u9635<\/p>\n<p>\u96c6\u7fa4\u4e0d\u662f\u88ab\u6570\u636e\u538b\u57ae\u7684&#xff0c;\u662f\u88ab&#034;\u53d1\u53f7&#034;\u7684\u901f\u5ea6\u9650\u5236\u4f4f\u7684\u3002<\/p>\n<hr \/>\n<h3>\u4e03\u3001\u4e09\u4e2a\u53c2\u6570\u9501\u6b7b\u4e86\u8c03\u5ea6\u901f\u7387<\/h3>\n<p>\u8fd9\u4e09\u4e2a\u6570\u5b57\u90fd\u4e0d\u662f\u62cd\u8111\u888b&#xff0c;\u5168\u90e8\u6765\u81ea\u914d\u7f6e\u548c\u6e90\u7801\u3002<\/p>\n<h4>7.1 maximum-offswitch-assignments &#061; 5<\/h4>\n<p>yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments &#061; 5<\/p>\n<p>\u8fd9\u4e2a\u53c2\u6570\u7684\u542b\u4e49\u662f&#xff1a;\u6bcf\u4e2a\u8282\u70b9\u3001\u6bcf\u6b21\u5fc3\u8df3&#xff0c;\u6700\u591a\u5206\u914d\u591a\u5c11\u4e2a&#034;\u975e\u672c\u5730&#xff08;off-switch&#xff09;&#034;\u5bb9\u5668\u3002<\/p>\n<p>\u9ed8\u8ba4\u503c\u5c31\u662f 5\u3002<\/p>\n<p>\u8fd9\u662f Capacity Scheduler \u7684\u4fdd\u62a4\u673a\u5236&#xff1a;\u9632\u6b62\u4e00\u6b21\u5fc3\u8df3\u628a\u5927\u91cf\u5bb9\u5668\u585e\u7ed9\u4e00\u4e2a\u8282\u70b9&#xff0c;\u9020\u6210\u8282\u70b9\u8fc7\u8f7d\u548c\u8c03\u5ea6\u6296\u52a8\u3002<\/p>\n<p>\u5b83\u5c31\u662f\u90a3\u4e2a&#034;\u6bcf\u79d2\u6700\u591a\u9886 5 \u4e2a\u53f7&#034;\u7684\u6765\u6e90\u3002<\/p>\n<h4>7.2 heartbeat-interval-ms &#061; 1000<\/h4>\n<p>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms &#061; 1000<\/p>\n<p>NodeManager \u6bcf\u79d2\u5411 RM \u6c47\u62a5\u4e00\u6b21\u3002<\/p>\n<p>\u6e90\u7801\u91cc\u5b83\u662f\u786c\u7f16\u7801\u7684\u9ed8\u8ba4\u503c&#xff1a;<\/p>\n<p><span class=\"token comment\">\/\/ YarnConfiguration.java<\/span><br \/>\n<span class=\"token keyword\">public<\/span> <span class=\"token keyword\">static<\/span> <span class=\"token keyword\">final<\/span> <span class=\"token keyword\">int<\/span> <span class=\"token constant\">DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1000<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<h4>7.3 heartbeat-interval-scaling-enable &#061; false&#xff08;\u6700\u5173\u952e\u7684\u4e00\u4e2a&#xff09;<\/h4>\n<p>yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable &#061; false<\/p>\n<p>\u8fd9\u4e00\u9879\u624d\u662f\u8ba9\u4eba\u7edd\u671b\u7684\u5730\u65b9\u3002<\/p>\n<p>YARN \u5176\u5b9e\u63d0\u4f9b\u4e86\u4e00\u4e2a&#034;\u52a8\u6001\u5fc3\u8df3&#034;\u673a\u5236&#xff1a;\u5f53\u961f\u5217\u5806\u79ef\u65f6&#xff0c;\u53ef\u4ee5\u8ba9\u8282\u70b9\u52a0\u5feb\u6c47\u62a5\u9891\u7387\u3002<\/p>\n<p># \u52a8\u6001\u5fc3\u8df3\u7684\u76f8\u5173\u53c2\u6570<br \/>\nyarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable &#061; false<br \/>\nyarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms &#061; 1000<br \/>\nyarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms &#061; 1000<\/p>\n<p>\u4f46 scaling-enable \u9ed8\u8ba4\u662f false\u3002<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf4&#xff1a;<\/p>\n<p>\u7cfb\u7edf\u5b8c\u5168\u5ffd\u7565 min-ms \/ max-ms \u7684\u8c03\u6574\u903b\u8f91&#xff0c;\u5f3a\u5236\u4f7f\u7528\u56fa\u5b9a\u7684 1000ms\u3002<\/p>\n<p>NodeManager \u6ca1\u6709\u4efb\u4f55\u52a0\u901f\u6c47\u62a5\u72b6\u6001\u7684\u53ef\u80fd\u2014\u2014\u54ea\u6015\u4efb\u52a1\u961f\u5217\u5806\u5230\u4e24\u4e07\u4e2a&#xff0c;\u5b83\u4f9d\u7136\u4fdd\u6301&#034;\u6bcf\u79d2\u4e00\u6b21&#034;\u7684\u8282\u594f\u3002<\/p>\n<h4>7.4 \u4e09\u4e2a\u53c2\u6570\u5408\u8d77\u6765<\/h4>\n<p>scaling-enable &#061; false        \u2192 \u5fc3\u8df3\u9891\u7387\u9501\u6b7b\u5728 1 Hz&#xff0c;\u4e0d\u80fd\u52a0\u901f<br \/>\nheartbeat-interval-ms &#061; 1000  \u2192 \u6bcf\u79d2\u53ea\u6709 1 \u6b21\u673a\u4f1a<br \/>\nmaximum-offswitch-assignments &#061; 5 \u2192 \u6bcf\u6b21\u673a\u4f1a\u6700\u591a\u9886 5 \u4e2a<br \/>\n        \u2193<br \/>\n\u6bcf\u4e2a\u8282\u70b9\u6bcf\u79d2\u6700\u591a 5 \u4e2a\u5bb9\u5668<br \/>\n        \u2193<br \/>\n\u6574\u4e2a\u96c6\u7fa4\u6bcf\u79d2\u6700\u591a 15 \u4e2a&#xff08;3 \u4e2a\u8ba1\u7b97\u8282\u70b9&#xff09;<\/p>\n<hr \/>\n<h3>\u516b\u30011333 \u79d2\u53ea\u662f\u4e0b\u9650<\/h3>\n<p>1333 \u79d2\u662f&#034;\u4efb\u52a1\u5168\u90e8\u5c31\u7eea\u3001\u8d44\u6e90\u5145\u8db3\u3001\u5b8c\u7f8e\u5339\u914d&#034;\u60c5\u51b5\u4e0b\u7684\u7406\u8bba\u6700\u5c0f\u503c\u3002<\/p>\n<p>\u5b9e\u9645\u8fd8\u8981\u53e0\u52a0\u4e09\u5757\u5f00\u9500&#xff1a;<\/p>\n<table>\n<tr>\u5f00\u9500\u91cf\u7ea7\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>AM \u2194 RM \u4ea4\u4e92\u5ef6\u8fdf<\/td>\n<td>\u6bcf\u6b21\u7ea6 2 \u79d2<\/td>\n<td>ApplicationMaster \u5fc3\u8df3\u4e0e\u8d44\u6e90\u534f\u5546<\/td>\n<\/tr>\n<tr>\n<td>JVM \u542f\u52a8\/\u9500\u6bc1<\/td>\n<td>1\u20133 \u79d2\/\u4efb\u52a1<\/td>\n<td>\u6bcf\u4e2a map task \u90fd\u8981\u8d77\u4e00\u4e2a JVM<\/td>\n<\/tr>\n<tr>\n<td>\u8d44\u6e90\u788e\u7247\u7b49\u5f85<\/td>\n<td>\u4e0d\u786e\u5b9a<\/td>\n<td>\u5076\u53d1\u6027\u7684\u7b49\u5f85&#xff0c;\u53ef\u80fd\u91cd\u8bd5&#xff08;speculative \/ retry&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6240\u4ee5\u771f\u5b9e\u7684&#034;\u4efb\u52a1\u7ea7\u56fa\u5b9a\u5f00\u9500&#034;\u8fdc\u5927\u4e8e 1333 \u79d2\u3002<\/p>\n<p>\u8fd9\u4e5f\u89e3\u91ca\u4e86\u4e3a\u4ec0\u4e48 11572 \u79d2\u4f1a\u51fa\u73b0\u5728\u8fd9\u4e48&#034;\u6574\u9f50&#034;\u7684\u533a\u95f4\u91cc\u2014\u2014\u56e0\u4e3a\u5b83\u4e0d\u662f\u88ab I\/O \u51b3\u5b9a\u7684&#xff0c;\u662f\u88ab\u8fd9\u6761\u6d41\u6c34\u7ebf\u7684\u8282\u594f\u51b3\u5b9a\u7684\u3002<\/p>\n<hr \/>\n<h3>\u4e5d\u300190% \u7684\u65f6\u95f4&#xff0c;\u78c1\u76d8\u5728\u7a7a\u8f6c<\/h3>\n<h4>9.1 \u6bcf\u4e2a task \u5230\u5e95\u5e72\u4e86\u591a\u5c11\u6d3b<\/h4>\n<p>\u5355\u6587\u4ef6\u5927\u5c0f : 100 MB<br \/>\n\u5355\u4e2a task  : \u5199 100 MB<br \/>\n\u6309\u5355\u76d8 100 MB\/s \u4f30\u7b97 : \u7ea6 1 \u79d2<\/p>\n<p>\u6bcf\u4e2a task \u5b9e\u9645\u642c\u6570\u636e\u7684\u65f6\u95f4&#xff0c;\u53ea\u6709 1 \u79d2\u5de6\u53f3\u3002<\/p>\n<p>\u4f46\u6bcf\u4e2a task \u7684\u5b8c\u6574\u751f\u547d\u5468\u671f\u662f&#xff1a;<\/p>\n<p>ResourceManager \u5206\u914d\u5bb9\u5668<br \/>\n    \u2193<br \/>\nNodeManager \u521b\u5efa\u5bb9\u5668\u76ee\u5f55<br \/>\n    \u2193<br \/>\n\u542f\u52a8 JVM<br \/>\n    \u2193<br \/>\n\u52a0\u8f7d Hadoop \u914d\u7f6e<br \/>\n    \u2193<br \/>\n\u5efa\u7acb HDFS \u8fde\u63a5&#xff08;\u542b Kerberos \u8ba4\u8bc1&#xff09;<br \/>\n    \u2193<br \/>\n\u5199\u5165 100MB   \u2190 \u5e72\u6d3b\u7684\u90e8\u5206&#xff0c;\u7ea6 1 \u79d2<br \/>\n    \u2193<br \/>\n\u5173\u95ed\u6587\u4ef6\u3001\u6c47\u62a5\u72b6\u6001\u3001\u56de\u6536\u5bb9\u5668<\/p>\n<p>\u5f00\u9500\u662f\u5e72\u6d3b\u7684\u6570\u500d\u3002<\/p>\n<h4>9.2 \u90a3\u771f\u5b9e\u5e26\u5bbd\u662f\u591a\u5c11<\/h4>\n<p>\u6309\u7ecf\u9a8c\u4f30\u7b97&#xff0c;\u8fd9\u7c7b&#034;\u5c0f\u6587\u4ef6 &#043; \u6d77\u91cf\u4efb\u52a1&#034;\u7684\u8bc4\u6d4b\u91cc&#xff0c;\u771f\u6b63\u5728\u5199\u6570\u636e\u7684\u65f6\u95f4\u901a\u5e38\u53ea\u5360\u603b\u65f6\u957f\u7684 5%\u201310%\u3002<\/p>\n<p>\u53cd\u63a8\u4e00\u4e0b&#xff1a;<\/p>\n<p>\u603b\u65f6\u957f 11572 s<br \/>\n\u6709\u6548 I\/O \u65f6\u95f4  \u2248 1157 s&#xff08;\u6309 10% \u4f30&#xff09;<br \/>\n\u771f\u5b9e\u5199\u5165\u5e26\u5bbd   \u2248 2048000 MB \u00f7 1157 s \u2248 1770 MB\/s \u2248 1.7 GB\/s<\/p>\n<p>\u6ce8\u610f\u8fd9\u662f\u63a8\u7b97\u503c&#xff0c;\u4e0d\u662f\u5b9e\u6d4b\u503c\u3002 \u4f46\u5b83\u6307\u51fa\u4e86\u4e00\u4e2a\u91cf\u7ea7\u5dee&#xff1a;<\/p>\n<p>\u5982\u679c\u628a\u8fd9\u4e2a\u6d4b\u8bd5\u6539\u5bf9&#xff0c;\u91cf\u51fa\u6765\u7684\u7ed3\u679c\u53ef\u80fd\u662f 1.7 GB\/s \u91cf\u7ea7&#xff0c;\u800c\u4e0d\u662f\u62a5\u544a\u4e0a\u7684 177 MB\/s\u3002 \u5dee\u4e86\u5c06\u8fd1\u5341\u500d\u3002<\/p>\n<p>\u800c\u90a3 90% \u7684\u65f6\u95f4\u91cc&#xff0c;\u78c1\u76d8\u5728\u5e72\u4ec0\u4e48&#xff1f;<\/p>\n<p>\u5728\u7b49\u3002<\/p>\n<hr \/>\n<h3>\u5341\u3001\u8bfb\u53d6\u4e3a\u4ec0\u4e48\u5feb 2.1 \u500d<\/h3>\n<p>\u8bfb\u53d6 5332s&#xff0c;\u5199\u5165 11572s&#xff0c;\u6bd4\u4f8b 2.16 \u500d\u3002<\/p>\n<p>\u8fd9\u4e2a\u6bd4\u4f8b\u540c\u65f6\u88ab\u4e09\u5c42\u56e0\u7d20\u5f71\u54cd&#xff1a;<\/p>\n<table>\n<tr>\u5c42\u5199\u5165\u8bfb\u53d6<\/tr>\n<tbody>\n<tr>\n<td>\u8c03\u5ea6\u5f00\u9500&#xff08;\u5171\u540c\u4e0b\u9650&#xff09;<\/td>\n<td>\u7ea6 800\u20131333s<\/td>\n<td>\u7ea6 800\u20131333s<\/td>\n<\/tr>\n<tr>\n<td>\u5355\u4efb\u52a1\u5f00\u9500<\/td>\n<td>\u5efa\u6587\u4ef6\u3001\u5206\u914d\u5757\u3001\u5199 pipeline<\/td>\n<td>\u53ea\u8bfb&#xff0c;\u6d41\u7a0b\u66f4\u77ed<\/td>\n<\/tr>\n<tr>\n<td>\u5b9e\u9645 I\/O<\/td>\n<td>2TB \u5199\u5165<\/td>\n<td>2TB \u8bfb\u53d6<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>HDFS \u672c\u8eab&#034;\u8bfb\u5feb\u5199\u6162&#034;\u662f\u6b63\u5e38\u7684&#xff08;\u5199\u5165\u8981\u8d70 3 \u526f\u672c pipeline&#xff0c;\u8bfb\u53d6\u53ef\u4ee5\u76f4\u63a5\u4ece\u6700\u8fd1\u526f\u672c\u8bfb&#xff09;\u3002<\/p>\n<p>\u4f46\u6709\u4e2a\u53cd\u76f4\u89c9\u7684\u70b9&#xff1a;<\/p>\n<p>\u56e0\u4e3a\u4e24\u8fb9\u90fd\u57ab\u7740\u540c\u4e00\u5757\u56fa\u5b9a\u7684\u8c03\u5ea6\u6210\u672c&#xff0c;&#034;2.1 \u500d&#034;\u8fd9\u4e2a\u6bd4\u503c\u662f\u88ab\u7a00\u91ca\u8fc7\u7684\u3002<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf4&#xff0c;\u771f\u5b9e\u7684\u78c1\u76d8\u8bfb\u5199\u6027\u80fd\u5dee\u8ddd&#xff0c;\u5f88\u53ef\u80fd\u6bd4 2.1 \u500d\u66f4\u5927\u2014\u2014\u628a\u8fd9\u4e2a\u56fa\u5b9a\u6210\u672c\u6263\u6389\u4e4b\u540e&#xff0c;\u5dee\u8ddd\u624d\u4f1a\u663e\u51fa\u6765\u3002<\/p>\n<hr \/>\n<h3>\u5341\u4e00\u3001\u540c\u4e00\u6279\u6d4b\u8bd5\u91cc\u7684 TeraSort<\/h3>\n<p>\u540c\u4e00\u6279\u57fa\u51c6\u91cc\u8fd8\u8dd1\u4e86 MapReduce \u8ba1\u7b97\u6027\u80fd&#xff1a;<\/p>\n<table>\n<tr>\u6d4b\u8bd5\u53c2\u6570\u8017\u65f6<\/tr>\n<tbody>\n<tr>\n<td>TeraGen<\/td>\n<td>10,000,000,000 \u884c<\/td>\n<td>4714 \u79d2&#xff08;\u7ea6 1.3 \u5c0f\u65f6&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>TeraSort<\/td>\n<td>1 TB<\/td>\n<td>741 \u79d2&#xff08;\u7ea6 12.3 \u5206\u949f&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u4e24\u7ec4\u6570\u636e\u7684\u4ef7\u503c&#xff0c;\u4e0d\u5728\u4e8e\u548c TestDFSIO \u505a\u6027\u80fd\u5bf9\u6bd4&#xff08;\u6d4b\u8bd5\u5bf9\u8c61\u4e0d\u540c&#xff09;&#xff0c;\u800c\u5728\u4e8e&#xff1a;<\/p>\n<p>\u96c6\u7fa4\u5728\u8fde\u7eed\u6570\u5c0f\u65f6\u7684\u538b\u529b\u6d4b\u8bd5\u4e0b&#xff0c;\u6ca1\u6709\u51fa\u73b0\u670d\u52a1\u4e2d\u65ad\u3001\u8282\u70b9\u6389\u7ebf\u3001\u4efb\u52a1\u5927\u9762\u79ef\u5931\u8d25\u3002<\/p>\n<p>\u4e00\u6b21\u6d4b\u9519\u7684\u57fa\u51c6&#xff0c;\u6570\u636e\u4e0d\u80fd\u7528\u6765\u8bc4\u4f30\u6027\u80fd&#xff0c;\u4f46\u53ef\u4ee5\u8bc1\u660e&#034;\u80fd\u6491\u4f4f&#034;\u3002<\/p>\n<p>\u8fd9\u4e5f\u662f\u8fd9\u6b21\u6d4b\u8bd5\u552f\u4e00\u53ef\u9760\u7684\u4ea7\u51fa\u3002<\/p>\n<hr \/>\n<h3>\u5341\u4e8c\u3001\u600e\u4e48\u6539&#xff1a;\u5c11\u4efb\u52a1&#xff0c;\u5927\u6587\u4ef6<\/h3>\n<p>\u6838\u5fc3\u539f\u5219\u53ea\u6709\u4e00\u6761&#xff1a;<\/p>\n<p>\u5355\u4efb\u52a1 I\/O \u65f6\u95f4 &gt;&gt; \u5355\u4efb\u52a1\u542f\u52a8\u5f00\u9500<\/p>\n<h4>12.1 \u53c2\u6570\u600e\u4e48\u6539<\/h4>\n<table>\n<tr>\u8c03\u6574\u9879\u539f\u6765\u6539\u6210\u7406\u7531<\/tr>\n<tbody>\n<tr>\n<td>\u6587\u4ef6\u6570<\/td>\n<td>20000<\/td>\n<td>100<\/td>\n<td>\u4efb\u52a1\u6570\u964d\u4e24\u4e2a\u6570\u91cf\u7ea7&#xff0c;\u8c03\u5ea6\u51e0\u79d2\u5185\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>\u5355\u6587\u4ef6\u5927\u5c0f<\/td>\n<td>100 MB<\/td>\n<td>10 GB<\/td>\n<td>\u5355\u4efb\u52a1 I\/O \u4ece 1s \u63d0\u5230 100s<\/td>\n<\/tr>\n<tr>\n<td>\u603b\u6570\u636e\u91cf<\/td>\n<td>2 TB<\/td>\n<td>1 TB<\/td>\n<td>\u6d4b\u8bd5\u65f6\u957f\u53ef\u63a7<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>12.2 \u65b0\u547d\u4ee4<\/h4>\n<p><span class=\"token comment\"># \u5199\u5165<\/span><br \/>\nhadoop jar hadoop-mapreduce-client-jobclient-3.3.3-tests.jar TestDFSIO <span class=\"token parameter variable\">-write<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">100<\/span> <span class=\"token parameter variable\">-fileSize<\/span> 10000MB<\/p>\n<p><span class=\"token comment\"># \u8bfb\u53d6<\/span><br \/>\nhadoop jar hadoop-mapreduce-client-jobclient-3.3.3-tests.jar TestDFSIO <span class=\"token parameter variable\">-read<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">100<\/span> <span class=\"token parameter variable\">-fileSize<\/span> 10000MB<\/p>\n<h4>12.3 \u4e3a\u4ec0\u4e48\u8fd9\u6837\u5c31\u5bf9\u4e86<\/h4>\n<p>\u539f\u6765&#xff1a;20000 \u00d7 100MB<br \/>\n  \u5355\u4efb\u52a1 I\/O \u2248 1s&#xff0c;\u542f\u52a8\u5f00\u9500 \u2248 2s<br \/>\n  \u2192 \u5f00\u9500\u5360\u6bd4 \u2248 66%&#xff0c;\u8c03\u5ea6\u8017\u65f6 \u2248 1333s<\/p>\n<p>\u6539\u540e&#xff1a;100 \u00d7 10GB<br \/>\n  \u5355\u4efb\u52a1 I\/O \u2248 100s&#xff0c;\u542f\u52a8\u5f00\u9500 \u2248 2s<br \/>\n  \u2192 \u5f00\u9500\u5360\u6bd4 \u2248 2%&#xff0c;\u8c03\u5ea6\u8017\u65f6 \u2248 7s<\/p>\n<p>\u540c\u6837\u7684\u601d\u8def&#xff0c;\u4ece&#034;\u6d4b\u4efb\u52a1\u541e\u5410&#034;\u53d8\u6210&#034;\u6d4b\u5b57\u8282\u541e\u5410&#034;\u3002<\/p>\n<h4>12.4 \u9884\u671f\u7ed3\u679c\u4e0e\u5224\u5b9a\u6807\u51c6<\/h4>\n<table>\n<tr>\u6307\u6807\u9884\u671f<\/tr>\n<tbody>\n<tr>\n<td>\u8c03\u5ea6\u5f00\u9500<\/td>\n<td>&lt; 10 \u79d2<\/td>\n<\/tr>\n<tr>\n<td>\u6d4b\u8bd5\u603b\u65f6\u957f<\/td>\n<td>\u5b8c\u5168\u7531\u78c1\u76d8\u5199\u5165\u901f\u5ea6\u51b3\u5b9a<\/td>\n<\/tr>\n<tr>\n<td>\u541e\u5410<\/td>\n<td>\u5927\u5e45\u63d0\u5347&#xff0c;\u63a5\u8fd1\u786c\u4ef6\u7269\u7406\u6781\u9650<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5224\u5b9a\u65b9\u6cd5&#xff1a;<\/p>\n<ul>\n<li>\u5982\u679c\u541e\u5410\u663e\u8457\u63d0\u5347 \u2192 \u4e4b\u524d\u7684\u74f6\u9888\u786e\u5b9e\u662f\u8c03\u5ea6&#xff0c;\u7ed3\u8bba\u6210\u7acb<\/li>\n<li>\u5982\u679c\u541e\u5410\u51e0\u4e4e\u6ca1\u53d8 \u2192 \u8fd9\u65f6\u5019\u624d\u8f6e\u5230\u67e5\u78c1\u76d8\u3001\u67e5\u7f51\u7edc\u3001\u67e5\u526f\u672c\u7b56\u7565<\/li>\n<\/ul>\n<hr \/>\n<h3>\u5341\u4e09\u3001\u5982\u679c\u771f\u7684\u8981\u538b\u5c0f\u6587\u4ef6<\/h3>\n<p>\u6709\u4e00\u79cd\u573a\u666f\u662f\u5408\u7406\u7684&#xff1a;\u4f60\u5c31\u662f\u8981\u8bc4\u4f30&#034;\u6d77\u91cf\u5c0f\u6587\u4ef6&#034;\u5bf9\u96c6\u7fa4\u7684\u51b2\u51fb\u3002<\/p>\n<p>\u8fd9\u65f6\u5019\u8981\u6d4b\u7684\u4e0d\u662f HDFS \u5e26\u5bbd&#xff0c;\u800c\u662f YARN \u8c03\u5ea6\u5668 &#043; NameNode \u5143\u6570\u636e\u7684\u80fd\u529b\u3002<\/p>\n<h4>13.1 \u4e34\u65f6\u653e\u5f00\u8c03\u5ea6\u9650\u5236&#xff08;\u6d4b\u8bd5\u540e\u5fc5\u987b\u8fd8\u539f&#xff09;<\/h4>\n<p># \u5f00\u542f\u52a8\u6001\u5fc3\u8df3<br \/>\nyarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable &#061; true<\/p>\n<p># \u5141\u8bb8\u6700\u5feb 100ms \u4e00\u6b21\u5fc3\u8df3<br \/>\nyarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms &#061; 100<\/p>\n<p># \u5355\u6b21\u5fc3\u8df3\u653e\u5f00\u5230 100 \u4e2a\u5bb9\u5668<br \/>\nyarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments &#061; 100<\/p>\n<p>\u6539\u5b8c\u4e4b\u540e&#xff0c;\u5206\u914d\u901f\u7387\u4ece&#xff1a;<\/p>\n<p>3 \u00d7 1 \u00d7 5 &#061; 15 \/s<\/p>\n<p>\u53d8\u6210&#xff1a;<\/p>\n<p>3 \u00d7 10 \u00d7 100 &#061; 3000 \/s<\/p>\n<p>\u7406\u8bba\u4e0a\u5feb\u4e86 200 \u500d\u3002<\/p>\n<h4>13.2 \u4e09\u4e2a\u5fc5\u987b\u63d0\u9192\u7684\u5730\u65b9<\/h4>\n<p>\u2460 \u8fd9\u7ec4\u53c2\u6570\u6709\u98ce\u9669<\/p>\n<p>maximum-offswitch-assignments \u8c03\u5927&#xff0c;\u4f1a\u8ba9\u5355\u8282\u70b9\u5728\u6781\u77ed\u65f6\u95f4\u5185\u88ab\u585e\u5165\u5927\u91cf\u5bb9\u5668&#xff0c;\u53ef\u80fd\u9020\u6210&#xff1a;<\/p>\n<ul>\n<li>NodeManager \u672c\u5730\u78c1\u76d8 IO \u6253\u6ee1<\/li>\n<li>\u5bb9\u5668\u542f\u52a8\u98ce\u66b4&#xff0c;JVM \u4e89\u62a2 CPU<\/li>\n<li>\u8c03\u5ea6\u5668\u6296\u52a8&#xff0c;\u5f71\u54cd\u5176\u4ed6\u961f\u5217<\/li>\n<\/ul>\n<p>\u2461 \u5fc5\u987b\u8fd8\u539f<\/p>\n<p>\u538b\u6d4b\u5b8c\u7acb\u523b\u6539\u56de\u9ed8\u8ba4\u503c\u3002\u8fd9\u7c7b\u53c2\u6570\u4e00\u65e6\u9057\u7559\u5728\u751f\u4ea7\u73af\u5883&#xff0c;\u4e0b\u4e00\u6b21\u6d41\u91cf\u9ad8\u5cf0\u53ef\u80fd\u5c31\u662f\u4e8b\u6545\u3002<\/p>\n<p>\u2462 \u4e0d\u8981\u7528\u6539\u53c2\u6570\u6765&#034;\u7f8e\u5316&#034; HDFS \u5e26\u5bbd\u6d4b\u8bd5<\/p>\n<p>\u4f60\u5982\u679c\u53ea\u662f\u60f3\u6d4b\u78c1\u76d8&#xff0c;\u6539\u53c2\u6570\u662f\u9519\u8bef\u7684\u65b9\u5411\u2014\u2014\u90a3\u662f\u5728\u6539\u88ab\u6d4b\u7cfb\u7edf&#xff0c;\u800c\u4e0d\u662f\u6539\u6d4b\u8bd5\u65b9\u6cd5\u3002<\/p>\n<hr \/>\n<h3>\u5341\u56db\u3001\u590d\u76d8<\/h3>\n<h4>14.1 \u516d\u4e2a\u5224\u65ad<\/h4>\n<p>\u2460 \u57fa\u51c6\u6d4b\u8bd5\u7684\u7b2c\u4e00\u6b65&#xff0c;\u662f\u7b97\u6e05\u695a&#034;\u8fd9\u5957\u6d4b\u8bd5\u6700\u5feb\u80fd\u8dd1\u591a\u5feb&#034;<\/p>\n<p>\u7406\u8bba\u4e0b\u9650 1333 \u79d2&#xff0c;\u5b9e\u6d4b 11572 \u79d2 \u2014\u2014 \u5148\u7b97\u5929\u82b1\u677f&#xff0c;\u624d\u77e5\u9053\u5927\u5934\u5728\u54ea\u3002<\/p>\n<p>\u2461 &#034;\u7ed3\u679c\u592a\u7a33\u5b9a&#034;\u672c\u8eab\u5c31\u662f\u4e00\u4e2a\u4fe1\u53f7<\/p>\n<p>\u8bfb\u53d6 4 \u8f6e\u6ce2\u52a8 1%\u3001\u5199\u5165 5 \u8f6e\u6ce2\u52a8 4.2%\u3002\u5f53\u4e00\u4e2a\u672c\u8be5\u6709\u6296\u52a8\u7684\u6307\u6807\u53d8\u5f97\u6781\u5176\u7a33\u5b9a\u65f6&#xff0c;\u8bf4\u660e\u74f6\u9888\u662f\u786e\u5b9a\u6027\u7684&#xff0c;\u4e0d\u662f\u7269\u7406\u7684\u3002<\/p>\n<p>\u2462 \u62c9\u53d6\u5f0f\u8c03\u5ea6\u7cfb\u7edf&#xff0c;\u5929\u7136\u6709&#034;\u53d1\u53f7\u4e0a\u9650&#034;<\/p>\n<p>YARN\u3001K8s\u3001Mesos \u90fd\u662f\u62c9\u53d6\u5f0f\u3002<\/p>\n<p>\u5b83\u4eec\u7684\u80fd\u529b\u4e0a\u9650\u4e0d\u662f&#034;\u80fd\u8dd1\u591a\u5feb&#034;&#xff0c;\u800c\u662f&#034;\u6bcf\u79d2\u80fd\u6d3e\u51fa\u53bb\u591a\u5c11&#034;\u3002<\/p>\n<p>\u2463 \u52a8\u6001\u5fc3\u8df3\u9ed8\u8ba4\u5173\u95ed&#xff0c;\u662f\u5f88\u591a\u4eba\u4e0d\u77e5\u9053\u7684\u4e00\u5c42<\/p>\n<p>heartbeat-interval-scaling-enable&#061;false \u610f\u5473\u7740\u5806\u79ef\u4e0d\u4f1a\u89e6\u53d1\u52a0\u901f\u3002<\/p>\n<p>\u5f88\u591a\u4eba\u4ee5\u4e3a&#034;\u961f\u5217\u957f\u4e86\u7cfb\u7edf\u4f1a\u81ea\u5df1\u52a0\u5feb&#034;&#xff0c;\u4e0d\u4f1a\u3002<\/p>\n<p>\u2464 \u4efb\u52a1\u7c92\u5ea6\u51b3\u5b9a\u4e86\u4f60\u6d4b\u7684\u662f&#034;\u4efb\u52a1\u541e\u5410&#034;\u8fd8\u662f&#034;\u5b57\u8282\u541e\u5410&#034;<\/p>\n<p>20000 \u00d7 100MB \u548c 100 \u00d7 10GB&#xff0c;\u524d\u8005\u6d4b\u8c03\u5ea6\u5668&#xff0c;\u540e\u8005\u6d4b\u78c1\u76d8\u3002<\/p>\n<p>\u4efb\u52a1\u6570\u8d85\u8fc7\u67d0\u4e2a\u91cf\u7ea7&#xff0c;\u88ab\u6d4b\u5bf9\u8c61\u5c31\u81ea\u52a8\u6362\u4e86\u3002<\/p>\n<p>\u2465 \u6d4b\u9519\u7684\u57fa\u51c6\u4e0d\u662f\u5e9f\u6570\u636e<\/p>\n<p>\u8fd9\u6b21 11500 \u79d2\u4e0d\u80fd\u8bc4\u4f30 HDFS \u6027\u80fd&#xff0c;\u4f46\u8bc1\u660e\u4e86\u96c6\u7fa4\u5728 3 \u5c0f\u65f6\u8fde\u7eed\u538b\u529b\u4e0b\u7a33\u5b9a\u8fd0\u884c\u3002<\/p>\n<p>\u5f88\u591a\u6d4b\u8bd5\u7684\u4ef7\u503c&#xff0c;\u4e0d\u5728\u5b83\u539f\u672c\u8981\u6d4b\u7684\u65b9\u5411\u4e0a\u3002<\/p>\n<h4>14.2 \u4e0b\u6b21\u505a\u57fa\u51c6\u6d4b\u8bd5\u524d\u7684\u68c0\u67e5\u6e05\u5355<\/h4>\n<table>\n<tr>\u68c0\u67e5\u9879\u600e\u4e48\u5224\u65ad<\/tr>\n<tbody>\n<tr>\n<td>\u4efb\u52a1\u6570\u662f\u5426\u5408\u9002<\/td>\n<td>\u603b\u4efb\u52a1\u6570 \u00f7 \u8c03\u5ea6\u901f\u7387 \u5e94\u8fdc\u5c0f\u4e8e\u9884\u4f30\u603b\u8017\u65f6<\/td>\n<\/tr>\n<tr>\n<td>\u5355\u4efb\u52a1 I\/O \u662f\u5426\u591f\u5927<\/td>\n<td>\u5355\u4efb\u52a1 I\/O \u65f6\u95f4\u5e94 \u2265 \u542f\u52a8\u5f00\u9500\u7684 10 \u500d<\/td>\n<\/tr>\n<tr>\n<td>\u662f\u5426\u5df2\u77e5\u8c03\u5ea6\u901f\u7387<\/td>\n<td>\u8282\u70b9\u6570 \u00d7 \u5fc3\u8df3\u9891\u7387 \u00d7 \u5355\u6b21\u5206\u914d\u6570<\/td>\n<\/tr>\n<tr>\n<td>\u52a8\u6001\u5fc3\u8df3\u662f\u5426\u5f00\u542f<\/td>\n<td>heartbeat-interval-scaling-enable<\/td>\n<\/tr>\n<tr>\n<td>\u7ed3\u679c\u6ce2\u52a8\u662f\u5426\u5408\u7406<\/td>\n<td>\u6ce2\u52a8\u8fc7\u5c0f \u2192 \u6000\u7591\u786e\u5b9a\u6027\u74f6\u9888<\/td>\n<\/tr>\n<tr>\n<td>\u8bfb\/\u5199\u662f\u5426\u5206\u5f00<\/td>\n<td>\u5206\u5f00\u8dd1&#xff0c;\u4e0d\u8981\u6df7\u5728\u4e00\u8f6e\u91cc<\/td>\n<\/tr>\n<tr>\n<td>\u662f\u5426\u52a8\u4e86\u751f\u4ea7\u53c2\u6570<\/td>\n<td>\u4e0d\u5e94\u8be5\u52a8<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u9644&#xff1a;\u6539\u8fdb\u7248\u811a\u672c<\/h3>\n<p><span class=\"token shebang important\">#!\/bin\/bash<\/span><br \/>\n<span class=\"token comment\"># hadoop_benchmark.sh \u2014\u2014 \u4e24\u6bb5\u5f0f&#xff1a;\u5927\u6587\u4ef6\u6d4b I\/O&#xff0c;\u5c0f\u6587\u4ef6\u6d4b\u8c03\u5ea6<\/span><br \/>\n<span class=\"token assign-left variable\">HADOOP_BIN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/bin\/hadoop&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">TEST_JAR<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-client-jobclient-3.3.3-tests.jar&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">EXAMPLE_JAR<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\/opt\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-3.3.3.jar&#034;<\/span><\/p>\n<p><span class=\"token assign-left variable\">STAMP<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%Y%m%d-%H%M%S<span class=\"token variable\">)<\/span><\/span><br \/>\n<span class=\"token assign-left variable\">LOG_FILE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;.\/benchmark_<span class=\"token variable\">${STAMP}<\/span>.log&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">CSV_FILE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;.\/benchmark_result_<span class=\"token variable\">${STAMP}<\/span>.csv&#034;<\/span><br \/>\n<span class=\"token assign-left variable\">RUN_TIMES<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">${1<span class=\"token operator\">:-<\/span>1}<\/span><br \/>\n<span class=\"token assign-left variable\">MODE<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">${2<span class=\"token operator\">:-<\/span>&#034;dfsio-write-big&#034;}<\/span><\/p>\n<p><span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;round,test_type,start_time,end_time,cost_seconds&#034;<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$CSV_FILE<\/span>&#034;<\/span><\/p>\n<p><span class=\"token function-name function\">clean_hdfs<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;[INFO] cleaning HDFS&#8230;&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/test\/terasort-input   <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/test\/terasort-output  <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n    <span class=\"token variable\">$HADOOP_BIN<\/span> fs <span class=\"token parameter variable\">-rm<\/span> <span class=\"token parameter variable\">-r<\/span> <span class=\"token parameter variable\">-skipTrash<\/span> \/benchmarks\/TestDFSIO* <span class=\"token operator\">&gt;<\/span>\/dev\/null <span class=\"token operator\"><span class=\"token file-descriptor important\">2<\/span>&gt;<\/span><span class=\"token file-descriptor important\">&amp;1<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p><span class=\"token function-name function\">run_test<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token builtin class-name\">local<\/span> <span class=\"token assign-left variable\">i<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\">$1<\/span><br \/>\n    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><br \/>\n    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;&#8212;&#8212;&#8212;- Round <span class=\"token variable\">$i<\/span> (<span class=\"token variable\">$MODE<\/span>) &#8212;&#8212;&#8212;-&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><br \/>\n    clean_hdfs<\/p>\n<p>    <span class=\"token builtin class-name\">local<\/span> START_TIME START_HUMAN END_TIME END_HUMAN COST<br \/>\n    <span class=\"token assign-left variable\">START_TIME<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%s<span class=\"token variable\">)<\/span><\/span><br \/>\n    <span class=\"token assign-left variable\">START_HUMAN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> <span class=\"token string\">&#034;&#043;%Y-%m-%d %H:%M:%S&#034;<\/span><span class=\"token variable\">)<\/span><\/span><\/p>\n<p>    <span class=\"token keyword\">case<\/span> <span class=\"token variable\">$MODE<\/span> <span class=\"token keyword\">in<\/span><br \/>\n        <span class=\"token comment\"># \u5927\u6587\u4ef6&#xff1a;\u6d4b\u771f\u5b9e I\/O \u5e26\u5bbd&#xff08;\u63a8\u8350&#xff09;<\/span><br \/>\n        dfsio-write-big<span class=\"token punctuation\">)<\/span> <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-write<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">100<\/span> <span class=\"token parameter variable\">-fileSize<\/span> 10000MB <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        dfsio-read-big<span class=\"token punctuation\">)<\/span>  <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-read<\/span>  <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">100<\/span> <span class=\"token parameter variable\">-fileSize<\/span> 10000MB <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        <span class=\"token comment\"># \u5c0f\u6587\u4ef6&#xff1a;\u6d4b\u8c03\u5ea6\u4e0e\u5143\u6570\u636e\u538b\u529b&#xff08;\u5bf9\u6bd4\u7528&#xff09;<\/span><br \/>\n        dfsio-write-small<span class=\"token punctuation\">)<\/span> <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-write<\/span> <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">20000<\/span> <span class=\"token parameter variable\">-fileSize<\/span> <span class=\"token number\">100<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        dfsio-read-small<span class=\"token punctuation\">)<\/span>  <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$TEST_JAR<\/span> TestDFSIO <span class=\"token parameter variable\">-read<\/span>  <span class=\"token parameter variable\">-nrFiles<\/span> <span class=\"token number\">20000<\/span> <span class=\"token parameter variable\">-fileSize<\/span> <span class=\"token number\">100<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        <span class=\"token comment\"># \u8ba1\u7b97\u6027\u80fd<\/span><br \/>\n        teragen<span class=\"token punctuation\">)<\/span>  <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$EXAMPLE_JAR<\/span> teragen <span class=\"token number\">10000000000<\/span> \/test\/terasort-input <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        terasort<span class=\"token punctuation\">)<\/span> <span class=\"token variable\">$HADOOP_BIN<\/span> jar <span class=\"token variable\">$EXAMPLE_JAR<\/span> terasort \/test\/terasort-input \/test\/terasort-output <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n        *<span class=\"token punctuation\">)<\/span> <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;[ERROR] Unknown mode: <span class=\"token variable\">$MODE<\/span>&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><span class=\"token punctuation\">;<\/span> <span class=\"token builtin class-name\">exit<\/span> <span class=\"token number\">1<\/span> <span class=\"token punctuation\">;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n    <span class=\"token keyword\">esac<\/span><\/p>\n<p>    <span class=\"token assign-left variable\">END_TIME<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> &#043;%s<span class=\"token variable\">)<\/span><\/span><br \/>\n    <span class=\"token assign-left variable\">END_HUMAN<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">date<\/span> <span class=\"token string\">&#034;&#043;%Y-%m-%d %H:%M:%S&#034;<\/span><span class=\"token variable\">)<\/span><\/span><br \/>\n    <span class=\"token assign-left variable\">COST<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token variable\"><span class=\"token variable\">$((<\/span>END_TIME <span class=\"token operator\">&#8211;<\/span> START_TIME<span class=\"token variable\">))<\/span><\/span><\/p>\n<p>    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;[RESULT] Round <span class=\"token variable\">$i<\/span> cost: <span class=\"token variable\">${COST}<\/span>s&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><br \/>\n    <span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$i<\/span>,<span class=\"token variable\">$MODE<\/span>,<span class=\"token variable\">$START_HUMAN<\/span>,<span class=\"token variable\">$END_HUMAN<\/span>,<span class=\"token variable\">$COST<\/span>&#034;<\/span> <span class=\"token operator\">&gt;&gt;<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$CSV_FILE<\/span>&#034;<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p><span class=\"token keyword\">for<\/span> <span class=\"token variable\"><span class=\"token punctuation\">((<\/span>i<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">;<\/span> i<span class=\"token operator\">&lt;&#061;<\/span>RUN_TIMES<span class=\"token punctuation\">;<\/span> i<span class=\"token operator\">&#043;&#043;<\/span><span class=\"token punctuation\">))<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span><br \/>\n    run_test <span class=\"token string\">&#034;<span class=\"token variable\">$i<\/span>&#034;<\/span><br \/>\n<span class=\"token keyword\">done<\/span><\/p>\n<p><span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><br \/>\n<span class=\"token builtin class-name\">echo<\/span> <span class=\"token string\">&#034;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; Completed. CSV: <span class=\"token variable\">$CSV_FILE<\/span> &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#034;<\/span> <span class=\"token operator\">|<\/span> <span class=\"token function\">tee<\/span> <span class=\"token parameter variable\">-a<\/span> <span class=\"token string\">&#034;<span class=\"token variable\">$LOG_FILE<\/span>&#034;<\/span><\/p>\n<p>\u7528\u6cd5&#xff1a;<\/p>\n<p><span class=\"token comment\"># \u5927\u6587\u4ef6\u5199\u5165&#xff08;\u6d4b I\/O \u5e26\u5bbd&#xff09;<\/span><br \/>\n.\/hadoop_benchmark.sh <span class=\"token number\">1<\/span> dfsio-write-big<\/p>\n<p><span class=\"token comment\"># \u5c0f\u6587\u4ef6\u5199\u5165&#xff08;\u5bf9\u7167&#xff1a;\u770b\u8c03\u5ea6\u5f71\u54cd&#xff09;<\/span><br \/>\n.\/hadoop_benchmark.sh <span class=\"token number\">1<\/span> dfsio-write-small<\/p>\n<p><span class=\"token comment\"># TeraGen &#043; TeraSort<\/span><br \/>\n.\/hadoop_benchmark.sh <span class=\"token number\">1<\/span> teragen <span class=\"token operator\">&amp;&amp;<\/span> .\/hadoop_benchmark.sh <span class=\"token number\">1<\/span> terasort<\/p>\n<hr \/>\n<p>\u6700\u540e\u4e00\u53e5<\/p>\n<p>\u8fd9\u6b21\u6d4b\u8bd5\u6700\u6709\u4ef7\u503c\u7684\u4ea7\u51fa&#xff0c;\u4e0d\u662f\u90a3 177 MB\/s&#xff0c;\u800c\u662f\u60f3\u660e\u767d\u4e86\u4e00\u4ef6\u4e8b&#xff1a;<\/p>\n<p>\u5f53\u4f60\u6d4b\u4e00\u4e2a\u7cfb\u7edf\u65f6&#xff0c;\u5982\u679c\u6d4b\u8bd5\u672c\u8eab\u7684\u5f00\u9500\u8d85\u8fc7\u4e86\u88ab\u6d4b\u5bf9\u8c61\u7684\u5f00\u9500&#xff0c; \u4f60\u6d4b\u5230\u7684\u5c31\u662f\u6d4b\u8bd5\u6846\u67b6&#xff0c;\u4e0d\u662f\u90a3\u4e2a\u7cfb\u7edf\u3002<\/p>\n<p>\u4e0b\u6b21\u8dd1\u57fa\u51c6\u4e4b\u524d&#xff0c;\u5148\u82b1\u4e94\u5206\u949f\u7b97\u4e00\u4e0b&#034;\u8fd9\u5957\u6d4b\u8bd5\u6700\u5feb\u80fd\u8dd1\u591a\u5feb&#034;\u3002<\/p>\n<p>\u4e94\u5206\u949f\u7684\u7b97\u672f&#xff0c;\u80fd\u7701\u4e0b\u4e09\u5c0f\u65f6\u7684\u767d\u8dd1\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2&#xff1a;\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668&#xff0c;\u4e0d\u662f HDFS \u5173\u952e\u8bcd&#xff1a;TestDFSIO\u3001HDFS \u57fa\u51c6\u6d4b\u8bd5\u3001YARN \u8c03\u5ea6\u5668\u3001maximum-offswitch-assignments\u3001\u5fc3\u8df3\u95f4\u9694\u3001\u5bb9\u5668\u5206\u914d\u901f\u7387\u3001Hadoop \u6027\u80fd\u6d4b\u8bd5\u3001TeraSort\u3001\u5c0f\u6587\u4ef6\u3001\u57fa\u51c6\u6d4b\u8bd5\u8bbe\u8ba1 \u76ee\u5f55<br \/>\n\u4e00\u3001\u5148\u8bf4\u7ed3\u8bba\u4e8c\u3001\u6d4b\u8bd5\u73b0\u573a\u4e09\u3001\u7ed3\u679c&#xff1a;\u4e94\u8f6e\u5199\u5165&#xff0c;\u7a33\u5b9a\u5f97\u4e0d\u50cf I\/O\u56db\u3001\u7b2c\u4e00\u53cd\u5e94&#xff1a;\u662f\u4e0d\u662f\u78c1\u76d8\u4e0d\u884c<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[779,2828,323],"topic":[],"class_list":["post-121717","post","type-post","status-publish","format-standard","hentry","category-server","tag-hadoop","tag-hdfs","tag-323"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/121717.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2&#xff1a;\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668&#xff0c;\u4e0d\u662f HDFS \u5173\u952e\u8bcd&#xff1a;TestDFSIO\u3001HDFS \u57fa\u51c6\u6d4b\u8bd5\u3001YARN \u8c03\u5ea6\u5668\u3001maximum-offswitch-assignments\u3001\u5fc3\u8df3\u95f4\u9694\u3001\u5bb9\u5668\u5206\u914d\u901f\u7387\u3001Hadoop \u6027\u80fd\u6d4b\u8bd5\u3001TeraSort\u3001\u5c0f\u6587\u4ef6\u3001\u57fa\u51c6\u6d4b\u8bd5\u8bbe\u8ba1 \u76ee\u5f55 \u4e00\u3001\u5148\u8bf4\u7ed3\u8bba\u4e8c\u3001\u6d4b\u8bd5\u73b0\u573a\u4e09\u3001\u7ed3\u679c&#xff1a;\u4e94\u8f6e\u5199\u5165&#xff0c;\u7a33\u5b9a\u5f97\u4e0d\u50cf I\/O\u56db\u3001\u7b2c\u4e00\u53cd\u5e94&#xff1a;\u662f\u4e0d\u662f\u78c1\u76d8\u4e0d\u884c\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/121717.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-11-03T05:16:40+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"6 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/121717.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/121717.html\",\"name\":\"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-11-03T05:16:40+00:00\",\"dateModified\":\"2026-11-03T05:16:40+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/121717.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/121717.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/121717.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/121717.html","og_locale":"zh_CN","og_type":"article","og_title":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2&#xff1a;\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668&#xff0c;\u4e0d\u662f HDFS \u5173\u952e\u8bcd&#xff1a;TestDFSIO\u3001HDFS \u57fa\u51c6\u6d4b\u8bd5\u3001YARN \u8c03\u5ea6\u5668\u3001maximum-offswitch-assignments\u3001\u5fc3\u8df3\u95f4\u9694\u3001\u5bb9\u5668\u5206\u914d\u901f\u7387\u3001Hadoop \u6027\u80fd\u6d4b\u8bd5\u3001TeraSort\u3001\u5c0f\u6587\u4ef6\u3001\u57fa\u51c6\u6d4b\u8bd5\u8bbe\u8ba1 \u76ee\u5f55 \u4e00\u3001\u5148\u8bf4\u7ed3\u8bba\u4e8c\u3001\u6d4b\u8bd5\u73b0\u573a\u4e09\u3001\u7ed3\u679c&#xff1a;\u4e94\u8f6e\u5199\u5165&#xff0c;\u7a33\u5b9a\u5f97\u4e0d\u50cf I\/O\u56db\u3001\u7b2c\u4e00\u53cd\u5e94&#xff1a;\u662f\u4e0d\u662f\u78c1\u76d8\u4e0d\u884c","og_url":"https:\/\/www.wsisp.com\/helps\/121717.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-11-03T05:16:40+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"6 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/121717.html","url":"https:\/\/www.wsisp.com\/helps\/121717.html","name":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-11-03T05:16:40+00:00","dateModified":"2026-11-03T05:16:40+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/121717.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/121717.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/121717.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"20000 \u4e2a\u6587\u4ef6\u8dd1\u4e86 11500 \u79d2\uff1a\u4f60\u538b\u7684\u662f YARN \u8c03\u5ea6\u5668\uff0c\u4e0d\u662f HDFS"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/121717","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=121717"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/121717\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=121717"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=121717"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=121717"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=121717"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}