![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093706-68021d42cc48b.jpg\")
<\/p>\n<\/p>\n
\u76ee\u5f55<\/p>\n
\u4e00\u3001\u6982\u8ff0<\/p>\n
1.1\u5386\u53f2\u670d\u52a1\u5668JobHistory<\/p>\n
1.2 Spark\u5386\u53f2\u670d\u52a1\u5668HistoryServer<\/p>\n
\u4e8c\u3001\u96c6\u6210\u914d\u7f6e<\/p>\n
\u4e3a\u4e86\u67e5\u770b\u7a0b\u5e8f\u7684\u5386\u53f2\u8fd0\u884c\u60c5\u51b5，\u9700\u8981\u914d\u7f6e\u4e00\u4e0b\u5386\u53f2\u670d\u52a1\u5668\u3002\u65b9\u4fbf\u5728xxxx:8088\u67e5\u770b\u5386\u53f2\u4efb\u52a1\u8fd0\u884c\u65e5\u5fd7\u4fe1\u606f\u3002<\/p>\n
HistoryServer\u670d\u52a1\u53ef\u4ee5\u8ba9\u7528\u6237\u901a\u8fc7Spark UI\u754c\u9762，\u67e5\u770b\u5386\u53f2\u5e94\u7528（\u5df2\u7ecf\u6267\u884c\u5b8c\u7684\u5e94\u7528）\u7684\u6267\u884c\u7ec6\u8282，\u6bd4\u5982job\u4fe1\u606f\u3001stage\u4fe1\u606f\u3001task\u4fe1\u606f\u7b49，\u8be5\u529f\u80fd\u662f\u57fa\u4e8espark eventlogs\u65e5\u5fd7\u6587\u4ef6\u7684，\u6240\u4ee5\u5fc5\u987b\u6253\u5f00eventlogs\u65e5\u5fd7\u5f00\u5173\u3002\u65b9\u4fbf\u5728xxxx:8088\u67e5\u770b\u5386\u53f2\u4efb\u52a1\u8fd0\u884c\u65e5\u5fd7\u4fe1\u606f\u3002![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093707-68021d435b117.png\")
<\/p>\n
yarn-site.xml<\/p>\n
<property>
\n <name>yarn.log.server.url<\/name>
\n <value>http:\/\/xxxxx:19888\/jobhistory\/logs<\/value>
\n<\/property> <\/p>\n
mapred-site.xml<\/p>\n
<property>
\n <name>mapreduce.jobhistory.address<\/name>
\n <value>xxxxx:10020<\/value>
\n <\/property><\/p>\n
<property>
\n <name>mapreduce.jobhistory.webapp.address<\/name>
\n <value>xxxxx19888<\/value>
\n <\/property><\/p>\n
\u5982\u4e0b\u662fYARN\u5b8c\u6574\u7684service_ddl.json<\/p>\n
cat YARN\/service_ddl.json<\/p>\n
{
\n "name": "YARN",
\n "label": "YARN",
\n "description": "\u5206\u5e03\u5f0f\u8d44\u6e90\u8c03\u5ea6\u4e0e\u7ba1\u7406\u5e73\u53f0",
\n "version": "3.3.3",
\n "sortNum": 2,
\n "dependencies":["HDFS"],
\n "packageName": "hadoop-3.3.3.tar.gz",
\n "decompressPackageName": "hadoop-3.3.3",
\n "roles": [
\n {
\n "name": "ResourceManager",
\n "label": "ResourceManager",
\n "roleType": "master",
\n "cardinality": "1",
\n "runAs": {
\n "user": "yarn",
\n "group": "hadoop"
\n },
\n "sortNum": 1,
\n "logFile": "logs\/hadoop-yarn-resourcemanager-${host}.log",
\n "jmxPort": 9323,
\n "startRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "start",
\n "resourcemanager"
\n ]
\n },
\n "stopRunner": {
\n "timeout": "600",
\n "program": "control_hadoop.sh",
\n "args": [
\n "stop",
\n "resourcemanager"
\n ]
\n },
\n "statusRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "status",
\n "resourcemanager"
\n ]
\n },
\n "restartRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "restart",
\n "resourcemanager"
\n ]
\n },
\n "externalLink": {
\n "name": "ResourceManager UI",
\n "label": "ResourceManager UI",
\n "url": "http:\/\/${host}:8088\/ui2"
\n }
\n },
\n {
\n "name": "NodeManager",
\n "label": "NodeManager",
\n "roleType": "worker",
\n "cardinality": "1",
\n "runAs": {
\n "user": "yarn",
\n "group": "hadoop"
\n },
\n "sortNum": 2,
\n "logFile": "logs\/hadoop-yarn-nodemanager-${host}.log",
\n "jmxPort": 9324,
\n "startRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "start",
\n "nodemanager"
\n ]
\n },
\n "stopRunner": {
\n "timeout": "600",
\n "program": "control_hadoop.sh",
\n "args": [
\n "stop",
\n "nodemanager"
\n ]
\n },
\n "statusRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "status","nodemanager"
\n ]
\n }
\n },
\n {
\n "name": "HistoryServer",
\n "label": "HistoryServer",
\n "roleType": "master",
\n "cardinality": "1",
\n "runAs": {
\n "user": "mapred",
\n "group": "hadoop"
\n },
\n "sortNum": 3,
\n "logFile": "logs\/hadoop-mapred-historyserver-${host}.log",
\n "jmxPort": 9325,
\n "startRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "start",
\n "historyserver"
\n ]
\n },
\n "stopRunner": {
\n "timeout": "600",
\n "program": "control_hadoop.sh",
\n "args": [
\n "stop",
\n "historyserver"
\n ]
\n },
\n "statusRunner": {
\n "timeout": "60",
\n "program": "control_hadoop.sh",
\n "args": [
\n "status","historyserver"
\n ]
\n }
\n },{
\n "name": "YarnClient",
\n "label": "YarnClient",
\n "roleType": "client",
\n "cardinality": "1",
\n "runAs": {
\n "user": "yarn",
\n "group": "hadoop"
\n }
\n }
\n ],
\n "configWriter": {
\n "generators": [
\n {
\n "filename": "yarn-site.xml",
\n "configFormat": "xml",
\n "outputDirectory": "etc\/hadoop\/",
\n "includeParams": [
\n "yarn.nodemanager.resource.cpu-vcores",
\n "yarn.nodemanager.resource.memory-mb",
\n "yarn.scheduler.minimum-allocation-mb",
\n "yarn.scheduler.minimum-allocation-vcores",
\n "yarn.log.server.url",
\n "yarn.nodemanager.aux-services",
\n "yarn.log-aggregation-enable",
\n "yarn.resourcemanager.ha.enabled",
\n "yarn.resourcemanager.hostname",
\n "yarn.resourcemanager.address",
\n "yarn.resourcemanager.webapp.address",
\n "yarn.resourcemanager.scheduler.address",
\n "yarn.resourcemanager.resource-tracker.address",
\n "yarn.resourcemanager.store.class",
\n "yarn.application.classpath",
\n "yarn.nodemanager.local-dirs",
\n "yarn.nodemanager.log-dirs",
\n "yarn.nodemanager.address",
\n "yarn.nodemanager.resource.count-logical-processors-as-cores",
\n "yarn.nodemanager.resource.detect-hardware-capabilities",
\n "yarn.nodemanager.resource.pcores-vcores-multiplier",
\n "yarn.resourcemanager.am.max-attempts",
\n "yarn.node-labels.enabled",
\n "yarn.node-labels.fs-store.root-dir",
\n "yarn.resourcemanager.principal",
\n "yarn.resourcemanager.keytab",
\n "yarn.nodemanager.principal",
\n "yarn.nodemanager.keytab",
\n "yarn.nodemanager.container-executor.class",
\n "yarn.nodemanager.linux-container-executor.group",
\n "yarn.nodemanager.linux-container-executor.path",
\n "custom.yarn.site.xml"
\n ]
\n },
\n {
\n "filename": "mapred-site.xml",
\n "configFormat": "xml",
\n "outputDirectory": "etc\/hadoop\/",
\n "includeParams": [
\n "mapreduce.jobhistory.keytab",
\n "mapreduce.jobhistory.principal",
\n "mapreduce.cluster.local.dir",
\n "mapreduce.jobhistory.address",
\n "mapreduce.jobhistory.webapp.address",
\n "custom.mapred.site.xml"
\n ]
\n }
\n ]
\n },
\n "parameters": [
\n {
\n "name": "yarn.node-labels.fs-store.root-dir",
\n "label": "Node Label\u5b58\u50a8\u76ee\u5f55",
\n "description": "Node Label\u5b58\u50a8\u76ee\u5f55",
\n "required": true,
\n "type": "input",
\n "value": "hdfs:\/\/${dfs.nameservices}\/user\/yarn\/nodeLabels",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "hdfs:\/\/${dfs.nameservices}\/user\/yarn\/nodeLabels"
\n },
\n {
\n "name": "yarn.nodemanager.resource.cpu-vcores",
\n "label": "nodemanager\u865a\u62df\u6838\u6570",
\n "description": "nodemanager\u865a\u62df\u6838\u6570",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "-1"
\n },
\n {
\n "name": "yarn.nodemanager.resource.memory-mb",
\n "label": "Nodemanaer\u8282\u70b9\u4e0aYARN\u53ef\u4f7f\u7528\u7684\u7269\u7406\u5185\u5b58\u603b\u91cf",
\n "description": "Nodemanaer\u8282\u70b9\u4e0aYARN\u53ef\u4f7f\u7528\u7684\u7269\u7406\u5185\u5b58\u603b\u91cf",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "2048"
\n },
\n {
\n "name": "yarn.scheduler.minimum-allocation-mb",
\n "label": "\u6700\u5c0f\u53ef\u5206\u914d\u5bb9\u5668\u7684\u5927\u5c0f",
\n "description": "\u6700\u5c0f\u53ef\u5206\u914d\u5bb9\u5668\u7684\u5927\u5c0f",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "1024"
\n },
\n {
\n "name": "yarn.scheduler.minimum-allocation-vcores",
\n "label": "nodemanager\u6700\u5c0f\u865a\u62df\u6838\u6570",
\n "description": "nodemanager\u6700\u5c0f\u865a\u62df\u6838\u6570",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "1"
\n },
\n {
\n "name": "yarn.nodemanager.resource.count-logical-processors-as-cores",
\n "label": "\u662f\u5426\u5c06\u7269\u7406\u6838\u6570\u4f5c\u4e3a\u865a\u62df\u6838\u6570",
\n "description": "\u662f\u5426\u5c06\u7269\u7406\u6838\u6570\u4f5c\u4e3a\u865a\u62df\u6838\u6570",
\n "required": true,
\n "type": "switch",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": true
\n },
\n {
\n "name": "yarn.nodemanager.resource.detect-hardware-capabilities",
\n "label": "\u662f\u5426\u8ba9yarn\u81ea\u5df1\u68c0\u6d4b\u786c\u4ef6\u8fdb\u884c\u914d\u7f6e",
\n "description": "\u662f\u5426\u8ba9yarn\u81ea\u5df1\u68c0\u6d4b\u786c\u4ef6\u8fdb\u884c\u914d\u7f6e",
\n "required": true,
\n "type": "switch",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": true
\n },
\n {
\n "name": "yarn.nodemanager.resource.pcores-vcores-multiplier",
\n "label": "\u865a\u62df\u6838\u6570\u4e0e\u7269\u7406\u6838\u6570\u6bd4\u4f8b",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "0.75"
\n },
\n {
\n "name": "yarn.resourcemanager.am.max-attempts",
\n "label": "AM\u91cd\u8bd5\u6b21\u6570",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "4"
\n },
\n {
\n "name": "yarn.nodemanager.aux-services",
\n "label": "yarn\u670d\u52a1\u673a\u5236",
\n "description": "yarn\u670d\u52a1\u673a\u5236",
\n "required": true,
\n "type": "input",
\n "value": "mapreduce_shuffle",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "mapreduce_shuffle"
\n },
\n {
\n "name": "yarn.log-aggregation-enable",
\n "label": "\u662f\u5426\u5f00\u542fyarn\u65e5\u5fd7\u805a\u5408",
\n "description": "\u5f00\u542fyarn\u65e5\u5fd7\u805a\u5408",
\n "required": true,
\n "type": "switch",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": true
\n },
\n {
\n "name": "yarn.resourcemanager.ha.enabled",
\n "label": "\u662f\u5426\u542f\u7528resourcemanager ha",
\n "description": "\u662f\u5426\u542f\u7528resourcemanager ha",
\n "configType": "ha",
\n "required": true,
\n "type": "switch",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": false
\n },
\n {
\n "name": "yarn.resourcemanager.hostname",
\n "label": "rm\u4e3b\u673a\u540d",
\n "description": "",
\n "configType": "ha",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${host}"
\n },
\n {
\n "name": "yarn.resourcemanager.webapp.address",
\n "label": "rm web\u5730\u5740",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${host}:8088"
\n },
\n {
\n "name": "yarn.resourcemanager.address",
\n "label": "ResourceManager\u5bf9\u5ba2\u6237\u7aef\u66b4\u9732\u7684\u5730\u5740rm",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "${host}:8032"
\n },
\n {
\n "name": "yarn.resourcemanager.scheduler.address",
\n "label": "ResourceManager\u5bf9ApplicationMaster\u66b4\u9732\u7684\u8bbf\u95ee\u5730\u5740rm",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "${host}:8030"
\n },
\n {
\n "name": "yarn.resourcemanager.resource-tracker.address",
\n "label": "ResourceManager\u5bf9NodeManager\u66b4\u9732\u7684\u5730\u5740rm",
\n "description": "",
\n "configType": "ha",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "${host}:8031"
\n },
\n {
\n "name": "yarn.resourcemanager.store.class",
\n "label": "yarn\u72b6\u6001\u4fe1\u606f\u5b58\u50a8\u7c7b",
\n "description": "",
\n "configType": "ha",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore"
\n },
\n {
\n "name": "yarn.nodemanager.address",
\n "label": "nodemanager\u5730\u5740",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "0.0.0.0:45454"
\n },
\n {
\n "name": "yarn.log.server.url",
\n "label": "historyserver\u5730\u5740",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "http:\/\/${historyserverHost}:19888\/jobhistory\/logs"
\n },
\n {
\n "name": "yarn.log-aggregation.retain-seconds",
\n "label": "\u65e5\u5fd7\u4fdd\u7559\u65f6\u957f",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "2592000"
\n },
\n {
\n "name": "yarn.nodemanager.remote-app-log-dir",
\n "label": "\u65e5\u5fd7\u4fdd\u7559\u4f4d\u7f6e",
\n "description": "",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "hdfs:\/\/${dfs.nameservices}\/user\/yarn\/yarn-logs\/"
\n },
\n {
\n "name": "yarn.nodemanager.local-dirs",
\n "label": "NodeManager\u672c\u5730\u5b58\u50a8\u76ee\u5f55",
\n "description": "NodeManager\u672c\u5730\u5b58\u50a8\u76ee\u5f55，\u53ef\u914d\u7f6e\u591a\u4e2a，\u6309\u9017\u53f7\u5206\u9694",
\n "required": true,
\n "configType": "path",
\n "separator": ",",
\n "type": "multiple",
\n "value": [
\n "\/data\/nm"
\n ],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n },
\n {
\n "name": "yarn.nodemanager.log-dirs",
\n "label": "NodeManager\u65e5\u5fd7\u5b58\u50a8\u76ee\u5f55",
\n "description": "NodeManager\u65e5\u5fd7\u5b58\u50a8\u76ee\u5f55，\u53ef\u914d\u7f6e\u591a\u4e2a，\u6309\u9017\u53f7\u5206\u9694",
\n "required": true,
\n "configType": "path",
\n "separator": ",",
\n "type": "multiple",
\n "value": [
\n "\/data\/nm\/userlogs"
\n ],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n },
\n {
\n "name": "mapreduce.cluster.local.dir",
\n "label": "MapReduce\u672c\u5730\u5b58\u50a8\u76ee\u5f55",
\n "description": "MapReduce\u672c\u5730\u5b58\u50a8\u76ee\u5f55",
\n "required": true,
\n "configType": "path",
\n "separator": ",",
\n "type": "input",
\n "value": "\/data\/mapred\/local",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "\/data\/mapred\/local"
\n },
\n {
\n "name": "mapreduce.jobhistory.address",
\n "label": "MapReduceJobHistoryServer\u5730\u5740",
\n "description": "MapReduceJobHistoryServer\u5730\u5740",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${host}:10020"
\n },
\n {
\n "name": "mapreduce.jobhistory.webapp.address",
\n "label": "MapReduceJobHistoryServerWeb-UI\u5730\u5740",
\n "description": "MapReduceJobHistoryServerWeb-UI\u5730\u5740",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${host}:19888"
\n },
\n {
\n "name": "yarn.application.classpath",
\n "label": "yarn\u5e94\u7528\u7a0b\u5e8f\u52a0\u8f7d\u7684classpath",
\n "description": "yarn\u5e94\u7528\u7a0b\u5e8f\u52a0\u8f7d\u7684classpath",
\n "required": true,
\n "separator": ",",
\n "type": "multiple",
\n "value":["${HADOOP_HOME}\/etc\/hadoop","${HADOOP_HOME}\/share\/hadoop\/common\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/common\/*","${HADOOP_HOME}\/share\/hadoop\/hdfs","${HADOOP_HOME}\/share\/hadoop\/hdfs\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/hdfs\/*","${HADOOP_HOME}\/share\/hadoop\/mapreduce\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/mapreduce\/*","${HADOOP_HOME}\/share\/hadoop\/yarn","${HADOOP_HOME}\/share\/hadoop\/yarn\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/yarn\/*"],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ["${HADOOP_HOME}\/etc\/hadoop","${HADOOP_HOME}\/share\/hadoop\/common\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/common\/*","${HADOOP_HOME}\/share\/hadoop\/hdfs","${HADOOP_HOME}\/share\/hadoop\/hdfs\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/hdfs\/*","${HADOOP_HOME}\/share\/hadoop\/mapreduce\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/mapreduce\/*","${HADOOP_HOME}\/share\/hadoop\/yarn","${HADOOP_HOME}\/share\/hadoop\/yarn\/lib\/*","${HADOOP_HOME}\/share\/hadoop\/yarn\/*"]
\n },
\n {
\n "name": "yarn.resourcemanager.principal",
\n "label": "ResourceManager\u670d\u52a1\u7684Kerberos\u4e3b\u4f53",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "rm\/_HOST@HADOOP.COM",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "rm\/_HOST@HADOOP.COM"
\n },
\n {
\n "name": "yarn.resourcemanager.keytab",
\n "label": "ResourceManager\u670d\u52a1\u7684Kerberos\u5bc6\u94a5\u6587\u4ef6\u8def\u5f84",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "\/etc\/security\/keytab\/rm.service.keytab",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "\/etc\/security\/keytab\/rm.service.keytab"
\n },
\n {
\n "name": "yarn.nodemanager.principal",
\n "label": "NodeManager\u670d\u52a1\u7684Kerberos\u4e3b\u4f53",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "nm\/_HOST@HADOOP.COM",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "nm\/_HOST@HADOOP.COM"
\n },
\n {
\n "name": "yarn.nodemanager.keytab",
\n "label": "NodeManager\u670d\u52a1\u7684Kerberos\u5bc6\u94a5\u6587\u4ef6\u8def\u5f84",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "\/etc\/security\/keytab\/nm.service.keytab",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "\/etc\/security\/keytab\/nm.service.keytab"
\n },
\n {
\n "name": "mapreduce.jobhistory.principal",
\n "label": "JobHistory\u670d\u52a1\u7684Kerberos\u4e3b\u4f53",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "jhs\/_HOST@HADOOP.COM",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "jhs\/_HOST@HADOOP.COM"
\n },
\n {
\n "name": "mapreduce.jobhistory.keytab",
\n "label": "JobHistory\u670d\u52a1\u7684Kerberos\u5bc6\u94a5\u6587\u4ef6\u8def\u5f84",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "\/etc\/security\/keytab\/jhs.service.keytab",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "\/etc\/security\/keytab\/jhs.service.keytab"
\n },
\n {
\n "name": "yarn.nodemanager.container-executor.class",
\n "label": "\u4f7f\u7528LinuxContainerExecutor\u7ba1\u7406Container",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"
\n },
\n {
\n "name": "yarn.nodemanager.linux-container-executor.group",
\n "label": "NodeManager\u7684\u542f\u52a8\u7528\u6237\u7684\u6240\u5c5e\u7ec4",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "hadoop",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "hadoop"
\n },
\n {
\n "name": "yarn.nodemanager.linux-container-executor.path",
\n "label": "LinuxContainerExecutor\u811a\u672c\u8def\u5f84",
\n "description": "",
\n "required": false,
\n "configType": "kb",
\n "configWithKerberos": true,
\n "type": "input",
\n "value": "${HADOOP_HOME}\/bin\/container-executor",
\n "configurableInWizard": true,
\n "hidden": true,
\n "defaultValue": "${HADOOP_HOME}\/bin\/container-executor"
\n },
\n {
\n "name": "yarn.node-labels.enabled",
\n "label": "\u542f\u7528YARN\u6807\u7b7e\u8c03\u5ea6",
\n "description": "\u5f00\u542f YARN Node Labels",
\n "required": true,
\n "type": "switch",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": false
\n },
\n {
\n "name": "enableKerberos",
\n "label": "\u542f\u7528Kerberos\u8ba4\u8bc1",
\n "description": "\u542f\u7528Kerberos\u8ba4\u8bc1",
\n "required": true,
\n "type": "switch",
\n "value": false,
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": false
\n },
\n {
\n "name": "custom.yarn.site.xml",
\n "label": "\u81ea\u5b9a\u4e49\u914d\u7f6eyarn-site.xml",
\n "description": "\u81ea\u5b9a\u4e49\u914d\u7f6e",
\n "configType": "custom",
\n "required": false,
\n "type": "multipleWithKey",
\n "value": [],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n },
\n {
\n "name": "custom.mapred.site.xml",
\n "label": "\u81ea\u5b9a\u4e49\u914d\u7f6emapred-site.xml",
\n "description": "\u81ea\u5b9a\u4e49\u914d\u7f6e",
\n "configType": "custom",
\n "required": false,
\n "type": "multipleWithKey",
\n "value": [],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n }
\n ]
\n} <\/p>\n
\u5982\u4e0b\u662fSPARK3\u5b8c\u6574\u7684service_ddl.json<\/p>\n
cat SPARK3\/service_ddl.json<\/p>\n
{
\n "name": "SPARK3",
\n "label": "Spark3",
\n "description": "\u5206\u5e03\u5f0f\u8ba1\u7b97\u7cfb\u7edf",
\n "version": "3.1.3",
\n "sortNum": 7,
\n "dependencies":[],
\n "packageName": "spark-3.1.3.tar.gz",
\n "decompressPackageName": "spark-3.1.3",
\n "roles": [
\n {
\n "name": "SparkClient3",
\n "label": "SparkClient3",
\n "roleType": "client",
\n "cardinality": "1",
\n "logFile": "logs\/hadoop-${user}-datanode-${host}.log"
\n },
\n {
\n "name": "SparkHistoryServer",
\n "label": "SparkHistoryServer",
\n "roleType": "master",
\n "runAs": {
\n "user": "hdfs",
\n "group": "hadoop"
\n },
\n "cardinality": 1,
\n "logFile": "logs\/spark-HistoryServer-${host}.out",
\n "jmxPort": 9094,
\n "startRunner": {
\n "timeout": "60",
\n "program": "control_histroy_server.sh",
\n "args": [
\n "start",
\n "histroyserver"
\n ]
\n },
\n "stopRunner": {
\n "timeout": "600",
\n "program": "control_histroy_server.sh",
\n "args": [
\n "stop",
\n "histroyserver"
\n ]
\n },
\n "statusRunner": {
\n "timeout": "60",
\n "program": "control_histroy_server.sh",
\n "args": [
\n "status",
\n "histroyserver"
\n ]
\n },
\n "restartRunner": {
\n "timeout": "60",
\n "program": "control_histroy_server.sh",
\n "args": [
\n "restart",
\n "histroyserver"
\n ]
\n },
\n "externalLink": {
\n "name": "SparkHistoryServer Ui",
\n "label": "SparkHistoryServer Ui",
\n "url": "http:\/\/${host}:18081"
\n }
\n }
\n ],
\n "configWriter": {
\n "generators": [
\n {
\n "filename": "spark-env.sh",
\n "configFormat": "custom",
\n "templateName": "spark-env.ftl",
\n "outputDirectory": "conf",
\n "includeParams": [
\n "SPARK_DIST_CLASSPATH",
\n "HADOOP_CONF_DIR",
\n "YARN_CONF_DIR",
\n "custom.spark.env.sh"
\n ]
\n },
\n {
\n "filename": "spark-defaults.conf",
\n "configFormat": "properties2",
\n "outputDirectory": "conf",
\n "includeParams": [
\n "spark.eventLog.dir",
\n "spark.history.fs.logDirectory",
\n "spark.eventLog.enabled",
\n "spark.master",
\n "spark.history.ui.port",
\n "spark.yarn.historyServer.address",
\n "spark.yarn.queue",
\n "spark.history.fs.cleaner.enabled",
\n "spark.history.fs.cleaner.interval",
\n "spark.history.fs.cleaner.maxAge",
\n "custom.spark.defaults.conf"
\n ]
\n }
\n ]
\n },
\n "parameters": [
\n {
\n "name": "spark.eventLog.dir",
\n "label": "eventLog\u8f93\u51fa\u7684hdfs\u8def\u5f84",
\n "description": "eventLog\u8f93\u51fa\u7684hdfs\u8def\u5f84",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "hdfs:\/\/${host}:8020\/spark3-history"
\n },
\n {
\n "name": "spark.history.fs.logDirectory",
\n "label": "Spark\u5386\u53f2\u65e5\u5fd7HDFS\u76ee\u5f55",
\n "description": "Spark\u5386\u53f2\u65e5\u5fd7HDFS\u76ee\u5f55",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "hdfs:\/\/${host}:8020\/spark3-history"
\n },
\n {
\n "name": "spark.eventLog.enabled",
\n "label": "\u5f00\u542fspark\u4e8b\u4ef6\u65e5\u5fd7",
\n "description": "\u5f00\u542fspark\u4e8b\u4ef6\u65e5\u5fd7",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "true"
\n },
\n {
\n "name": "spark.master",
\n "label": "\u5f00\u542fsparkmaster",
\n "description": "\u5f00\u542fsparkmaster",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "yarn"
\n },
\n {
\n "name": "spark.history.ui.port",
\n "label": "sparkhistoryweb\u7aef\u53e3",
\n "description": "sparkhistoryweb\u7aef\u53e3",
\n "required": true,
\n "type": "input",
\n "value": "18081",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "18081"
\n },
\n {
\n "name": "spark.yarn.queue",
\n "label": "\u6307\u5b9a\u63d0\u4ea4\u5230Yarn\u7684\u8d44\u6e90\u6c60",
\n "description": "\u6307\u5b9a\u63d0\u4ea4\u5230Yarn\u7684\u8d44\u6e90\u6c60",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "default"
\n },
\n {
\n "name": "spark.yarn.historyServer.address",
\n "label": "application\u7684\u65e5\u5fd7\u8bbf\u95ee\u5730\u5740",
\n "description": "application\u7684\u65e5\u5fd7\u8bbf\u95ee\u5730\u5740",
\n "required": true,
\n "type": "input",
\n "value":"",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${host}:18081"
\n },
\n {
\n "name": "spark.history.fs.cleaner.enabled",
\n "label": "sparkhistory\u65e5\u5fd7\u662f\u5426\u5b9a\u65f6\u6e05\u9664",
\n "description": "sparkhistory\u65e5\u5fd7\u662f\u5426\u5b9a\u65f6\u6e05\u9664",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "true"
\n },
\n {
\n "name": "spark.history.fs.cleaner.interval",
\n "label": "history-server\u7684\u65e5\u5fd7\u68c0\u67e5\u95f4\u9694",
\n "description": "history-server\u7684\u65e5\u5fd7\u68c0\u67e5\u95f4\u9694",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "1d"
\n },
\n {
\n "name": "spark.history.fs.cleaner.maxAge",
\n "label": "history-server\u65e5\u5fd7\u751f\u547d\u5468\u671f",
\n "description": "history-server\u65e5\u5fd7\u751f\u547d\u5468\u671f",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "30d"
\n },
\n {
\n "name": "SPARK_DIST_CLASSPATH",
\n "label": "spark\u52a0\u8f7dClasspath\u8def\u5f84",
\n "description": "",
\n "required": true,
\n "configType": "map",
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "$(${HADOOP_HOME}\/bin\/hadoop classpath)"
\n },
\n {
\n "name": "HADOOP_CONF_DIR",
\n "label": "Hadoop\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55",
\n "description": "",
\n "configType": "map",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${HADOOP_HOME}\/etc\/hadoop"
\n },{
\n "name": "YARN_CONF_DIR",
\n "label": "Yarn\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55",
\n "description": "",
\n "configType": "map",
\n "required": true,
\n "type": "input",
\n "value": "",
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": "${HADOOP_HOME}\/etc\/hadoop"
\n },
\n {
\n "name": "custom.spark.env.sh",
\n "label": "\u81ea\u5b9a\u4e49\u914d\u7f6espark-env.sh",
\n "description": "\u81ea\u5b9a\u4e49\u914d\u7f6espark-env.sh",
\n "configType": "custom",
\n "required": false,
\n "type": "multipleWithKey",
\n "value": [{"SPARK_CLASSPATH":"${INSTALL_PATH}\/spark-3.1.3\/carbonlib\/*"}],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n },
\n {
\n "name": "custom.spark.defaults.conf",
\n "label": "\u81ea\u5b9a\u4e49\u914d\u7f6espark-defaults.conf",
\n "description": "\u81ea\u5b9a\u4e49\u914d\u7f6e",
\n "configType": "custom",
\n "required": false,
\n "type": "multipleWithKey",
\n "value": [],
\n "configurableInWizard": true,
\n "hidden": false,
\n "defaultValue": ""
\n }
\n ]
\n}<\/p>\n
sparhistroyServer\u542f\u505c\u811a\u672c<\/p>\n
cat spark-3.1.3\/control_histroy_server.sh<\/p>\n
#!\/bin\/bash
\n#
\n# Licensed to the Apache Software Foundation (ASF) under one or more
\n# contributor license agreements. See the NOTICE file distributed with
\n# this work for additional information regarding copyright ownership.
\n# The ASF licenses this file to You under the Apache License, Version 2.0
\n# (the "License"); you may not use this file except in compliance with
\n# the License. You may obtain a copy of the License at
\n#
\n# http:\/\/www.apache.org\/licenses\/LICENSE-2.0
\n#
\n# Unless required by applicable law or agreed to in writing, software
\n# distributed under the License is distributed on an "AS IS" BASIS,
\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\n# See the License for the specific language governing permissions and
\n# limitations under the License.
\n#<\/p>\n
usage="Usage: start.sh (start|stop|status|restart) <command> "<\/p>\n
# if no args specified, show usage
\nif [ $# -le 1 ]; then
\n echo $usage
\n exit 1
\nfi
\nstartStop=$1
\nshift
\ncommand=$1
\nSH_DIR=`dirname $0`
\nexport LOG_DIR=$SH_DIR\/logs
\nexport PID_DIR=$SH_DIR\/pid<\/p>\n
export HOSTNAME=`hostname`<\/p>\n
if [ ! -d "$LOG_DIR" ]; then
\n mkdir $LOG_DIR
\nfi<\/p>\n
source \/etc\/profile.d\/datasophon-env.sh
\n# \u521b\u5efa\u65e5\u5fd7\u8def\u5f84
\nsudo -u hdfs \/opt\/datasophon\/hdfs\/bin\/hdfs dfs -test -d \/spark3-history
\nif [ $? -ne 0 ] ;then
\n source \/etc\/profile.d\/datasophon-env.sh
\n sudo -u hdfs \/opt\/datasophon\/hdfs\/bin\/hdfs dfs -mkdir -p \/spark3-history
\n sudo -u hdfs \/opt\/datasophon\/hdfs\/bin\/hdfs dfs -chown -R hdfs:hadoop \/spark3-history\/
\n sudo -u hdfs \/opt\/datasophon\/hdfs\/bin\/hdfs dfs -chmod -R 777 \/spark3-history\/<\/p>\n
fi<\/p>\n
start(){
\n[ -w "$PID_DIR" ] || mkdir -p "$PID_DIR"
\n ifOrNot=`ps -ef |grep HistoryServer | grep spark |grep -v "grep" |wc -l`
\n if [ 1 == $ifOrNot ]
\n then
\n echo "$command is running "
\n exit 1
\n else
\n echo "$command is not running"
\n fi<\/p>\n
echo starting $command, logging
\n exec_command="$SH_DIR\/sbin\/start-history-server.sh"
\n echo " $exec_command"
\n $exec_command<\/p>\n
}
\nstop(){
\n ifOrNot=`ps -ef |grep HistoryServer | grep spark |grep -v "grep" |wc -l`
\n if [ 1 == $ifOrNot ];then
\n ifStop=`jps | grep -E 'HistoryServer' | grep -v 'Job*' | awk '{print $1}'`
\n if [ ! -z $ifStop ]; then
\n echo "stop $command "
\n kill -9 $ifStop
\n fi
\n else
\n echo "$command is not running"
\n exit 1
\n fi<\/p>\n
}
\nstatus(){
\n ifOrNot=`ps -ef |grep HistoryServer | grep spark |grep -v "grep" |wc -l`
\n if [ 1 == $ifOrNot ]
\n then
\n echo "$command is running "
\n else
\n echo "$command is not running"
\n exit 1
\n fi
\n}
\nrestart(){
\nstop
\nsleep 10
\nstart
\n}
\ncase $startStop in
\n (start)
\n start
\n ;;
\n (stop)
\n stop
\n ;;
\n (status)
\n status
\n;;
\n (restart)
\n restart
\n ;;
\n (*)
\n echo $usage
\n exit 1
\n ;;
\nesac<\/p>\n
echo "End $startStop $command."<\/p>\n
\n
spark\u91cd\u65b0\u6253\u5305\u6211\u4eec\u90e8\u7f72\u5b89\u88c5<\/p>\n
\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093707-68021d43c2f2d.png\")
\u9009\u62e9SparkhistoryServer\u5b89\u88c5\u8282\u70b9![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093708-68021d440ec3f.png\")
<\/p>\n
\n
\u9009\u62e9Sparkclient![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093708-68021d444398e.png\")
<\/p>\n
\n
\u670d\u52a1\u914d\u7f6e![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093708-68021d4477dd4.png\")
<\/p>\n
\n
\u5b89\u88c5\u5b8c\u6210<\/p>\n
\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093708-68021d44c1976.png\")
<\/p>\n
\u5b89\u88c5\u5b8c\u540e\u7684spark-defaults.conf \u914d\u7f6e\u6587\u4ef6<\/p>\n
cat spark3\/conf\/spark-defaults.conf<\/p>\n
spark.eventLog.dir hdfs:\/\/windp-aio:8020\/spark3-history
\nspark.history.fs.logDirectory hdfs:\/\/windp-aio:8020\/spark3-history
\nspark.eventLog.enabled true
\nspark.master yarn
\nspark.history.ui.port 18081
\nspark.yarn.historyServer.address windp-aio:18081
\nspark.yarn.queue default
\nspark.history.fs.cleaner.enabled true
\nspark.history.fs.cleaner.interval 1d
\nspark.history.fs.cleaner.maxAge 30d<\/p>\n
datasophon\u9875\u9762<\/p>\n
\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093709-68021d4501279.png\")
<\/p>\n
\u63d0\u4ea4spark-examples\u6d4b\u8bd5\u4efb\u52a1<\/p>\n
spark-submit \\\\
\n–class org.apache.spark.examples.SparkPi \\\\
\n–master yarn \\\\
\n–executor-memory 1G \\\\
\n–num-executors 2 \\\\
\n\/opt\/datasophon\/spark3\/examples\/jars\/spark-examples_2.12-3.1.3.jar \\\\
\n10 <\/p>\n
\u67e5\u770bYARN<\/p>\n
\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093709-68021d455c09b.png\")
<\/p>\n
\u67e5\u770bSPARK<\/p>\n
\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2025\/04\/20250418093709-68021d45c8439.png\")
<\/p>\n
\u81f3\u6b64\u96c6\u6210\u5b8c\u6210\u3002\u00a0<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6587\u7ae0\u6d4f\u89c8\u9605\u8bfb6.6k\u6b21\uff0c\u70b9\u8d5e78\u6b21\uff0c\u6536\u85cf72\u6b21\u3002HistoryServer\u670d\u52a1\u53ef\u4ee5\u8ba9\u7528\u6237\u901a\u8fc7Spark UI\u754c\u9762\uff0c\u67e5\u770b\u5386\u53f2\u5e94\u7528\uff08\u5df2\u7ecf\u6267\u884c\u5b8c\u7684\u5e94\u7528\uff09\u7684\u6267\u884c\u7ec6\u8282\uff0c\u6bd4\u5982job\u4fe1\u606f\u3001stage\u4fe1\u606f\u3001task\u4fe1\u606f\u7b49\uff0c\u8be5\u529f\u80fd\u662f\u57fa\u4e8espark eventlogs\u65e5\u5fd7\u6587\u4ef6\u7684\uff0c\u6240\u4ee5\u5fc5\u987b\u6253\u5f00eventlogs\u65e5\u5fd7\u5f00\u5173\u3002\u65b9\u4fbf\u5728xxxx:8088\u67e5\u770b\u5386\u53f2\u4efb\u52a1\u8fd0\u884c\u65e5\u5fd7\u4fe1\u606f\u3002_spark\u5386\u53f2\u670d\u52a1\u5668<\/p>\n","protected":false},"author":2,"featured_media":9729,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[366,368,365,367,363,364],"topic":[],"class_list":["post-9739","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-historyserver","tag-jobhistory","tag-spark","tag-yarn","tag-amp","tag-364"],"yoast_head":"\n