{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Spark集群监控",
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 5602,
"iteration": 1630058663789,
"links": [],
"panels": [
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 38,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 1
},
"id": 48,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "jvm_memory_usage{application=\"$application_Id\",executor_id=\"driver\",mem_type=\"total\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器内存总量分配使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 12
},
"id": 46,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "jvm_memory_usage{application=\"$application_Id\",executor_id=\"driver\",mem_type=\"non-heap\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器非堆内存分配使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 44,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "jvm_memory_usage{application=\"$application_Id\",executor_id=\"driver\",mem_type=\"PS-MarkSweep\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "jvm_memory_usage{application=\"application_1629181908892_0087\",executor_id=\"driver\",mem_type=\"PS-Scavenge\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "JVM垃圾收集情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Driver Heap Usage",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 32
},
"id": 12,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "{application=\"$application_Id\",executor_id=\"driver\",mem_type=\"heap\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器堆内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 42
},
"id": 20,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"memUsed_MB\",type=\"memory\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "avg(block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"memUsed_MB\",type=\"memory\"})",
"format": "time_series",
"intervalFactor": 1,
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器blockManager内存使用",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Executor Heap Usage",
"fill": 1,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 51
},
"id": 10,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "jvm_memory_usage{mem_type=\"heap\", qty=\"usage\", application=\"$application_Id\", executor_id=\"driver\"} ",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器堆内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 59
},
"id": 40,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"diskSpaceUsed_MB\",type=\"disk\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器内blockmanager磁盘空间使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 68
},
"id": 42,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"maxMem_MB\",type=\"memory\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"memUsed_MB\",type=\"memory\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "B"
},
{
"expr": "block_manager{application=\"$application_Id\",executor_id=\"driver\",qty=\"remainingMem_MB\",type=\"memory\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "驱动器blockmanager内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "当前application驱动器运行情况",
"type": "row"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 1
},
"id": 36,
"panels": [],
"title": "application下各executor资源使用情况",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Executor Old-Gen",
"fill": 1,
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 2
},
"id": 24,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",mem_type=\"PS-Old-Gen\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "执行器堆内存-老年代使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Executor Eden-Space",
"fill": 1,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 13
},
"id": 26,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "{application=\"$application_Id\",executor_id!~\"driver\", mem_type=\"PS-Eden-Space\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "执行器年轻代内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "HiveExternalCatalog相关指标",
"fill": 1,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"id": 22,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"fileCacheHits\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "E"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"filesDiscovered\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"parallelListingJobCount\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "B"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"partitionsFetched\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "C"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"disk\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "D"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"memory\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "F"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",type=~\"threadpool\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "G"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HiveExternalCatalog相关指标",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Executor HDFS reads",
"fill": 1,
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 37
},
"id": 30,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"read_bytes\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "avg({application=\"$application_ID\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"read_bytes\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "B"
},
{
"expr": "{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"read_ops\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "C"
},
{
"expr": "{application=\"$application_ID\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"largeRead_ops\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "执行器读取hdfs数据情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "HDFS write Rate / s",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 48
},
"id": 28,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "filesystem_usage{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"write_bytes\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "avg(filesystem_usage{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"write_bytes\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "B"
},
{
"expr": "filesystem_usage{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"write_ops\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "C"
},
{
"expr": "avg(filesystem_usage{application=\"$application_Id\",executor_id!~\"driver\",fs_type=\"hdfs\",qty=\"write_ops\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "写入hdfs数据速率",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 58
},
"id": 34,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "当前application下Driver和各Executor内存使用情况",
"fill": 1,
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 3
},
"id": 8,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "jvm_memory_usage{application=\"$application_Id\",job=\"$spark\",mem_type=\"total\",qty=\"used\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{executor_id}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "当前application下Driver和各Executor内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 14
},
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"dimension": "createTime",
"type": "regex"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"druidDS": "bullseye_report_update",
"expr": "count(executor_info{application=\"$application_Id\",executor_id!~\"driver\"})/4",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "当前application可用executor数量",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "application内存使用总量和executor总量",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 59
},
"id": 32,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "当前application下job、stage运行情况",
"fill": 1,
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 4
},
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "DAG_scheduler{application=\"$application_Id\",qty=~\"job\",type=\"job\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "DAG_scheduler{application=\"$application_Id\",qty=~\"activeJobs\",type=\"job\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "B"
},
{
"expr": "DAG_scheduler{application=\"$application_Id\",type=~\"stage\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "当前application下job-stage运行情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "DAG调度消息处理耗时情况",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 18
},
"id": 54,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "DAG_scheduler{application=\"$application_Id\",type=~\"messageProcessingTime\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "DAG调度消息处理耗时情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"fill": 1,
"gridPos": {
"h": 21,
"w": 24,
"x": 0,
"y": 27
},
"id": 52,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "task_info{application=\"$application_Id\",qty=\"committed\",task=\"jvm\"}",
"format": "time_series",
"intervalFactor": 1,
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "task_info{application=\"$application_Id\",qty=\"init\",task=\"jvm\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "B"
},
{
"expr": "task_info{application=\"$application_Id\",qty=\"max\",task=\"jvm\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "C"
},
{
"expr": "task_info{application=\"$application_Id\",qty=\"usage\",task=\"jvm\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "D"
},
{
"expr": "task_info{application=\"$application_Id\",qty=\"used\",task=\"jvm\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "当前application任务执行时内存使用情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "Tasks",
"fill": 1,
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 48
},
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "sum(executor_info{application=\"$application_Id\", qty=\"activeTasks\"}) by (qty)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{qty}}",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
},
{
"expr": "sum(executor_info{application=\"$application_Id\", qty=\"completeTasks\"}) by (qty)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{qty}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "当前application任务执行情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus-qsh-k2",
"description": "各执行器线程池任务执行[活跃、完毕]和内存信息情况",
"fill": 1,
"gridPos": {
"h": 13,
"w": 24,
"x": 0,
"y": 57
},
"id": 50,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"currentAggregator": {
"type": "count"
},
"currentFilter": {
"type": "selector"
},
"currentPostAggregator": {
"fn": "+",
"type": "arithmetic"
},
"currentSelect": {
"dimension": "",
"metric": ""
},
"customGranularity": "minute",
"expr": "executor_info{application=\"$application_Id\",executor_id!~\"driver\",type=\"threadpool\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"limit": 5,
"queryType": "timeseries",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "各执行器线程池任务执行[活跃、完毕]和内存信息情况",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "application下job运行情况",
"type": "row"
}
],
"refresh": "10s",
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {
"text": "spark-sql-context-10-1-144-31",
"value": "spark-sql-context-10-1-144-31"
},
"datasource": "Prometheus-qsh-k2",
"definition": "label_values(job)",
"hide": 0,
"includeAll": false,
"label": "",
"multi": true,
"name": "spark",
"options": [],
"query": "label_values(job)",
"refresh": 2,
"regex": "/spark-.*-context-.*/",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"selected": true,
"text": "application_1568613312062_8621262",
"value": "application_1568613312062_8621262"
},
"datasource": "Prometheus-qsh-k2",
"definition": "label_values({job=~\"${spark:regex}\"},application)",
"hide": 0,
"includeAll": false,
"label": null,
"multi": true,
"name": "application_Id",
"options": [],
"query": "label_values({job=~\"${spark:regex}\"},application)",
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "spark集群监控",
"uid": "NWHhCnVnk",
"version": 57
}
|