Additional mempool metrics (#2878)

* Rename tx downloader & verifier metrics

* Add version to mempool metrics

* Add new metrics

* Make sure mempool gauges are zeroed when instances are dropped

* Updated mempool grafana dashboard

* Removed transaction verification dashboard; moved to mempool

* Update mempool dashboard

* Add reason to error labels in mempool dashboard

* Rename some metrics per review

Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>
This commit is contained in:
Conrado Gouvea 2021-10-19 14:10:25 -03:00 committed by GitHub
parent 4b8b65a627
commit a5d1467624
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 247 additions and 667 deletions

View File

@ -8,6 +8,12 @@
"hide": true, "hide": true,
"iconColor": "rgba(0, 211, 255, 1)", "iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts", "name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard" "type": "dashboard"
} }
] ]
@ -16,8 +22,8 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 9, "id": 15,
"iteration": 1634083651443, "iteration": 1634239984015,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -59,7 +65,7 @@
"fillGradient": 1, "fillGradient": 1,
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 6, "w": 24,
"x": 0, "x": 0,
"y": 0 "y": 0
}, },
@ -82,18 +88,11 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeat": "job", "repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [ "seriesOverrides": [
{ {
"$$hashKey": "object:232", "$$hashKey": "object:232",
@ -104,6 +103,10 @@
"$$hashKey": "object:239", "$$hashKey": "object:239",
"alias": "serialized bytes", "alias": "serialized bytes",
"yaxis": 2 "yaxis": 2
},
{
"alias": "rejected serialized bytes",
"yaxis": 2
} }
], ],
"spaceLength": 10, "spaceLength": 10,
@ -124,6 +127,14 @@
"interval": "", "interval": "",
"legendFormat": " serialized bytes", "legendFormat": " serialized bytes",
"refId": "Serialized Bytes" "refId": "Serialized Bytes"
},
{
"exemplar": true,
"expr": "mempool_currently_queued_transactions{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "queued transactions",
"refId": "Queued Transactions"
} }
], ],
"thresholds": [], "thresholds": [],
@ -209,160 +220,9 @@
"fillGradient": 1, "fillGradient": 1,
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 6, "w": 24,
"x": 6, "x": 0,
"y": 0 "y": 9
},
"hiddenSeries": false,
"id": 8,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1634083651443,
"repeatPanelId": 2,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet-tmp",
"value": "zebrad-mainnet-tmp"
}
},
"seriesOverrides": [
{
"$$hashKey": "object:232",
"alias": "transactions",
"yaxis": 1
},
{
"$$hashKey": "object:239",
"alias": "serialized bytes",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "zcash_mempool_size_transactions{job=\"$job\"}",
"interval": "",
"legendFormat": "transactions",
"refId": "Transactions"
},
{
"exemplar": true,
"expr": "zcash_mempool_size_bytes{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": " serialized bytes",
"refId": "Serialized Bytes"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Mempool Storage - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "none",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"decimals": null,
"format": "decbytes",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "Transactions"
},
"properties": [
{
"id": "displayName",
"value": "transactions"
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "Serialized Bytes"
},
"properties": [
{
"id": "displayName",
"value": "serialized bytes"
}
]
}
]
},
"fill": 1,
"fillGradient": 1,
"gridPos": {
"h": 9,
"w": 6,
"x": 12,
"y": 0
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 9, "id": 9,
@ -383,19 +243,10 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1634083651443,
"repeatPanelId": 2,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [ "seriesOverrides": [
{ {
"$$hashKey": "object:232", "$$hashKey": "object:232",
@ -406,6 +257,10 @@
"$$hashKey": "object:239", "$$hashKey": "object:239",
"alias": "serialized bytes", "alias": "serialized bytes",
"yaxis": 2 "yaxis": 2
},
{
"alias": "rejected serialized bytes",
"yaxis": 2
} }
], ],
"spaceLength": 10, "spaceLength": 10,
@ -414,25 +269,26 @@
"targets": [ "targets": [
{ {
"exemplar": true, "exemplar": true,
"expr": "zcash_mempool_size_transactions{job=\"$job\"}", "expr": "mempool_rejected_transaction_ids{job=\"$job\"}",
"hide": false,
"interval": "", "interval": "",
"legendFormat": "transactions", "legendFormat": "rejected transactions",
"refId": "Transactions" "refId": "Rejected Transactions IDs"
}, },
{ {
"exemplar": true, "exemplar": true,
"expr": "zcash_mempool_size_bytes{job=\"$job\"}", "expr": "mempool_rejected_transaction_ids_bytes{job=\"$job\"}",
"hide": false, "hide": false,
"interval": "", "interval": "",
"legendFormat": " serialized bytes", "legendFormat": "rejected serialized bytes",
"refId": "Serialized Bytes" "refId": "Rejected Serialized TXID Bytes"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Mempool Storage - $job", "title": "Mempool Rejected Storage - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -478,168 +334,13 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "Transactions"
},
"properties": [
{
"id": "displayName",
"value": "transactions"
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "Serialized Bytes"
},
"properties": [
{
"id": "displayName",
"value": "serialized bytes"
}
]
}
]
},
"fill": 1,
"fillGradient": 1,
"gridPos": {
"h": 9,
"w": 6,
"x": 18,
"y": 0
},
"hiddenSeries": false,
"id": 10,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatIteration": 1634083651443,
"repeatPanelId": 2,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet-tmp",
"value": "zebrad-testnet-tmp"
}
},
"seriesOverrides": [
{
"$$hashKey": "object:232",
"alias": "transactions",
"yaxis": 1
},
{
"$$hashKey": "object:239",
"alias": "serialized bytes",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "zcash_mempool_size_transactions{job=\"$job\"}",
"interval": "",
"legendFormat": "transactions",
"refId": "Transactions"
},
{
"exemplar": true,
"expr": "zcash_mempool_size_bytes{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": " serialized bytes",
"refId": "Serialized Bytes"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Mempool Storage - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:65",
"format": "none",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:66",
"decimals": null,
"format": "decbytes",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 6, "w": 24,
"x": 0, "x": 0,
"y": 9 "y": 18
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 7, "id": 7,
@ -659,18 +360,11 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeat": "job", "repeat": "job",
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-mainnet",
"value": "zebrad-mainnet"
}
},
"seriesOverrides": [], "seriesOverrides": [],
"spaceLength": 10, "spaceLength": 10,
"stack": false, "stack": false,
@ -742,25 +436,23 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "description": "",
"defaults": {},
"overrides": []
},
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 1,
"gridPos": { "gridPos": {
"h": 8, "h": 9,
"w": 6, "w": 24,
"x": 6, "x": 0,
"y": 9 "y": 26
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 11, "id": 8,
"legend": { "legend": {
"avg": false, "avg": false,
"current": false, "current": false,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": false,
"show": true, "show": true,
"total": false, "total": false,
"values": false "values": false
@ -772,46 +464,87 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1634083651443, "seriesOverrides": [
"repeatPanelId": 7, {
"scopedVars": { "$$hashKey": "object:232",
"job": { "alias": "transactions",
"selected": false, "yaxis": 1
"text": "zebrad-mainnet-tmp",
"value": "zebrad-mainnet-tmp"
}
}, },
"seriesOverrides": [], {
"$$hashKey": "object:239",
"alias": "serialized bytes",
"yaxis": 2
}
],
"spaceLength": 10, "spaceLength": 10,
"stack": false, "stack": false,
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"exemplar": true, "exemplar": true,
"expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1m])", "expr": "mempool_queued_transactions_total{job=\"$job\"}",
"hide": false, "hide": false,
"instant": false,
"interval": "", "interval": "",
"legendFormat": "download", "legendFormat": "queued",
"refId": "B" "refId": "Queued"
}, },
{ {
"exemplar": true, "exemplar": true,
"expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])", "expr": "mempool_downloaded_transactions_total{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "commit", "legendFormat": "downloaded",
"refId": "A" "refId": "Downloaded"
},
{
"exemplar": true,
"expr": "mempool_pushed_transactions_total{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "pushed",
"refId": "Pushed"
},
{
"exemplar": true,
"expr": "mempool_verified_transactions_total{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "verified",
"refId": "Verified"
},
{
"exemplar": true,
"expr": "mempool_cancelled_verify_tasks_total{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "cancelled",
"refId": "Cancelled"
},
{
"exemplar": true,
"expr": "mempool_failed_verify_tasks_total{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "failed - {{reason}}",
"refId": "Failed"
},
{
"exemplar": true,
"expr": "mempool_gossiped_transactions_total{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "gossiped",
"refId": "Gossiped"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Rates", "title": "Transaction Downloader and Verifier, Gossiper - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -827,18 +560,19 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:80", "$$hashKey": "object:65",
"format": "short", "format": "none",
"label": null, "label": "",
"logBase": 1, "logBase": 1,
"max": null, "max": null,
"min": null, "min": null,
"show": true "show": true
}, },
{ {
"$$hashKey": "object:81", "$$hashKey": "object:66",
"format": "short", "decimals": null,
"label": null, "format": "decbytes",
"label": "",
"logBase": 1, "logBase": 1,
"max": null, "max": null,
"min": null, "min": null,
@ -856,17 +590,13 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 0,
"gridPos": { "gridPos": {
"h": 8, "h": 7,
"w": 6, "w": 24,
"x": 12, "x": 0,
"y": 9 "y": 35
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 12, "id": 12,
@ -886,19 +616,11 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1634083651443, "repeatDirection": "h",
"repeatPanelId": 7,
"scopedVars": {
"job": {
"selected": false,
"text": "zebrad-testnet",
"value": "zebrad-testnet"
}
},
"seriesOverrides": [], "seriesOverrides": [],
"spaceLength": 10, "spaceLength": 10,
"stack": false, "stack": false,
@ -906,26 +628,31 @@
"targets": [ "targets": [
{ {
"exemplar": true, "exemplar": true,
"expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1m])", "expr": "rate(mempool_downloaded_transactions_total{job=\"$job\"}[1m]) * 60",
"hide": false,
"instant": false,
"interval": "", "interval": "",
"legendFormat": "download", "legendFormat": "downloaded per min",
"refId": "B" "refId": "Downloaded"
}, },
{ {
"exemplar": true, "exemplar": true,
"expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])", "expr": "rate(mempool_verified_transactions_total{job=\"$job\"}[1m]) * 60",
"interval": "", "interval": "",
"legendFormat": "commit", "legendFormat": "verified per min",
"refId": "A" "refId": "Verified"
},
{
"exemplar": true,
"expr": "rate(mempool_queued_transactions_total{job=\"$job\"}[1m]) * 60",
"interval": "",
"legendFormat": "queued per min",
"refId": "Queued"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Rates", "title": "Transaction Downloader and Verifier (Rates) - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -941,7 +668,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:80", "$$hashKey": "object:1174",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -950,7 +677,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:81", "$$hashKey": "object:1175",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -970,25 +697,23 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "description": "",
"defaults": {},
"overrides": []
},
"fill": 1, "fill": 1,
"fillGradient": 0, "fillGradient": 1,
"gridPos": { "gridPos": {
"h": 8, "h": 9,
"w": 6, "w": 24,
"x": 18, "x": 0,
"y": 9 "y": 42
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 13, "id": 10,
"legend": { "legend": {
"avg": false, "avg": false,
"current": false, "current": false,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": false,
"show": true, "show": true,
"total": false, "total": false,
"values": false "values": false
@ -1000,46 +725,40 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.5.7", "pluginVersion": "8.1.2",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1634083651443, "seriesOverrides": [
"repeatPanelId": 7, {
"scopedVars": { "$$hashKey": "object:232",
"job": { "alias": "transactions",
"selected": false, "yaxis": 1
"text": "zebrad-testnet-tmp",
"value": "zebrad-testnet-tmp"
}
}, },
"seriesOverrides": [], {
"$$hashKey": "object:239",
"alias": "serialized bytes",
"yaxis": 2
}
],
"spaceLength": 10, "spaceLength": 10,
"stack": false, "stack": false,
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"exemplar": true, "exemplar": true,
"expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1m])", "expr": "sum by (version) (mempool_downloaded_transactions_total{job=\"$job\"})",
"hide": false, "hide": false,
"instant": false,
"interval": "", "interval": "",
"legendFormat": "download", "legendFormat": "{{version}}",
"refId": "B" "refId": "Downloaded"
},
{
"exemplar": true,
"expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1m])",
"interval": "",
"legendFormat": "commit",
"refId": "A"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Rates", "title": "Downloaded Txs by Version - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -1055,18 +774,19 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:80", "$$hashKey": "object:65",
"format": "short", "format": "none",
"label": null, "label": "",
"logBase": 1, "logBase": 1,
"max": null, "max": null,
"min": null, "min": null,
"show": true "show": true
}, },
{ {
"$$hashKey": "object:81", "$$hashKey": "object:66",
"format": "short", "decimals": null,
"label": null, "format": "decbytes",
"label": "",
"logBase": 1, "logBase": 1,
"max": null, "max": null,
"min": null, "min": null,
@ -1080,7 +800,7 @@
} }
], ],
"refresh": "5s", "refresh": "5s",
"schemaVersion": 27, "schemaVersion": 30,
"style": "dark", "style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
@ -1115,7 +835,6 @@
"skipUrlSync": false, "skipUrlSync": false,
"sort": 1, "sort": 1,
"tagValuesQuery": "", "tagValuesQuery": "",
"tags": [],
"tagsQuery": "", "tagsQuery": "",
"type": "query", "type": "query",
"useTags": false "useTags": false
@ -1129,6 +848,6 @@
"timepicker": {}, "timepicker": {},
"timezone": "", "timezone": "",
"title": "mempool", "title": "mempool",
"uid": "WFuaZ3Dnz", "uid": "wVXGE6v7z",
"version": 21 "version": 8
} }

View File

@ -1,207 +0,0 @@
{
"__inputs": [
{
"name": "DS_PROMETHEUS-ZEBRA",
"label": "Prometheus-Zebra",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "8.1.2"
},
{
"type": "panel",
"id": "graph",
"name": "Graph (old)",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": null,
"iteration": 1630092146360,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeatDirection": "h",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "rate(gossip_downloaded_transaction_count{job=\"$job\"}[1m]) * 60",
"interval": "",
"legendFormat": "gossip_downloaded_transaction_count per min",
"refId": "C"
},
{
"exemplar": true,
"expr": "rate(gossip_verified_transaction_count{job=\"$job\"}[1m]) * 60",
"interval": "",
"legendFormat": "gossip_verified_transaction_count per min",
"refId": "D"
},
{
"exemplar": true,
"expr": "gossip_queued_transaction_count{job=\"$job\"}",
"interval": "",
"legendFormat": "gossip_queued_transaction_count",
"refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Transaction Verifier Gossip Count - $job",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "5s",
"schemaVersion": 30,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {},
"datasource": "${DS_PROMETHEUS-ZEBRA}",
"definition": "label_values(zcash_chain_verified_block_height, job)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "job",
"options": [],
"query": {
"query": "label_values(zcash_chain_verified_block_height, job)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "transaction verification",
"uid": "oBEHvS4nz",
"version": 2
}

View File

@ -387,6 +387,7 @@ impl Service<Request> for Mempool {
} }
} }
Err((txid, e)) => { Err((txid, e)) => {
metrics::counter!("mempool.failed.verify.tasks.total", 1, "reason" => e.to_string());
storage.reject_if_needed(txid, e); storage.reject_if_needed(txid, e);
// TODO: should we also log the result? // TODO: should we also log the result?
} }

View File

@ -33,7 +33,7 @@ use futures::{
ready, ready,
stream::{FuturesUnordered, Stream}, stream::{FuturesUnordered, Stream},
}; };
use pin_project::pin_project; use pin_project::{pin_project, pinned_drop};
use thiserror::Error; use thiserror::Error;
use tokio::{sync::oneshot, task::JoinHandle}; use tokio::{sync::oneshot, task::JoinHandle};
use tower::{Service, ServiceExt}; use tower::{Service, ServiceExt};
@ -141,7 +141,7 @@ impl From<UnminedTx> for Gossip {
} }
/// Represents a [`Stream`] of download and verification tasks. /// Represents a [`Stream`] of download and verification tasks.
#[pin_project] #[pin_project(PinnedDrop)]
#[derive(Debug)] #[derive(Debug)]
pub struct Downloads<ZN, ZV, ZS> pub struct Downloads<ZN, ZV, ZS>
where where
@ -312,11 +312,19 @@ where
_ => unreachable!("wrong response to transaction request"), _ => unreachable!("wrong response to transaction request"),
}; };
metrics::counter!("gossip.downloaded.transaction.count", 1); metrics::counter!(
"mempool.downloaded.transactions.total",
1,
"version" => format!("{}",tx.transaction.version()),
);
tx tx
} }
Gossip::Tx(tx) => { Gossip::Tx(tx) => {
metrics::counter!("gossip.pushed.transaction.count", 1); metrics::counter!(
"mempool.pushed.transactions.total",
1,
"version" => format!("{}",tx.transaction.version()),
);
tx tx
} }
}; };
@ -337,7 +345,11 @@ where
result.map_err(|e| TransactionDownloadVerifyError::Invalid(e.into())) result.map_err(|e| TransactionDownloadVerifyError::Invalid(e.into()))
} }
.map_ok(|tx| { .map_ok(|tx| {
metrics::counter!("gossip.verified.transaction.count", 1); metrics::counter!(
"mempool.verified.transactions.total",
1,
"version" => format!("{}", tx.transaction.transaction.version()),
);
tx tx
}) })
// Tack the hash onto the error so we can remove the cancel handle // Tack the hash onto the error so we can remove the cancel handle
@ -351,7 +363,7 @@ where
tokio::select! { tokio::select! {
_ = &mut cancel_rx => { _ = &mut cancel_rx => {
tracing::trace!("task cancelled prior to completion"); tracing::trace!("task cancelled prior to completion");
metrics::counter!("gossip.cancelled.count", 1); metrics::counter!("mempool.cancelled.verify.tasks.total", 1);
Err((TransactionDownloadVerifyError::Cancelled, txid)) Err((TransactionDownloadVerifyError::Cancelled, txid))
} }
verification = fut => verification, verification = fut => verification,
@ -370,7 +382,11 @@ where
?MAX_INBOUND_CONCURRENCY, ?MAX_INBOUND_CONCURRENCY,
"queued transaction hash for download" "queued transaction hash for download"
); );
metrics::gauge!("gossip.queued.transaction.count", self.pending.len() as _); metrics::gauge!(
"mempool.currently.queued.transactions",
self.pending.len() as _
);
metrics::counter!("mempool.queued.transactions.total", 1);
Ok(()) Ok(())
} }
@ -407,6 +423,10 @@ where
} }
assert!(self.pending.is_empty()); assert!(self.pending.is_empty());
assert!(self.cancel_handles.is_empty()); assert!(self.cancel_handles.is_empty());
metrics::gauge!(
"mempool.currently.queued.transactions",
self.pending.len() as _
);
} }
/// Get the number of currently in-flight download tasks. /// Get the number of currently in-flight download tasks.
@ -438,3 +458,18 @@ where
Ok(()) Ok(())
} }
} }
#[pinned_drop]
impl<ZN, ZV, ZS> PinnedDrop for Downloads<ZN, ZV, ZS>
where
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + 'static,
ZN::Future: Send,
ZV: Service<tx::Request, Response = tx::Response, Error = BoxError> + Send + Clone + 'static,
ZV::Future: Send,
ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
ZS::Future: Send,
{
fn drop(self: Pin<&mut Self>) {
metrics::gauge!("mempool.currently.queued.transactions", 0 as _);
}
}

View File

@ -38,11 +38,14 @@ where
receiver.changed().await?; receiver.changed().await?;
let txs = receiver.borrow().clone(); let txs = receiver.borrow().clone();
let txs_len = txs.len();
let request = zn::Request::AdvertiseTransactionIds(txs); let request = zn::Request::AdvertiseTransactionIds(txs);
info!(?request, "sending mempool transaction broadcast"); info!(?request, "sending mempool transaction broadcast");
// broadcast requests don't return errors, and we'd just want to ignore them anyway // broadcast requests don't return errors, and we'd just want to ignore them anyway
let _ = broadcast_network.ready_and().await?.call(request).await; let _ = broadcast_network.ready_and().await?.call(request).await;
metrics::counter!("mempool.gossiped.transactions.total", txs_len as _);
} }
} }

View File

@ -7,7 +7,10 @@
//! [`Storage`] does not expose a service so it can only be used by other code directly. //! [`Storage`] does not expose a service so it can only be used by other code directly.
//! Only code inside the [`crate::components::mempool`] module has access to it. //! Only code inside the [`crate::components::mempool`] module has access to it.
use std::collections::{HashMap, HashSet}; use std::{
collections::{HashMap, HashSet},
mem::size_of,
};
use thiserror::Error; use thiserror::Error;
@ -126,6 +129,12 @@ pub struct Storage {
HashMap<SameEffectsChainRejectionError, HashSet<transaction::Hash>>, HashMap<SameEffectsChainRejectionError, HashSet<transaction::Hash>>,
} }
impl Drop for Storage {
fn drop(&mut self) {
self.clear();
}
}
impl Storage { impl Storage {
/// Insert a [`VerifiedUnminedTx`] into the mempool, caching any rejections. /// Insert a [`VerifiedUnminedTx`] into the mempool, caching any rejections.
/// ///
@ -231,12 +240,14 @@ impl Storage {
self.tip_rejected_exact.clear(); self.tip_rejected_exact.clear();
self.tip_rejected_same_effects.clear(); self.tip_rejected_same_effects.clear();
self.chain_rejected_same_effects.clear(); self.chain_rejected_same_effects.clear();
self.update_rejected_metrics();
} }
/// Clears rejections that only apply to the current tip. /// Clears rejections that only apply to the current tip.
pub fn clear_tip_rejections(&mut self) { pub fn clear_tip_rejections(&mut self) {
self.tip_rejected_exact.clear(); self.tip_rejected_exact.clear();
self.tip_rejected_same_effects.clear(); self.tip_rejected_same_effects.clear();
self.update_rejected_metrics();
} }
/// Clears rejections that only apply to the current tip. /// Clears rejections that only apply to the current tip.
@ -258,6 +269,7 @@ impl Storage {
map.clear(); map.clear();
} }
} }
self.update_rejected_metrics();
} }
/// Returns the set of [`UnminedTxId`]s in the mempool. /// Returns the set of [`UnminedTxId`]s in the mempool.
@ -454,4 +466,21 @@ impl Storage {
} }
Ok(()) Ok(())
} }
/// Update metrics related to the rejected lists.
///
/// Must be called every time the rejected lists change.
fn update_rejected_metrics(&self) {
metrics::gauge!(
"mempool.rejected.transaction.ids",
self.rejected_transaction_count() as _
);
// This is just an approximation.
// TODO: make it more accurate #2869
let item_size = size_of::<(transaction::Hash, SameEffectsTipRejectionError)>();
metrics::gauge!(
"mempool.rejected.transaction.ids.bytes",
(self.rejected_transaction_count() * item_size) as _
);
}
} }