octo
/
Zebra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make block metrics more accurate (#2835) 

* Check for state errors before updating metrics

Previously, the metrics would be updated for some rejected blocks.

* Clarify and expand block verification metrics

Rename checkpoint-specific metrics to clarify their purpose.

Add metrics for:
- finalized blocks on disk
- blocks verified using the full block verifier
  (this metric was previously incorrectly called `zcash_chain_verified_block_height`)

* Update dashboard metric names

Also:
- add some extra block height metrics
- fix a dashboard name

* Add exact block heights to Grafana dashboards

* Add a missing comment

* grafana: use 0 decimals for metrics

Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

* grafana: show the entire height instead of abbreviated

Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

* grafana: show the entire height instead of abbreviated

Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

* Fix typo in metric name

Co-authored-by: Conrado Gouvea <conrado@zfnd.org>

* Move height gauges to the state, so they are correct

If we update height gauges in futures, they can execute out of order,
so the metrics can be incorrect.

Instead:
- move the height gauges to the state, and update them based on the best tip
- move the verified block counts to the state
- continue to include all verified blocks on all non-finalized chains
  (not just the best chain)

* Show exact checkpoint heights in the dashboard

Co-authored-by: Deirdre Connolly <deirdre@zfnd.org>
Co-authored-by: Conrado Gouvea <conrado@zfnd.org>
This commit is contained in:
teor 2021-10-07 23:42:38 +10:00 committed by GitHub
parent 04d2cfb3d0
commit f4118dadda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 221 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
grafana/block_verification.json
View File

@ -15,8 +15,8 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 5, "id": 1,
"iteration": 1614572681383, "iteration": 1633497866311,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -25,10 +25,9 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"decimals": 0,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -42,13 +41,16 @@
"hiddenSeries": false, "hiddenSeries": false,
"id": 4, "id": 4,
"legend": { "legend": {
"alignAsTable": false,
"avg": false, "avg": false,
"current": false, "current": true,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": true,
"show": true, "show": true,
"sideWidth": null,
"total": false, "total": false,
"values": false "values": true
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
@ -57,7 +59,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -75,17 +77,42 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"exemplar": true,
"expr": "zcash_chain_verified_block_height{job=\"$job\"}", "expr": "zcash_chain_verified_block_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "verified block height", "legendFormat": "committed block height",
"refId": "A" "refId": "A"
},
{
"exemplar": true,
"expr": "state_full_verifier_committed_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "full block verifier height",
"refId": "B"
},
{
"exemplar": true,
"expr": "state_checkpoint_finalized_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "checkpoint verifier height",
"refId": "C"
},
{
"exemplar": true,
"expr": "state_finalized_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "finalized block height",
"refId": "D"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Block Height - $job", "title": "Verified Block Height - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -101,7 +128,8 @@
}, },
"yaxes": [ "yaxes": [
{ {
"format": "short", "$$hashKey": "object:84",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -109,7 +137,8 @@
"show": true "show": true
}, },
{ {
"format": "short", "$$hashKey": "object:85",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -128,10 +157,9 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"decimals": 6,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -145,13 +173,16 @@
"hiddenSeries": false, "hiddenSeries": false,
"id": 6, "id": 6,
"legend": { "legend": {
"alignAsTable": false,
"avg": false, "avg": false,
"current": false, "current": true,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": true,
"show": true, "show": true,
"sideWidth": null,
"total": false, "total": false,
"values": false "values": true
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
@ -160,11 +191,11 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1614572681383, "repeatIteration": 1633497866311,
"repeatPanelId": 4, "repeatPanelId": 4,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -179,17 +210,42 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"exemplar": true,
"expr": "zcash_chain_verified_block_height{job=\"$job\"}", "expr": "zcash_chain_verified_block_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "verified block height", "legendFormat": "committed block height",
"refId": "A" "refId": "A"
},
{
"exemplar": true,
"expr": "state_full_verifier_committed_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "full block verifier height",
"refId": "B"
},
{
"exemplar": true,
"expr": "state_checkpoint_finalized_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "checkpoint verifier height",
"refId": "C"
},
{
"exemplar": true,
"expr": "state_finalized_block_height{job=\"$job\"}",
"hide": false,
"interval": "",
"legendFormat": "finalized block height",
"refId": "D"
} }
], ],
"thresholds": [], "thresholds": [],
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Block Height - $job", "title": "Verified Block Height - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -205,6 +261,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:84",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -213,6 +270,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:85",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -233,9 +291,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -264,7 +320,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -311,7 +367,7 @@
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Sync Count - $job", "title": "Block Sync Count - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -327,6 +383,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:167",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -335,6 +392,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:168",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -355,9 +413,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -386,12 +442,12 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatDirection": "h", "repeatDirection": "h",
"repeatIteration": 1614572681383, "repeatIteration": 1633497866311,
"repeatPanelId": 5, "repeatPanelId": 5,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -434,7 +490,7 @@
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Sync Count - $job", "title": "Block Sync Count - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -450,6 +506,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:167",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -458,6 +515,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:168",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -478,9 +536,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -509,7 +565,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -556,7 +612,7 @@
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Gossip Count - $job", "title": "Block Gossip Count - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -572,6 +628,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:252",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -580,6 +637,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:253",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -600,9 +658,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -631,12 +687,12 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatDirection": "h", "repeatDirection": "h",
"repeatIteration": 1614572681383, "repeatIteration": 1633497866311,
"repeatPanelId": 2, "repeatPanelId": 2,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -679,7 +735,7 @@
"timeFrom": null, "timeFrom": null,
"timeRegions": [], "timeRegions": [],
"timeShift": null, "timeShift": null,
"title": "Block Verifier Gossip Count - $job", "title": "Block Gossip Count - $job",
"tooltip": { "tooltip": {
"shared": true, "shared": true,
"sort": 0, "sort": 0,
@ -695,6 +751,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:252",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -703,6 +760,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:253",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -718,7 +776,7 @@
} }
], ],
"refresh": "5s", "refresh": "5s",
"schemaVersion": 26, "schemaVersion": 27,
"style": "dark", "style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
@ -726,7 +784,7 @@
{ {
"allValue": null, "allValue": null,
"current": { "current": {
"selected": false, "selected": true,
"text": [ "text": [
"All" "All"
], ],
@ -736,13 +794,18 @@
}, },
"datasource": "Prometheus-Zebra", "datasource": "Prometheus-Zebra",
"definition": "label_values(zcash_chain_verified_block_height, job)", "definition": "label_values(zcash_chain_verified_block_height, job)",
"description": null,
"error": null,
"hide": 0, "hide": 0,
"includeAll": true, "includeAll": true,
"label": null, "label": null,
"multi": true, "multi": true,
"name": "job", "name": "job",
"options": [], "options": [],
"query": "label_values(zcash_chain_verified_block_height, job)", "query": {
"query": "label_values(zcash_chain_verified_block_height, job)",
"refId": "Prometheus-Zebra-job-Variable-Query"
},
"refresh": 1, "refresh": 1,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
@ -763,5 +826,5 @@
"timezone": "", "timezone": "",
"title": "block verification", "title": "block verification",
"uid": "rO_Cl5tGz", "uid": "rO_Cl5tGz",
"version": 19 "version": 8
} }

115
grafana/checkpoint_verification.json
View File

@ -15,8 +15,8 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 3, "id": 2,
"iteration": 1614572686856, "iteration": 1633574016226,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -25,10 +25,9 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"decimals": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -43,12 +42,13 @@
"id": 2, "id": 2,
"legend": { "legend": {
"avg": false, "avg": false,
"current": false, "current": true,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": true,
"show": true, "show": true,
"total": false, "total": false,
"values": false "values": true
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
@ -57,7 +57,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -100,13 +100,15 @@
"refId": "D" "refId": "D"
}, },
{ {
"expr": "state_finalized_committed_block_height{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_committed_block_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_committed_block_height", "legendFormat": "state_finalized_committed_block_height",
"refId": "E" "refId": "E"
}, },
{ {
"expr": "state_finalized_queued_max_height{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_queued_max_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_queued_max_height", "legendFormat": "state_finalized_queued_max_height",
"refId": "F" "refId": "F"
@ -132,7 +134,8 @@
}, },
"yaxes": [ "yaxes": [
{ {
"format": "short", "$$hashKey": "object:84",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -140,7 +143,8 @@
"show": true "show": true
}, },
{ {
"format": "short", "$$hashKey": "object:85",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -159,10 +163,9 @@
"dashLength": 10, "dashLength": 10,
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"decimals": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -177,12 +180,13 @@
"id": 9, "id": 9,
"legend": { "legend": {
"avg": false, "avg": false,
"current": false, "current": true,
"max": false, "max": false,
"min": false, "min": false,
"rightSide": true,
"show": true, "show": true,
"total": false, "total": false,
"values": false "values": true
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
@ -191,12 +195,12 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatDirection": "h", "repeatDirection": "h",
"repeatIteration": 1614572686856, "repeatIteration": 1633574016226,
"repeatPanelId": 2, "repeatPanelId": 2,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -235,13 +239,15 @@
"refId": "D" "refId": "D"
}, },
{ {
"expr": "state_finalized_committed_block_height{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_committed_block_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_committed_block_height", "legendFormat": "state_finalized_committed_block_height",
"refId": "E" "refId": "E"
}, },
{ {
"expr": "state_finalized_queued_max_height{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_queued_max_height{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_queued_max_height", "legendFormat": "state_finalized_queued_max_height",
"refId": "F" "refId": "F"
@ -267,7 +273,8 @@
}, },
"yaxes": [ "yaxes": [
{ {
"format": "short", "$$hashKey": "object:84",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -275,7 +282,8 @@
"show": true "show": true
}, },
{ {
"format": "short", "$$hashKey": "object:85",
"format": "none",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@ -295,9 +303,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -326,7 +332,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -351,7 +357,8 @@
"refId": "A" "refId": "A"
}, },
{ {
"expr": "rate(state_finalized_committed_block_count[1s])", "exemplar": true,
"expr": "rate(state_checkpoint_committed_block_count[1s])",
"interval": "", "interval": "",
"legendFormat": "state commit rate [1s]", "legendFormat": "state commit rate [1s]",
"refId": "C" "refId": "C"
@ -389,6 +396,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:252",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -397,6 +405,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:253",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -417,9 +426,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -448,12 +455,12 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatDirection": "h", "repeatDirection": "h",
"repeatIteration": 1614572686856, "repeatIteration": 1633574016226,
"repeatPanelId": 8, "repeatPanelId": 8,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -474,7 +481,8 @@
"refId": "A" "refId": "A"
}, },
{ {
"expr": "rate(state_finalized_committed_block_count[1s])", "exemplar": true,
"expr": "rate(state_checkpoint_committed_block_count[1s])",
"interval": "", "interval": "",
"legendFormat": "state commit rate [1s]", "legendFormat": "state commit rate [1s]",
"refId": "C" "refId": "C"
@ -512,6 +520,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:252",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -520,6 +529,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:253",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -540,9 +550,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -571,7 +579,7 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -596,7 +604,8 @@
"refId": "A" "refId": "A"
}, },
{ {
"expr": "state_finalized_queued_block_count{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_queued_block_count{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_queued_block_count", "legendFormat": "state_finalized_queued_block_count",
"refId": "B" "refId": "B"
@ -658,6 +667,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:337",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -666,6 +676,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:338",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -686,9 +697,7 @@
"dashes": false, "dashes": false,
"datasource": null, "datasource": null,
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {},
"custom": {}
},
"overrides": [] "overrides": []
}, },
"fill": 1, "fill": 1,
@ -717,12 +726,12 @@
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "7.2.2", "pluginVersion": "7.5.7",
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatDirection": "h", "repeatDirection": "h",
"repeatIteration": 1614572686856, "repeatIteration": 1633574016226,
"repeatPanelId": 4, "repeatPanelId": 4,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -743,7 +752,8 @@
"refId": "A" "refId": "A"
}, },
{ {
"expr": "state_finalized_queued_block_count{job=\"$job\"}", "exemplar": true,
"expr": "state_checkpoint_queued_block_count{job=\"$job\"}",
"interval": "", "interval": "",
"legendFormat": "state_finalized_queued_block_count", "legendFormat": "state_finalized_queued_block_count",
"refId": "B" "refId": "B"
@ -805,6 +815,7 @@
}, },
"yaxes": [ "yaxes": [
{ {
"$$hashKey": "object:337",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -813,6 +824,7 @@
"show": true "show": true
}, },
{ {
"$$hashKey": "object:338",
"format": "short", "format": "short",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
@ -828,7 +840,7 @@
} }
], ],
"refresh": "5s", "refresh": "5s",
"schemaVersion": 26, "schemaVersion": 27,
"style": "dark", "style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
@ -836,7 +848,7 @@
{ {
"allValue": null, "allValue": null,
"current": { "current": {
"selected": false, "selected": true,
"text": [ "text": [
"All" "All"
], ],
@ -846,13 +858,18 @@
}, },
"datasource": "Prometheus-Zebra", "datasource": "Prometheus-Zebra",
"definition": "label_values(sync_prospective_tips_len, job)", "definition": "label_values(sync_prospective_tips_len, job)",
"description": null,
"error": null,
"hide": 0, "hide": 0,
"includeAll": true, "includeAll": true,
"label": null, "label": null,
"multi": true, "multi": true,
"name": "job", "name": "job",
"options": [], "options": [],
"query": "label_values(sync_prospective_tips_len, job)", "query": {
"query": "label_values(sync_prospective_tips_len, job)",
"refId": "Prometheus-Zebra-job-Variable-Query"
},
"refresh": 1, "refresh": 1,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
@ -886,5 +903,5 @@
"timezone": "", "timezone": "",
"title": "checkpoint verification", "title": "checkpoint verification",
"uid": "o4LmN_OMk", "uid": "o4LmN_OMk",
"version": 42 "version": 6
} }

12
grafana/syncer.json
View File

@ -17,7 +17,7 @@
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 5, "id": 5,
"iteration": 1628659147689, "iteration": 1633496321106,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -172,7 +172,7 @@
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1628659147689, "repeatIteration": 1633496321106,
"repeatPanelId": 2, "repeatPanelId": 2,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -363,7 +363,7 @@
"show": false "show": false
}, },
"pluginVersion": "7.5.7", "pluginVersion": "7.5.7",
"repeatIteration": 1628659147689, "repeatIteration": 1633496321106,
"repeatPanelId": 4, "repeatPanelId": 4,
"reverseYBuckets": false, "reverseYBuckets": false,
"scopedVars": { "scopedVars": {
@ -602,7 +602,7 @@
"pointradius": 2, "pointradius": 2,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
"repeatIteration": 1628659147689, "repeatIteration": 1633496321106,
"repeatPanelId": 7, "repeatPanelId": 7,
"scopedVars": { "scopedVars": {
"job": { "job": {
@ -757,7 +757,7 @@
}, },
"timepicker": {}, "timepicker": {},
"timezone": "", "timezone": "",
"title": "Syncer", "title": "syncer",
"uid": "Sl3h19Gnk", "uid": "Sl3h19Gnk",
"version": 9 "version": 11
} }

5
zebra-consensus/src/block.rs
View File

@ -198,11 +198,6 @@ where
.map_err(VerifyBlockError::Transaction)?; .map_err(VerifyBlockError::Transaction)?;
} }
// Update the metrics after all the validation is finished
tracing::trace!("verified block");
metrics::gauge!("zcash.chain.verified.block.height", height.0 as _);
metrics::counter!("zcash.chain.verified.block.total", 1);
let new_outputs = Arc::try_unwrap(known_utxos) let new_outputs = Arc::try_unwrap(known_utxos)
.expect("all verification tasks using known_utxos are complete"); .expect("all verification tasks using known_utxos are complete");

19
zebra-state/src/service.rs
View File

@ -203,6 +203,20 @@ impl StateService {
self.queued_blocks.prune_by_height(finalized_tip_height); self.queued_blocks.prune_by_height(finalized_tip_height);
let tip_block = self.mem.best_tip_block().map(ChainTipBlock::from); let tip_block = self.mem.best_tip_block().map(ChainTipBlock::from);
// update metrics using the best non-finalized tip
if let Some(tip_block) = tip_block.as_ref() {
metrics::gauge!(
"state.full_verifier.committed.block.height",
tip_block.height.0 as _
);
// This height gauge is updated for both fully verified and checkpoint blocks.
// These updates can't conflict, because the state makes sure that blocks
// are committed in order.
metrics::gauge!("zcash.chain.verified.block.height", tip_block.height.0 as _);
}
self.chain_tip_sender.set_best_non_finalized_tip(tip_block); self.chain_tip_sender.set_best_non_finalized_tip(tip_block);
tracing::trace!("finished processing queued block"); tracing::trace!("finished processing queued block");
@ -260,6 +274,11 @@ impl StateService {
} else { } else {
tracing::trace!(?child_hash, "validating queued child"); tracing::trace!(?child_hash, "validating queued child");
result = self.validate_and_commit(child).map_err(CloneError::from); result = self.validate_and_commit(child).map_err(CloneError::from);
if result.is_ok() {
// Update the metrics if semantic and contextual validation passes
metrics::counter!("state.full_verifier.committed.block.count", 1);
metrics::counter!("zcash.chain.verified.block.total", 1);
}
} }
let _ = rsp_tx.send(result.clone().map(|()| child_hash).map_err(BoxError::from)); let _ = rsp_tx.send(result.clone().map(|()| child_hash).map_err(BoxError::from));

31
zebra-state/src/service/finalized_state.rs
View File

@ -180,9 +180,9 @@ impl FinalizedState {
self.max_queued_height = height.0 as _; self.max_queued_height = height.0 as _;
} }
metrics::gauge!("state.finalized.queued.max.height", self.max_queued_height); metrics::gauge!("state.checkpoint.queued.max.height", self.max_queued_height);
metrics::gauge!( metrics::gauge!(
"state.finalized.queued.block.count", "state.checkpoint.queued.block.count",
self.queued_by_prev_hash.len() as f64 self.queued_by_prev_hash.len() as f64
); );
@ -228,8 +228,6 @@ impl FinalizedState {
finalized: FinalizedBlock, finalized: FinalizedBlock,
source: &str, source: &str,
) -> Result<block::Hash, BoxError> { ) -> Result<block::Hash, BoxError> {
block_precommit_metrics(&finalized);
let finalized_tip_height = self.finalized_tip_height(); let finalized_tip_height = self.finalized_tip_height();
let hash_by_height = self.db.cf_handle("hash_by_height").unwrap(); let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
@ -443,6 +441,9 @@ impl FinalizedState {
// In case of errors, propagate and do not write the batch. // In case of errors, propagate and do not write the batch.
let batch = prepare_commit()?; let batch = prepare_commit()?;
// The block has passed contextual validation, so update the metrics
block_precommit_metrics(&block, hash, height);
let result = self.db.write(batch).map(|()| hash); let result = self.db.write(batch).map(|()| hash);
tracing::trace!(?source, "committed block from"); tracing::trace!(?source, "committed block from");
@ -473,17 +474,23 @@ impl FinalizedState {
let block_result; let block_result;
if result.is_ok() { if result.is_ok() {
metrics::counter!("state.finalized.committed.block.count", 1); metrics::counter!("state.checkpoint.finalized.block.count", 1);
metrics::gauge!( metrics::gauge!(
"state.finalized.committed.block.height", "state.checkpoint.finalized.block.height",
finalized.height.0 as _ finalized.height.0 as _
); );
// This height gauge is updated for both fully verified and checkpoint blocks.
// These updates can't conflict, because the state makes sure that blocks
// are committed in order.
metrics::gauge!("zcash.chain.verified.block.height", finalized.height.0 as _);
metrics::counter!("zcash.chain.verified.block.total", 1);
block_result = Ok(finalized); block_result = Ok(finalized);
} else { } else {
metrics::counter!("state.finalized.error.block.count", 1); metrics::counter!("state.checkpoint.error.block.count", 1);
metrics::gauge!( metrics::gauge!(
"state.finalized.error.block.height", "state.checkpoint.error.block.height",
finalized.height.0 as _ finalized.height.0 as _
); );
@ -677,9 +684,7 @@ impl Drop for FinalizedState {
} }
} }
fn block_precommit_metrics(finalized: &FinalizedBlock) { fn block_precommit_metrics(block: &Block, hash: block::Hash, height: block::Height) {
let (hash, height, block) = (finalized.hash, finalized.height, finalized.block.as_ref());
let transaction_count = block.transactions.len(); let transaction_count = block.transactions.len();
let transparent_prevout_count = block let transparent_prevout_count = block
.transactions .transactions
@ -723,6 +728,10 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) {
orchard_nullifier_count, orchard_nullifier_count,
"preparing to commit finalized block" "preparing to commit finalized block"
); );
metrics::counter!("state.finalized.block.count", 1);
metrics::gauge!("state.finalized.block.height", height.0 as _);
metrics::counter!( metrics::counter!(
"state.finalized.cumulative.transactions", "state.finalized.cumulative.transactions",
transaction_count as u64 transaction_count as u64