From c064fb02a5e8e56641d769f1ac32456b14dce55f Mon Sep 17 00:00:00 2001 From: Ray Slakinski Date: Mon, 26 Apr 2021 09:45:14 -0400 Subject: [PATCH] don't crash on error, just log it and move on. update snapshot dashboard script added --- contrib/grafana/dashboard.json | 201 ++++---------- .../ontvacstats-snapshot.json | 262 ++++++++++++++++++ contrib/update_dashboard/update_dashboard.sh | 7 + main.go | 12 +- 4 files changed, 331 insertions(+), 151 deletions(-) create mode 100644 contrib/update_dashboard/ontvacstats-snapshot.json create mode 100644 contrib/update_dashboard/update_dashboard.sh diff --git a/contrib/grafana/dashboard.json b/contrib/grafana/dashboard.json index c60bc67..3cb73e0 100644 --- a/contrib/grafana/dashboard.json +++ b/contrib/grafana/dashboard.json @@ -21,10 +21,9 @@ { "cacheTimeout": null, "datasource": null, - "description": "Days left until everyone 18+ Is fully vaccinated", + "description": "Days left until everyone 18+ Is partially vaccinated.\n\nThis divides the remaining population without any vaccination by the average doses administered over the past 3 days.", "fieldConfig": { "defaults": { - "custom": {}, "mappings": [ { "id": 0, @@ -34,7 +33,6 @@ "value": "null" } ], - "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ @@ -68,22 +66,24 @@ "maxDataPoints": 100, "options": { "colorMode": "value", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "/^{instance=\"127.0.0.1:8080\", job=\"ontvacstat\"}$/", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.1", "targets": [ { - "expr": "(ontvacstat_eligible_population - ontvacstat_total_doses_administered) / avg_over_time(ontvacstat_previous_day_doses_administered[3d])", + "exemplar": true, + "expr": "((ontvacstat_eligible_population - ontvacstat_total_individuals_fully_vaccinated) - (ontvacstat_total_doses_administered - ontvacstat_total_doses_in_fully_vaccinated_individuals)) / avg_over_time(ontvacstat_previous_day_doses_administered[3d])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -95,123 +95,54 @@ "type": "stat" }, { - "cacheTimeout": null, "datasource": null, - "description": "Current rate of vaccination", "fieldConfig": { "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "#EAB839", + "value": 10000 + }, + { + "color": "green", + "value": 110000 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { "h": 6, - "w": 5, + "w": 6, "x": 5, "y": 0 }, - "id": 9, - "interval": null, + "id": 4, "links": [], - "maxDataPoints": 100, "options": { - "orientation": "horizontal", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], - "fields": "/^{instance=\"127.0.0.1:8080\", job=\"ontvacstat\"}$/", + "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "text": {}, + "textMode": "auto" }, - "pluginVersion": "7.3.4", - "targets": [ - { - "expr": " rate(ontvacstat_previous_day_doses_administered[1d])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Daily Rate of Vaccination", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 10, - "y": 0 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "7.5.1", "targets": [ { "expr": "ontvacstat_previous_day_doses_administered", @@ -222,59 +153,31 @@ "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Doses Administered/day", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Doses Administered Yesterday", + "type": "stat" }, { "datasource": null, + "description": "% of Ontario Population partially or Fully Vaccinated.", "fieldConfig": { "defaults": { - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 75 } ] }, @@ -284,14 +187,14 @@ }, "gridPos": { "h": 8, - "w": 18, + "w": 11, "x": 0, "y": 6 }, "id": 2, "links": [], "options": { - "displayMode": "basic", + "displayMode": "lcd", "orientation": "horizontal", "reduceOptions": { "calcs": [ @@ -300,24 +203,27 @@ "fields": "", "values": false }, - "showUnfilled": true + "showUnfilled": true, + "text": {} }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.1", "targets": [ { - "expr": "ontvacstat_total_doses_administered / (ontvacstat_eligible_population - ontvacstat_total_doses_in_fully_vaccinated_individuals) * 100", + "exemplar": true, + "expr": "(ontvacstat_total_doses_administered - ontvacstat_total_doses_in_fully_vaccinated_individuals) / (ontvacstat_eligible_population - ontvacstat_total_individuals_fully_vaccinated) * 100", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "% Partially Vaccinated", + "legendFormat": "Partially Vaccinated", "refId": "A" }, { - "expr": "ontvacstat_total_doses_in_fully_vaccinated_individuals / ontvacstat_eligible_population * 100", + "exemplar": true, + "expr": "ontvacstat_total_individuals_fully_vaccinated / ontvacstat_eligible_population * 100", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "% Fully Vaccinated", + "legendFormat": "Fully Vaccinated", "refId": "B" } ], @@ -327,14 +233,15 @@ "type": "bargauge" } ], - "schemaVersion": 26, + "refresh": false, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -360,5 +267,5 @@ "timezone": "", "title": "Ontario Vaccination Stats", "uid": "vhnVv3lMk", - "version": 8 + "version": 17 } \ No newline at end of file diff --git a/contrib/update_dashboard/ontvacstats-snapshot.json b/contrib/update_dashboard/ontvacstats-snapshot.json new file mode 100644 index 0000000..dacfb21 --- /dev/null +++ b/contrib/update_dashboard/ontvacstats-snapshot.json @@ -0,0 +1,262 @@ +{ + "dashboard": { + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": 25, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "datasource": null, + "description": "Days left until everyone 18+ Is partially vaccinated.\n\nThis divides the remaining population without any vaccination by theaverage doses administered over the past 3 days.", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 8, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^{instance=\"127.0.0.1:8080\", job=\"ontvacstat\"}$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.1", + "targets": [ + { + "exemplar": true, + "expr": "((ontvacstat_eligible_population - ontvacstat_total_individuals_fully_vaccinated) - (ontvacstat_total_doses_administered - ontvacstat_total_doses_in_fully_vaccinated_individuals)) / avg_over_time(ontvacstat_previous_day_doses_administered[3d])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "~ Days Left", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 10000 + }, + { + "color": "green", + "value": 110000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 5, + "y": 0 + }, + "id": 4, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.1", + "targets": [ + { + "expr": "ontvacstat_previous_day_doses_administered", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Previous Day Doses Administered", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Doses Administered Yesterday", + "type": "stat" + }, + { + "datasource": null, + "description": "% of Ontario Population partially or Fully Vaccinated.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 75 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 6 + }, + "id": 2, + "links": [], + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "7.5.1", + "targets": [ + { + "exemplar": true, + "expr": "(ontvacstat_total_doses_administered - ontvacstat_total_doses_in_fully_vaccinated_individuals) / (ontvacstat_eligible_population - ontvacstat_total_individuals_fully_vaccinated) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Partially Vaccinated", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ontvacstat_total_individuals_fully_vaccinated / ontvacstat_eligible_population * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Fully Vaccinated", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Vaccinations & Fully Vaccinated", + "type": "bargauge" + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ontario Vaccination Stats", + "uid": "vhnVv3lMk", + "version": 17 + }, + "expires": 86400, + "key": "ontvacstats" +} \ No newline at end of file diff --git a/contrib/update_dashboard/update_dashboard.sh b/contrib/update_dashboard/update_dashboard.sh new file mode 100644 index 0000000..b5d4c7a --- /dev/null +++ b/contrib/update_dashboard/update_dashboard.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +export API_KEY="XXXXX" + +curl --keepalive-time 60 -X DELETE -s 'http://localhost:3000/api/snapshots/ontvacstats' -H "Content-Type: application/json" -H "Authorization: Bearer $API_KEY" + +curl --keepalive-time 60 -s 'http://localhost:3000/api/snapshots' -H "Content-Type: application/json" -H "Authorization: Bearer $API_KEY" --data @ontvacstats-snapshot.json \ No newline at end of file diff --git a/main.go b/main.go index 7dc6ad0..134a05d 100644 --- a/main.go +++ b/main.go @@ -65,21 +65,25 @@ func init() { } func updateMetrics() { - data_url := "https://data.ontario.ca/api/3/action/datastore_search?sort=report_date+desc&limit=1&resource_id=8a89caa9-511c-4568-af89-7f2174b4378c" + data_url := "https://data.ontario.ca/api/3/action/datasstore_search?sort=report_date+desc&limit=1&resource_id=8a89caa9-511c-4568-af89-7f2174b4378c" eligiblePopulation := int64(11971129) resp, getErr := http.Get(data_url) if getErr != nil { - log.Fatalln(getErr) + log.Println(getErr) + return } body, readErr := ioutil.ReadAll(resp.Body) if readErr != nil { - log.Fatalln(readErr) + log.Println(readErr) + return } data := OntCovidData{} err := json.Unmarshal(body, &data) if err != nil { - fmt.Println(err) + log.Println(err) + return } + totalDosesAdministeredString := strings.Replace(data.Result.Records[0].TotalDosesAdministered, ",", "", -1) totalDosesAdministered, _ := strconv.ParseInt(totalDosesAdministeredString, 0, 64) records.TotalDosesAdministered().Set(float64(totalDosesAdministered))