From 409f02194877495f2c55729ee4744d0aed884161 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:29:23 -0500 Subject: [PATCH 1/7] dist/spec: move access.json dashboard to metrics-access package. --- dist/package/openSUSE-release-tools.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 0ef8db51..318d4821 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -493,6 +493,7 @@ fi %{_bindir}/osrt-metrics %{_datadir}/%{source_dir}/metrics %exclude %{_datadir}/%{source_dir}/metrics/access +%exclude %{_datadir}/%{source_dir}/metrics/grafana/access.json %{_datadir}/%{source_dir}/metrics.py %{_datadir}/%{source_dir}/metrics_release.py # To avoid adding grafana as BuildRequires since it does not live in same repo. @@ -512,6 +513,7 @@ fi %{_bindir}/osrt-metrics-access-aggregate %{_bindir}/osrt-metrics-access-ingest %{_datadir}/%{source_dir}/metrics/access +%{_datadir}/%{source_dir}/metrics/grafana/access.json %{_unitdir}/osrt-metrics-access.service %{_unitdir}/osrt-metrics-access.timer From 625fa382fa3ab5833f8d1a2d74cb99b9fd02852b Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:30:20 -0500 Subject: [PATCH 2/7] metrics/access/aggregate: skip the current day since the logs are incomplete. --- metrics/access/aggregate.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/access/aggregate.php b/metrics/access/aggregate.php index f09ff4c0..e184e435 100755 --- a/metrics/access/aggregate.php +++ b/metrics/access/aggregate.php @@ -14,6 +14,8 @@ const IPV6_PREFIX = 'ipv6.'; const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0]|tumbleweed)$/'; $begin = new DateTime(); +// Skip the current day since the logs are incomplete and not compressed yet. +$begin->sub(date_interval_create_from_date_string('1 day')); $source_map = [ 'ipv4' => [ '2010-01-03' => false, From 2e93b6c3bd6fe8123a6135e2be0bd825f83fdcd7 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:32:43 -0500 Subject: [PATCH 3/7] metrics/access/aggregate: remove left over sprintf() from PONTIFEX lines. The directory was broken out from PONTIFEX and LANGLEY which made sprintf() no longer necessary. Still works, but a waste. --- metrics/access/aggregate.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/access/aggregate.php b/metrics/access/aggregate.php index e184e435..4d45d2b6 100755 --- a/metrics/access/aggregate.php +++ b/metrics/access/aggregate.php @@ -22,13 +22,13 @@ $source_map = [ '2014-04-14' => sprintf(LANGLEY, 2) . '/' . VHOST, '2017-12-04' => sprintf(LANGLEY, 3) . '/' . VHOST, // 2017-12-05 has bad permissions on langley and is still on origin. - $begin->format('Y-m-d') => sprintf(PONTIFEX) . '/' . VHOST, + $begin->format('Y-m-d') => PONTIFEX . '/' . VHOST, 'filename' => FILENAME, ], 'ipv6' => [ '2012-12-31' => false, '2017-12-04' => sprintf(LANGLEY, 3) . '/' . IPV6_PREFIX . VHOST, - $begin->format('Y-m-d') => sprintf(PONTIFEX) . '/' . IPV6_PREFIX . VHOST, + $begin->format('Y-m-d') => PONTIFEX . '/' . IPV6_PREFIX . VHOST, 'filename' => IPV6_PREFIX . FILENAME, ], ]; From 138facc9610a9d9dd8953d16a05950f58c028cca Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:34:18 -0500 Subject: [PATCH 4/7] metrics/grafana/access: remove broken datasource templating from export. Unresolved upstream issue that makes the export unusable on a new system. --- metrics/grafana/access.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/metrics/grafana/access.json b/metrics/grafana/access.json index 134e444b..324b7fb3 100644 --- a/metrics/grafana/access.json +++ b/metrics/grafana/access.json @@ -1,7 +1,7 @@ { "__inputs": [ { - "name": "DS_OSRT_ACCESS", + "name": "osrt_access", "label": "osrt_access", "description": "", "type": "datasource", @@ -9,7 +9,7 @@ "pluginName": "InfluxDB" }, { - "name": "DS_OPENSUSE:LEAP:42.2", + "name": "openSUSE:Leap:42.2", "label": "openSUSE:Leap:42.2", "description": "", "type": "datasource", @@ -17,7 +17,7 @@ "pluginName": "InfluxDB" }, { - "name": "DS_OPENSUSE:LEAP:42.3", + "name": "openSUSE:Leap:42.3", "label": "openSUSE:Leap:42.3", "description": "", "type": "datasource", @@ -25,7 +25,7 @@ "pluginName": "InfluxDB" }, { - "name": "DS_OPENSUSE:LEAP:15.0", + "name": "openSUSE:Leap:15.0", "label": "openSUSE:Leap:15.0", "description": "", "type": "datasource", @@ -79,7 +79,7 @@ "type": "dashboard" }, { - "datasource": "${DS_OPENSUSE:LEAP:42.2}", + "datasource": "openSUSE:Leap:42.2", "enable": false, "hide": false, "iconColor": "rgba(255, 96, 96, 1)", @@ -91,7 +91,7 @@ "type": "tags" }, { - "datasource": "${DS_OPENSUSE:LEAP:42.3}", + "datasource": "openSUSE:Leap:42.3", "enable": false, "hide": false, "iconColor": "rgba(255, 96, 96, 1)", @@ -103,7 +103,7 @@ "type": "tags" }, { - "datasource": "${DS_OPENSUSE:LEAP:15.0}", + "datasource": "openSUSE:Leap:15.0", "enable": false, "hide": false, "iconColor": "rgba(255, 96, 96, 1)", @@ -156,7 +156,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_OSRT_ACCESS}", + "datasource": "osrt_access", "fill": 1, "gridPos": { "h": 10, @@ -266,7 +266,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_OSRT_ACCESS}", + "datasource": "osrt_access", "fill": 1, "gridPos": { "h": 10, @@ -376,7 +376,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_OSRT_ACCESS}", + "datasource": "osrt_access", "fill": 1, "gridPos": { "h": 10, @@ -483,7 +483,7 @@ }, { "columns": [], - "datasource": "${DS_OSRT_ACCESS}", + "datasource": "osrt_access", "fontSize": "100%", "gridPos": { "h": 10, @@ -571,7 +571,7 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_OSRT_ACCESS}", + "datasource": "osrt_access", "fill": 1, "gridPos": { "h": 10, From 8a59f4a862496a52e9c61321e9e458f87a35fb4c Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:35:04 -0500 Subject: [PATCH 5/7] metrics/access/aggregate: change PONTIFEX base URL based on poo#36862. --- metrics/access/aggregate.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/access/aggregate.php b/metrics/access/aggregate.php index 4d45d2b6..2fb44d82 100755 --- a/metrics/access/aggregate.php +++ b/metrics/access/aggregate.php @@ -6,7 +6,7 @@ use InfluxDB\Database; $CACHE_DIR = $_SERVER['HOME'] . '/.cache/openSUSE-release-tools-access'; const PROTOCOLS = ['ipv4', 'ipv6']; -const PONTIFEX = 'http://downloadlogs.infra.opensuse.org'; +const PONTIFEX = 'http://pontifex.infra.opensuse.org/logs'; const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de'; const VHOST = 'download.opensuse.org'; const FILENAME = 'download.opensuse.org-%s-access_log.xz'; From 2ab5065f93d25383a6202a29bd149874a4441628 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 16:45:55 -0500 Subject: [PATCH 6/7] metrics/grafana/access: add explanation about loss of custom log format. --- metrics/grafana/access.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/metrics/grafana/access.json b/metrics/grafana/access.json index 324b7fb3..4bca5062 100644 --- a/metrics/grafana/access.json +++ b/metrics/grafana/access.json @@ -125,9 +125,9 @@ "links": [], "panels": [ { - "content": "These metrics were generated by parsing the Apache access logs of [download.opensuse.org](http://download.opensuse.org/).\n\n- A unique identifier (UUID) is included in logs when zypper includes it in the request. Multiple requests by the same UUID within the given aggregation frequency are counted as a single unique user.\n- The target product is determined by looking for specially formatted path elements representing build repositories for a particular product and the main repositories for that product.", + "content": "These metrics were generated by parsing the Apache access logs of [download.opensuse.org](http://download.opensuse.org/).\n\n- A unique identifier (UUID) is included in logs when zypper includes it in the request. Multiple requests by the same UUID within the given aggregation frequency are counted as a single unique user.\n- The target product is determined by looking for specially formatted path elements representing build repositories for a particular product and the main repositories for that product.\n\n__The logs are entirely invalid, due to a config mistake, from _2017-12-07_ to _2018-03-08_ and the custom log format, which includes the UUID from zypper, was not present from _2018-03-08_ until _2018-05-29_ which is the cause of the large \"hole\" in the graphs.__", "gridPos": { - "h": 3, + "h": 4, "w": 24, "x": 0, "y": 0 @@ -144,7 +144,7 @@ "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 4 }, "id": 26, "panels": [], @@ -162,7 +162,7 @@ "h": 10, "w": 24, "x": 0, - "y": 4 + "y": 5 }, "id": 10, "legend": { @@ -272,7 +272,7 @@ "h": 10, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 12, "legend": { @@ -382,7 +382,7 @@ "h": 10, "w": 24, "x": 0, - "y": 24 + "y": 25 }, "id": 17, "legend": { @@ -489,7 +489,7 @@ "h": 10, "w": 5, "x": 0, - "y": 34 + "y": 35 }, "id": 15, "links": [], @@ -577,7 +577,7 @@ "h": 10, "w": 19, "x": 5, - "y": 34 + "y": 35 }, "id": 13, "legend": { @@ -689,7 +689,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 45 }, "id": 28, "panels": [ @@ -1049,7 +1049,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 46 }, "id": 30, "panels": [ @@ -1474,7 +1474,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 47 }, "id": 32, "panels": [ From 6df56b1e08f62f460de658ad153d738b428d8e9c Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 20 Jun 2018 22:56:00 -0500 Subject: [PATCH 7/7] metrics/access/ingest: update pattern to handle most recent log format. --- metrics/access/ingest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/access/ingest.php b/metrics/access/ingest.php index 53526b1c..57c1a1a9 100755 --- a/metrics/access/ingest.php +++ b/metrics/access/ingest.php @@ -1,7 +1,7 @@ #!/usr/bin/php