yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1588 from jberry-suse/metrics-access-followup

Browse Source 
Metrics access followup for live data final location and small cleanups
This commit is contained in:
Jimmy Berry 2018-06-21 09:41:58 -05:00 committed by GitHub
commit 58f5c91121
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dist/package/openSUSE-release-tools.spec vendored
View File

@ -493,6 +493,7 @@ fi
%{_bindir}/osrt-metrics
%{_datadir}/%{source_dir}/metrics
%exclude %{_datadir}/%{source_dir}/metrics/access
%exclude %{_datadir}/%{source_dir}/metrics/grafana/access.json
%{_datadir}/%{source_dir}/metrics.py
%{_datadir}/%{source_dir}/metrics_release.py
# To avoid adding grafana as BuildRequires since it does not live in same repo.
@ -512,6 +513,7 @@ fi
%{_bindir}/osrt-metrics-access-aggregate
%{_bindir}/osrt-metrics-access-ingest
%{_datadir}/%{source_dir}/metrics/access
%{_datadir}/%{source_dir}/metrics/grafana/access.json
%{_unitdir}/osrt-metrics-access.service
%{_unitdir}/osrt-metrics-access.timer

8
metrics/access/aggregate.php
View File

@ -6,7 +6,7 @@ use InfluxDB\Database;
$CACHE_DIR = $_SERVER['HOME'] . '/.cache/openSUSE-release-tools-access';
const PROTOCOLS = ['ipv4', 'ipv6'];
const PONTIFEX = 'http://downloadlogs.infra.opensuse.org';
const PONTIFEX = 'http://pontifex.infra.opensuse.org/logs';
const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de';
const VHOST = 'download.opensuse.org';
const FILENAME = 'download.opensuse.org-%s-access_log.xz';
@ -14,19 +14,21 @@ const IPV6_PREFIX = 'ipv6.';
const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0]|tumbleweed)$/';
$begin = new DateTime();
// Skip the current day since the logs are incomplete and not compressed yet.
$begin->sub(date_interval_create_from_date_string('1 day'));
$source_map = [
'ipv4' => [
'2010-01-03' => false,
'2014-04-14' => sprintf(LANGLEY, 2) . '/' . VHOST,
'2017-12-04' => sprintf(LANGLEY, 3) . '/' . VHOST,
// 2017-12-05 has bad permissions on langley and is still on origin.
$begin->format('Y-m-d') => sprintf(PONTIFEX) . '/' . VHOST,
$begin->format('Y-m-d') => PONTIFEX . '/' . VHOST,
'filename' => FILENAME,
],
'ipv6' => [
'2012-12-31' => false,
'2017-12-04' => sprintf(LANGLEY, 3) . '/' . IPV6_PREFIX . VHOST,
$begin->format('Y-m-d') => sprintf(PONTIFEX) . '/' . IPV6_PREFIX . VHOST,
$begin->format('Y-m-d') => PONTIFEX . '/' . IPV6_PREFIX . VHOST,
'filename' => IPV6_PREFIX . FILENAME,
],
];

2
metrics/access/ingest.php
View File

@ -1,7 +1,7 @@
#!/usr/bin/php
<?php
const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: "?(\S+-\S+-\S+-\S+-\S+|-)"? "?(dvd|ftp|-)"?)?/';
const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: +"?(\S+-\S+-\S+-\S+-[^\s"]+|-)"? "?(dvd|ftp|-)"?)?/';
const REGEX_PRODUCT = '#/(?:(tumbleweed)|distribution/(?:leap/)?(\d+\.\d+)|openSUSE(?:_|:/)(?:leap(?:_|:/))?(factory|tumbleweed|\d+\.\d+))#i';
$total = 0;

46
metrics/grafana/access.json
View File

@ -1,7 +1,7 @@
{
"__inputs": [
{
"name": "DS_OSRT_ACCESS",
"name": "osrt_access",
"label": "osrt_access",
"description": "",
"type": "datasource",
@ -9,7 +9,7 @@
"pluginName": "InfluxDB"
},
{
"name": "DS_OPENSUSE:LEAP:42.2",
"name": "openSUSE:Leap:42.2",
"label": "openSUSE:Leap:42.2",
"description": "",
"type": "datasource",
@ -17,7 +17,7 @@
"pluginName": "InfluxDB"
},
{
"name": "DS_OPENSUSE:LEAP:42.3",
"name": "openSUSE:Leap:42.3",
"label": "openSUSE:Leap:42.3",
"description": "",
"type": "datasource",
@ -25,7 +25,7 @@
"pluginName": "InfluxDB"
},
{
"name": "DS_OPENSUSE:LEAP:15.0",
"name": "openSUSE:Leap:15.0",
"label": "openSUSE:Leap:15.0",
"description": "",
"type": "datasource",
@ -79,7 +79,7 @@
"type": "dashboard"
},
{
"datasource": "${DS_OPENSUSE:LEAP:42.2}",
"datasource": "openSUSE:Leap:42.2",
"enable": false,
"hide": false,
"iconColor": "rgba(255, 96, 96, 1)",
@ -91,7 +91,7 @@
"type": "tags"
},
{
"datasource": "${DS_OPENSUSE:LEAP:42.3}",
"datasource": "openSUSE:Leap:42.3",
"enable": false,
"hide": false,
"iconColor": "rgba(255, 96, 96, 1)",
@ -103,7 +103,7 @@
"type": "tags"
},
{
"datasource": "${DS_OPENSUSE:LEAP:15.0}",
"datasource": "openSUSE:Leap:15.0",
"enable": false,
"hide": false,
"iconColor": "rgba(255, 96, 96, 1)",
@ -125,9 +125,9 @@
"links": [],
"panels": [
{
"content": "These metrics were generated by parsing the Apache access logs of [download.opensuse.org](http://download.opensuse.org/).\n\n- A unique identifier (UUID) is included in logs when zypper includes it in the request. Multiple requests by the same UUID within the given aggregation frequency are counted as a single unique user.\n- The target product is determined by looking for specially formatted path elements representing build repositories for a particular product and the main repositories for that product.",
"content": "These metrics were generated by parsing the Apache access logs of [download.opensuse.org](http://download.opensuse.org/).\n\n- A unique identifier (UUID) is included in logs when zypper includes it in the request. Multiple requests by the same UUID within the given aggregation frequency are counted as a single unique user.\n- The target product is determined by looking for specially formatted path elements representing build repositories for a particular product and the main repositories for that product.\n\n__The logs are entirely invalid, due to a config mistake, from _2017-12-07_ to _2018-03-08_ and the custom log format, which includes the UUID from zypper, was not present from _2018-03-08_ until _2018-05-29_ which is the cause of the large \"hole\" in the graphs.__",
"gridPos": {
"h": 3,
"h": 4,
"w": 24,
"x": 0,
"y": 0
@ -144,7 +144,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 3
"y": 4
},
"id": 26,
"panels": [],
@ -156,13 +156,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_OSRT_ACCESS}",
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 4
"y": 5
},
"id": 10,
"legend": {
@ -266,13 +266,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_OSRT_ACCESS}",
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 14
"y": 15
},
"id": 12,
"legend": {
@ -376,13 +376,13 @@
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_OSRT_ACCESS}",
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 24
"y": 25
},
"id": 17,
"legend": {
@ -483,13 +483,13 @@
},
{
"columns": [],
"datasource": "${DS_OSRT_ACCESS}",
"datasource": "osrt_access",
"fontSize": "100%",
"gridPos": {
"h": 10,
"w": 5,
"x": 0,
"y": 34
"y": 35
},
"id": 15,
"links": [],
@ -571,13 +571,13 @@
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_OSRT_ACCESS}",
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 19,
"x": 5,
"y": 34
"y": 35
},
"id": 13,
"legend": {
@ -689,7 +689,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 44
"y": 45
},
"id": 28,
"panels": [
@ -1049,7 +1049,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 45
"y": 46
},
"id": 30,
"panels": [
@ -1474,7 +1474,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 46
"y": 47
},
"id": 32,
"panels": [