yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1695 from jberry-suse/metrics-images

Browse Source 
metrics: add image totals by product to access dashboard.
This commit is contained in:
Jimmy Berry 2018-09-14 16:09:10 -05:00 committed by GitHub
commit 628c1a107d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 548 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
metrics/access/aggregate.php
View File

@ -11,7 +11,7 @@ const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de';
const VHOST = 'download.opensuse.org';
const FILENAME = 'download.opensuse.org-%s-access_log.xz';
const IPV6_PREFIX = 'ipv6.';
const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0]|tumbleweed)$/';
const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0-1]|tumbleweed)$/';
$begin = new DateTime();
// Skip the current day since the logs are incomplete and not compressed yet.
@ -174,6 +174,7 @@ function aggregate_all($period)
if (!isset($merged_protocol[$protocol])) $merged_protocol[$protocol] = [];
$data_new['days'] = 1;
normalize($data_new);
aggregate($intervals, $merged_protocol[$protocol], $date, $date_previous, $data_new,
['protocol' => $protocol], 'protocol');
@ -220,6 +221,11 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [
$count = write_summary($interval, $date_previous, $summary, $tags, $prefix);
if ($prefix == 'access') {
$summary = summarize_product_plus_key($merged[$interval]['data']['total_image_product']);
$count += write_summary_product_plus_key($interval, $date_previous, $summary, 'image');
}
error_log("[$prefix] [$interval] [{$merged[$interval]['value']}] wrote $count points at " .
$date_previous->format('Y-m-d') . " spanning " . $merged[$interval]['data']['days'] . ' day(s)');
}
@ -237,6 +243,14 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [
}
}
function normalize(&$data)
{
// Ensure fields added later, that are not present in all data, are available.
if (!isset($data['total_image_product'])) {
$data['total_image_product'] = [];
}
}
function merge(&$data1, $data2)
{
$data1['days'] += $data2['days'];
@ -248,22 +262,28 @@ function merge(&$data1, $data2)
$data1['total_product'][$product] += $data2['total_product'][$product];
}
foreach ($data2['unique_product'] as $product => $unqiue) {
if (empty($data1['unique_product'][$product]))
$data1['unique_product'][$product] = [];
foreach ($unqiue as $uuid => $count) {
if (empty($data1['unique_product'][$product][$uuid]))
$data1['unique_product'][$product][$uuid] = 0;
$data1['unique_product'][$product][$uuid] += $data2['unique_product'][$product][$uuid];
}
}
merge_product_plus_key($data1['unique_product'], $data2['unique_product']);
merge_product_plus_key($data1['total_image_product'], $data2['total_image_product']);
$data1['total_invalid'] += $data2['total_invalid'];
$data1['bytes'] += $data2['bytes'];
}
function merge_product_plus_key(&$data1, $data2)
{
foreach ($data2 as $product => $pairs) {
if (empty($data1[$product]))
$data1[$product] = [];
foreach ($pairs as $key => $value) {
if (empty($data1[$product][$key]))
$data1[$product][$key] = 0;
$data1[$product][$key] += $data2[$product][$key];
}
}
}
function summarize($data)
{
static $products = [];
@ -319,6 +339,29 @@ function summarize($data)
return $summary;
}
function summarize_product_plus_key($data)
{
static $keys = [];
$summary = [];
$products = array_merge(array_keys($keys), array_keys($data));
foreach ($products as $product) {
if (!product_filter($product)) continue;
$keys_keys = isset($keys[$product]) ? array_keys($keys[$product]) : [];
$data_keys = isset($data[$product]) ? array_keys($data[$product]) : [];
$product_keys = array_merge($keys_keys, $data_keys);
$summary[$product] = [];
foreach ($product_keys as $key) {
// Fill empty data with zeros to achieve appropriate result in graph.
$summary[$product][$key] = isset($data[$product][$key]) ? $data[$product][$key] : 0;
}
}
return $summary;
}
function product_filter($product)
{
return (bool) preg_match(PRODUCT_PATTERN, $product);
@ -343,6 +386,20 @@ function write_summary($interval, DateTime $value, $summary, $tags = [], $prefix
return count($points);
}
function write_summary_product_plus_key($interval, DateTime $date, $summary, $prefix)
{
$measurement = $prefix . '_' . $interval;
$points = [];
foreach ($summary as $product => $pairs) {
foreach ($pairs as $key => $value) {
$points[] = new Point($measurement, null,
['product' => $product, 'key' => $key], ['value' => $value], $date->getTimestamp());
}
}
write($points);
return count($points);
}
function write($points)
{
static $database = null;

12
metrics/access/ingest.php
View File

@ -3,11 +3,13 @@
const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: +"?(\S+-\S+-\S+-\S+-[^\s"]+|-)"? "?(dvd|ftp|-)"?)?/';
const REGEX_PRODUCT = '#/(?:(tumbleweed)|distribution/(?:leap/)?(\d+\.\d+)|openSUSE(?:_|:/)(?:leap(?:_|:/))?(factory|tumbleweed|\d+\.\d+))#i';
const REGEX_IMAGE = '#(?:/(?:iso|live)/[^/]+-(DVD|NET|GNOME-Live|KDE-Live|Rescue-CD|Kubic-DVD)-[^/]+\.iso(?:\.torrent)?|/jeos/[^/]+-(JeOS)\.[^/]+\.(?:qcow2|vhdx|vmdk|vmx)$)#';
$total = 0;
$total_invalid = 0;
$total_product = [];
$unique_product = [];
$total_image_product = [];
$file = $argc == 2 ? $argv[1] : 'php://stdin';
$handle = fopen($file, 'r');
@ -40,6 +42,15 @@ while (($line = fgets($handle)) !== false) {
if (!isset($unique_product[$product][$uuid])) $unique_product[$product][$uuid] = 0;
$unique_product[$product][$uuid] += 1;
}
if (preg_match(REGEX_IMAGE, $match[3], $match_image)) {
// Remove empty match groups and select non-all match.
$values = array_filter($match_image);
$image = next($values);
if (!isset($total_image_product[$product])) $total_image_product[$product] = [];
if (!isset($total_image_product[$product][$image])) $total_image_product[$product][$image] = 0;
$total_image_product[$product][$image] += 1;
}
}
$position = ftell($handle);
fclose($handle);
@ -54,6 +65,7 @@ echo json_encode([
'total' => $total,
'total_product' => $total_product,
'unique_product' => $unique_product,
'total_image_product' => $total_image_product,
'total_invalid' => $total_invalid,
'bytes' => $position,
]) . "\n"; // JSON_PRETTY_PRINT for debugging.

477
metrics/grafana/access.json
View File

@ -201,8 +201,6 @@
"measurement": "/^access_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@ -311,8 +309,6 @@
"measurement": "/^access_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(11\\.[1-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@ -421,8 +417,6 @@
"measurement": "/^access_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@ -535,8 +529,6 @@
"measurement": "/^access_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"measurement\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
"rawQuery": false,
"refId": "A",
"resultFormat": "table",
"select": [
@ -617,8 +609,6 @@
"measurement": "/^access_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT MAX(\"unique\") FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ group by product",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@ -1903,6 +1893,453 @@
],
"title": "Tool Feedback",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 48
},
"id": 43,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 9
},
"id": 40,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"alias": "$tag_key",
"groupBy": [
{
"params": [
"key"
],
"type": "tag"
}
],
"hide": false,
"measurement": "/^image_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
}
]
],
"slimit": "",
"tags": [
{
"key": "product",
"operator": "=~",
"value": "/^$product$/"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Total by Image (stacked)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 19
},
"id": 45,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "$tag_key",
"groupBy": [
{
"params": [
"key"
],
"type": "tag"
}
],
"hide": false,
"measurement": "/^image_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
}
]
],
"slimit": "",
"tags": [
{
"key": "product",
"operator": "=~",
"value": "/^$product$/"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Total by Image",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 29
},
"id": 44,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": true,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"alias": "$tag_key",
"groupBy": [
{
"params": [
"key"
],
"type": "tag"
}
],
"hide": false,
"measurement": "/^image_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
}
]
],
"slimit": "",
"tags": [
{
"key": "product",
"operator": "=~",
"value": "/^$product$/"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Total by Image (percentage)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": "100",
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "osrt_access",
"fill": 1,
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 39
},
"id": 46,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": true,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"alias": "$tag_key",
"groupBy": [
{
"params": [
"key"
],
"type": "tag"
}
],
"hide": false,
"measurement": "/^image_$frequency$/",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
}
]
],
"slimit": "",
"tags": [
{
"key": "product",
"operator": "=~",
"value": "/^$product$/"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Total by Image (percentage - zoomed)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": "100",
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"title": "Image Media - $product",
"type": "row"
}
],
"refresh": false,
@ -1942,6 +2379,26 @@
],
"query": "day, week, month",
"type": "custom"
},
{
"allValue": null,
"current": {},
"datasource": "osrt_access",
"hide": 0,
"includeAll": false,
"label": "Product",
"multi": false,
"name": "product",
"options": [],
"query": "show tag values from image_month with key = \"product\"",
"refresh": 1,
"regex": "",
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},