diff --git a/metrics/access/aggregate.php b/metrics/access/aggregate.php index 0c0f5de5..2259d89c 100755 --- a/metrics/access/aggregate.php +++ b/metrics/access/aggregate.php @@ -11,7 +11,7 @@ const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de'; const VHOST = 'download.opensuse.org'; const FILENAME = 'download.opensuse.org-%s-access_log.xz'; const IPV6_PREFIX = 'ipv6.'; -const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0]|tumbleweed)$/'; +const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0-1]|tumbleweed)$/'; $begin = new DateTime(); // Skip the current day since the logs are incomplete and not compressed yet. @@ -174,6 +174,7 @@ function aggregate_all($period) if (!isset($merged_protocol[$protocol])) $merged_protocol[$protocol] = []; $data_new['days'] = 1; + normalize($data_new); aggregate($intervals, $merged_protocol[$protocol], $date, $date_previous, $data_new, ['protocol' => $protocol], 'protocol'); @@ -220,6 +221,11 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [ $count = write_summary($interval, $date_previous, $summary, $tags, $prefix); + if ($prefix == 'access') { + $summary = summarize_product_plus_key($merged[$interval]['data']['total_image_product']); + $count += write_summary_product_plus_key($interval, $date_previous, $summary, 'image'); + } + error_log("[$prefix] [$interval] [{$merged[$interval]['value']}] wrote $count points at " . $date_previous->format('Y-m-d') . " spanning " . $merged[$interval]['data']['days'] . ' day(s)'); } @@ -237,6 +243,14 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [ } } +function normalize(&$data) +{ + // Ensure fields added later, that are not present in all data, are available. + if (!isset($data['total_image_product'])) { + $data['total_image_product'] = []; + } +} + function merge(&$data1, $data2) { $data1['days'] += $data2['days']; @@ -248,22 +262,28 @@ function merge(&$data1, $data2) $data1['total_product'][$product] += $data2['total_product'][$product]; } - foreach ($data2['unique_product'] as $product => $unqiue) { - if (empty($data1['unique_product'][$product])) - $data1['unique_product'][$product] = []; - - foreach ($unqiue as $uuid => $count) { - if (empty($data1['unique_product'][$product][$uuid])) - $data1['unique_product'][$product][$uuid] = 0; - - $data1['unique_product'][$product][$uuid] += $data2['unique_product'][$product][$uuid]; - } - } + merge_product_plus_key($data1['unique_product'], $data2['unique_product']); + merge_product_plus_key($data1['total_image_product'], $data2['total_image_product']); $data1['total_invalid'] += $data2['total_invalid']; $data1['bytes'] += $data2['bytes']; } +function merge_product_plus_key(&$data1, $data2) +{ + foreach ($data2 as $product => $pairs) { + if (empty($data1[$product])) + $data1[$product] = []; + + foreach ($pairs as $key => $value) { + if (empty($data1[$product][$key])) + $data1[$product][$key] = 0; + + $data1[$product][$key] += $data2[$product][$key]; + } + } +} + function summarize($data) { static $products = []; @@ -319,6 +339,29 @@ function summarize($data) return $summary; } +function summarize_product_plus_key($data) +{ + static $keys = []; + + $summary = []; + $products = array_merge(array_keys($keys), array_keys($data)); + foreach ($products as $product) { + if (!product_filter($product)) continue; + + $keys_keys = isset($keys[$product]) ? array_keys($keys[$product]) : []; + $data_keys = isset($data[$product]) ? array_keys($data[$product]) : []; + $product_keys = array_merge($keys_keys, $data_keys); + + $summary[$product] = []; + foreach ($product_keys as $key) { + // Fill empty data with zeros to achieve appropriate result in graph. + $summary[$product][$key] = isset($data[$product][$key]) ? $data[$product][$key] : 0; + } + } + + return $summary; +} + function product_filter($product) { return (bool) preg_match(PRODUCT_PATTERN, $product); @@ -343,6 +386,20 @@ function write_summary($interval, DateTime $value, $summary, $tags = [], $prefix return count($points); } +function write_summary_product_plus_key($interval, DateTime $date, $summary, $prefix) +{ + $measurement = $prefix . '_' . $interval; + $points = []; + foreach ($summary as $product => $pairs) { + foreach ($pairs as $key => $value) { + $points[] = new Point($measurement, null, + ['product' => $product, 'key' => $key], ['value' => $value], $date->getTimestamp()); + } + } + write($points); + return count($points); +} + function write($points) { static $database = null; diff --git a/metrics/access/ingest.php b/metrics/access/ingest.php index 57c1a1a9..5582a537 100755 --- a/metrics/access/ingest.php +++ b/metrics/access/ingest.php @@ -3,11 +3,13 @@ const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: +"?(\S+-\S+-\S+-\S+-[^\s"]+|-)"? "?(dvd|ftp|-)"?)?/'; const REGEX_PRODUCT = '#/(?:(tumbleweed)|distribution/(?:leap/)?(\d+\.\d+)|openSUSE(?:_|:/)(?:leap(?:_|:/))?(factory|tumbleweed|\d+\.\d+))#i'; +const REGEX_IMAGE = '#(?:/(?:iso|live)/[^/]+-(DVD|NET|GNOME-Live|KDE-Live|Rescue-CD|Kubic-DVD)-[^/]+\.iso(?:\.torrent)?|/jeos/[^/]+-(JeOS)\.[^/]+\.(?:qcow2|vhdx|vmdk|vmx)$)#'; $total = 0; $total_invalid = 0; $total_product = []; $unique_product = []; +$total_image_product = []; $file = $argc == 2 ? $argv[1] : 'php://stdin'; $handle = fopen($file, 'r'); @@ -40,6 +42,15 @@ while (($line = fgets($handle)) !== false) { if (!isset($unique_product[$product][$uuid])) $unique_product[$product][$uuid] = 0; $unique_product[$product][$uuid] += 1; } + + if (preg_match(REGEX_IMAGE, $match[3], $match_image)) { + // Remove empty match groups and select non-all match. + $values = array_filter($match_image); + $image = next($values); + if (!isset($total_image_product[$product])) $total_image_product[$product] = []; + if (!isset($total_image_product[$product][$image])) $total_image_product[$product][$image] = 0; + $total_image_product[$product][$image] += 1; + } } $position = ftell($handle); fclose($handle); @@ -54,6 +65,7 @@ echo json_encode([ 'total' => $total, 'total_product' => $total_product, 'unique_product' => $unique_product, + 'total_image_product' => $total_image_product, 'total_invalid' => $total_invalid, 'bytes' => $position, ]) . "\n"; // JSON_PRETTY_PRINT for debugging. diff --git a/metrics/grafana/access.json b/metrics/grafana/access.json index 4bca5062..1268b8c5 100644 --- a/metrics/grafana/access.json +++ b/metrics/grafana/access.json @@ -201,8 +201,6 @@ "measurement": "/^access_$frequency$/", "orderByTime": "ASC", "policy": "default", - "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"", - "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -311,8 +309,6 @@ "measurement": "/^access_$frequency$/", "orderByTime": "ASC", "policy": "default", - "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(11\\.[1-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"", - "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -421,8 +417,6 @@ "measurement": "/^access_$frequency$/", "orderByTime": "ASC", "policy": "default", - "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"", - "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -535,8 +529,6 @@ "measurement": "/^access_$frequency$/", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"measurement\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": false, "refId": "A", "resultFormat": "table", "select": [ @@ -617,8 +609,6 @@ "measurement": "/^access_$frequency$/", "orderByTime": "ASC", "policy": "default", - "query": "SELECT MAX(\"unique\") FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ group by product", - "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -1903,6 +1893,453 @@ ], "title": "Tool Feedback", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 43, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osrt_access", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_key", + "groupBy": [ + { + "params": [ + "key" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "/^image_$frequency$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "slimit": "", + "tags": [ + { + "key": "product", + "operator": "=~", + "value": "/^$product$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total by Image (stacked)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osrt_access", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_key", + "groupBy": [ + { + "params": [ + "key" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "/^image_$frequency$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "slimit": "", + "tags": [ + { + "key": "product", + "operator": "=~", + "value": "/^$product$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total by Image", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osrt_access", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_key", + "groupBy": [ + { + "params": [ + "key" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "/^image_$frequency$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "slimit": "", + "tags": [ + { + "key": "product", + "operator": "=~", + "value": "/^$product$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total by Image (percentage)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "osrt_access", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_key", + "groupBy": [ + { + "params": [ + "key" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "/^image_$frequency$/", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "slimit": "", + "tags": [ + { + "key": "product", + "operator": "=~", + "value": "/^$product$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total by Image (percentage - zoomed)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": "100", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "title": "Image Media - $product", + "type": "row" } ], "refresh": false, @@ -1942,6 +2379,26 @@ ], "query": "day, week, month", "type": "custom" + }, + { + "allValue": null, + "current": {}, + "datasource": "osrt_access", + "hide": 0, + "includeAll": false, + "label": "Product", + "multi": false, + "name": "product", + "options": [], + "query": "show tag values from image_month with key = \"product\"", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } ] },