Merge pull request #1695 from jberry-suse/metrics-images

metrics: add image totals by product to access dashboard.

metrics/access/aggregate.php

@@ -11,7 +11,7 @@ const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de';
 const VHOST = 'download.opensuse.org';
 const FILENAME = 'download.opensuse.org-%s-access_log.xz';
 const IPV6_PREFIX = 'ipv6.';
-const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0]|tumbleweed)$/';
+const PRODUCT_PATTERN = '/^(10\.[2-3]|11\.[0-4]|12\.[1-3]|13\.[1-2]|42\.[1-3]|15\.[0-1]|tumbleweed)$/';
 
 $begin = new DateTime();
 // Skip the current day since the logs are incomplete and not compressed yet.
@@ -174,6 +174,7 @@ function aggregate_all($period)
 
       if (!isset($merged_protocol[$protocol])) $merged_protocol[$protocol] = [];
       $data_new['days'] = 1;
+      normalize($data_new);
       aggregate($intervals, $merged_protocol[$protocol], $date, $date_previous, $data_new,
         ['protocol' => $protocol], 'protocol');
 
@@ -220,6 +221,11 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [
 
         $count = write_summary($interval, $date_previous, $summary, $tags, $prefix);
 
+        if ($prefix == 'access') {
+          $summary = summarize_product_plus_key($merged[$interval]['data']['total_image_product']);
+          $count += write_summary_product_plus_key($interval, $date_previous, $summary, 'image');
+        }
+
         error_log("[$prefix] [$interval] [{$merged[$interval]['value']}] wrote $count points at " .
           $date_previous->format('Y-m-d') . " spanning " . $merged[$interval]['data']['days'] . ' day(s)');
       }
@@ -237,6 +243,14 @@ function aggregate($intervals, &$merged, $date, $date_previous, $data, $tags = [
   }
 }
 
+function normalize(&$data)
+{
+  // Ensure fields added later, that are not present in all data, are available.
+  if (!isset($data['total_image_product'])) {
+    $data['total_image_product'] = [];
+  }
+}
+
 function merge(&$data1, $data2)
 {
   $data1['days'] += $data2['days'];
@@ -248,22 +262,28 @@ function merge(&$data1, $data2)
     $data1['total_product'][$product] += $data2['total_product'][$product];
   }
 
-  foreach ($data2['unique_product'] as $product => $unqiue) {
+  merge_product_plus_key($data1['unique_product'], $data2['unique_product']);
-    if (empty($data1['unique_product'][$product]))
+  merge_product_plus_key($data1['total_image_product'], $data2['total_image_product']);
-      $data1['unique_product'][$product] = [];
-
-    foreach ($unqiue as $uuid => $count) {
-      if (empty($data1['unique_product'][$product][$uuid]))
-        $data1['unique_product'][$product][$uuid] = 0;
-
-      $data1['unique_product'][$product][$uuid] += $data2['unique_product'][$product][$uuid];
-    }
-  }
 
   $data1['total_invalid'] += $data2['total_invalid'];
   $data1['bytes'] += $data2['bytes'];
 }
 
+function merge_product_plus_key(&$data1, $data2)
+{
+  foreach ($data2 as $product => $pairs) {
+    if (empty($data1[$product]))
+      $data1[$product] = [];
+
+    foreach ($pairs as $key => $value) {
+      if (empty($data1[$product][$key]))
+        $data1[$product][$key] = 0;
+
+      $data1[$product][$key] += $data2[$product][$key];
+    }
+  }
+}
+
 function summarize($data)
 {
   static $products = [];
@@ -319,6 +339,29 @@ function summarize($data)
   return $summary;
 }
 
+function summarize_product_plus_key($data)
+{
+  static $keys = [];
+
+  $summary = [];
+  $products = array_merge(array_keys($keys), array_keys($data));
+  foreach ($products as $product) {
+    if (!product_filter($product)) continue;
+
+    $keys_keys = isset($keys[$product]) ? array_keys($keys[$product]) : [];
+    $data_keys = isset($data[$product]) ? array_keys($data[$product]) : [];
+    $product_keys = array_merge($keys_keys, $data_keys);
+
+    $summary[$product] = [];
+    foreach ($product_keys as $key) {
+      // Fill empty data with zeros to achieve appropriate result in graph.
+      $summary[$product][$key] = isset($data[$product][$key]) ? $data[$product][$key] : 0;
+    }
+  }
+
+  return $summary;
+}
+
 function product_filter($product)
 {
   return (bool) preg_match(PRODUCT_PATTERN, $product);
@@ -343,6 +386,20 @@ function write_summary($interval, DateTime $value, $summary, $tags = [], $prefix
   return count($points);
 }
 
+function write_summary_product_plus_key($interval, DateTime $date, $summary, $prefix)
+{
+  $measurement = $prefix . '_' . $interval;
+  $points = [];
+  foreach ($summary as $product => $pairs) {
+    foreach ($pairs as $key => $value) {
+      $points[] = new Point($measurement, null,
+        ['product' => $product, 'key' => $key], ['value' => $value], $date->getTimestamp());
+    }
+  }
+  write($points);
+  return count($points);
+}
+
 function write($points)
 {
   static $database = null;

metrics/access/ingest.php

@@ -3,11 +3,13 @@
 
 const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: +"?(\S+-\S+-\S+-\S+-[^\s"]+|-)"? "?(dvd|ftp|-)"?)?/';
 const REGEX_PRODUCT = '#/(?:(tumbleweed)|distribution/(?:leap/)?(\d+\.\d+)|openSUSE(?:_|:/)(?:leap(?:_|:/))?(factory|tumbleweed|\d+\.\d+))#i';
+const REGEX_IMAGE = '#(?:/(?:iso|live)/[^/]+-(DVD|NET|GNOME-Live|KDE-Live|Rescue-CD|Kubic-DVD)-[^/]+\.iso(?:\.torrent)?|/jeos/[^/]+-(JeOS)\.[^/]+\.(?:qcow2|vhdx|vmdk|vmx)$)#';
 
 $total = 0;
 $total_invalid = 0;
 $total_product = [];
 $unique_product = [];
+$total_image_product = [];
 
 $file = $argc == 2 ? $argv[1] : 'php://stdin';
 $handle = fopen($file, 'r');
@@ -40,6 +42,15 @@ while (($line = fgets($handle)) !== false) {
     if (!isset($unique_product[$product][$uuid])) $unique_product[$product][$uuid] = 0;
     $unique_product[$product][$uuid] += 1;
   }
+
+  if (preg_match(REGEX_IMAGE, $match[3], $match_image)) {
+    // Remove empty match groups and select non-all match.
+    $values = array_filter($match_image);
+    $image = next($values);
+    if (!isset($total_image_product[$product])) $total_image_product[$product] = [];
+    if (!isset($total_image_product[$product][$image])) $total_image_product[$product][$image] = 0;
+    $total_image_product[$product][$image] += 1;
+  }
 }
 $position = ftell($handle);
 fclose($handle);
@@ -54,6 +65,7 @@ echo json_encode([
   'total' => $total,
   'total_product' => $total_product,
   'unique_product' => $unique_product,
+  'total_image_product' => $total_image_product,
   'total_invalid' => $total_invalid,
   'bytes' => $position,
 ]) . "\n"; // JSON_PRETTY_PRINT for debugging.

metrics/grafana/access.json

@@ -201,8 +201,6 @@
           "measurement": "/^access_$frequency$/",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
-          "rawQuery": false,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -311,8 +309,6 @@
           "measurement": "/^access_$frequency$/",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(11\\.[1-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
-          "rawQuery": false,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -421,8 +417,6 @@
           "measurement": "/^access_$frequency$/",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT \"unique\" FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ GROUP BY \"product\"",
-          "rawQuery": false,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -535,8 +529,6 @@
           "measurement": "/^access_$frequency$/",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT mean(\"value\") FROM \"measurement\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
-          "rawQuery": false,
           "refId": "A",
           "resultFormat": "table",
           "select": [
@@ -617,8 +609,6 @@
           "measurement": "/^access_$frequency$/",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT MAX(\"unique\") FROM /^access_$frequency$/ WHERE $timeFilter AND product =~ /^(10\\.[2-3]|11\\.[0-4]|12\\.[1-3]|13\\.[1-2]|42\\.[1-3]|15\\.[0]|tumbleweed)$/ group by product",
-          "rawQuery": false,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -1903,6 +1893,453 @@
       ],
       "title": "Tool Feedback",
       "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 48
+      },
+      "id": 43,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "osrt_access",
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 24,
+            "x": 0,
+            "y": 9
+          },
+          "id": 40,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "$tag_key",
+              "groupBy": [
+                {
+                  "params": [
+                    "key"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "hide": false,
+              "measurement": "/^image_$frequency$/",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "slimit": "",
+              "tags": [
+                {
+                  "key": "product",
+                  "operator": "=~",
+                  "value": "/^$product$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total by Image (stacked)",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "osrt_access",
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 24,
+            "x": 0,
+            "y": 19
+          },
+          "id": 45,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "$tag_key",
+              "groupBy": [
+                {
+                  "params": [
+                    "key"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "hide": false,
+              "measurement": "/^image_$frequency$/",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "slimit": "",
+              "tags": [
+                {
+                  "key": "product",
+                  "operator": "=~",
+                  "value": "/^$product$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total by Image",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "osrt_access",
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 24,
+            "x": 0,
+            "y": 29
+          },
+          "id": 44,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": true,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "$tag_key",
+              "groupBy": [
+                {
+                  "params": [
+                    "key"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "hide": false,
+              "measurement": "/^image_$frequency$/",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "slimit": "",
+              "tags": [
+                {
+                  "key": "product",
+                  "operator": "=~",
+                  "value": "/^$product$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total by Image (percentage)",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": "100",
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "osrt_access",
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 24,
+            "x": 0,
+            "y": 39
+          },
+          "id": 46,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": true,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "alias": "$tag_key",
+              "groupBy": [
+                {
+                  "params": [
+                    "key"
+                  ],
+                  "type": "tag"
+                }
+              ],
+              "hide": false,
+              "measurement": "/^image_$frequency$/",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  }
+                ]
+              ],
+              "slimit": "",
+              "tags": [
+                {
+                  "key": "product",
+                  "operator": "=~",
+                  "value": "/^$product$/"
+                }
+              ]
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total by Image (percentage - zoomed)",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": "100",
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "title": "Image Media - $product",
+      "type": "row"
     }
   ],
   "refresh": false,
@@ -1942,6 +2379,26 @@
         ],
         "query": "day, week, month",
         "type": "custom"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "osrt_access",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Product",
+        "multi": false,
+        "name": "product",
+        "options": [],
+        "query": "show tag values from image_month with key = \"product\"",
+        "refresh": 1,
+        "regex": "",
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
       }
     ]
   },