yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

metrics/access: initial ingest script.

Browse Source
This commit is contained in:
Jimmy Berry 2018-06-14 22:45:58 -05:00
parent 3b991374ae
commit 6fecfcb128
1 changed files with 59 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
metrics/access/ingest.php Executable file
View File

@ -0,0 +1,59 @@
#!/usr/bin/php
<?php
const REGEX_LINE = '/\S+ \S+ \S+ \[([^:]+:\d+:\d+:\d+ [^\]]+)\] "(\S+)(?: (\S+) \S+)?" (\S+) (\S+) "[^"]*" "[^"]*" .* size:(\S+) \S+(?: "?(\S+-\S+-\S+-\S+-\S+|-)"? "?(dvd|ftp|-)"?)?/';
const REGEX_PRODUCT = '#/(?:(tumbleweed)|distribution/(?:leap/)?(\d+\.\d+)|openSUSE(?:_|:/)(?:leap(?:_|:/))?(factory|tumbleweed|\d+\.\d+))#i';
$total = 0;
$total_invalid = 0;
$total_product = [];
$unique_product = [];
$file = $argc == 2 ? $argv[1] : 'php://stdin';
$handle = fopen($file, 'r');
while (($line = fgets($handle)) !== false) {
if (!preg_match(REGEX_LINE, $line, $match)) {
error_log('[failed to parse] ' . rtrim($line));
$total_invalid++;
continue;
}
// Only interested in GET or HEAD requests, others are invalid.
if ($match[2] != 'GET' && $match[2] != 'HEAD') continue;
$total++;
// Attempt to determine for which product was the request.
if (!preg_match(REGEX_PRODUCT, $match[3], $match_product)) {
continue;
}
// Remove empty match groups and select non-all match.
$values = array_filter($match_product);
$product = str_replace('factory', 'tumbleweed', strtolower(next($values)));
if (!isset($total_product[$product])) $total_product[$product] = 0;
$total_product[$product] += 1;
if (count($match) == 9 && $match[7] != '-') {
$uuid = $match[7];
if (!isset($unique_product[$product])) $unique_product[$product] = [];
if (!isset($unique_product[$product][$uuid])) $unique_product[$product][$uuid] = 0;
$unique_product[$product][$uuid] += 1;
}
}
$position = ftell($handle);
fclose($handle);
error_log('processed ' . number_format($position) . ' bytes');
error_log('found ' . number_format($total) . ' requests across ' .
number_format(count($total_product)) . ' products');
ksort($total_product);
ksort($unique_product);
echo json_encode([
'total' => $total,
'total_product' => $total_product,
'unique_product' => $unique_product,
'total_invalid' => $total_invalid,
'bytes' => $position,
]) . "\n"; // JSON_PRETTY_PRINT for debugging.