From 1dd9b64480d7dfee18c761f95e82f68ff5657672 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Fri, 9 Mar 2018 16:53:26 -0600 Subject: [PATCH] metrics: do not drop entire database, but instead measurements. This ensures that release data, which is ingest separately, is not wiped out. Additionally, it is less destructive during update. --- metrics.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/metrics.py b/metrics.py index 03c1051d..39acbead 100755 --- a/metrics.py +++ b/metrics.py @@ -267,15 +267,18 @@ def who_workaround(request, review, relax=False): # allocating memory for entire incoming data set at once. def walk_points(points, target): global client - # Wait until just before writing to drop database. - client.drop_database(client._database) - client.create_database(client._database) + measurements = set() counters = {} final = [] time_last = None wrote = 0 for point in sorted(points, key=lambda l: l.time): + if point.measurement not in measurements: + # Wait until just before writing to drop measurement. + client.drop_measurement(point.measurement) + measurements.add(point.measurement) + if point.time != time_last and len(final) >= 1000: # Write final point in batches of ~1000, but guard against writing # when in the middle of points at the same time as they may end up @@ -351,6 +354,9 @@ def main(args): osc.conf.get_config(override_apiurl=args.apiurl) osc.conf.config['debug'] = args.debug + # Ensure database exists. + client.create_database(client._database) + metrics_release.ingest(client) if args.release_only: return