Mercurial > gemma
changeset 5538:ff95d3603e4d aggregate-gm-import-logging
WIP: More aggregation.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Mon, 25 Oct 2021 23:05:45 +0200 |
parents | 2e5e51288a6c |
children | 90ba92820b26 |
files | schema/updates/1469/aggregate-gm-logs.sql |
diffstat | 1 files changed, 34 insertions(+), 13 deletions(-) [+] |
line wrap: on
line diff
--- a/schema/updates/1469/aggregate-gm-logs.sql Mon Oct 25 21:35:54 2021 +0200 +++ b/schema/updates/1469/aggregate-gm-logs.sql Mon Oct 25 23:05:45 2021 +0200 @@ -8,7 +8,7 @@ last_id integer; last_time timestamp with time zone; curr_gid text; - gauge text; + value text; joined text; num integer; BEGIN @@ -85,9 +85,22 @@ END IF; - -- TODO: ""Missing mandatory values: " - -- TODO: "Cannot convert units: " - -- TODO: "Ignored measure codes:" + -- TODO: ""Missing mandatory values: " + -- TODO: "Cannot convert units: " + + -- ignored measure codes + SELECT INTO joined string_agg( + gid || ' (' + || array_to_string(ARRAY(SELECT DISTINCT unnest(ign_meas_codes)), '; ') + || ')', ', ') + FROM agg_tracker + WHERE ign_meas_codes IS NOT NULL; + + IF joined IS NOT NULL THEN + INSERT INTO filtered_logs VALUES ( + last_id, last_time, 'warn'::log_type, + 'Ignored measure codes: ' || joined); + END IF; -- predictions SELECT INTO joined string_agg( @@ -135,10 +148,10 @@ ELSIF line.msg ~ '^Inserted \d+ measurements for ....................$' THEN num := substring( line.msg from '^Inserted (\d+)')::integer; - gauge := substring( + value := substring( line.msg from '^Inserted \d+ measurements for (....................)$'); - INSERT INTO agg_tracker (gid, measurements) VALUES (gauge, num) + INSERT INTO agg_tracker (gid, measurements) VALUES (value, num) ON CONFLICT (gid) DO UPDATE SET measurements = EXCLUDED.measurements + num; @@ -153,39 +166,47 @@ ELSIF line.msg ~ '^Inserted \d+ predictions for ....................$' THEN num := substring( line.msg from '^Inserted (\d+)')::integer; - gauge := substring( + value := substring( line.msg from '^Inserted \d+ predictions for (....................)$'); - INSERT INTO agg_tracker (gid, predictions) VALUES (gauge, num) + INSERT INTO agg_tracker (gid, predictions) VALUES (value, num) ON CONFLICT (gid) DO UPDATE SET predictions = EXCLUDED.predictions + num; ELSIF line.msg ~ '^''Reference_code'' not specified. Assuming ''ZPG''$' THEN - INSERT INTO agg_tracker (gid, assume_zpg) VALUES (gauge, true) + INSERT INTO agg_tracker (gid, assume_zpg) VALUES (curr_gid, true) ON CONFLICT (gid) DO UPDATE SET assume_zpg = true; ELSIF line.msg ~ '^Cannot find gauge "...................." for import$' THEN - gauge := substring( + value := substring( line.msg from '^Cannot find gauge "(....................)" for import$'); - INSERT INTO agg_tracker (gid, unknown) VALUES (gauge, true) + INSERT INTO agg_tracker (gid, unknown) VALUES (value, true) ON CONFLICT (gid) DO UPDATE SET unknown = true; ELSIF line.msg ~ '^''Unit'' not specified. Assuming ''cm''$' THEN - INSERT INTO agg_tracker (gid, assume_cm) VALUES (gauge, 1) + INSERT INTO agg_tracker (gid, assume_cm) VALUES (curr_gid, 1) ON CONFLICT (gid) DO UPDATE SET assume_cm = EXCLUDED.assume_cm + 1; + ELSIF line.msg ~ '^Ignored message with measure_code .+' THEN + value := substring( + line.msg from '^Ignored message with measure_code (.+)$'); + + INSERT INTO agg_tracker (gid, ign_meas_codes) VALUES (curr_gid, array[value]) + ON CONFLICT (gid) + DO UPDATE SET ign_meas_codes = + array_cat(agg_tracker.ign_meas_codes, EXCLUDED.ign_meas_codes); + ELSIF line.msg ~ '^Importing gauge measurements took ' THEN -- TODO: Flush aggregation because its likely the last entry. ELSE -- TODO: Handle 'Missing mandatory value at %s. Ignored (bad service)' - -- TODO: Handle 'Ignored message with measure_code %s' -- TODO: Handle "unknown unit '%s'" -- Not handled, copy through .. INSERT INTO filtered_logs VALUES (line.import_id, line.time, line.kind, line.msg);