Mercurial > gemma
changeset 5541:29804c8e817d aggregate-gm-import-logging
WIP: Handle rest correctly.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Tue, 26 Oct 2021 02:02:23 +0200 |
parents | 4d815f295e57 |
children | 0acb06fc77e6 |
files | schema/updates/1469/aggregate-gm-logs.sql |
diffstat | 1 files changed, 35 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/schema/updates/1469/aggregate-gm-logs.sql Tue Oct 26 01:15:30 2021 +0200 +++ b/schema/updates/1469/aggregate-gm-logs.sql Tue Oct 26 02:02:23 2021 +0200 @@ -11,6 +11,9 @@ value text; joined text; num integer; + line record; + hold record; + done boolean; BEGIN CREATE TEMP TABLE filtered_logs ( @@ -32,13 +35,24 @@ measurements integer NOT NULL DEFAULT 0, predictions integer NOT NULL DEFAULT 0 ); + last_id := -1; + done := false; + hold := NULL; + + OPEN logs; - FOR line IN logs LOOP + LOOP + + FETCH logs INTO line; + + IF NOT FOUND THEN + done := true; + END IF; -- RAISE NOTICE '%', line.msg; - IF last_id <> line.import_id THEN + IF done OR last_id <> line.import_id THEN -- unknown SELECT INTO joined string_agg(gid, ', ') @@ -86,7 +100,6 @@ 'Ignored measurements with value -99999: ' || joined); END IF; - -- missing mandatory values SELECT INTO joined string_agg( gid || ' (' @@ -164,6 +177,14 @@ 'No changes for: ' || joined); END IF; + IF hold is NOT NULL THEN + INSERT INTO filtered_logs VALUES (hold.import_id, hold.time, hold.kind, hold.msg); + hold := NULL; + END IF; + + IF done THEN + EXIT; + END IF; -- reset aggregate table TRUNCATE agg_tracker; @@ -183,14 +204,6 @@ ON CONFLICT (gid) DO UPDATE SET measurements = EXCLUDED.measurements + num; - ELSIF line.msg ~ '^Ignored \d+ measurements with value -99999$' THEN - num := substring( - line.msg from '^Ignored (\d+)')::integer; - - INSERT INTO agg_tracker (gid, bad_values) VALUES (curr_gid, num) - ON CONFLICT (gid) - DO UPDATE SET bad_values = EXCLUDED.bad_values + num; - ELSIF line.msg ~ '^Inserted \d+ predictions for ....................$' THEN num := substring( line.msg from '^Inserted (\d+)')::integer; @@ -207,6 +220,14 @@ ON CONFLICT (gid) DO UPDATE SET assume_zpg = true; + ELSIF line.msg ~ '^Ignored \d+ measurements with value -99999$' THEN + num := substring( + line.msg from '^Ignored (\d+)')::integer; + + INSERT INTO agg_tracker (gid, bad_values) VALUES (curr_gid, num) + ON CONFLICT (gid) + DO UPDATE SET bad_values = EXCLUDED.bad_values + num; + ELSIF line.msg ~ '^Cannot find gauge "...................." for import$' THEN value := substring( line.msg from '^Cannot find gauge "(....................)" for import$'); @@ -250,11 +271,11 @@ array_cat(agg_tracker.rescale_errors, EXCLUDED.rescale_errors); ELSIF line.msg ~ '^Importing gauge measurements took ' THEN - - -- TODO: Flush aggregation because its likely the last entry. + -- Likely the last entry of this import. + hold := line; ELSE -- Not handled, copy through .. - RAISE NOTICE '%', line.msg; + -- RAISE NOTICE '%', line.msg; INSERT INTO filtered_logs VALUES (line.import_id, line.time, line.kind, line.msg); END IF; @@ -262,8 +283,6 @@ last_time := line.time; END LOOP; - -- TODO: Handle remains from agg_tracker - -- DELETE FROM import.import_logs WHERE import_id IN ( -- SELECT id FROM import.imports WHERE kind = 'gm')