--- a/schema/updates/1469/aggregate-gm-logs.sql	Tue Oct 26 01:15:30 2021 +0200
+++ b/schema/updates/1469/aggregate-gm-logs.sql	Tue Oct 26 02:02:23 2021 +0200
@@ -11,6 +11,9 @@
   value text;
   joined text;
   num integer;
+  line record;
+  hold record;
+  done boolean;
 BEGIN
 
   CREATE TEMP TABLE filtered_logs (
@@ -32,13 +35,24 @@
       measurements   integer NOT NULL DEFAULT 0,
       predictions    integer NOT NULL DEFAULT 0
   );
+
   last_id := -1;
+  done := false;
+  hold := NULL;
+
+  OPEN logs;
 
-  FOR line IN logs LOOP
+  LOOP
+
+    FETCH logs INTO line;
+
+    IF NOT FOUND THEN
+        done := true;
+    END IF;
 
     -- RAISE NOTICE '%', line.msg;
 
-    IF last_id <> line.import_id THEN
+    IF done OR last_id <> line.import_id THEN
 
         -- unknown
         SELECT INTO joined string_agg(gid, ', ')
@@ -86,7 +100,6 @@
                 'Ignored measurements with value -99999: ' || joined);
         END IF;
 
-
         -- missing mandatory values
         SELECT INTO joined string_agg(
             gid || ' ('
@@ -164,6 +177,14 @@
                 'No changes for: ' || joined);
         END IF;
 
+        IF hold is NOT NULL THEN
+            INSERT INTO filtered_logs VALUES (hold.import_id, hold.time, hold.kind, hold.msg);
+            hold := NULL;
+        END IF;
+
+        IF done THEN
+            EXIT;
+        END IF;
 
         -- reset aggregate table
         TRUNCATE agg_tracker;
@@ -183,14 +204,6 @@
           ON CONFLICT (gid)
           DO UPDATE SET measurements = EXCLUDED.measurements + num;
 
-    ELSIF line.msg ~ '^Ignored \d+ measurements with value -99999$' THEN
-        num := substring(
-            line.msg from '^Ignored (\d+)')::integer;
-
-        INSERT INTO agg_tracker (gid, bad_values) VALUES (curr_gid, num)
-          ON CONFLICT (gid)
-          DO UPDATE SET bad_values = EXCLUDED.bad_values + num;
-
     ELSIF line.msg ~ '^Inserted \d+ predictions for ....................$' THEN
         num := substring(
             line.msg from '^Inserted (\d+)')::integer;
@@ -207,6 +220,14 @@
           ON CONFLICT (gid)
           DO UPDATE SET assume_zpg = true;
 
+    ELSIF line.msg ~ '^Ignored \d+ measurements with value -99999$' THEN
+        num := substring(
+            line.msg from '^Ignored (\d+)')::integer;
+
+        INSERT INTO agg_tracker (gid, bad_values) VALUES (curr_gid, num)
+          ON CONFLICT (gid)
+          DO UPDATE SET bad_values = EXCLUDED.bad_values + num;
+
     ELSIF line.msg ~ '^Cannot find gauge "...................." for import$' THEN
         value := substring(
             line.msg from '^Cannot find gauge "(....................)" for import$');
@@ -250,11 +271,11 @@
             array_cat(agg_tracker.rescale_errors, EXCLUDED.rescale_errors);
 
     ELSIF line.msg ~ '^Importing gauge measurements took ' THEN
-
-        -- TODO: Flush aggregation because its likely the last entry.
+        -- Likely the last entry of this import.
+        hold := line;
     ELSE
         -- Not handled, copy through ..
-        RAISE NOTICE '%', line.msg;
+        -- RAISE NOTICE '%', line.msg;
         INSERT INTO filtered_logs VALUES (line.import_id, line.time, line.kind, line.msg);
 
     END IF;
@@ -262,8 +283,6 @@
     last_time := line.time;
   END LOOP;
 
-  -- TODO: Handle remains from agg_tracker
-
   -- DELETE FROM import.import_logs WHERE import_id IN (
   --  SELECT id FROM import.imports WHERE kind = 'gm')