Mercurial > gemma

changeset 5670:b75d0b303328

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Various fixes and improvements of gauges import: - Allow update of erased data (and thereby set erased to false) - Fix source_organization to work with ERDMS2 - Give ISRS of new and updated gauges in summary - Fixed reference of null pointers if revlevels are missing - Fixed reference of null pointer on update errors - Added ISRS to reference_code warning
author Sascha Wilde <wilde@sha-bang.de>
date Fri, 08 Dec 2023 17:29:56 +0100
parents 6bf942f486e9
children 4bfcbb747768
files pkg/imports/wg.go
diffstat 1 files changed, 48 insertions(+), 29 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/wg.go	Thu Dec 07 16:56:24 2023 +0100
+++ b/pkg/imports/wg.go	Fri Dec 08 17:29:56 2023 +0100
@@ -136,19 +136,20 @@
 
 	updateGaugeSQL = `
 UPDATE waterway.gauges SET
-  objname = $6,
-  geom = ST_SetSRID(ST_MakePoint($7, $8), 4326),
-  applicability_from_km = $9,
-  applicability_to_km = $10,
-  zero_point = $11,
-  geodref = $12,
-  date_info = $13,
-  source_organization = $14,
-  lastupdate = $15,
-  validity = $16
-WHERE location = ($1::char(2), $2::char(3), $3::char(5), $4::char(5), $5::int)
-  AND NOT erased
-  AND $15 > lastupdate
+  objname = $2,
+  geom = ST_SetSRID(ST_MakePoint($3, $4), 4326),
+  applicability_from_km = $5,
+  applicability_to_km = $6,
+  zero_point = $7,
+  geodref = $8,
+  date_info = $9,
+  source_organization = $10,
+  lastupdate = $11,
+  validity = $12,
+  erased = false
+WHERE isrs_astext(location) = $1
+  AND validity <@ $12
+  AND $11 > lastupdate
 RETURNING 1
 `
 
@@ -230,7 +231,7 @@
 	var gauges []string
 	var unchanged int
 
-	var invalidISRS, startEndOrder, missingObjname, missingZeropoint []string
+	var invalidISRS, startEndOrder, missingObjname, missingZeropoint, updatedGauges, newGauges []string
 
 	databaseErrors := map[string][]string{}
 
@@ -336,9 +337,9 @@
 			}
 
 			var source sql.NullString
-			if dr.Source != nil {
+			if dr.RisidxSource != nil {
 				source = sql.NullString{
-					String: string(*dr.Source),
+					String: string(*dr.RisidxSource),
 					Valid:  true,
 				}
 			}
@@ -389,6 +390,7 @@
 						databaseErrors[key] = append(databaseErrors[key], isrs)
 						return continueErr
 					}
+					newGauges = append(newGauges, isrs)
 					//feedback.Info("insert new version")
 				case !isNew:
 					var lu *time.Time
@@ -399,11 +401,7 @@
 					// try to update
 					var dummy int
 					err2 := tx.StmtContext(ctx, updateStmt).QueryRowContext(ctx,
-						code.CountryCode,
-						code.LoCode,
-						code.FairwaySection,
-						code.Orc,
-						code.Hectometre,
+						code.String(),
 						dr.Objname.Loc,
 						dr.Lon, dr.Lat,
 						from,
@@ -421,21 +419,31 @@
 						unchanged++
 						return continueErr
 					case err2 != nil:
-						key := pgxutils.ReadableError{Err: err}.Error()
+						key := pgxutils.ReadableError{Err: err2}.Error()
 						databaseErrors[key] = append(databaseErrors[key], isrs)
 						unchanged++
 						return continueErr
 					default:
+						updatedGauges = append(updatedGauges, isrs)
 						//feedback.Info("update")
 					}
 
 					// Remove obsolete reference water levels
 					var currLevels pgtype.VarcharArray
-					currLevels.Set([]string{
-						string(*dr.Reflevel1code),
-						string(*dr.Reflevel2code),
-						string(*dr.Reflevel3code),
-					})
+					var reflevels []string
+					if dr.Reflevel1code != nil {
+						reflevels = append(reflevels,
+							string(*dr.Reflevel1code))
+					}
+					if dr.Reflevel2code != nil {
+						reflevels = append(reflevels,
+							string(*dr.Reflevel2code))
+					}
+					if dr.Reflevel3code != nil {
+						reflevels = append(reflevels,
+							string(*dr.Reflevel3code))
+					}
+					currLevels.Set(reflevels)
 					rwls, err := tx.StmtContext(ctx,
 						deleteReferenceWaterLevelsStmt).QueryContext(ctx,
 						code.String(),
@@ -496,9 +504,9 @@
 					}
 					if !isNtSDepthRef {
 						feedback.Warn(
-							"Reference level code '%s' is not in line "+
+							"Reference level code '%s' of %s is not in line "+
 								"with the NtS reference_code table",
-							string(**wl.level))
+							string(**wl.level), isrs)
 					}
 
 					if _, err := tx.StmtContext(
@@ -550,6 +558,14 @@
 		feedback.Error("Missing objname: %s", strings.Join(missingObjname, ", "))
 	}
 
+	if len(updatedGauges) > 0 {
+		feedback.Info("Updated gauges: %s", strings.Join(updatedGauges, ", "))
+	}
+
+	if len(newGauges) > 0 {
+		feedback.Info("New gauges: %s", strings.Join(newGauges, ", "))
+	}
+
 	if len(gauges) == 0 {
 		return nil, UnchangedError("No gauges returned from ERDMS")
 	}
@@ -579,6 +595,9 @@
 
 	if unchanged == len(gauges) {
 		return nil, UnchangedError("All gauges unchanged")
+	} else {
+		feedback.Info("Unchanged gauges: %d",
+			unchanged)
 	}
 
 	feedback.Info("Importing gauges took %s",