Mercurial > gemma

changeset 5528:133dc5b3076a aggregate-gm-import-logging

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
WIP: Started to log GM imports in a more agregated way.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Sun, 24 Oct 2021 16:54:00 +0200
parents 35966741e45e
children f95f268a83bd
files pkg/imports/gm.go
diffstat 1 files changed, 104 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/gm.go	Fri Oct 22 20:11:19 2021 +0200
+++ b/pkg/imports/gm.go	Sun Oct 24 16:54:00 2021 +0200
@@ -321,6 +321,73 @@
 	return fn, nil
 }
 
+type gmLog struct {
+	isrs                *models.Isrs
+	unknown             bool
+	assumedZPG          bool
+	ignoredMeasureCodes []nts.Measure_code_enum
+	assumedCM           int
+	missingValues       []time.Time
+	badValue            int
+	newM                int
+	newP                int
+}
+
+type gmLogs []*gmLog
+
+func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) {
+	for _, m := range gl.ignoredMeasureCodes {
+		if m == mc {
+			return
+		}
+	}
+	gl.ignoredMeasureCodes = append(gl.ignoredMeasureCodes, mc)
+}
+
+func (gls gmLogs) logging(feedback Feedback) {
+	gls.logUnknown(feedback)
+	gls.logAssumedZPG(feedback)
+	// TODO: assumed CM
+	// TODO: bad values
+	// TODO: missing values
+	// TODO: new predictions
+	// TODO: new measurements
+}
+
+func (gls gmLogs) logUnknown(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if gl.assumedZPG {
+			if sb.Len() == 0 {
+				sb.WriteString("Cannot find following gauges: ")
+			} else {
+				sb.WriteString(", ")
+			}
+			sb.WriteString(gl.isrs.String())
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Warn(sb.String())
+	}
+}
+
+func (gls gmLogs) logAssumedZPG(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if gl.assumedZPG {
+			if sb.Len() == 0 {
+				sb.WriteString("'Reference_code' not specified. Assuming 'ZPG': ")
+			} else {
+				sb.WriteString(", ")
+			}
+			sb.WriteString(gl.isrs.String())
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Info(sb.String())
+	}
+}
+
 func doForGM(
 	ctx context.Context,
 	gauges []string,
@@ -353,6 +420,12 @@
 	}
 
 	var gids []string
+
+	// To prevent spamming the log actual logging
+	// is defered to be presented in an aggregated way.
+	var logs gmLogs
+	defer logs.logging(feedback)
+
 	for _, msg := range result {
 		for _, wrm := range msg.Wrm {
 			curr := string(*wrm.Geo_object.Id)
@@ -362,44 +435,54 @@
 				feedback.Warn("Invalid ISRS code %v", err)
 				continue
 			}
-			feedback.Info("Found measurements/predictions for %s", curr)
+			log := &gmLog{isrs: currIsrs}
+			logs = append(logs, log)
+			gids = append(gids, curr)
+
+			//feedback.Info("Found measurements/predictions for %s", curr)
 			if !isKnown(curr) {
-				feedback.Warn("Cannot find gauge %q for import", curr)
+				log.unknown = true
+				// feedback.Warn("Cannot find gauge %q for import", curr)
 				continue
 			}
 
 			var referenceCode string
 			if wrm.Reference_code == nil {
-				feedback.Info("'Reference_code' not specified. Assuming 'ZPG'")
+				//feedback.Info("'Reference_code' not specified. Assuming 'ZPG'")
+				log.assumedZPG = true
 				referenceCode = "ZPG"
 			} else {
 				referenceCode = string(*wrm.Reference_code)
 			}
 
-			badValue := 0
-			newM, newP := 0, 0
+			//badValue := 0
+			//newM, newP := 0, 0
 			for _, measure := range wrm.Measure {
 				var unit string
 				if *measure.Measure_code != nts.Measure_code_enumWAL {
-					feedback.Warn("Ignored message with measure_code %s",
-						*measure.Measure_code)
+					log.ignoreMeasureCode(*measure.Measure_code)
+					//feedback.Warn("Ignored message with measure_code %s",
+					//		*measure.Measure_code)
 					continue
 				}
 				if measure.Unit == nil {
-					feedback.Info("'Unit' not specified. Assuming 'cm'")
+					//feedback.Info("'Unit' not specified. Assuming 'cm'")
+					log.assumedCM++
 					unit = "cm"
 				} else {
 					unit = string(*measure.Unit)
 				}
 
 				if measure.Value == nil {
-					feedback.Warn("Missing mandatory value at %s. Ignored (bad service)",
-						measure.Measuredate.Format(time.RFC3339))
+					log.missingValues = append(log.missingValues, measure.Measuredate.Time)
+					//feedback.Warn("Missing mandatory value at %s. Ignored (bad service)",
+					//	measure.Measuredate.Format(time.RFC3339))
 					continue
 				}
 
 				convert, err := rescale(unit)
 				if err != nil {
+					// TODO: log rescale error
 					feedback.Error(err.Error())
 					continue
 				}
@@ -409,7 +492,7 @@
 
 				// -99999 is used by some gauges to signal an error
 				if *measure.Value == -99999 {
-					badValue++
+					log.badValue++
 					continue
 				}
 
@@ -459,7 +542,7 @@
 					case err != nil:
 						feedback.Error(pgxutils.ReadableError{Err: err}.Error())
 					default:
-						newP++
+						log.newP++
 					}
 				} else {
 					err = insertGMStmt.QueryRowContext(
@@ -485,19 +568,18 @@
 					case err != nil:
 						feedback.Error(pgxutils.ReadableError{Err: err}.Error())
 					default:
-						newM++
+						log.newM++
 					}
 				}
 			}
-			if badValue > 0 {
-				feedback.Warn("Ignored %d measurements with value -99999",
-					badValue)
-			}
-			feedback.Info("Inserted %d measurements for %s",
-				newM, curr)
-			feedback.Info("Inserted %d predictions for %s",
-				newP, curr)
-			gids = append(gids, curr)
+			//if badValue > 0 {
+			//	feedback.Warn("Ignored %d measurements with value -99999",
+			//		badValue)
+			//}
+			//feedback.Info("Inserted %d measurements for %s",
+			//	newM, curr)
+			//feedback.Info("Inserted %d predictions for %s",
+			//	newP, curr)
 		}
 	}
 	return gids, nil