changeset 5529:f95f268a83bd aggregate-gm-import-logging

WIP: Continued work on aggregating GM logs.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Sun, 24 Oct 2021 20:16:41 +0200
parents 133dc5b3076a
children 3ccbb6162112
files pkg/imports/gm.go
diffstat 1 files changed, 69 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/gm.go	Sun Oct 24 16:54:00 2021 +0200
+++ b/pkg/imports/gm.go	Sun Oct 24 20:16:41 2021 +0200
@@ -322,10 +322,11 @@
 }
 
 type gmLog struct {
-	isrs                *models.Isrs
+	gid                 string
 	unknown             bool
 	assumedZPG          bool
-	ignoredMeasureCodes []nts.Measure_code_enum
+	ignoredMeasureCodes []string
+	rescaleErrors       []string
 	assumedCM           int
 	missingValues       []time.Time
 	badValue            int
@@ -335,25 +336,60 @@
 
 type gmLogs []*gmLog
 
-func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) {
-	for _, m := range gl.ignoredMeasureCodes {
-		if m == mc {
-			return
+func extend(haystack []string, needle string) []string {
+	for _, straw := range haystack {
+		if straw == needle {
+			return haystack
 		}
 	}
-	gl.ignoredMeasureCodes = append(gl.ignoredMeasureCodes, mc)
+	return append(haystack, needle)
+}
+
+func (gl *gmLog) addRescaleError(err error) {
+	gl.rescaleErrors = extend(gl.rescaleErrors, err.Error())
+}
+
+func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) {
+	gl.ignoredMeasureCodes = extend(gl.ignoredMeasureCodes, string(mc))
+}
+
+func (gls gmLogs) find(gid string) *gmLog {
+	for _, gl := range gls {
+		if gl.gid == gid {
+			return gl
+		}
+	}
+	return nil
 }
 
 func (gls gmLogs) logging(feedback Feedback) {
 	gls.logUnknown(feedback)
 	gls.logAssumedZPG(feedback)
-	// TODO: assumed CM
+	gls.logAssumedCM(feedback)
 	// TODO: bad values
 	// TODO: missing values
+	// TODO: rescale errors
 	// TODO: new predictions
 	// TODO: new measurements
 }
 
+func (gls gmLogs) logAssumedCM(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if gl.assumedCM > 0 {
+			if sb.Len() == 0 {
+				sb.WriteString("'Unit' not specified. Assuming 'cm': ")
+			} else {
+				sb.WriteString(", ")
+			}
+			fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.assumedCM)
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Warn(sb.String())
+	}
+}
+
 func (gls gmLogs) logUnknown(feedback Feedback) {
 	var sb strings.Builder
 	for _, gl := range gls {
@@ -363,7 +399,7 @@
 			} else {
 				sb.WriteString(", ")
 			}
-			sb.WriteString(gl.isrs.String())
+			sb.WriteString(gl.gid)
 		}
 	}
 	if sb.Len() > 0 {
@@ -380,7 +416,7 @@
 			} else {
 				sb.WriteString(", ")
 			}
-			sb.WriteString(gl.isrs.String())
+			sb.WriteString(gl.gid)
 		}
 	}
 	if sb.Len() > 0 {
@@ -388,6 +424,25 @@
 	}
 }
 
+// logFinder is a helper to search recently used logs
+// or create a new one if no log for a given gauge
+// existed before.
+func logFinder(logs *gmLogs) func(string) *gmLog {
+	var lastLog *gmLog
+	return func(gid string) *gmLog {
+		if lastLog != nil && lastLog.gid == gid {
+			return lastLog
+		}
+		if ll := logs.find(gid); ll != nil {
+			lastLog = ll
+			return ll
+		}
+		lastLog = &gmLog{gid: gid}
+		*logs = append(*logs, lastLog)
+		return lastLog
+	}
+}
+
 func doForGM(
 	ctx context.Context,
 	gauges []string,
@@ -426,6 +481,8 @@
 	var logs gmLogs
 	defer logs.logging(feedback)
 
+	findLog := logFinder(&logs)
+
 	for _, msg := range result {
 		for _, wrm := range msg.Wrm {
 			curr := string(*wrm.Geo_object.Id)
@@ -435,8 +492,7 @@
 				feedback.Warn("Invalid ISRS code %v", err)
 				continue
 			}
-			log := &gmLog{isrs: currIsrs}
-			logs = append(logs, log)
+			log := findLog(curr)
 			gids = append(gids, curr)
 
 			//feedback.Info("Found measurements/predictions for %s", curr)
@@ -482,8 +538,7 @@
 
 				convert, err := rescale(unit)
 				if err != nil {
-					// TODO: log rescale error
-					feedback.Error(err.Error())
+					log.addRescaleError(err)
 					continue
 				}
 				convert(measure.Value)