Mercurial > gemma

changeset 5531:84f0c53e6ffe aggregate-gm-import-logging

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Completed GM logging aggregation. TODO: cleanup and dedup.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Sun, 24 Oct 2021 23:57:14 +0200
parents 3ccbb6162112
children 7e21a9e6353d
files pkg/imports/gm.go
diffstat 1 files changed, 74 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/gm.go	Sun Oct 24 23:06:24 2021 +0200
+++ b/pkg/imports/gm.go	Sun Oct 24 23:57:14 2021 +0200
@@ -327,11 +327,11 @@
 	assumedZPG          bool
 	ignoredMeasureCodes []string
 	rescaleErrors       []string
+	missingValues       []string
 	assumedCM           int
-	missingValues       []time.Time
 	badValues           int
-	newM                int
-	newP                int
+	measurements        int
+	predictions         int
 }
 
 type gmLogs []*gmLog
@@ -369,9 +369,67 @@
 	gls.logBadValues(feedback)
 	gls.logMissingValues(feedback)
 	gls.logRescaleErrors(feedback)
-	// TODO: log ignoredMeasureCodes
-	// TODO: new predictions
-	// TODO: new measurements
+	gls.logIgnoredMeasureCodes(feedback)
+	gls.logPredictions(feedback)
+	gls.logMeasurements(feedback)
+}
+
+func (gls gmLogs) logMeasurements(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if gl.measurements > 0 {
+			if sb.Len() == 0 {
+				sb.WriteString("New measurements: ")
+			} else {
+				sb.WriteString("; ")
+			}
+			fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.measurements)
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Info(sb.String())
+	}
+}
+
+func (gls gmLogs) logPredictions(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if gl.predictions > 0 {
+			if sb.Len() == 0 {
+				sb.WriteString("New predictions: ")
+			} else {
+				sb.WriteString("; ")
+			}
+			fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.predictions)
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Info(sb.String())
+	}
+}
+
+func (gls gmLogs) logIgnoredMeasureCodes(feedback Feedback) {
+	var sb strings.Builder
+	for _, gl := range gls {
+		if len(gl.ignoredMeasureCodes) > 0 {
+			if sb.Len() == 0 {
+				sb.WriteString("Ignored measure codes: ")
+			} else {
+				sb.WriteString("; ")
+			}
+			fmt.Fprintf(&sb, "%s (", gl.gid)
+			for i, mc := range gl.ignoredMeasureCodes {
+				if i > 0 {
+					sb.WriteString(", ")
+				}
+				sb.WriteString(mc)
+			}
+			sb.WriteByte(')')
+		}
+	}
+	if sb.Len() > 0 {
+		feedback.Warn(sb.String())
+	}
 }
 
 func (gls gmLogs) logRescaleErrors(feedback Feedback) {
@@ -412,7 +470,7 @@
 				if i > 0 {
 					sb.WriteString(", ")
 				}
-				sb.WriteString(m.Format(time.RFC3339))
+				sb.WriteString(m)
 			}
 			sb.WriteByte(')')
 		}
@@ -429,7 +487,7 @@
 			if sb.Len() == 0 {
 				sb.WriteString("Ignored measurements with value -99999: ")
 			} else {
-				sb.WriteString(", ")
+				sb.WriteString("; ")
 			}
 			fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.badValues)
 		}
@@ -446,7 +504,7 @@
 			if sb.Len() == 0 {
 				sb.WriteString("'Unit' not specified. Assuming 'cm': ")
 			} else {
-				sb.WriteString(", ")
+				sb.WriteString("; ")
 			}
 			fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.assumedCM)
 		}
@@ -463,7 +521,7 @@
 			if sb.Len() == 0 {
 				sb.WriteString("Cannot find following gauges: ")
 			} else {
-				sb.WriteString(", ")
+				sb.WriteString("; ")
 			}
 			sb.WriteString(gl.gid)
 		}
@@ -480,7 +538,7 @@
 			if sb.Len() == 0 {
 				sb.WriteString("'Reference_code' not specified. Assuming 'ZPG': ")
 			} else {
-				sb.WriteString(", ")
+				sb.WriteString("; ")
 			}
 			sb.WriteString(gl.gid)
 		}
@@ -596,9 +654,9 @@
 				}
 
 				if measure.Value == nil {
-					log.missingValues = append(log.missingValues, measure.Measuredate.Time)
-					//feedback.Warn("Missing mandatory value at %s. Ignored (bad service)",
-					//	measure.Measuredate.Format(time.RFC3339))
+					log.missingValues = append(
+						log.missingValues,
+						measure.Measuredate.Time.Format(time.RFC3339))
 					continue
 				}
 
@@ -663,7 +721,7 @@
 					case err != nil:
 						feedback.Error(pgxutils.ReadableError{Err: err}.Error())
 					default:
-						log.newP++
+						log.predictions++
 					}
 				} else {
 					err = insertGMStmt.QueryRowContext(
@@ -689,7 +747,7 @@
 					case err != nil:
 						feedback.Error(pgxutils.ReadableError{Err: err}.Error())
 					default:
-						log.newM++
+						log.measurements++
 					}
 				}
 			}