Mercurial > gemma
changeset 5528:133dc5b3076a aggregate-gm-import-logging
WIP: Started to log GM imports in a more agregated way.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Sun, 24 Oct 2021 16:54:00 +0200 |
parents | 35966741e45e |
children | f95f268a83bd |
files | pkg/imports/gm.go |
diffstat | 1 files changed, 104 insertions(+), 22 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/imports/gm.go Fri Oct 22 20:11:19 2021 +0200 +++ b/pkg/imports/gm.go Sun Oct 24 16:54:00 2021 +0200 @@ -321,6 +321,73 @@ return fn, nil } +type gmLog struct { + isrs *models.Isrs + unknown bool + assumedZPG bool + ignoredMeasureCodes []nts.Measure_code_enum + assumedCM int + missingValues []time.Time + badValue int + newM int + newP int +} + +type gmLogs []*gmLog + +func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) { + for _, m := range gl.ignoredMeasureCodes { + if m == mc { + return + } + } + gl.ignoredMeasureCodes = append(gl.ignoredMeasureCodes, mc) +} + +func (gls gmLogs) logging(feedback Feedback) { + gls.logUnknown(feedback) + gls.logAssumedZPG(feedback) + // TODO: assumed CM + // TODO: bad values + // TODO: missing values + // TODO: new predictions + // TODO: new measurements +} + +func (gls gmLogs) logUnknown(feedback Feedback) { + var sb strings.Builder + for _, gl := range gls { + if gl.assumedZPG { + if sb.Len() == 0 { + sb.WriteString("Cannot find following gauges: ") + } else { + sb.WriteString(", ") + } + sb.WriteString(gl.isrs.String()) + } + } + if sb.Len() > 0 { + feedback.Warn(sb.String()) + } +} + +func (gls gmLogs) logAssumedZPG(feedback Feedback) { + var sb strings.Builder + for _, gl := range gls { + if gl.assumedZPG { + if sb.Len() == 0 { + sb.WriteString("'Reference_code' not specified. Assuming 'ZPG': ") + } else { + sb.WriteString(", ") + } + sb.WriteString(gl.isrs.String()) + } + } + if sb.Len() > 0 { + feedback.Info(sb.String()) + } +} + func doForGM( ctx context.Context, gauges []string, @@ -353,6 +420,12 @@ } var gids []string + + // To prevent spamming the log actual logging + // is defered to be presented in an aggregated way. + var logs gmLogs + defer logs.logging(feedback) + for _, msg := range result { for _, wrm := range msg.Wrm { curr := string(*wrm.Geo_object.Id) @@ -362,44 +435,54 @@ feedback.Warn("Invalid ISRS code %v", err) continue } - feedback.Info("Found measurements/predictions for %s", curr) + log := &gmLog{isrs: currIsrs} + logs = append(logs, log) + gids = append(gids, curr) + + //feedback.Info("Found measurements/predictions for %s", curr) if !isKnown(curr) { - feedback.Warn("Cannot find gauge %q for import", curr) + log.unknown = true + // feedback.Warn("Cannot find gauge %q for import", curr) continue } var referenceCode string if wrm.Reference_code == nil { - feedback.Info("'Reference_code' not specified. Assuming 'ZPG'") + //feedback.Info("'Reference_code' not specified. Assuming 'ZPG'") + log.assumedZPG = true referenceCode = "ZPG" } else { referenceCode = string(*wrm.Reference_code) } - badValue := 0 - newM, newP := 0, 0 + //badValue := 0 + //newM, newP := 0, 0 for _, measure := range wrm.Measure { var unit string if *measure.Measure_code != nts.Measure_code_enumWAL { - feedback.Warn("Ignored message with measure_code %s", - *measure.Measure_code) + log.ignoreMeasureCode(*measure.Measure_code) + //feedback.Warn("Ignored message with measure_code %s", + // *measure.Measure_code) continue } if measure.Unit == nil { - feedback.Info("'Unit' not specified. Assuming 'cm'") + //feedback.Info("'Unit' not specified. Assuming 'cm'") + log.assumedCM++ unit = "cm" } else { unit = string(*measure.Unit) } if measure.Value == nil { - feedback.Warn("Missing mandatory value at %s. Ignored (bad service)", - measure.Measuredate.Format(time.RFC3339)) + log.missingValues = append(log.missingValues, measure.Measuredate.Time) + //feedback.Warn("Missing mandatory value at %s. Ignored (bad service)", + // measure.Measuredate.Format(time.RFC3339)) continue } convert, err := rescale(unit) if err != nil { + // TODO: log rescale error feedback.Error(err.Error()) continue } @@ -409,7 +492,7 @@ // -99999 is used by some gauges to signal an error if *measure.Value == -99999 { - badValue++ + log.badValue++ continue } @@ -459,7 +542,7 @@ case err != nil: feedback.Error(pgxutils.ReadableError{Err: err}.Error()) default: - newP++ + log.newP++ } } else { err = insertGMStmt.QueryRowContext( @@ -485,19 +568,18 @@ case err != nil: feedback.Error(pgxutils.ReadableError{Err: err}.Error()) default: - newM++ + log.newM++ } } } - if badValue > 0 { - feedback.Warn("Ignored %d measurements with value -99999", - badValue) - } - feedback.Info("Inserted %d measurements for %s", - newM, curr) - feedback.Info("Inserted %d predictions for %s", - newP, curr) - gids = append(gids, curr) + //if badValue > 0 { + // feedback.Warn("Ignored %d measurements with value -99999", + // badValue) + //} + //feedback.Info("Inserted %d measurements for %s", + // newM, curr) + //feedback.Info("Inserted %d predictions for %s", + // newP, curr) } } return gids, nil