Mercurial > gemma
changeset 5529:f95f268a83bd aggregate-gm-import-logging
WIP: Continued work on aggregating GM logs.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Sun, 24 Oct 2021 20:16:41 +0200 |
parents | 133dc5b3076a |
children | 3ccbb6162112 |
files | pkg/imports/gm.go |
diffstat | 1 files changed, 69 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/imports/gm.go Sun Oct 24 16:54:00 2021 +0200 +++ b/pkg/imports/gm.go Sun Oct 24 20:16:41 2021 +0200 @@ -322,10 +322,11 @@ } type gmLog struct { - isrs *models.Isrs + gid string unknown bool assumedZPG bool - ignoredMeasureCodes []nts.Measure_code_enum + ignoredMeasureCodes []string + rescaleErrors []string assumedCM int missingValues []time.Time badValue int @@ -335,25 +336,60 @@ type gmLogs []*gmLog -func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) { - for _, m := range gl.ignoredMeasureCodes { - if m == mc { - return +func extend(haystack []string, needle string) []string { + for _, straw := range haystack { + if straw == needle { + return haystack } } - gl.ignoredMeasureCodes = append(gl.ignoredMeasureCodes, mc) + return append(haystack, needle) +} + +func (gl *gmLog) addRescaleError(err error) { + gl.rescaleErrors = extend(gl.rescaleErrors, err.Error()) +} + +func (gl *gmLog) ignoreMeasureCode(mc nts.Measure_code_enum) { + gl.ignoredMeasureCodes = extend(gl.ignoredMeasureCodes, string(mc)) +} + +func (gls gmLogs) find(gid string) *gmLog { + for _, gl := range gls { + if gl.gid == gid { + return gl + } + } + return nil } func (gls gmLogs) logging(feedback Feedback) { gls.logUnknown(feedback) gls.logAssumedZPG(feedback) - // TODO: assumed CM + gls.logAssumedCM(feedback) // TODO: bad values // TODO: missing values + // TODO: rescale errors // TODO: new predictions // TODO: new measurements } +func (gls gmLogs) logAssumedCM(feedback Feedback) { + var sb strings.Builder + for _, gl := range gls { + if gl.assumedCM > 0 { + if sb.Len() == 0 { + sb.WriteString("'Unit' not specified. Assuming 'cm': ") + } else { + sb.WriteString(", ") + } + fmt.Fprintf(&sb, "%s (%d)", gl.gid, gl.assumedCM) + } + } + if sb.Len() > 0 { + feedback.Warn(sb.String()) + } +} + func (gls gmLogs) logUnknown(feedback Feedback) { var sb strings.Builder for _, gl := range gls { @@ -363,7 +399,7 @@ } else { sb.WriteString(", ") } - sb.WriteString(gl.isrs.String()) + sb.WriteString(gl.gid) } } if sb.Len() > 0 { @@ -380,7 +416,7 @@ } else { sb.WriteString(", ") } - sb.WriteString(gl.isrs.String()) + sb.WriteString(gl.gid) } } if sb.Len() > 0 { @@ -388,6 +424,25 @@ } } +// logFinder is a helper to search recently used logs +// or create a new one if no log for a given gauge +// existed before. +func logFinder(logs *gmLogs) func(string) *gmLog { + var lastLog *gmLog + return func(gid string) *gmLog { + if lastLog != nil && lastLog.gid == gid { + return lastLog + } + if ll := logs.find(gid); ll != nil { + lastLog = ll + return ll + } + lastLog = &gmLog{gid: gid} + *logs = append(*logs, lastLog) + return lastLog + } +} + func doForGM( ctx context.Context, gauges []string, @@ -426,6 +481,8 @@ var logs gmLogs defer logs.logging(feedback) + findLog := logFinder(&logs) + for _, msg := range result { for _, wrm := range msg.Wrm { curr := string(*wrm.Geo_object.Id) @@ -435,8 +492,7 @@ feedback.Warn("Invalid ISRS code %v", err) continue } - log := &gmLog{isrs: currIsrs} - logs = append(logs, log) + log := findLog(curr) gids = append(gids, curr) //feedback.Info("Found measurements/predictions for %s", curr) @@ -482,8 +538,7 @@ convert, err := rescale(unit) if err != nil { - // TODO: log rescale error - feedback.Error(err.Error()) + log.addRescaleError(err) continue } convert(measure.Value)