Mercurial > gemma

diff pkg/imports/agm.go @ 3277:232fc90e6ee2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Disentangle gauge measurements and predictions Representing both in one table has led to the necessity to make the distinction at many places such as statements, definitions of partial indexes and application code. At least in one place in the AGM import the distinction in application code was too late and measurements matching an approved measurement could have been missed.
author Tom Gottfried <tom@intevation.de>
date Wed, 15 May 2019 19:08:49 +0200
parents 56b297592c0a
children 831193935739
line wrap: on
line diff
--- a/pkg/imports/agm.go	Wed May 15 17:55:38 2019 +0200
+++ b/pkg/imports/agm.go	Wed May 15 19:08:49 2019 +0200
@@ -78,7 +78,6 @@
     ON n.fk_gauge_id = o.fk_gauge_id AND n.measure_date = o.measure_date
     WHERE n.id IN (SELECT key FROM staged)
 	  AND o.id NOT IN (SELECT key FROM staged)
-          AND NOT o.predicted
 )
 DELETE FROM waterway.gauge_measurements WHERE id IN (SELECT id from to_delete)`
 
@@ -120,38 +119,23 @@
 }
 
 type agmLine struct {
-	CountryCode        string   `json:"country-code"`
-	Sender             string   `json:"sender"`
-	LanguageCode       string   `json:"language-code"`
-	DateIssue          timetz   `json:"date-issue"`
-	ReferenceCode      string   `json:"reference-code"`
-	WaterLevel         float64  `json:"water-level"`
-	Predicted          bool     `json:"predicted"`
-	ValueMin           *float64 `json:"value-min"`
-	ValueMax           *float64 `json:"value-max"`
-	DateInfo           timetz   `json:"date-info"`
-	SourceOrganization string   `json:"source-organization"`
+	CountryCode        string  `json:"country-code"`
+	Sender             string  `json:"sender"`
+	LanguageCode       string  `json:"language-code"`
+	DateIssue          timetz  `json:"date-issue"`
+	ReferenceCode      string  `json:"reference-code"`
+	WaterLevel         float64 `json:"water-level"`
+	DateInfo           timetz  `json:"date-info"`
+	SourceOrganization string  `json:"source-organization"`
 }
 
 func (a *agmLine) hasDiff(b *agmLine) bool {
 	const eps = 0.00001
-	fdiff := func(x, y *float64) bool {
-		if x == nil && y == nil {
-			return false
-		}
-		if (x == nil && y != nil) || (x != nil && y == nil) {
-			return true
-		}
-		return math.Abs(*x-*y) > eps
-	}
 	return a.CountryCode != b.CountryCode ||
 		a.Sender != b.Sender ||
 		a.LanguageCode != b.LanguageCode ||
 		a.ReferenceCode != b.ReferenceCode ||
 		math.Abs(a.WaterLevel-b.WaterLevel) > eps ||
-		a.Predicted != b.Predicted ||
-		fdiff(a.ValueMin, b.ValueMin) ||
-		fdiff(a.ValueMax, b.ValueMax) ||
 		a.SourceOrganization != b.SourceOrganization
 }
 
@@ -171,9 +155,6 @@
   date_issue,
   reference_code,
   water_level,
-  predicted,
-  value_min,
-  value_max,
   date_info,
   source_organization
 FROM waterway.gauge_measurements
@@ -191,9 +172,6 @@
   date_issue,
   reference_code,
   water_level,
-  predicted,
-  value_min,
-  value_max,
   date_info,
   source_organization,
   is_waterlevel,
@@ -209,9 +187,6 @@
   $12,
   $13,
   $14,
-  $15,
-  $16,
-  $17,
   true,
   false
 )
@@ -382,9 +357,6 @@
 			oldDateIssue          time.Time
 			oldReferenceCode      string
 			oldValue              float64
-			oldPredicted          bool
-			oldValueMin           sql.NullFloat64
-			oldValueMax           sql.NullFloat64
 			oldDateInfo           time.Time
 			oldSourceOrganization string
 		)
@@ -405,9 +377,6 @@
 			&oldDateIssue,
 			&oldReferenceCode,
 			&oldValue,
-			&oldPredicted,
-			&oldValueMin,
-			&oldValueMax,
 			&oldDateInfo,
 			&oldSourceOrganization,
 		)
@@ -433,17 +402,6 @@
 		}
 		newValue := value
 
-		newPredicted := false
-
-		newValueMin := sql.NullFloat64{
-			Float64: 0,
-			Valid:   true,
-		}
-		newValueMax := sql.NullFloat64{
-			Float64: 0,
-			Valid:   true,
-		}
-
 		newDateInfo := newDateIssue
 
 		newSourceOrganization := newSender
@@ -470,9 +428,6 @@
 			newDateIssue,
 			newReferenceCode,
 			newValue,
-			newPredicted,
-			newValueMin,
-			newValueMax,
 			newDateInfo,
 			newSourceOrganization,
 		).Scan(&newID); err != nil {
@@ -499,9 +454,6 @@
 			newDateIssue,
 			newReferenceCode,
 			newValue,
-			newPredicted,
-			newValueMin,
-			newValueMax,
 			newDateInfo,
 			newSourceOrganization,
 		)
@@ -521,14 +473,11 @@
 				oldDateIssue,
 				oldReferenceCode,
 				oldValue,
-				oldPredicted,
-				oldValueMin,
-				oldValueMax,
 				oldDateInfo,
 				oldSourceOrganization,
 			)
 			// Ignore if there is no diff.
-			if o.Predicted || !n.hasDiff(o) {
+			if !n.hasDiff(o) {
 				continue
 			}
 			ase.Versions = []*agmLine{o, n}
@@ -549,19 +498,9 @@
 	dateIssue time.Time,
 	referenceCode string,
 	waterLevel float64,
-	predicted bool,
-	valueMin sql.NullFloat64,
-	valueMax sql.NullFloat64,
 	dateInfo time.Time,
 	sourceOrganization string,
 ) *agmLine {
-	nilFloat := func(v sql.NullFloat64) *float64 {
-		var p *float64
-		if v.Valid {
-			p = &v.Float64
-		}
-		return p
-	}
 	return &agmLine{
 		CountryCode:        countryCode,
 		Sender:             sender,
@@ -569,9 +508,6 @@
 		DateIssue:          timetz{dateIssue},
 		ReferenceCode:      referenceCode,
 		WaterLevel:         waterLevel,
-		Predicted:          predicted,
-		ValueMin:           nilFloat(valueMin),
-		ValueMax:           nilFloat(valueMax),
 		DateInfo:           timetz{dateInfo},
 		SourceOrganization: sourceOrganization,
 	}