Mercurial > gemma

changeset 2562:ce39e9954e85

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Make upload of AGM require only "fk_gauge_id" "measure_date" and "value"
author Sascha Wilde <wilde@intevation.de>
date Fri, 08 Mar 2019 18:57:58 +0100
parents 4d5f419a2318
children dc4fae4bdb8f
files pkg/controllers/uploadedimports.go pkg/imports/agm.go pkg/misc/lang.go
diffstat 3 files changed, 56 insertions(+), 89 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/controllers/uploadedimports.go	Fri Mar 08 16:49:40 2019 +0100
+++ b/pkg/controllers/uploadedimports.go	Fri Mar 08 18:57:58 2019 +0100
@@ -73,8 +73,16 @@
 	return uploadedImport(
 		imports.AGMJobKind,
 		"agm.csv",
-		func(_ *http.Request, dir string) (imports.Job, error) {
-			return &imports.ApprovedGaugeMeasurements{Dir: dir}, nil
+		func(req *http.Request, dir string) (imports.Job, error) {
+			originator := req.FormValue("originator")
+			if originator == "" {
+				return nil, BadUploadParameterError("missing 'originator' parameter")
+			}
+
+			return &imports.ApprovedGaugeMeasurements{
+				Dir:        dir,
+				Originator: originator,
+			}, nil
 		},
 	)
 }
--- a/pkg/imports/agm.go	Fri Mar 08 16:49:40 2019 +0100
+++ b/pkg/imports/agm.go	Fri Mar 08 18:57:58 2019 +0100
@@ -10,6 +10,7 @@
 //
 // Author(s):
 //  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+//  * Sascha Wilde <wilde@intevation.de>
 
 package imports
 
@@ -33,7 +34,8 @@
 )
 
 type ApprovedGaugeMeasurements struct {
-	Dir string `json:"dir"`
+	Dir        string `json:"dir"`
+	Originator string `json:"originator"`
 }
 
 // GMAPJobKind is the unique name of an approved gauge measurements import job.
@@ -144,13 +146,11 @@
 	return a.CountryCode != b.CountryCode ||
 		a.Sender != b.Sender ||
 		a.LanguageCode != b.LanguageCode ||
-		!a.DateIssue.Time.Equal(b.DateIssue.Time) ||
 		a.ReferenceCode != b.ReferenceCode ||
 		math.Abs(a.WaterLevel-b.WaterLevel) > eps ||
 		a.Predicted != b.Predicted ||
 		fdiff(a.ValueMin, b.ValueMin) ||
 		fdiff(a.ValueMax, b.ValueMax) ||
-		!a.DateInfo.Time.Equal(b.DateInfo.Time) ||
 		a.SourceOrganization != b.SourceOrganization
 }
 
@@ -249,20 +249,9 @@
 	}
 
 	var (
-		fkGaugeIDIdx     = -1
-		measureDateIdx   = -1
-		fromIdx          = -1
-		languageCodeIdx  = -1
-		countryCodeIdx   = -1
-		dateIssueIdx     = -1
-		referenceCodeIdx = -1
-		valueIdx         = -1
-		predictedIdx     = -1
-		valueMinIdx      = -1
-		valueMaxIdx      = -1
-		dateInfoIdx      = -1
-		originatorIdx    = -1
-		unitIdx          = -1
+		fkGaugeIDIdx   = -1
+		measureDateIdx = -1
+		valueIdx       = -1
 	)
 
 	headerFields := []struct {
@@ -271,19 +260,7 @@
 	}{
 		{&fkGaugeIDIdx, "fk_gauge_id"},
 		{&measureDateIdx, "measure_date"},
-		{&fromIdx, "from"}, // "sender",
-		{&languageCodeIdx, "language_code"},
-		{&countryCodeIdx, "country_code"},
-		{&dateIssueIdx, "date_issue"},
-		{&referenceCodeIdx, "reference_code"},
 		{&valueIdx, "value"}, // "water_level",
-		{&predictedIdx, "predicted"},
-		// "is_waterlevel",
-		{&valueMinIdx, "value_min"},
-		{&valueMaxIdx, "value_max"},
-		{&dateInfoIdx, "date_info"},
-		{&originatorIdx, "originator"}, // "source_organization",
-		{&unitIdx, "unit"},
 	}
 
 nextHeader:
@@ -313,19 +290,6 @@
 		return nil, fmt.Errorf("Missing columns: %s", strings.Join(missing, ", "))
 	}
 
-	inCm, _ := rescale("cm")
-	scaler := func(row []string) (func(float32) float32, error) {
-		if unitIdx == -1 {
-			return inCm, nil
-		}
-		unit := row[unitIdx]
-		if unit == "cm" {
-			return inCm, nil
-		}
-		s, err := rescale(unit)
-		return s, err
-	}
-
 	tx, err := conn.BeginTx(ctx, nil)
 	if err != nil {
 		return nil, err
@@ -367,10 +331,6 @@
 		case err != nil:
 			return nil, fmt.Errorf("CSV parsing failed: %v", err)
 		}
-		convert, err := scaler(row)
-		if err != nil {
-			return nil, fmt.Errorf("line %d: %v", line, err)
-		}
 
 		gids := row[fkGaugeIDIdx]
 		gid, err := models.IsrsFromString(gids)
@@ -453,57 +413,32 @@
 			return nil, err
 		}
 
-		newSender := row[fromIdx]
-		newLanguageCode := row[languageCodeIdx]
-		newCountryCode := row[countryCodeIdx]
-
-		dis, err := guessDate(row[dateIssueIdx])
-		if err != nil {
-			return nil, fmt.Errorf("Invalid 'date_issue' line %d: %v", line, err)
-		}
-		newDateIssue := dis
-
-		newReferenceCode := row[referenceCodeIdx]
+		newSender := agm.Originator
+		newCountryCode := gid.CountryCode
+		newLanguageCode := misc.CCtoLang[gid.CountryCode]
+		newDateIssue := time.Now()
+		newReferenceCode := "ZPG"
 
 		value, err := strconv.ParseFloat(row[valueIdx], 32)
 		if err != nil {
 			return nil, fmt.Errorf("Invalid 'value' line %d: %v", line, err)
 		}
-		newValue := float64(convert(float32(value)))
+		newValue := value
 
-		newPredicted := strings.ToLower(row[predictedIdx]) == "true"
+		newPredicted := false
 
-		var newValueMin sql.NullFloat64
-		if vm := row[valueMinIdx]; vm != "" {
-			valueMin, err := strconv.ParseFloat(vm, 32)
-			if err != nil {
-				return nil, fmt.Errorf("Invalid 'value_min' line %d: %v", line, err)
-			}
-			newValueMin = sql.NullFloat64{
-				Float64: float64(convert(float32(valueMin))),
-				Valid:   true,
-			}
+		newValueMin := sql.NullFloat64{
+			Float64: 0,
+			Valid:   true,
+		}
+		newValueMax := sql.NullFloat64{
+			Float64: 0,
+			Valid:   true,
 		}
 
-		var newValueMax sql.NullFloat64
-		if vm := row[valueMaxIdx]; vm != "" {
-			valueMax, err := strconv.ParseFloat(vm, 32)
-			if err != nil {
-				return nil, fmt.Errorf("Invalid 'value_max' line %d: %v", line, err)
-			}
-			newValueMax = sql.NullFloat64{
-				Float64: float64(convert(float32(valueMax))),
-				Valid:   true,
-			}
-		}
+		newDateInfo := newDateIssue
 
-		din, err := guessDate(row[dateInfoIdx])
-		if err != nil {
-			return nil, fmt.Errorf("Invalid 'date_info' line %d: %v", line, err)
-		}
-		newDateInfo := din
-
-		newSourceOrganization := row[originatorIdx]
+		newSourceOrganization := newSender
 
 		var newID int64
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/misc/lang.go	Fri Mar 08 18:57:58 2019 +0100
@@ -0,0 +1,24 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2019 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Sascha Wilde <wilde@intevation.de>
+
+package misc
+
+var CCtoLang = map[string]string{
+	"GB": "EN",
+	"AT": "DE",
+	"SK": "SK",
+	"HU": "HU",
+	"HR": "HR",
+	"BG": "BG",
+	"RO": "RO",
+}