Mercurial > gemma

changeset 2633:eb1d119f253f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fetch data from ERDMS for all allowed countries This will enable a sys_admin to import data for all countries while a waterway_admin still can only import data for his own country.
author Tom Gottfried <tom@intevation.de>
date Wed, 13 Mar 2019 18:19:44 +0100
parents 0b14de0bb85f
children 49ce3c11ca72
files pkg/imports/dmv.go pkg/imports/erdms.go pkg/imports/wg.go pkg/models/imports.go
diffstat 4 files changed, 185 insertions(+), 153 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/dmv.go	Wed Mar 13 17:13:15 2019 +0100
+++ b/pkg/imports/dmv.go	Wed Mar 13 18:19:44 2019 +0100
@@ -17,14 +17,9 @@
 	"context"
 	"database/sql"
 	"errors"
-	"fmt"
-	"log"
-	"strings"
 	"time"
 
 	"gemma.intevation.de/gemma/pkg/models"
-	"gemma.intevation.de/gemma/pkg/soap"
-	"gemma.intevation.de/gemma/pkg/soap/erdms"
 )
 
 type DistanceMarksVirtual struct {
@@ -97,42 +92,19 @@
 	}
 	defer tx.Rollback()
 
-	var country string
-	err = tx.QueryRowContext(ctx, selectCurrentUserCountrySQL).Scan(&country)
-	switch {
-	case err == sql.ErrNoRows:
-		return nil, errors.New("Cannot figure out user country")
-	case err != nil:
+	responseData, err := getRisData(
+		tx,
+		ctx,
+		feedback,
+		dmv.Username,
+		dmv.Password,
+		dmv.URL,
+		dmv.Insecure,
+		"dismar")
+	if err != nil {
 		return nil, err
 	}
 
-	country = strings.ToUpper(country)
-	feedback.Info("Using country '%s'.", country)
-
-	var auth *soap.BasicAuth
-	if dmv.Username != "" {
-		auth = &soap.BasicAuth{
-			Login:    dmv.Username,
-			Password: dmv.Password,
-		}
-	}
-
-	client := erdms.NewRefService(dmv.URL, dmv.Insecure, auth)
-
-	request := &erdms.GetRisDataXML{
-		GetRisDataXMLType: &erdms.GetRisDataXMLType{
-			Subcode: erdms.NoNS{Text: country + "%"},
-			Funcode: erdms.NoNS{Text: "DISMAR"},
-		},
-	}
-
-	data, err := client.GetRisDataXML(request)
-
-	if err != nil {
-		log.Printf("error: %v\n", err)
-		return nil, fmt.Errorf("Error requesting ERDMS service: %v", err)
-	}
-
 	insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksVirtualSQL)
 	if err != nil {
 		return nil, err
@@ -141,44 +113,46 @@
 
 	var ignored, features int
 
-	for _, dr := range data.RisdataReturn {
-		if dr.RisidxCode == nil {
-			ignored++
-			continue
-		}
+	for _, data := range responseData {
+		for _, dr := range data.RisdataReturn {
+			if dr.RisidxCode == nil {
+				ignored++
+				continue
+			}
 
-		code, err := models.IsrsFromString(string(*dr.RisidxCode))
-		if err != nil {
-			feedback.Warn("invalid ISRS code %v", err)
-			ignored++
-			continue
-		}
+			code, err := models.IsrsFromString(string(*dr.RisidxCode))
+			if err != nil {
+				feedback.Warn("invalid ISRS code %v", err)
+				ignored++
+				continue
+			}
 
-		if dr.Lat == nil || dr.Lon == nil {
-			feedback.Warn("missing lat/lon: %s", code)
-			ignored++
-			continue
-		}
+			if dr.Lat == nil || dr.Lon == nil {
+				feedback.Warn("missing lat/lon: %s", code)
+				ignored++
+				continue
+			}
 
-		if dr.Relenc == nil {
-			feedback.Warn("missing relenc: %s", code)
-			ignored++
-			continue
-		}
+			if dr.Relenc == nil {
+				feedback.Warn("missing relenc: %s", code)
+				ignored++
+				continue
+			}
 
-		if _, err := insertStmt.ExecContext(
-			ctx,
-			code.CountryCode,
-			code.LoCode,
-			code.FairwaySection,
-			code.Orc,
-			code.Hectometre,
-			float64(*dr.Lon), float64(*dr.Lat),
-			string(*dr.Relenc),
-		); err != nil {
-			return nil, err
+			if _, err := insertStmt.ExecContext(
+				ctx,
+				code.CountryCode,
+				code.LoCode,
+				code.FairwaySection,
+				code.Orc,
+				code.Hectometre,
+				float64(*dr.Lon), float64(*dr.Lat),
+				string(*dr.Relenc),
+			); err != nil {
+				return nil, err
+			}
+			features++
 		}
-		features++
 	}
 	feedback.Info("ignored: %d", ignored)
 	feedback.Info("features: %d", features)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/imports/erdms.go	Wed Mar 13 18:19:44 2019 +0100
@@ -0,0 +1,82 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2018, 2019 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+//  * Tom Gottfried <tom.gottfried@intevation.de>
+
+package imports
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"gemma.intevation.de/gemma/pkg/soap"
+	"gemma.intevation.de/gemma/pkg/soap/erdms"
+)
+
+const selectUserCountriesSQL = `SELECT DISTINCT country FROM users.list_users`
+
+func getRisData(
+	tx *sql.Tx,
+	ctx context.Context,
+	feedback Feedback,
+	username string,
+	password string,
+	URL string,
+	insecure bool,
+	funcode string,
+) ([]*erdms.GetRisDataXMLResponse, error) {
+
+	var auth *soap.BasicAuth
+	if username != "" {
+		auth = &soap.BasicAuth{
+			Login:    username,
+			Password: password,
+		}
+	}
+
+	client := erdms.NewRefService(URL, insecure, auth)
+
+	rows, err := tx.QueryContext(ctx, selectUserCountriesSQL)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var country string
+	var countries []string
+	var responseData []*erdms.GetRisDataXMLResponse
+	for rows.Next() {
+		err = rows.Scan(&country)
+		if err != nil {
+			return nil, err
+		}
+		countries = append(countries, country)
+
+		request := &erdms.GetRisDataXML{
+			GetRisDataXMLType: &erdms.GetRisDataXMLType{
+				Subcode: erdms.NoNS{Text: country + "%"},
+				Funcode: erdms.NoNS{Text: funcode},
+			},
+		}
+
+		data, err := client.GetRisDataXML(request)
+		if err != nil {
+			return nil, fmt.Errorf("Error requesting ERDMS service: %v", err)
+		}
+		responseData = append(responseData, data)
+	}
+	feedback.Info("Import data for countries: %s.",
+		strings.Join(countries, ", "))
+	return responseData, nil
+}
--- a/pkg/imports/wg.go	Wed Mar 13 17:13:15 2019 +0100
+++ b/pkg/imports/wg.go	Wed Mar 13 18:19:44 2019 +0100
@@ -18,14 +18,11 @@
 	"context"
 	"database/sql"
 	"errors"
-	"fmt"
-	"strings"
 	"time"
 
 	"github.com/jackc/pgx/pgtype"
 
 	"gemma.intevation.de/gemma/pkg/models"
-	"gemma.intevation.de/gemma/pkg/soap"
 	"gemma.intevation.de/gemma/pkg/soap/erdms"
 )
 
@@ -69,8 +66,6 @@
 func (*WaterwayGauge) CleanUp() error { return nil }
 
 const (
-	selectCurrentUserCountrySQL = `SELECT users.current_user_country()`
-
 	hasGaugeSQL = `
 SELECT true
 FROM waterway.gauges
@@ -145,41 +140,19 @@
 	}
 	defer tx.Rollback()
 
-	var country string
-	err = tx.QueryRowContext(ctx, selectCurrentUserCountrySQL).Scan(&country)
-	switch {
-	case err == sql.ErrNoRows:
-		return nil, errors.New("Cannot figure out user country")
-	case err != nil:
+	responseData, err := getRisData(
+		tx,
+		ctx,
+		feedback,
+		wg.Username,
+		wg.Password,
+		wg.URL,
+		wg.Insecure,
+		"wtwgag")
+	if err != nil {
 		return nil, err
 	}
 
-	country = strings.ToUpper(country)
-	feedback.Info("Using country '%s'.", country)
-
-	var auth *soap.BasicAuth
-	if wg.Username != "" {
-		auth = &soap.BasicAuth{
-			Login:    wg.Username,
-			Password: wg.Password,
-		}
-	}
-
-	client := erdms.NewRefService(wg.URL, wg.Insecure, auth)
-
-	request := &erdms.GetRisDataXML{
-		GetRisDataXMLType: &erdms.GetRisDataXMLType{
-			Subcode: erdms.NoNS{Text: country + "%"},
-			Funcode: erdms.NoNS{Text: "wtwgag"},
-		},
-	}
-
-	data, err := client.GetRisDataXML(request)
-
-	if err != nil {
-		return nil, fmt.Errorf("Error requesting ERDMS service: %v", err)
-	}
-
 	hasGaugeStmt, err := tx.PrepareContext(ctx, hasGaugeSQL)
 	if err != nil {
 		return nil, err
@@ -189,59 +162,62 @@
 	var ignored int
 
 	type idxCode struct {
+		jdx  int
 		idx  int
 		code *models.Isrs
 	}
 
 	var news, olds []idxCode
 
-	for i, dr := range data.RisdataReturn {
-		if dr.RisidxCode == nil {
-			ignored++
-			continue
-		}
-		code, err := models.IsrsFromString(string(*dr.RisidxCode))
-		if err != nil {
-			feedback.Warn("invalid ISRS code %v", err)
-			ignored++
-			continue
-		}
+	for j, data := range responseData {
+		for i, dr := range data.RisdataReturn {
+			if dr.RisidxCode == nil {
+				ignored++
+				continue
+			}
+			code, err := models.IsrsFromString(string(*dr.RisidxCode))
+			if err != nil {
+				feedback.Warn("invalid ISRS code %v", err)
+				ignored++
+				continue
+			}
 
-		if dr.Objname.Loc == nil {
-			feedback.Warn("missing objname: %s", code)
-			ignored++
-			continue
-		}
+			if dr.Objname.Loc == nil {
+				feedback.Warn("missing objname: %s", code)
+				ignored++
+				continue
+			}
 
-		if dr.Lat == nil || dr.Lon == nil {
-			feedback.Warn("missing lat/lon: %s", code)
-			ignored++
-			continue
-		}
+			if dr.Lat == nil || dr.Lon == nil {
+				feedback.Warn("missing lat/lon: %s", code)
+				ignored++
+				continue
+			}
 
-		if dr.Zeropoint == nil {
-			feedback.Warn("missing zeropoint: %s", code)
-			ignored++
-			continue
-		}
+			if dr.Zeropoint == nil {
+				feedback.Warn("missing zeropoint: %s", code)
+				ignored++
+				continue
+			}
 
-		var dummy bool
-		err = hasGaugeStmt.QueryRowContext(ctx,
-			code.CountryCode,
-			code.LoCode,
-			code.FairwaySection,
-			code.Orc,
-			code.Hectometre,
-		).Scan(&dummy)
-		switch {
-		case err == sql.ErrNoRows:
-			news = append(news, idxCode{idx: i, code: code})
-		case err != nil:
-			return nil, err
-		case !dummy:
-			return nil, errors.New("Unexpected result")
-		default:
-			olds = append(olds, idxCode{idx: i, code: code})
+			var dummy bool
+			err = hasGaugeStmt.QueryRowContext(ctx,
+				code.CountryCode,
+				code.LoCode,
+				code.FairwaySection,
+				code.Orc,
+				code.Hectometre,
+			).Scan(&dummy)
+			switch {
+			case err == sql.ErrNoRows:
+				news = append(news, idxCode{jdx: j, idx: i, code: code})
+			case err != nil:
+				return nil, err
+			case !dummy:
+				return nil, errors.New("Unexpected result")
+			default:
+				olds = append(olds, idxCode{jdx: j, idx: i, code: code})
+			}
 		}
 	}
 	feedback.Info("ignored gauges: %d", ignored)
@@ -298,7 +274,7 @@
 	// insert/update the gauges
 	for i := range news {
 		ic := &news[i]
-		dr := data.RisdataReturn[ic.idx]
+		dr := responseData[ic.jdx].RisdataReturn[ic.idx]
 
 		feedback.Info("insert/update %s", ic.code)
 
--- a/pkg/models/imports.go	Wed Mar 13 17:13:15 2019 +0100
+++ b/pkg/models/imports.go	Wed Mar 13 18:19:44 2019 +0100
@@ -44,7 +44,7 @@
 		ConfigurableURLImport
 	}
 
-	// WaterwayAxisImport specifies an import of waterway gauges.
+	// WaterwayGaugeImport specifies an import of waterway gauges.
 	WaterwayGaugeImport struct {
 		ConfigurableURLImport
 	}