Mercurial > gemma
changeset 2633:eb1d119f253f
Fetch data from ERDMS for all allowed countries
This will enable a sys_admin to import data for all countries while
a waterway_admin still can only import data for his own country.
author | Tom Gottfried <tom@intevation.de> |
---|---|
date | Wed, 13 Mar 2019 18:19:44 +0100 |
parents | 0b14de0bb85f |
children | 49ce3c11ca72 |
files | pkg/imports/dmv.go pkg/imports/erdms.go pkg/imports/wg.go pkg/models/imports.go |
diffstat | 4 files changed, 185 insertions(+), 153 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/imports/dmv.go Wed Mar 13 17:13:15 2019 +0100 +++ b/pkg/imports/dmv.go Wed Mar 13 18:19:44 2019 +0100 @@ -17,14 +17,9 @@ "context" "database/sql" "errors" - "fmt" - "log" - "strings" "time" "gemma.intevation.de/gemma/pkg/models" - "gemma.intevation.de/gemma/pkg/soap" - "gemma.intevation.de/gemma/pkg/soap/erdms" ) type DistanceMarksVirtual struct { @@ -97,42 +92,19 @@ } defer tx.Rollback() - var country string - err = tx.QueryRowContext(ctx, selectCurrentUserCountrySQL).Scan(&country) - switch { - case err == sql.ErrNoRows: - return nil, errors.New("Cannot figure out user country") - case err != nil: + responseData, err := getRisData( + tx, + ctx, + feedback, + dmv.Username, + dmv.Password, + dmv.URL, + dmv.Insecure, + "dismar") + if err != nil { return nil, err } - country = strings.ToUpper(country) - feedback.Info("Using country '%s'.", country) - - var auth *soap.BasicAuth - if dmv.Username != "" { - auth = &soap.BasicAuth{ - Login: dmv.Username, - Password: dmv.Password, - } - } - - client := erdms.NewRefService(dmv.URL, dmv.Insecure, auth) - - request := &erdms.GetRisDataXML{ - GetRisDataXMLType: &erdms.GetRisDataXMLType{ - Subcode: erdms.NoNS{Text: country + "%"}, - Funcode: erdms.NoNS{Text: "DISMAR"}, - }, - } - - data, err := client.GetRisDataXML(request) - - if err != nil { - log.Printf("error: %v\n", err) - return nil, fmt.Errorf("Error requesting ERDMS service: %v", err) - } - insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksVirtualSQL) if err != nil { return nil, err @@ -141,44 +113,46 @@ var ignored, features int - for _, dr := range data.RisdataReturn { - if dr.RisidxCode == nil { - ignored++ - continue - } + for _, data := range responseData { + for _, dr := range data.RisdataReturn { + if dr.RisidxCode == nil { + ignored++ + continue + } - code, err := models.IsrsFromString(string(*dr.RisidxCode)) - if err != nil { - feedback.Warn("invalid ISRS code %v", err) - ignored++ - continue - } + code, err := models.IsrsFromString(string(*dr.RisidxCode)) + if err != nil { + feedback.Warn("invalid ISRS code %v", err) + ignored++ + continue + } - if dr.Lat == nil || dr.Lon == nil { - feedback.Warn("missing lat/lon: %s", code) - ignored++ - continue - } + if dr.Lat == nil || dr.Lon == nil { + feedback.Warn("missing lat/lon: %s", code) + ignored++ + continue + } - if dr.Relenc == nil { - feedback.Warn("missing relenc: %s", code) - ignored++ - continue - } + if dr.Relenc == nil { + feedback.Warn("missing relenc: %s", code) + ignored++ + continue + } - if _, err := insertStmt.ExecContext( - ctx, - code.CountryCode, - code.LoCode, - code.FairwaySection, - code.Orc, - code.Hectometre, - float64(*dr.Lon), float64(*dr.Lat), - string(*dr.Relenc), - ); err != nil { - return nil, err + if _, err := insertStmt.ExecContext( + ctx, + code.CountryCode, + code.LoCode, + code.FairwaySection, + code.Orc, + code.Hectometre, + float64(*dr.Lon), float64(*dr.Lat), + string(*dr.Relenc), + ); err != nil { + return nil, err + } + features++ } - features++ } feedback.Info("ignored: %d", ignored) feedback.Info("features: %d", features)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/erdms.go Wed Mar 13 18:19:44 2019 +0100 @@ -0,0 +1,82 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018, 2019 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann <sascha.teichmann@intevation.de> +// * Tom Gottfried <tom.gottfried@intevation.de> + +package imports + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "gemma.intevation.de/gemma/pkg/soap" + "gemma.intevation.de/gemma/pkg/soap/erdms" +) + +const selectUserCountriesSQL = `SELECT DISTINCT country FROM users.list_users` + +func getRisData( + tx *sql.Tx, + ctx context.Context, + feedback Feedback, + username string, + password string, + URL string, + insecure bool, + funcode string, +) ([]*erdms.GetRisDataXMLResponse, error) { + + var auth *soap.BasicAuth + if username != "" { + auth = &soap.BasicAuth{ + Login: username, + Password: password, + } + } + + client := erdms.NewRefService(URL, insecure, auth) + + rows, err := tx.QueryContext(ctx, selectUserCountriesSQL) + if err != nil { + return nil, err + } + defer rows.Close() + + var country string + var countries []string + var responseData []*erdms.GetRisDataXMLResponse + for rows.Next() { + err = rows.Scan(&country) + if err != nil { + return nil, err + } + countries = append(countries, country) + + request := &erdms.GetRisDataXML{ + GetRisDataXMLType: &erdms.GetRisDataXMLType{ + Subcode: erdms.NoNS{Text: country + "%"}, + Funcode: erdms.NoNS{Text: funcode}, + }, + } + + data, err := client.GetRisDataXML(request) + if err != nil { + return nil, fmt.Errorf("Error requesting ERDMS service: %v", err) + } + responseData = append(responseData, data) + } + feedback.Info("Import data for countries: %s.", + strings.Join(countries, ", ")) + return responseData, nil +}
--- a/pkg/imports/wg.go Wed Mar 13 17:13:15 2019 +0100 +++ b/pkg/imports/wg.go Wed Mar 13 18:19:44 2019 +0100 @@ -18,14 +18,11 @@ "context" "database/sql" "errors" - "fmt" - "strings" "time" "github.com/jackc/pgx/pgtype" "gemma.intevation.de/gemma/pkg/models" - "gemma.intevation.de/gemma/pkg/soap" "gemma.intevation.de/gemma/pkg/soap/erdms" ) @@ -69,8 +66,6 @@ func (*WaterwayGauge) CleanUp() error { return nil } const ( - selectCurrentUserCountrySQL = `SELECT users.current_user_country()` - hasGaugeSQL = ` SELECT true FROM waterway.gauges @@ -145,41 +140,19 @@ } defer tx.Rollback() - var country string - err = tx.QueryRowContext(ctx, selectCurrentUserCountrySQL).Scan(&country) - switch { - case err == sql.ErrNoRows: - return nil, errors.New("Cannot figure out user country") - case err != nil: + responseData, err := getRisData( + tx, + ctx, + feedback, + wg.Username, + wg.Password, + wg.URL, + wg.Insecure, + "wtwgag") + if err != nil { return nil, err } - country = strings.ToUpper(country) - feedback.Info("Using country '%s'.", country) - - var auth *soap.BasicAuth - if wg.Username != "" { - auth = &soap.BasicAuth{ - Login: wg.Username, - Password: wg.Password, - } - } - - client := erdms.NewRefService(wg.URL, wg.Insecure, auth) - - request := &erdms.GetRisDataXML{ - GetRisDataXMLType: &erdms.GetRisDataXMLType{ - Subcode: erdms.NoNS{Text: country + "%"}, - Funcode: erdms.NoNS{Text: "wtwgag"}, - }, - } - - data, err := client.GetRisDataXML(request) - - if err != nil { - return nil, fmt.Errorf("Error requesting ERDMS service: %v", err) - } - hasGaugeStmt, err := tx.PrepareContext(ctx, hasGaugeSQL) if err != nil { return nil, err @@ -189,59 +162,62 @@ var ignored int type idxCode struct { + jdx int idx int code *models.Isrs } var news, olds []idxCode - for i, dr := range data.RisdataReturn { - if dr.RisidxCode == nil { - ignored++ - continue - } - code, err := models.IsrsFromString(string(*dr.RisidxCode)) - if err != nil { - feedback.Warn("invalid ISRS code %v", err) - ignored++ - continue - } + for j, data := range responseData { + for i, dr := range data.RisdataReturn { + if dr.RisidxCode == nil { + ignored++ + continue + } + code, err := models.IsrsFromString(string(*dr.RisidxCode)) + if err != nil { + feedback.Warn("invalid ISRS code %v", err) + ignored++ + continue + } - if dr.Objname.Loc == nil { - feedback.Warn("missing objname: %s", code) - ignored++ - continue - } + if dr.Objname.Loc == nil { + feedback.Warn("missing objname: %s", code) + ignored++ + continue + } - if dr.Lat == nil || dr.Lon == nil { - feedback.Warn("missing lat/lon: %s", code) - ignored++ - continue - } + if dr.Lat == nil || dr.Lon == nil { + feedback.Warn("missing lat/lon: %s", code) + ignored++ + continue + } - if dr.Zeropoint == nil { - feedback.Warn("missing zeropoint: %s", code) - ignored++ - continue - } + if dr.Zeropoint == nil { + feedback.Warn("missing zeropoint: %s", code) + ignored++ + continue + } - var dummy bool - err = hasGaugeStmt.QueryRowContext(ctx, - code.CountryCode, - code.LoCode, - code.FairwaySection, - code.Orc, - code.Hectometre, - ).Scan(&dummy) - switch { - case err == sql.ErrNoRows: - news = append(news, idxCode{idx: i, code: code}) - case err != nil: - return nil, err - case !dummy: - return nil, errors.New("Unexpected result") - default: - olds = append(olds, idxCode{idx: i, code: code}) + var dummy bool + err = hasGaugeStmt.QueryRowContext(ctx, + code.CountryCode, + code.LoCode, + code.FairwaySection, + code.Orc, + code.Hectometre, + ).Scan(&dummy) + switch { + case err == sql.ErrNoRows: + news = append(news, idxCode{jdx: j, idx: i, code: code}) + case err != nil: + return nil, err + case !dummy: + return nil, errors.New("Unexpected result") + default: + olds = append(olds, idxCode{jdx: j, idx: i, code: code}) + } } } feedback.Info("ignored gauges: %d", ignored) @@ -298,7 +274,7 @@ // insert/update the gauges for i := range news { ic := &news[i] - dr := data.RisdataReturn[ic.idx] + dr := responseData[ic.jdx].RisdataReturn[ic.idx] feedback.Info("insert/update %s", ic.code)
--- a/pkg/models/imports.go Wed Mar 13 17:13:15 2019 +0100 +++ b/pkg/models/imports.go Wed Mar 13 18:19:44 2019 +0100 @@ -44,7 +44,7 @@ ConfigurableURLImport } - // WaterwayAxisImport specifies an import of waterway gauges. + // WaterwayGaugeImport specifies an import of waterway gauges. WaterwayGaugeImport struct { ConfigurableURLImport }