Mercurial > gemma
changeset 5732:9dbef85c6bb4
Merged uploadwg branch.
author | Sascha Wilde <wilde@intevation.de> |
---|---|
date | Tue, 28 May 2024 18:22:34 +0200 |
parents | 5a0fbdda6e2d (current diff) 0500d76e074b (diff) |
children | 87b6dcfbab34 |
files | |
diffstat | 6 files changed, 181 insertions(+), 30 deletions(-) [+] |
line wrap: on
line diff
--- a/client/src/components/importconfiguration/ScheduledImports.vue Tue May 28 18:20:58 2024 +0200 +++ b/client/src/components/importconfiguration/ScheduledImports.vue Tue May 28 18:22:34 2024 +0200 @@ -144,6 +144,7 @@ :url="url" :username="username" :password="password" + :directImport="directImport" /> <Waterwayaxis v-if="import_ == $options.IMPORTTYPES.WATERWAYAXIS" @@ -585,6 +586,7 @@ case this.$options.IMPORTTYPES.BOTTLENECK: case this.$options.IMPORTTYPES.FAIRWAYAVAILABILITY: case this.$options.IMPORTTYPES.GAUGEMEASUREMENT: + case this.$options.IMPORTTYPES.WATERWAYGAUGES: return true; default: return false; @@ -859,6 +861,9 @@ case this.$options.IMPORTTYPES.GAUGEMEASUREMENT: routeParam = "ugm"; break; + case this.$options.IMPORTTYPES.WATERWAYGAUGES: + routeParam = "uwg"; + break; default: throw new Error("invalid importroute"); }
--- a/client/src/components/importconfiguration/types/Waterwaygauges.vue Tue May 28 18:20:58 2024 +0200 +++ b/client/src/components/importconfiguration/types/Waterwaygauges.vue Tue May 28 18:22:34 2024 +0200 @@ -2,25 +2,29 @@ <div> <div class="d-flex px-2"> <div class="flex-column w-100"> - <div class="flex-row text-left"> - <small class="text-muted"> <translate>URL</translate> </small> - </div> - <div class="w-100"> - <input - @input="urlChanged" - class="url form-control form-control-sm" - type="url" - :value="url" - /> - </div> + <template v-if="!directImport"> + <div class="flex-row text-left"> + <small class="text-muted"> + <translate>URL</translate> + </small> + </div> + <div class="w-100"> + <input + @input="urlChanged" + class="url form-control form-control-sm" + type="url" + :value="url" + /> + </div> + </template> </div> </div> - <div v-if="!url" class="d-flex px-2"> + <div v-if="!directImport && !url" class="d-flex px-2"> <small ><translate class="text-danger">Please enter a URL</translate></small > </div> - <div class="d-flex px-2"> + <div v-if="!directImport" class="d-flex px-2"> <div class="flex-column mt-2 mr-3 w-50"> <div class="flex-row text-left"> <small class="text-muted"> <translate>Username</translate> </small> @@ -81,19 +85,20 @@ * SPDX-License-Identifier: AGPL-3.0-or-later * License-Filename: LICENSES/AGPL-3.0.txt * - * Copyright (C) 2018 by via donau + * Copyright (C) 2018, 2024 by via donau * – Österreichische Wasserstraßen-Gesellschaft mbH * Software engineering by Intevation GmbH * * Author(s): * Thomas Junk <thomas.junk@intevation.de> + * Sascha Wilde <wilde@intevation.de> */ import { mapState } from "vuex"; export default { name: "waterwaygauges", - props: ["username", "password", "url"], + props: ["username", "password", "url", "directImport"], data() { return { passwordVisible: false
--- a/pkg/controllers/routes.go Tue May 28 18:20:58 2024 +0200 +++ b/pkg/controllers/routes.go Tue May 28 18:22:34 2024 +0200 @@ -234,6 +234,9 @@ api.Handle("/imports/ugm", waterwayAdmin( importUploadedGaugeMeasurement())).Methods(http.MethodPost) + api.Handle("/imports/uwg", waterwayAdmin( + importUploadedWaterwayGauge())).Methods(http.MethodPost) + api.Handle("/imports/stsh", sysAdmin( importUploadedStretchShape())).Methods(http.MethodPost)
--- a/pkg/controllers/uploadedimports.go Tue May 28 18:20:58 2024 +0200 +++ b/pkg/controllers/uploadedimports.go Tue May 28 18:22:34 2024 +0200 @@ -143,6 +143,17 @@ return &imports.StretchShape{Dir: dir}, nil }, ) + +} + +func importUploadedWaterwayGauge() http.HandlerFunc { + return uploadedImport( + imports.UWGJobKind, + "data.xml", + func(_ *http.Request, dir string) (imports.Job, error) { + return &imports.UploadedWaterwayGauge{Dir: dir}, nil + }, + ) } func (bup badUploadParameterError) Error() string {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/uwg.go Tue May 28 18:22:34 2024 +0200 @@ -0,0 +1,90 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018, 2024 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann <sascha.teichmann@intevation.de> +// * Sascha Wilde <wilde@intevation.de> + +package imports + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + + "gemma.intevation.de/gemma/pkg/soap" + erdms "gemma.intevation.de/gemma/pkg/soap/erdms2" +) + +// UploadedWaterwayGauge is an Job to extract gauge measurement data +// from an uploaded XML file and stores it into the database. +type UploadedWaterwayGauge struct { + Dir string `json:"dir"` +} + +// UWGJobKind is the unique name of this import job type. +const UWGJobKind JobKind = "uwg" + +type uwgJobCreator struct{} + +func init() { RegisterJobCreator(UWGJobKind, uwgJobCreator{}) } + +func (uwgJobCreator) Description() string { return "uploaded waterway gauges" } + +func (uwgJobCreator) Create() Job { return new(UploadedWaterwayGauge) } + +func (uwgJobCreator) Depends() [2][]string { return wgJobCreator{}.Depends() } + +func (uwgJobCreator) AutoAccept() bool { return true } + +// StageDone is a NOP for gauge measurements imports. +func (uwgJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error { return nil } + +// CleanUp removes the temporary files from the filesystem. +func (uwg *UploadedWaterwayGauge) CleanUp() error { return os.RemoveAll(uwg.Dir) } + +// Do executes the actual uploaded waterway gauges import. +func (uwg *UploadedWaterwayGauge) Do( + ctx context.Context, + _ int64, + conn *sql.Conn, + feedback Feedback, +) (any, error) { + + fetch := func( + _ context.Context, + _ *sql.Tx, + ) ([]*erdms.GetRisDataXMLResponse, error) { + + var dst []*erdms.GetRisDataXMLResponse + + if err := soap.ValidateFile( + filepath.Join(uwg.Dir, "data.xml"), + "ERDMS_WS_XSD_2.0-fixed.xsd", + &dst, + ); err != nil { + return nil, err + } + + if dst == nil { + return nil, errors.New("no gauges found") + } + return dst, nil + } + + return storeWaterwayGauges( + ctx, + conn, + feedback, + fetch, + ) +}
--- a/pkg/imports/wg.go Tue May 28 18:20:58 2024 +0200 +++ b/pkg/imports/wg.go Tue May 28 18:22:34 2024 +0200 @@ -4,12 +4,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // -// Copyright (C) 2018, 2019 by via donau +// Copyright (C) 2018, 2019, 2024 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Sascha L. Teichmann <sascha.teichmann@intevation.de> +// * Sascha Wilde <wilde@intevation.de> // * Tom Gottfried <tom.gottfried@intevation.de> package imports @@ -182,28 +183,19 @@ var errContinue = errors.New("continue") -// Do implements the actual import. -func (wg *WaterwayGauge) Do( +func storeWaterwayGauges( ctx context.Context, - _ int64, conn *sql.Conn, feedback Feedback, + fetch func(context.Context, *sql.Tx) ([]*erdms.GetRisDataXMLResponse, error), ) (any, error) { - start := time.Now() - responseData, countries, err := getRisData( - ctx, - conn, - feedback, - wg.Username, - wg.Password, - wg.URL, - wg.Insecure, - "wtwgag") + tx, err := conn.BeginTx(ctx, nil) if err != nil { return nil, err } + defer tx.Rollback() var eraseGaugeStmt, insertStmt, fixValidityStmt, updateStmt, @@ -235,7 +227,12 @@ databaseErrors := map[string][]string{} - for _, data := range responseData { + wgs, err := fetch(ctx, tx) + if err != nil { + return nil, err + } + + for _, data := range wgs { for _, dr := range data.RisdataReturn { isrs := string(*dr.RisidxCode) @@ -570,6 +567,11 @@ return nil, UnchangedError("No gauges returned from ERDMS") } + countries, err := userCountries(ctx, conn) + if err != nil { + return nil, err + } + var pgCountries, pgGauges pgtype.VarcharArray pgCountries.Set(countries) pgGauges.Set(gauges) @@ -604,3 +606,38 @@ return nil, err } + +// Do implements the actual import. +func (wg *WaterwayGauge) Do( + ctx context.Context, + _ int64, + conn *sql.Conn, + feedback Feedback, +) (any, error) { + + fetch := func( + ctx context.Context, + _ *sql.Tx, + ) ([]*erdms.GetRisDataXMLResponse, error) { + responseData, _, err := getRisData( + ctx, + conn, + feedback, + wg.Username, + wg.Password, + wg.URL, + wg.Insecure, + "wtwgag") + if err != nil { + return nil, err + } + return responseData, nil + } + + return storeWaterwayGauges( + ctx, + conn, + feedback, + fetch, + ) +}