Mercurial > gemma

changeset 5732:9dbef85c6bb4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged uploadwg branch.
author Sascha Wilde <wilde@intevation.de>
date Tue, 28 May 2024 18:22:34 +0200
parents 5a0fbdda6e2d (current diff) 0500d76e074b (diff)
children 87b6dcfbab34
files
diffstat 6 files changed, 181 insertions(+), 30 deletions(-) [+]
line wrap: on
line diff
--- a/client/src/components/importconfiguration/ScheduledImports.vue	Tue May 28 18:20:58 2024 +0200
+++ b/client/src/components/importconfiguration/ScheduledImports.vue	Tue May 28 18:22:34 2024 +0200
@@ -144,6 +144,7 @@
       :url="url"
       :username="username"
       :password="password"
+      :directImport="directImport"
     />
     <Waterwayaxis
       v-if="import_ == $options.IMPORTTYPES.WATERWAYAXIS"
@@ -585,6 +586,7 @@
         case this.$options.IMPORTTYPES.BOTTLENECK:
         case this.$options.IMPORTTYPES.FAIRWAYAVAILABILITY:
         case this.$options.IMPORTTYPES.GAUGEMEASUREMENT:
+        case this.$options.IMPORTTYPES.WATERWAYGAUGES:
           return true;
         default:
           return false;
@@ -859,6 +861,9 @@
         case this.$options.IMPORTTYPES.GAUGEMEASUREMENT:
           routeParam = "ugm";
           break;
+        case this.$options.IMPORTTYPES.WATERWAYGAUGES:
+          routeParam = "uwg";
+          break;
         default:
           throw new Error("invalid importroute");
       }
--- a/client/src/components/importconfiguration/types/Waterwaygauges.vue	Tue May 28 18:20:58 2024 +0200
+++ b/client/src/components/importconfiguration/types/Waterwaygauges.vue	Tue May 28 18:22:34 2024 +0200
@@ -2,25 +2,29 @@
   <div>
     <div class="d-flex px-2">
       <div class="flex-column w-100">
-        <div class="flex-row text-left">
-          <small class="text-muted"> <translate>URL</translate> </small>
-        </div>
-        <div class="w-100">
-          <input
-            @input="urlChanged"
-            class="url form-control form-control-sm"
-            type="url"
-            :value="url"
-          />
-        </div>
+        <template v-if="!directImport">
+          <div class="flex-row text-left">
+            <small class="text-muted">
+              <translate>URL</translate>
+            </small>
+          </div>
+          <div class="w-100">
+            <input
+              @input="urlChanged"
+              class="url form-control form-control-sm"
+              type="url"
+              :value="url"
+            />
+          </div>
+        </template>
       </div>
     </div>
-    <div v-if="!url" class="d-flex px-2">
+    <div v-if="!directImport && !url" class="d-flex px-2">
       <small
         ><translate class="text-danger">Please enter a URL</translate></small
       >
     </div>
-    <div class="d-flex px-2">
+    <div v-if="!directImport" class="d-flex px-2">
       <div class="flex-column mt-2 mr-3 w-50">
         <div class="flex-row text-left">
           <small class="text-muted"> <translate>Username</translate> </small>
@@ -81,19 +85,20 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  * License-Filename: LICENSES/AGPL-3.0.txt
  *
- * Copyright (C) 2018 by via donau
+ * Copyright (C) 2018, 2024 by via donau
  *   – Österreichische Wasserstraßen-Gesellschaft mbH
  * Software engineering by Intevation GmbH
  *
  * Author(s):
  * Thomas Junk <thomas.junk@intevation.de>
+ * Sascha Wilde <wilde@intevation.de>
  */
 
 import { mapState } from "vuex";
 
 export default {
   name: "waterwaygauges",
-  props: ["username", "password", "url"],
+  props: ["username", "password", "url", "directImport"],
   data() {
     return {
       passwordVisible: false
--- a/pkg/controllers/routes.go	Tue May 28 18:20:58 2024 +0200
+++ b/pkg/controllers/routes.go	Tue May 28 18:22:34 2024 +0200
@@ -234,6 +234,9 @@
 	api.Handle("/imports/ugm", waterwayAdmin(
 		importUploadedGaugeMeasurement())).Methods(http.MethodPost)
 
+	api.Handle("/imports/uwg", waterwayAdmin(
+		importUploadedWaterwayGauge())).Methods(http.MethodPost)
+
 	api.Handle("/imports/stsh", sysAdmin(
 		importUploadedStretchShape())).Methods(http.MethodPost)
 
--- a/pkg/controllers/uploadedimports.go	Tue May 28 18:20:58 2024 +0200
+++ b/pkg/controllers/uploadedimports.go	Tue May 28 18:22:34 2024 +0200
@@ -143,6 +143,17 @@
 			return &imports.StretchShape{Dir: dir}, nil
 		},
 	)
+
+}
+
+func importUploadedWaterwayGauge() http.HandlerFunc {
+	return uploadedImport(
+		imports.UWGJobKind,
+		"data.xml",
+		func(_ *http.Request, dir string) (imports.Job, error) {
+			return &imports.UploadedWaterwayGauge{Dir: dir}, nil
+		},
+	)
 }
 
 func (bup badUploadParameterError) Error() string {
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/imports/uwg.go	Tue May 28 18:22:34 2024 +0200
@@ -0,0 +1,90 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2018, 2024 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+//  * Sascha Wilde <wilde@intevation.de>
+
+package imports
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"os"
+	"path/filepath"
+
+	"gemma.intevation.de/gemma/pkg/soap"
+	erdms "gemma.intevation.de/gemma/pkg/soap/erdms2"
+)
+
+// UploadedWaterwayGauge is an Job to extract gauge measurement data
+// from an uploaded XML file and stores it into the database.
+type UploadedWaterwayGauge struct {
+	Dir string `json:"dir"`
+}
+
+// UWGJobKind is the unique name of this import job type.
+const UWGJobKind JobKind = "uwg"
+
+type uwgJobCreator struct{}
+
+func init() { RegisterJobCreator(UWGJobKind, uwgJobCreator{}) }
+
+func (uwgJobCreator) Description() string { return "uploaded waterway gauges" }
+
+func (uwgJobCreator) Create() Job { return new(UploadedWaterwayGauge) }
+
+func (uwgJobCreator) Depends() [2][]string { return wgJobCreator{}.Depends() }
+
+func (uwgJobCreator) AutoAccept() bool { return true }
+
+// StageDone is a NOP for gauge measurements imports.
+func (uwgJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error { return nil }
+
+// CleanUp removes the temporary files from the filesystem.
+func (uwg *UploadedWaterwayGauge) CleanUp() error { return os.RemoveAll(uwg.Dir) }
+
+// Do executes the actual uploaded waterway gauges import.
+func (uwg *UploadedWaterwayGauge) Do(
+	ctx context.Context,
+	_ int64,
+	conn *sql.Conn,
+	feedback Feedback,
+) (any, error) {
+
+	fetch := func(
+		_ context.Context,
+		_ *sql.Tx,
+	) ([]*erdms.GetRisDataXMLResponse, error) {
+
+		var dst []*erdms.GetRisDataXMLResponse
+
+		if err := soap.ValidateFile(
+			filepath.Join(uwg.Dir, "data.xml"),
+			"ERDMS_WS_XSD_2.0-fixed.xsd",
+			&dst,
+		); err != nil {
+			return nil, err
+		}
+
+		if dst == nil {
+			return nil, errors.New("no gauges found")
+		}
+		return dst, nil
+	}
+
+	return storeWaterwayGauges(
+		ctx,
+		conn,
+		feedback,
+		fetch,
+	)
+}
--- a/pkg/imports/wg.go	Tue May 28 18:20:58 2024 +0200
+++ b/pkg/imports/wg.go	Tue May 28 18:22:34 2024 +0200
@@ -4,12 +4,13 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // License-Filename: LICENSES/AGPL-3.0.txt
 //
-// Copyright (C) 2018, 2019 by via donau
+// Copyright (C) 2018, 2019, 2024 by via donau
 //   – Österreichische Wasserstraßen-Gesellschaft mbH
 // Software engineering by Intevation GmbH
 //
 // Author(s):
 //  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+//  * Sascha Wilde <wilde@intevation.de>
 //  * Tom Gottfried <tom.gottfried@intevation.de>
 
 package imports
@@ -182,28 +183,19 @@
 
 var errContinue = errors.New("continue")
 
-// Do implements the actual import.
-func (wg *WaterwayGauge) Do(
+func storeWaterwayGauges(
 	ctx context.Context,
-	_ int64,
 	conn *sql.Conn,
 	feedback Feedback,
+	fetch func(context.Context, *sql.Tx) ([]*erdms.GetRisDataXMLResponse, error),
 ) (any, error) {
-
 	start := time.Now()
 
-	responseData, countries, err := getRisData(
-		ctx,
-		conn,
-		feedback,
-		wg.Username,
-		wg.Password,
-		wg.URL,
-		wg.Insecure,
-		"wtwgag")
+	tx, err := conn.BeginTx(ctx, nil)
 	if err != nil {
 		return nil, err
 	}
+	defer tx.Rollback()
 
 	var eraseGaugeStmt, insertStmt,
 		fixValidityStmt, updateStmt,
@@ -235,7 +227,12 @@
 
 	databaseErrors := map[string][]string{}
 
-	for _, data := range responseData {
+	wgs, err := fetch(ctx, tx)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, data := range wgs {
 		for _, dr := range data.RisdataReturn {
 
 			isrs := string(*dr.RisidxCode)
@@ -570,6 +567,11 @@
 		return nil, UnchangedError("No gauges returned from ERDMS")
 	}
 
+	countries, err := userCountries(ctx, conn)
+	if err != nil {
+		return nil, err
+	}
+
 	var pgCountries, pgGauges pgtype.VarcharArray
 	pgCountries.Set(countries)
 	pgGauges.Set(gauges)
@@ -604,3 +606,38 @@
 
 	return nil, err
 }
+
+// Do implements the actual import.
+func (wg *WaterwayGauge) Do(
+	ctx context.Context,
+	_ int64,
+	conn *sql.Conn,
+	feedback Feedback,
+) (any, error) {
+
+	fetch := func(
+		ctx context.Context,
+		_ *sql.Tx,
+	) ([]*erdms.GetRisDataXMLResponse, error) {
+		responseData, _, err := getRisData(
+			ctx,
+			conn,
+			feedback,
+			wg.Username,
+			wg.Password,
+			wg.URL,
+			wg.Insecure,
+			"wtwgag")
+		if err != nil {
+			return nil, err
+		}
+		return responseData, nil
+	}
+
+	return storeWaterwayGauges(
+		ctx,
+		conn,
+		feedback,
+		fetch,
+	)
+}