Mercurial > gemma

changeset 4070:4332b9e26e2f historization_ng

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged default
author Sascha Wilde <wilde@intevation.de>
date Thu, 25 Jul 2019 13:20:32 +0200
parents 12f476e91c70 (current diff) 6ebe42af392d (diff)
children 1eb39e9e8ec2
files pkg/imports/agm.go pkg/imports/bn.go pkg/imports/errors.go pkg/imports/gm.go pkg/imports/wg.go schema/gemma.sql schema/version.sql
diffstat 19 files changed, 166 insertions(+), 147 deletions(-) [+]
line wrap: on
line diff
--- a/client/src/components/gauge/Waterlevel.vue	Thu Jul 25 12:54:30 2019 +0200
+++ b/client/src/components/gauge/Waterlevel.vue	Thu Jul 25 13:20:32 2019 +0200
@@ -460,8 +460,8 @@
         .attr("clip-path", "url(#waterlevel-clip)");
       svg.selectAll(".hdc-line").attr("stroke", "red");
       svg.selectAll(".ldc-line").attr("stroke", "green");
-      svg.selectAll(".mw-line").attr("stroke", "grey");
-      svg.selectAll(".rn-line").attr("stroke", "grey");
+      svg.selectAll(".mw-line").attr("stroke", "rgb(128,128,128)");
+      svg.selectAll(".rn-line").attr("stroke", "rgb(128,128,128)");
       svg
         .selectAll(".ref-waterlevel-label")
         .style("font-size", "10px")
--- a/client/src/components/importoverview/AGMLogItem.vue	Thu Jul 25 12:54:30 2019 +0200
+++ b/client/src/components/importoverview/AGMLogItem.vue	Thu Jul 25 13:20:32 2019 +0200
@@ -40,13 +40,15 @@
           {{ entry }}
         </div>
         <div :class="isNew(line) ? 'col-6' : 'col-4'">
-          {{ line.versions[0][entry] }}
+          <span :class="line.versions[1] ? '' : 'text-danger'">{{
+            line.versions[0][entry]
+          }}</span>
         </div>
         <div
           v-if="isOld(line) && isDifferent(line, entry)"
           :class="isNew(line) ? 'col-6' : 'col-4'"
         >
-          {{ line.versions[1][entry] }}
+          {{ line.versions[1] ? line.versions[1][entry] : $options.DELETED }}
         </div>
       </div>
     </div>
@@ -54,6 +56,22 @@
 </template>
 
 <script>
+/* This is Free Software under GNU Affero General Public License v >= 3.0
+ * without warranty, see README.md and license for details.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ * License-Filename: LICENSES/AGPL-3.0.txt
+ *
+ * Copyright (C) 2018, 2019 by via donau
+ *   – Österreichische Wasserstraßen-Gesellschaft mbH
+ * Software engineering by Intevation GmbH
+ *
+ * Author(s):
+ * * Thomas Junk <thomas.junk@intevation.de>
+ * * Markus Kottländer <markus.kottlaender@intevation.de>
+ * * Fadi Abbud <fadi.abbud@intevation.de>
+ */
+import app from "@/main";
 export default {
   props: ["line", "index", "showDiff"],
   computed: {
@@ -72,12 +90,14 @@
       return !this.isNew(result);
     },
     isDifferent(result, entry) {
+      if (!result.versions[1]) return true;
       return (
         this.isOld(result) &&
         result.versions[0][entry] != result.versions[1][entry]
       );
     }
-  }
+  },
+  DELETED: app.$gettext("deleted")
 };
 </script>
 
--- a/pkg/controllers/user.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/controllers/user.go	Thu Jul 25 13:20:32 2019 +0200
@@ -29,6 +29,7 @@
 	"gemma.intevation.de/gemma/pkg/auth"
 	"gemma.intevation.de/gemma/pkg/misc"
 	"gemma.intevation.de/gemma/pkg/models"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/scheduler"
 )
 
@@ -262,6 +263,7 @@
 	}
 
 	if err != nil {
+		err = pgxutils.HandleError(err)
 		return
 	}
 
--- a/pkg/imports/agm.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/agm.go	Thu Jul 25 13:20:32 2019 +0200
@@ -491,10 +491,9 @@
 	var removed int
 
 	// Issue deletes
-	for _, key := range sortByLocationCode(oldGMLines) {
-		old := oldGMLines[key]
+	for _, old := range oldGMLines {
 		removed += len(old)
-		for _, line := range sortByMeasureDate(old) {
+		for _, line := range old {
 			if _, err := txTrackStmt.ExecContext(
 				ctx, importID, "waterway.gauge_measurements",
 				line.id,
@@ -521,6 +520,12 @@
 		return nil, fmt.Errorf("Commit failed: %v", err)
 	}
 
+	// Sort here to mix the deletes right beside the matching inserts/updates.
+	// This also makes the output deterministic.
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].FKGaugeID.Less(&entries[j].FKGaugeID)
+	})
+
 	feedback.Info("Imported %d entries with changes", len(entries))
 	feedback.Info("Importing approved gauge measurements took %s",
 		time.Since(start))
@@ -528,32 +533,6 @@
 	return entries, nil
 }
 
-func sortByLocationCode(data map[models.Isrs]map[int64]*agmLine) []models.Isrs {
-	keys := make([]models.Isrs, len(data))
-	var i int
-	for key := range data {
-		keys[i] = key
-		i++
-	}
-	sort.Slice(keys, func(i, j int) bool {
-		return keys[i].Less(&keys[j])
-	})
-	return keys
-}
-
-func sortByMeasureDate(data map[int64]*agmLine) []*agmLine {
-	lines := make([]*agmLine, len(data))
-	var i int
-	for _, line := range data {
-		lines[i] = line
-		i++
-	}
-	sort.Slice(lines, func(i, j int) bool {
-		return lines[i].MeasureDate.Before(lines[j].MeasureDate.Time)
-	})
-	return lines
-}
-
 func getOldGMLines(
 	ctx context.Context,
 	stmt *sql.Stmt,
--- a/pkg/imports/bn.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/bn.go	Thu Jul 25 13:20:32 2019 +0200
@@ -24,6 +24,7 @@
 	"strings"
 	"time"
 
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/soap/ifbn"
 	"github.com/jackc/pgx/pgtype"
 )
@@ -581,7 +582,7 @@
 		)
 	}
 	if err != nil {
-		feedback.Warn(handleError(err).Error())
+		feedback.Warn(pgxutils.HandleError(err).Error())
 		return nil
 	}
 	defer bns.Close()
@@ -593,7 +594,7 @@
 		bnIds = append(bnIds, nid)
 	}
 	if err := bns.Err(); err != nil {
-		feedback.Warn(handleError(err).Error())
+		feedback.Warn(pgxutils.HandleError(err).Error())
 		return nil
 	}
 	if len(bnIds) == 0 {
@@ -610,7 +611,7 @@
 		&validity,
 		&pgBnIds,
 	); err != nil {
-		feedback.Warn(handleError(err).Error())
+		feedback.Warn(pgxutils.HandleError(err).Error())
 		if err2 := tx.Rollback(); err2 != nil {
 			return err2
 		}
@@ -623,7 +624,7 @@
 		bn.Bottleneck_id,
 		validity,
 	); err != nil {
-		feedback.Warn(handleError(err).Error())
+		feedback.Warn(pgxutils.HandleError(err).Error())
 		if err2 := tx.Rollback(); err2 != nil {
 			return err2
 		}
@@ -660,7 +661,7 @@
 			&pgMaterials,
 		); err != nil {
 			feedback.Warn("Failed to insert riverbed materials")
-			feedback.Warn(handleError(err).Error())
+			feedback.Warn(pgxutils.HandleError(err).Error())
 			return nil
 		}
 	}
--- a/pkg/imports/errors.go	Thu Jul 25 12:54:30 2019 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,84 +0,0 @@
-// This is Free Software under GNU Affero General Public License v >= 3.0
-// without warranty, see README.md and license for details.
-//
-// SPDX-License-Identifier: AGPL-3.0-or-later
-// License-Filename: LICENSES/AGPL-3.0.txt
-//
-// Copyright (C) 2019 by via donau
-//   – Österreichische Wasserstraßen-Gesellschaft mbH
-// Software engineering by Intevation GmbH
-//
-// Author(s):
-//  * Tom Gottfried <tom.gottfried@intevation.de>
-
-package imports
-
-import (
-	"strings"
-
-	"github.com/jackc/pgx"
-)
-
-func handleError(err error) error {
-	switch e := err.(type) {
-	case pgx.PgError:
-		return dbError(e)
-	}
-	return err
-}
-
-// Handle PostgreSQL error codes
-const (
-	notNullViolation         = "23502"
-	foreignKeyViolation      = "23503"
-	violatesRowLevelSecurity = "42501"
-	noDataFound              = "P0002"
-)
-
-type dbError pgx.PgError
-
-func (err dbError) Error() string {
-	switch err.Code {
-	case notNullViolation:
-		switch err.SchemaName {
-		case "waterway":
-			switch err.TableName {
-			case "gauges":
-				switch err.ColumnName {
-				case "objname":
-					return "Missing objname"
-				case "geom":
-					return "Missing lat/lon"
-				case "zero_point":
-					return "Missing zeropoint"
-				}
-			}
-		}
-	case foreignKeyViolation:
-		switch err.SchemaName {
-		case "waterway":
-			switch err.TableName {
-			case "gauge_measurements", "gauge_predictions", "bottlenecks":
-				switch err.ConstraintName {
-				case "gauge_key":
-					return "Referenced gauge with matching temporal validity not available"
-				}
-			}
-		}
-	case noDataFound:
-		// Most recent line from stacktrace contains name of failed function
-		recent := strings.SplitN(err.Where, "\n", 1)[0]
-		switch {
-		case strings.Contains(recent, "isrsrange_points"):
-			return "No distance mark found for at least one given ISRS Location Code"
-		case strings.Contains(recent, "isrsrange_axis"):
-			return "No contiguous axis found between given ISRS Location Codes"
-		case strings.Contains(recent, "isrsrange_area"):
-			return "No area around axis between given ISRS Location Codes"
-		}
-
-	case violatesRowLevelSecurity:
-		return "Could not save: Data outside the area of responsibility."
-	}
-	return "Unexpected database error: " + err.Message
-}
--- a/pkg/imports/fd.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/fd.go	Thu Jul 25 13:20:32 2019 +0200
@@ -22,6 +22,7 @@
 	"time"
 
 	"gemma.intevation.de/gemma/pkg/misc"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/wfs"
 )
 
@@ -310,7 +311,7 @@
 					// ignore -> filtered by responsibility_areas
 					continue features
 				case err != nil:
-					feedback.Warn(handleError(err).Error())
+					feedback.Warn(pgxutils.HandleError(err).Error())
 					continue features
 				}
 				// Store for potential later removal.
--- a/pkg/imports/gm.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/gm.go	Thu Jul 25 13:20:32 2019 +0200
@@ -25,6 +25,7 @@
 	"time"
 
 	"gemma.intevation.de/gemma/pkg/models"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/soap/nts"
 	"github.com/jackc/pgx/pgtype"
 )
@@ -442,7 +443,7 @@
 					case err == sql.ErrNoRows:
 						// thats expected, nothing to do
 					case err != nil:
-						feedback.Warn(handleError(err).Error())
+						feedback.Warn(pgxutils.HandleError(err).Error())
 					default:
 						newP++
 					}
@@ -473,7 +474,7 @@
 					case err == sql.ErrNoRows:
 						// thats expected, nothing to do
 					case err != nil:
-						feedback.Warn(handleError(err).Error())
+						feedback.Warn(pgxutils.HandleError(err).Error())
 					default:
 						newM++
 					}
--- a/pkg/imports/sec.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/sec.go	Thu Jul 25 13:20:32 2019 +0200
@@ -19,6 +19,7 @@
 	"time"
 
 	"gemma.intevation.de/gemma/pkg/models"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 )
 
 type Section struct {
@@ -181,7 +182,7 @@
 		sec.Source,
 		sec.Tolerance,
 	).Scan(&id); err != nil {
-		return nil, handleError(err)
+		return nil, pgxutils.HandleError(err)
 	}
 
 	if err := track(ctx, tx, importID, "waterway.sections", id); err != nil {
--- a/pkg/imports/st.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/st.go	Thu Jul 25 13:20:32 2019 +0200
@@ -20,6 +20,7 @@
 	"time"
 
 	"gemma.intevation.de/gemma/pkg/models"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 )
 
 type Stretch struct {
@@ -202,7 +203,7 @@
 		st.Source,
 		st.Tolerance,
 	).Scan(&id); err != nil {
-		return nil, handleError(err)
+		return nil, pgxutils.HandleError(err)
 	}
 
 	// store the associated countries.
--- a/pkg/imports/wa.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/wa.go	Thu Jul 25 13:20:32 2019 +0200
@@ -24,6 +24,7 @@
 	"strconv"
 	"time"
 
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/wfs"
 )
 
@@ -246,7 +247,7 @@
 					outside++
 					// ignore -> filtered by responsibility_areas
 				case err != nil:
-					feedback.Warn(handleError(err).Error())
+					feedback.Warn(pgxutils.HandleError(err).Error())
 				default:
 					features++
 				}
--- a/pkg/imports/wg.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/wg.go	Thu Jul 25 13:20:32 2019 +0200
@@ -22,6 +22,7 @@
 	"github.com/jackc/pgx/pgtype"
 
 	"gemma.intevation.de/gemma/pkg/models"
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/soap/erdms"
 )
 
@@ -326,7 +327,7 @@
 			).Scan(&isNew)
 			switch {
 			case err != nil:
-				feedback.Warn(handleError(err).Error())
+				feedback.Warn(pgxutils.HandleError(err).Error())
 				if err2 := tx.Rollback(); err2 != nil {
 					return nil, err2
 				}
@@ -351,7 +352,7 @@
 					source,
 					time.Time(*dr.Lastupdate),
 				); err != nil {
-					feedback.Warn(handleError(err).Error())
+					feedback.Warn(pgxutils.HandleError(err).Error())
 					if err2 := tx.Rollback(); err2 != nil {
 						return nil, err2
 					}
@@ -388,7 +389,7 @@
 					unchanged++
 					continue
 				case err2 != nil:
-					feedback.Warn(handleError(err2).Error())
+					feedback.Warn(pgxutils.HandleError(err2).Error())
 					if err3 := tx.Rollback(); err3 != nil {
 						return nil, err3
 					}
@@ -435,7 +436,7 @@
 				code.String(),
 				&validity,
 			); err != nil {
-				feedback.Warn(handleError(err).Error())
+				feedback.Warn(pgxutils.HandleError(err).Error())
 				if err2 := tx.Rollback(); err2 != nil {
 					return nil, err2
 				}
@@ -483,7 +484,7 @@
 					string(**wl.level),
 					int64(**wl.value),
 				); err != nil {
-					feedback.Warn(handleError(err).Error())
+					feedback.Warn(pgxutils.HandleError(err).Error())
 					tx.Rollback()
 					continue
 				}
--- a/pkg/imports/wx.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/imports/wx.go	Thu Jul 25 13:20:32 2019 +0200
@@ -23,6 +23,7 @@
 	"io"
 	"time"
 
+	"gemma.intevation.de/gemma/pkg/pgxutils"
 	"gemma.intevation.de/gemma/pkg/wfs"
 )
 
@@ -320,7 +321,7 @@
 		// ignore -> filtered by responsibility_areas
 		return nil
 	case err != nil:
-		feedback.Warn(handleError(err).Error())
+		feedback.Warn(pgxutils.HandleError(err).Error())
 	default:
 		*features++
 	}
--- a/pkg/models/isrs.go	Thu Jul 25 12:54:30 2019 +0200
+++ b/pkg/models/isrs.go	Thu Jul 25 13:20:32 2019 +0200
@@ -44,12 +44,6 @@
 }
 
 func (isrs *Isrs) Less(other *Isrs) bool {
-	if isrs.Hectometre < other.Hectometre {
-		return true
-	}
-	if isrs.Hectometre > other.Hectometre {
-		return false
-	}
 	if isrs.CountryCode < other.CountryCode {
 		return true
 	}
@@ -71,6 +65,12 @@
 	if isrs.Orc < other.Orc {
 		return true
 	}
+	if isrs.Orc > other.Orc {
+		return false
+	}
+	if isrs.Hectometre < other.Hectometre {
+		return true
+	}
 	return false
 }
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/pgxutils/errors.go	Thu Jul 25 13:20:32 2019 +0200
@@ -0,0 +1,95 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2019 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Tom Gottfried <tom.gottfried@intevation.de>
+
+package pgxutils
+
+import (
+	"strings"
+
+	"github.com/jackc/pgx"
+)
+
+// Handle PostgreSQL error codes
+func HandleError(err error) error {
+	switch e := err.(type) {
+	case pgx.PgError:
+		return dbError(e)
+	}
+	return err
+}
+
+const (
+	notNullViolation         = "23502"
+	foreignKeyViolation      = "23503"
+	uniqueViolation          = "23505"
+	violatesRowLevelSecurity = "42501"
+	noDataFound              = "P0002"
+)
+
+type dbError pgx.PgError
+
+func (err dbError) Error() string {
+	switch err.Code {
+	case notNullViolation:
+		switch err.SchemaName {
+		case "waterway":
+			switch err.TableName {
+			case "gauges":
+				switch err.ColumnName {
+				case "objname":
+					return "Missing objname"
+				case "geom":
+					return "Missing lat/lon"
+				case "zero_point":
+					return "Missing zeropoint"
+				}
+			}
+		}
+	case foreignKeyViolation:
+		switch err.SchemaName {
+		case "waterway":
+			switch err.TableName {
+			case "gauge_measurements", "gauge_predictions", "bottlenecks":
+				switch err.ConstraintName {
+				case "gauge_key":
+					return "Referenced gauge with matching temporal validity not available"
+				}
+			}
+		}
+	case uniqueViolation:
+		switch err.SchemaName {
+		case "internal":
+			switch err.TableName {
+			case "user_profiles":
+				switch err.ConstraintName {
+				case "user_profiles_pkey":
+					return "A user with that name already exists"
+				}
+			}
+		}
+	case noDataFound:
+		// Most recent line from stacktrace contains name of failed function
+		recent := strings.SplitN(err.Where, "\n", 1)[0]
+		switch {
+		case strings.Contains(recent, "isrsrange_points"):
+			return "No distance mark found for at least one given ISRS Location Code"
+		case strings.Contains(recent, "isrsrange_axis"):
+			return "No contiguous axis found between given ISRS Location Codes"
+		case strings.Contains(recent, "isrsrange_area"):
+			return "No area around axis between given ISRS Location Codes"
+		}
+	case violatesRowLevelSecurity:
+		return "Could not save: Data outside the area of responsibility."
+	}
+	return "Unexpected database error: " + err.Message
+}
--- a/schema/auth_tests.sql	Thu Jul 25 12:54:30 2019 +0200
+++ b/schema/auth_tests.sql	Thu Jul 25 13:20:32 2019 +0200
@@ -151,7 +151,7 @@
     log AS (
         INSERT INTO import.import_logs (import_id, msg)
             SELECT id, 'test' FROM job)
-    INSERT INTO import.track_imports
+    INSERT INTO import.track_imports (import_id, relation, key)
         SELECT id, 'waterway.bottlenecks', 0 FROM job
     $$,
     'Waterway admin can add import job and related data');
--- a/schema/gemma.sql	Thu Jul 25 12:54:30 2019 +0200
+++ b/schema/gemma.sql	Thu Jul 25 13:20:32 2019 +0200
@@ -729,12 +729,9 @@
         UNIQUE (bottleneck_id, surdat),
         -- additional_data xml -- Currently not relevant for GEMMA
         critical boolean,
-        date_info timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        date_info timestamp with time zone NOT NULL,
         source_organization varchar NOT NULL
     )
-    CREATE TRIGGER fairway_availability_date_info
-        BEFORE UPDATE ON fairway_availability
-        FOR EACH ROW EXECUTE PROCEDURE update_date_info()
     -- FIXME: From the DRC it is unclear what the exact semantics of
     --   surdat and Date_Info ar unclear.  Currently we assume that
     --   (fk_bn_fid,surdat) has to be unique, but that might be false.
--- a/schema/tap_tests_data.sql	Thu Jul 25 12:54:30 2019 +0200
+++ b/schema/tap_tests_data.sql	Thu Jul 25 13:20:32 2019 +0200
@@ -147,7 +147,7 @@
 log AS (
     INSERT INTO import.import_logs (import_id, msg)
         SELECT id, 'test' FROM job)
-INSERT INTO import.track_imports
+INSERT INTO import.track_imports (import_id, relation, key)
     SELECT id, 'waterway.bottlenecks', 1 FROM job;
 
 WITH
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/schema/updates/1009/01.drop-fa-date_info-default.sql	Thu Jul 25 13:20:32 2019 +0200
@@ -0,0 +1,2 @@
+ALTER TABLE waterway.fairway_availability ALTER COLUMN date_info DROP DEFAULT;
+DROP TRIGGER fairway_availability_date_info ON waterway.fairway_availability;