Mercurial > gemma
changeset 4070:4332b9e26e2f historization_ng
Merged default
author | Sascha Wilde <wilde@intevation.de> |
---|---|
date | Thu, 25 Jul 2019 13:20:32 +0200 |
parents | 12f476e91c70 (current diff) 6ebe42af392d (diff) |
children | 1eb39e9e8ec2 |
files | pkg/imports/agm.go pkg/imports/bn.go pkg/imports/errors.go pkg/imports/gm.go pkg/imports/wg.go schema/gemma.sql schema/version.sql |
diffstat | 19 files changed, 166 insertions(+), 147 deletions(-) [+] |
line wrap: on
line diff
--- a/client/src/components/gauge/Waterlevel.vue Thu Jul 25 12:54:30 2019 +0200 +++ b/client/src/components/gauge/Waterlevel.vue Thu Jul 25 13:20:32 2019 +0200 @@ -460,8 +460,8 @@ .attr("clip-path", "url(#waterlevel-clip)"); svg.selectAll(".hdc-line").attr("stroke", "red"); svg.selectAll(".ldc-line").attr("stroke", "green"); - svg.selectAll(".mw-line").attr("stroke", "grey"); - svg.selectAll(".rn-line").attr("stroke", "grey"); + svg.selectAll(".mw-line").attr("stroke", "rgb(128,128,128)"); + svg.selectAll(".rn-line").attr("stroke", "rgb(128,128,128)"); svg .selectAll(".ref-waterlevel-label") .style("font-size", "10px")
--- a/client/src/components/importoverview/AGMLogItem.vue Thu Jul 25 12:54:30 2019 +0200 +++ b/client/src/components/importoverview/AGMLogItem.vue Thu Jul 25 13:20:32 2019 +0200 @@ -40,13 +40,15 @@ {{ entry }} </div> <div :class="isNew(line) ? 'col-6' : 'col-4'"> - {{ line.versions[0][entry] }} + <span :class="line.versions[1] ? '' : 'text-danger'">{{ + line.versions[0][entry] + }}</span> </div> <div v-if="isOld(line) && isDifferent(line, entry)" :class="isNew(line) ? 'col-6' : 'col-4'" > - {{ line.versions[1][entry] }} + {{ line.versions[1] ? line.versions[1][entry] : $options.DELETED }} </div> </div> </div> @@ -54,6 +56,22 @@ </template> <script> +/* This is Free Software under GNU Affero General Public License v >= 3.0 + * without warranty, see README.md and license for details. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * License-Filename: LICENSES/AGPL-3.0.txt + * + * Copyright (C) 2018, 2019 by via donau + * – Österreichische Wasserstraßen-Gesellschaft mbH + * Software engineering by Intevation GmbH + * + * Author(s): + * * Thomas Junk <thomas.junk@intevation.de> + * * Markus Kottländer <markus.kottlaender@intevation.de> + * * Fadi Abbud <fadi.abbud@intevation.de> + */ +import app from "@/main"; export default { props: ["line", "index", "showDiff"], computed: { @@ -72,12 +90,14 @@ return !this.isNew(result); }, isDifferent(result, entry) { + if (!result.versions[1]) return true; return ( this.isOld(result) && result.versions[0][entry] != result.versions[1][entry] ); } - } + }, + DELETED: app.$gettext("deleted") }; </script>
--- a/pkg/controllers/user.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/controllers/user.go Thu Jul 25 13:20:32 2019 +0200 @@ -29,6 +29,7 @@ "gemma.intevation.de/gemma/pkg/auth" "gemma.intevation.de/gemma/pkg/misc" "gemma.intevation.de/gemma/pkg/models" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/scheduler" ) @@ -262,6 +263,7 @@ } if err != nil { + err = pgxutils.HandleError(err) return }
--- a/pkg/imports/agm.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/agm.go Thu Jul 25 13:20:32 2019 +0200 @@ -491,10 +491,9 @@ var removed int // Issue deletes - for _, key := range sortByLocationCode(oldGMLines) { - old := oldGMLines[key] + for _, old := range oldGMLines { removed += len(old) - for _, line := range sortByMeasureDate(old) { + for _, line := range old { if _, err := txTrackStmt.ExecContext( ctx, importID, "waterway.gauge_measurements", line.id, @@ -521,6 +520,12 @@ return nil, fmt.Errorf("Commit failed: %v", err) } + // Sort here to mix the deletes right beside the matching inserts/updates. + // This also makes the output deterministic. + sort.Slice(entries, func(i, j int) bool { + return entries[i].FKGaugeID.Less(&entries[j].FKGaugeID) + }) + feedback.Info("Imported %d entries with changes", len(entries)) feedback.Info("Importing approved gauge measurements took %s", time.Since(start)) @@ -528,32 +533,6 @@ return entries, nil } -func sortByLocationCode(data map[models.Isrs]map[int64]*agmLine) []models.Isrs { - keys := make([]models.Isrs, len(data)) - var i int - for key := range data { - keys[i] = key - i++ - } - sort.Slice(keys, func(i, j int) bool { - return keys[i].Less(&keys[j]) - }) - return keys -} - -func sortByMeasureDate(data map[int64]*agmLine) []*agmLine { - lines := make([]*agmLine, len(data)) - var i int - for _, line := range data { - lines[i] = line - i++ - } - sort.Slice(lines, func(i, j int) bool { - return lines[i].MeasureDate.Before(lines[j].MeasureDate.Time) - }) - return lines -} - func getOldGMLines( ctx context.Context, stmt *sql.Stmt,
--- a/pkg/imports/bn.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/bn.go Thu Jul 25 13:20:32 2019 +0200 @@ -24,6 +24,7 @@ "strings" "time" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/soap/ifbn" "github.com/jackc/pgx/pgtype" ) @@ -581,7 +582,7 @@ ) } if err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) return nil } defer bns.Close() @@ -593,7 +594,7 @@ bnIds = append(bnIds, nid) } if err := bns.Err(); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) return nil } if len(bnIds) == 0 { @@ -610,7 +611,7 @@ &validity, &pgBnIds, ); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) if err2 := tx.Rollback(); err2 != nil { return err2 } @@ -623,7 +624,7 @@ bn.Bottleneck_id, validity, ); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) if err2 := tx.Rollback(); err2 != nil { return err2 } @@ -660,7 +661,7 @@ &pgMaterials, ); err != nil { feedback.Warn("Failed to insert riverbed materials") - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) return nil } }
--- a/pkg/imports/errors.go Thu Jul 25 12:54:30 2019 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,84 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2019 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Tom Gottfried <tom.gottfried@intevation.de> - -package imports - -import ( - "strings" - - "github.com/jackc/pgx" -) - -func handleError(err error) error { - switch e := err.(type) { - case pgx.PgError: - return dbError(e) - } - return err -} - -// Handle PostgreSQL error codes -const ( - notNullViolation = "23502" - foreignKeyViolation = "23503" - violatesRowLevelSecurity = "42501" - noDataFound = "P0002" -) - -type dbError pgx.PgError - -func (err dbError) Error() string { - switch err.Code { - case notNullViolation: - switch err.SchemaName { - case "waterway": - switch err.TableName { - case "gauges": - switch err.ColumnName { - case "objname": - return "Missing objname" - case "geom": - return "Missing lat/lon" - case "zero_point": - return "Missing zeropoint" - } - } - } - case foreignKeyViolation: - switch err.SchemaName { - case "waterway": - switch err.TableName { - case "gauge_measurements", "gauge_predictions", "bottlenecks": - switch err.ConstraintName { - case "gauge_key": - return "Referenced gauge with matching temporal validity not available" - } - } - } - case noDataFound: - // Most recent line from stacktrace contains name of failed function - recent := strings.SplitN(err.Where, "\n", 1)[0] - switch { - case strings.Contains(recent, "isrsrange_points"): - return "No distance mark found for at least one given ISRS Location Code" - case strings.Contains(recent, "isrsrange_axis"): - return "No contiguous axis found between given ISRS Location Codes" - case strings.Contains(recent, "isrsrange_area"): - return "No area around axis between given ISRS Location Codes" - } - - case violatesRowLevelSecurity: - return "Could not save: Data outside the area of responsibility." - } - return "Unexpected database error: " + err.Message -}
--- a/pkg/imports/fd.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/fd.go Thu Jul 25 13:20:32 2019 +0200 @@ -22,6 +22,7 @@ "time" "gemma.intevation.de/gemma/pkg/misc" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/wfs" ) @@ -310,7 +311,7 @@ // ignore -> filtered by responsibility_areas continue features case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) continue features } // Store for potential later removal.
--- a/pkg/imports/gm.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/gm.go Thu Jul 25 13:20:32 2019 +0200 @@ -25,6 +25,7 @@ "time" "gemma.intevation.de/gemma/pkg/models" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/soap/nts" "github.com/jackc/pgx/pgtype" ) @@ -442,7 +443,7 @@ case err == sql.ErrNoRows: // thats expected, nothing to do case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) default: newP++ } @@ -473,7 +474,7 @@ case err == sql.ErrNoRows: // thats expected, nothing to do case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) default: newM++ }
--- a/pkg/imports/sec.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/sec.go Thu Jul 25 13:20:32 2019 +0200 @@ -19,6 +19,7 @@ "time" "gemma.intevation.de/gemma/pkg/models" + "gemma.intevation.de/gemma/pkg/pgxutils" ) type Section struct { @@ -181,7 +182,7 @@ sec.Source, sec.Tolerance, ).Scan(&id); err != nil { - return nil, handleError(err) + return nil, pgxutils.HandleError(err) } if err := track(ctx, tx, importID, "waterway.sections", id); err != nil {
--- a/pkg/imports/st.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/st.go Thu Jul 25 13:20:32 2019 +0200 @@ -20,6 +20,7 @@ "time" "gemma.intevation.de/gemma/pkg/models" + "gemma.intevation.de/gemma/pkg/pgxutils" ) type Stretch struct { @@ -202,7 +203,7 @@ st.Source, st.Tolerance, ).Scan(&id); err != nil { - return nil, handleError(err) + return nil, pgxutils.HandleError(err) } // store the associated countries.
--- a/pkg/imports/wa.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/wa.go Thu Jul 25 13:20:32 2019 +0200 @@ -24,6 +24,7 @@ "strconv" "time" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/wfs" ) @@ -246,7 +247,7 @@ outside++ // ignore -> filtered by responsibility_areas case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) default: features++ }
--- a/pkg/imports/wg.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/wg.go Thu Jul 25 13:20:32 2019 +0200 @@ -22,6 +22,7 @@ "github.com/jackc/pgx/pgtype" "gemma.intevation.de/gemma/pkg/models" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/soap/erdms" ) @@ -326,7 +327,7 @@ ).Scan(&isNew) switch { case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) if err2 := tx.Rollback(); err2 != nil { return nil, err2 } @@ -351,7 +352,7 @@ source, time.Time(*dr.Lastupdate), ); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) if err2 := tx.Rollback(); err2 != nil { return nil, err2 } @@ -388,7 +389,7 @@ unchanged++ continue case err2 != nil: - feedback.Warn(handleError(err2).Error()) + feedback.Warn(pgxutils.HandleError(err2).Error()) if err3 := tx.Rollback(); err3 != nil { return nil, err3 } @@ -435,7 +436,7 @@ code.String(), &validity, ); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) if err2 := tx.Rollback(); err2 != nil { return nil, err2 } @@ -483,7 +484,7 @@ string(**wl.level), int64(**wl.value), ); err != nil { - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) tx.Rollback() continue }
--- a/pkg/imports/wx.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/imports/wx.go Thu Jul 25 13:20:32 2019 +0200 @@ -23,6 +23,7 @@ "io" "time" + "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/wfs" ) @@ -320,7 +321,7 @@ // ignore -> filtered by responsibility_areas return nil case err != nil: - feedback.Warn(handleError(err).Error()) + feedback.Warn(pgxutils.HandleError(err).Error()) default: *features++ }
--- a/pkg/models/isrs.go Thu Jul 25 12:54:30 2019 +0200 +++ b/pkg/models/isrs.go Thu Jul 25 13:20:32 2019 +0200 @@ -44,12 +44,6 @@ } func (isrs *Isrs) Less(other *Isrs) bool { - if isrs.Hectometre < other.Hectometre { - return true - } - if isrs.Hectometre > other.Hectometre { - return false - } if isrs.CountryCode < other.CountryCode { return true } @@ -71,6 +65,12 @@ if isrs.Orc < other.Orc { return true } + if isrs.Orc > other.Orc { + return false + } + if isrs.Hectometre < other.Hectometre { + return true + } return false }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/pgxutils/errors.go Thu Jul 25 13:20:32 2019 +0200 @@ -0,0 +1,95 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2019 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried <tom.gottfried@intevation.de> + +package pgxutils + +import ( + "strings" + + "github.com/jackc/pgx" +) + +// Handle PostgreSQL error codes +func HandleError(err error) error { + switch e := err.(type) { + case pgx.PgError: + return dbError(e) + } + return err +} + +const ( + notNullViolation = "23502" + foreignKeyViolation = "23503" + uniqueViolation = "23505" + violatesRowLevelSecurity = "42501" + noDataFound = "P0002" +) + +type dbError pgx.PgError + +func (err dbError) Error() string { + switch err.Code { + case notNullViolation: + switch err.SchemaName { + case "waterway": + switch err.TableName { + case "gauges": + switch err.ColumnName { + case "objname": + return "Missing objname" + case "geom": + return "Missing lat/lon" + case "zero_point": + return "Missing zeropoint" + } + } + } + case foreignKeyViolation: + switch err.SchemaName { + case "waterway": + switch err.TableName { + case "gauge_measurements", "gauge_predictions", "bottlenecks": + switch err.ConstraintName { + case "gauge_key": + return "Referenced gauge with matching temporal validity not available" + } + } + } + case uniqueViolation: + switch err.SchemaName { + case "internal": + switch err.TableName { + case "user_profiles": + switch err.ConstraintName { + case "user_profiles_pkey": + return "A user with that name already exists" + } + } + } + case noDataFound: + // Most recent line from stacktrace contains name of failed function + recent := strings.SplitN(err.Where, "\n", 1)[0] + switch { + case strings.Contains(recent, "isrsrange_points"): + return "No distance mark found for at least one given ISRS Location Code" + case strings.Contains(recent, "isrsrange_axis"): + return "No contiguous axis found between given ISRS Location Codes" + case strings.Contains(recent, "isrsrange_area"): + return "No area around axis between given ISRS Location Codes" + } + case violatesRowLevelSecurity: + return "Could not save: Data outside the area of responsibility." + } + return "Unexpected database error: " + err.Message +}
--- a/schema/auth_tests.sql Thu Jul 25 12:54:30 2019 +0200 +++ b/schema/auth_tests.sql Thu Jul 25 13:20:32 2019 +0200 @@ -151,7 +151,7 @@ log AS ( INSERT INTO import.import_logs (import_id, msg) SELECT id, 'test' FROM job) - INSERT INTO import.track_imports + INSERT INTO import.track_imports (import_id, relation, key) SELECT id, 'waterway.bottlenecks', 0 FROM job $$, 'Waterway admin can add import job and related data');
--- a/schema/gemma.sql Thu Jul 25 12:54:30 2019 +0200 +++ b/schema/gemma.sql Thu Jul 25 13:20:32 2019 +0200 @@ -729,12 +729,9 @@ UNIQUE (bottleneck_id, surdat), -- additional_data xml -- Currently not relevant for GEMMA critical boolean, - date_info timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + date_info timestamp with time zone NOT NULL, source_organization varchar NOT NULL ) - CREATE TRIGGER fairway_availability_date_info - BEFORE UPDATE ON fairway_availability - FOR EACH ROW EXECUTE PROCEDURE update_date_info() -- FIXME: From the DRC it is unclear what the exact semantics of -- surdat and Date_Info ar unclear. Currently we assume that -- (fk_bn_fid,surdat) has to be unique, but that might be false.
--- a/schema/tap_tests_data.sql Thu Jul 25 12:54:30 2019 +0200 +++ b/schema/tap_tests_data.sql Thu Jul 25 13:20:32 2019 +0200 @@ -147,7 +147,7 @@ log AS ( INSERT INTO import.import_logs (import_id, msg) SELECT id, 'test' FROM job) -INSERT INTO import.track_imports +INSERT INTO import.track_imports (import_id, relation, key) SELECT id, 'waterway.bottlenecks', 1 FROM job; WITH
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/schema/updates/1009/01.drop-fa-date_info-default.sql Thu Jul 25 13:20:32 2019 +0200 @@ -0,0 +1,2 @@ +ALTER TABLE waterway.fairway_availability ALTER COLUMN date_info DROP DEFAULT; +DROP TRIGGER fairway_availability_date_info ON waterway.fairway_availability;