Mercurial > gemma
changeset 4089:d9c3eb32190f pdfscaling
merge with default
| author | Thomas Junk <thomas.junk@intevation.de> |
|---|---|
| date | Thu, 25 Jul 2019 17:22:31 +0200 |
| parents | 2db1124c32f7 (current diff) 1562a5fb36dc (diff) |
| children | 7e013ae05280 |
| files | pkg/misc/time.go |
| diffstat | 11 files changed, 69 insertions(+), 119 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/common/time.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/common/time.go Thu Jul 25 17:22:31 2019 +0200 @@ -27,8 +27,26 @@ DateFormat = "2006-01-02" ) +// TimeParser is a list of time formats. +type TimeParser []string + var utc0 = time.Unix(0, 0) +// Parse tries to parse a given string by the entries of the layout +// list one after another. The first matching time is returned. +// If no layout matches the last error is returned or time zero +// if the layout list is empty. +func (tg TimeParser) Parse(s string) (time.Time, error) { + var err error + var t time.Time + for _, layout := range tg { + if t, err = time.Parse(layout, s); err == nil { + break + } + } + return t, err +} + func InterpolateTime(t1 time.Time, m1 float64, t2 time.Time, m2 float64) func(float64) time.Time { // f(m1) = t1
--- a/pkg/controllers/gauges.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/controllers/gauges.go Thu Jul 25 17:22:31 2019 +0200 @@ -636,7 +636,7 @@ } } - cs[i].Value = common.NashSutcliffe(predicted, observed) + cs[i].Value = sanitizeFloat64(common.NashSutcliffe(predicted, observed)) cs[i].Samples = len(predicted) predicted = predicted[:0] @@ -652,6 +652,18 @@ return } +func sanitizeFloat64(x float64) float64 { + switch { + case math.IsNaN(x): + return 0 + case math.IsInf(x, +1): + return math.MaxFloat64 + case math.IsInf(x, -1): + return -math.MaxFloat64 + } + return x +} + func waterlevels(rw http.ResponseWriter, req *http.Request) { gauge := mux.Vars(req)["gauge"]
--- a/pkg/imports/agm.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/imports/agm.go Thu Jul 25 17:22:31 2019 +0200 @@ -31,6 +31,7 @@ "strings" "time" + "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/misc" "gemma.intevation.de/gemma/pkg/models" ) @@ -103,10 +104,10 @@ return os.RemoveAll(agm.Dir) } -var guessDate = misc.TimeGuesser([]string{ +var guessDate = common.TimeParser([]string{ "02.01.2006 15:04", "2006-01-02T15:04:05-07:00", -}).Guess +}).Parse type timetz struct{ time.Time }
--- a/pkg/imports/fa.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/imports/fa.go Thu Jul 25 17:22:31 2019 +0200 @@ -37,11 +37,6 @@ Insecure bool `json:"insecure"` } -type uniqueFairwayAvailability struct { - BottleneckId string - Surdat time.Time -} - // FAJobKind is import queue type identifier. const FAJobKind JobKind = "fa" @@ -61,14 +56,7 @@ FROM waterway.effective_fairway_availability ORDER BY measure_date DESC LIMIT 1 ` - listFairwayAvailabilitySQL = ` -SELECT - fa.id, - bn.bottleneck_id, - fa.surdat -FROM waterway.fairway_availability fa -JOIN waterway.bottlenecks bn ON bn.id = fa.bottleneck_id -` + insertFASQL = ` INSERT INTO waterway.fairway_availability ( position_code, @@ -87,7 +75,11 @@ $4, $5, $6 -) +) ON CONFLICT (bottleneck_id, surdat) DO UPDATE SET + position_code = EXCLUDED.position_code, + critical = EXCLUDED.critical, + date_info = EXCLUDED.date_info, + source_organization = EXCLUDED.source_organization RETURNING id` insertBnPdfsSQL = ` @@ -104,6 +96,7 @@ $4, $5 ) ON CONFLICT ON CONSTRAINT bottleneck_pdfs_pkey DO NOTHING` + insertEFASQL = ` INSERT INTO waterway.effective_fairway_availability ( fairway_availability_id, @@ -131,6 +124,7 @@ $9, $10 ) ON CONFLICT ON CONSTRAINT effective_fairway_availability_pkey DO NOTHING` + insertFAVSQL = ` INSERT INTO waterway.fa_reference_values ( fairway_availability_id, @@ -215,36 +209,6 @@ return bns, nil } -func loadFairwayAvailabilities(ctx context.Context, tx *sql.Tx) (map[uniqueFairwayAvailability]int64, error) { - rows, err := tx.QueryContext(ctx, listFairwayAvailabilitySQL) - if err != nil { - return nil, err - } - defer rows.Close() - fairwayAvailabilities := map[uniqueFairwayAvailability]int64{} - for rows.Next() { - var id int64 - var bnId string - var sd time.Time - if err = rows.Scan( - &id, - &bnId, - &sd, - ); err != nil { - return nil, err - } - key := uniqueFairwayAvailability{ - BottleneckId: bnId, - Surdat: sd, - } - fairwayAvailabilities[key] = id - } - if err = rows.Err(); err != nil { - return nil, err - } - return fairwayAvailabilities, nil -} - func latestDate(ctx context.Context, tx *sql.Tx) (pgtype.Timestamp, error) { var date pgtype.Timestamp err := tx.QueryRowContext(ctx, latestMeasureDateSQL).Scan(&date) @@ -285,12 +249,7 @@ return nil, err } - fairwayAvailabilities, err := loadFairwayAvailabilities(ctx, tx) - if err != nil { - return nil, err - } - - faids, err := doForFAs(ctx, bns, fairwayAvailabilities, fas, tx, feedback) + faids, err := doForFAs(ctx, bns, fas, tx, feedback) if err != nil { return nil, fmt.Errorf("Error processing data: %v", err) } @@ -320,7 +279,6 @@ func doForFAs( ctx context.Context, bnIds bottlenecks, - fairwayAvailabilities map[uniqueFairwayAvailability]int64, fas []*ifaf.FairwayAvailability, tx *sql.Tx, feedback Feedback, @@ -355,25 +313,17 @@ feedback.Warn("Bottleneck %s not found in database.", faRes.Bottleneck_id) continue } - uniqueFa := uniqueFairwayAvailability{ - BottleneckId: faRes.Bottleneck_id, - Surdat: faRes.SURDAT, - } - var found bool - if faID, found = fairwayAvailabilities[uniqueFa]; !found { - err = insertFAStmt.QueryRowContext( - ctx, - faRes.POSITION, - faRes.Bottleneck_id, - faRes.SURDAT, - faRes.Critical, - faRes.Date_Info, - faRes.Source, - ).Scan(&faID) - if err != nil { - return nil, err - } - fairwayAvailabilities[uniqueFa] = faID + err = insertFAStmt.QueryRowContext( + ctx, + faRes.POSITION, + faRes.Bottleneck_id, + faRes.SURDAT, + faRes.Critical, + faRes.Date_Info, + faRes.Source, + ).Scan(&faID) + if err != nil { + return nil, err } feedback.Info("Processing for Bottleneck %s", faRes.Bottleneck_id) faIDs = append(faIDs, faRes.Bottleneck_id)
--- a/pkg/imports/fd.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/imports/fd.go Thu Jul 25 17:22:31 2019 +0200 @@ -21,7 +21,7 @@ "io" "time" - "gemma.intevation.de/gemma/pkg/misc" + "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/pgxutils" "gemma.intevation.de/gemma/pkg/wfs" ) @@ -48,11 +48,11 @@ type fdTime struct{ time.Time } -var guessFDTime = misc.TimeGuesser([]string{ +var guessFDTime = common.TimeParser([]string{ "20060102", "2006", "", -}).Guess +}).Parse func (fdt *fdTime) UnmarshalJSON(data []byte) error { var s string
--- a/pkg/imports/wp.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/imports/wp.go Thu Jul 25 17:22:31 2019 +0200 @@ -30,7 +30,7 @@ "github.com/jackc/pgx/pgtype" - "gemma.intevation.de/gemma/pkg/misc" + "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/models" "gemma.intevation.de/gemma/pkg/wfs" ) @@ -418,7 +418,7 @@ feedback.Info( "Matching points to lines with a precision of %.4fm.", precision) - parseDate := misc.TimeGuesser([]string{"02.01.2006"}).Guess + parseDate := common.TimeParser([]string{"02.01.2006"}).Parse insertStmt, err := tx.PrepareContext(ctx, insertWaterwayProfileSQL) if err != nil {
--- a/pkg/misc/time.go Thu Jul 25 17:22:10 2019 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0.Reader. -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Sascha L. Teichmann <sascha.teichmann@intevation.de> - -package misc - -import "time" - -// TimeGuesser is a list of time formats. -type TimeGuesser []string - -// Guess tries to parse a given string by the entries of the layout -// list one after another. The first matching time is returned. -// If no layout matches the last error is returned or time zero -// if the layout list is empty. -func (tg TimeGuesser) Guess(s string) (time.Time, error) { - var err error - var t time.Time - for _, layout := range tg { - if t, err = time.Parse(layout, s); err == nil { - break - } - } - return t, err -}
--- a/pkg/soap/nts/service.go Thu Jul 25 17:22:10 2019 +0200 +++ b/pkg/soap/nts/service.go Thu Jul 25 17:22:31 2019 +0200 @@ -18,14 +18,14 @@ "encoding/xml" "time" - "gemma.intevation.de/gemma/pkg/misc" + "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/soap" ) -var guessDateTime = misc.TimeGuesser([]string{ +var guessDateTime = common.TimeParser([]string{ "2006-01-02T15:04:05", "2006-01-02T15:04:05-07:00", -}).Guess +}).Parse type DateTime struct{ time.Time }
--- a/schema/gemma.sql Thu Jul 25 17:22:10 2019 +0200 +++ b/schema/gemma.sql Thu Jul 25 17:22:31 2019 +0200 @@ -826,8 +826,8 @@ id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, state import_state NOT NULL DEFAULT 'queued', kind varchar NOT NULL, - enqueued timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - due timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + enqueued timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + due timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, retry_wait interval CHECK(retry_wait IS NULL OR retry_wait >= interval '0 microseconds'), @@ -850,7 +850,7 @@ CREATE TABLE import_logs ( import_id int NOT NULL REFERENCES imports(id) ON DELETE CASCADE, - time timestamp NOT NULL DEFAULT now(), + time timestamp with time zone NOT NULL DEFAULT now(), kind log_type NOT NULL DEFAULT 'info', msg TEXT NOT NULL )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/schema/updates/1010/01.timezones-imports.sql Thu Jul 25 17:22:31 2019 +0200 @@ -0,0 +1,3 @@ +ALTER TABLE import.imports ALTER COLUMN enqueued TYPE timestamp with time zone; +ALTER TABLE import.imports ALTER COLUMN due TYPE timestamp with time zone; +ALTER TABLE import.import_logs ALTER COLUMN time TYPE timestamp with time zone;