# HG changeset patch # User Tom Gottfried # Date 1558020153 -7200 # Node ID 6514b943654ef2826a5832a904dc251a9d13fd7a # Parent e37b8afd7bd95f76c61be5d8a5a4091d786a7bac Re-enable checking of gauge availability This partly reverts rev. 1cb6676d1510, which removed this code in favour of handling database errors later. The thinko in this was that possibly many NtS messages for the same gauge would lead to many errors for a single reason and the same amount of unnecessary database round-trips. Checking whether the current user is allowed to import data for a gauge now also handles sys_admin correctly and the import stops with status unchanged if no such gauge is available. diff -r e37b8afd7bd9 -r 6514b943654e pkg/imports/gm.go --- a/pkg/imports/gm.go Thu May 16 16:47:19 2019 +0200 +++ b/pkg/imports/gm.go Thu May 16 17:22:33 2019 +0200 @@ -19,6 +19,7 @@ "context" "database/sql" "fmt" + "sort" "strings" "time" @@ -41,6 +42,18 @@ const GMJobKind JobKind = "gm" const ( + listGaugesSQL = ` +SELECT + (location).country_code, + (location).locode, + (location).fairway_section, + (location).orc, + (location).hectometre +FROM waterway.gauges +WHERE (location).country_code = users.current_user_country() + OR pg_has_role('sys_admin', 'MEMBER') +` + // Note: we do not expect corrections of data through this service. So // any constraint conflicts are triggered by actual redundat data which // can be dropped. @@ -177,6 +190,44 @@ ) } +func loadGauges(ctx context.Context, conn *sql.Conn) ([]string, error) { + + rows, err := conn.QueryContext(ctx, listGaugesSQL) + if err != nil { + return nil, err + } + defer rows.Close() + + var gauges []string + + for rows.Next() { + var g models.Isrs + if err = rows.Scan( + &g.CountryCode, + &g.LoCode, + &g.FairwaySection, + &g.Orc, + &g.Hectometre, + ); err != nil { + return nil, err + } + gauges = append(gauges, g.String()) + } + + if err = rows.Err(); err != nil { + return nil, err + } + + if len(gauges) == 0 { + return nil, UnchangedError( + "No gauges for which measurements can be imported in database") + } + + sort.Strings(gauges) + + return gauges, nil +} + func storeGaugeMeasurements( ctx context.Context, importID int64, @@ -187,8 +238,13 @@ start := time.Now() - // TODO get date_issue for selected gauges - gids, err := doForGM(ctx, fetch, conn, feedback) + // Get gauges from database, for which user is allowed to import data + gauges, err := loadGauges(ctx, conn) + if err != nil { + return nil, err + } + + gids, err := doForGM(ctx, gauges, fetch, conn, feedback) if err != nil { feedback.Error("Error processing gauges: %v", err) return nil, err @@ -242,6 +298,7 @@ func doForGM( ctx context.Context, + gauges []string, fetch func() ([]*nts.RIS_Message_Type, error), conn *sql.Conn, feedback Feedback, @@ -259,6 +316,12 @@ } defer insertGMStmt.Close() + // lookup to see if data can be imported for gauge + isKnown := func(s string) bool { + idx := sort.SearchStrings(gauges, s) + return idx < len(gauges) && gauges[idx] == s + } + result, err := fetch() if err != nil { return nil, err @@ -275,6 +338,10 @@ continue } feedback.Info("Found measurements/predictions for %s", curr) + if !isKnown(curr) { + feedback.Warn("Cannot import data for %s", curr) + continue + } var referenceCode string if wrm.Reference_code == nil {