Mercurial > gemma
changeset 2084:ddbac0f22ffb
Waterway profiles import: Restructured code a bit in preparation of downloading Profile geometries from WFS first (needs to be done).
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Thu, 31 Jan 2019 16:32:18 +0100 |
parents | 6deafd6f7f86 |
children | bca8bda0b805 |
files | pkg/imports/wp.go schema/gemma.sql |
diffstat | 2 files changed, 35 insertions(+), 19 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/imports/wp.go Thu Jan 31 12:40:54 2019 +0100 +++ b/pkg/imports/wp.go Thu Jan 31 16:32:18 2019 +0100 @@ -127,8 +127,39 @@ conn *sql.Conn, feedback Feedback, ) (interface{}, error) { + start := time.Now() - start := time.Now() + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + // TODO: Download profile geometries from WFS. + + summary, err := wp.processCSV(ctx, importID, tx, start, feedback) + if err != nil { + return nil, fmt.Errorf("error processing CVS: %v", err) + } + + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf( + "Importing waterway profiles failed after %s: %v", + time.Since(start), err) + } + + feedback.Info("Importing waterway profiles took %s", + time.Since(start)) + return summary, nil +} + +func (wp *WaterwayProfiles) processCSV( + ctx context.Context, + importID int64, + tx *sql.Tx, + start time.Time, + feedback Feedback, +) (interface{}, error) { f, err := os.Open(filepath.Join(wp.Dir, "wp.csv")) if err != nil { @@ -158,12 +189,10 @@ sourceIdx = -1 ) - type headerField struct { + fields := []struct { idx *int substr string - } - - fields := []headerField{ + }{ {&locationIdx, "location"}, {&validFromIdx, "valid_from"}, {&validToIdx, "valid_to"}, @@ -206,12 +235,6 @@ parseDate := misc.TimeGuesser([]string{"02.01.2006"}).Guess - tx, err := conn.BeginTx(ctx, nil) - if err != nil { - return nil, err - } - defer tx.Rollback() - insertStmt, err := tx.PrepareContext(ctx, insertWaterwayProfileSQL) if err != nil { return nil, err @@ -348,15 +371,7 @@ return nil, UnchangedError("No new entries in waterway profiles.") } - if err := tx.Commit(); err != nil { - return nil, fmt.Errorf( - "Importing waterway profiles failed after %s: %v", - time.Since(start), err) - } - feedback.Info("%d new entries in waterway profiles.", len(ids)) - feedback.Info("Importing waterway profiles took %s", - time.Since(start)) summary := struct { IDs []int64 `json:"ids"`
--- a/schema/gemma.sql Thu Jan 31 12:40:54 2019 +0100 +++ b/schema/gemma.sql Thu Jan 31 16:32:18 2019 +0100 @@ -398,6 +398,7 @@ CREATE TABLE waterway_profiles ( id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, location isrs NOT NULL, + geom geography(linestring, 4326), -- TODO: NOT NIL validity tstzrange, EXCLUDE USING GIST (isrs_asText(location) WITH =, validity WITH &&), lnwl double precision,