Mercurial > gemma

annotate pkg/imports/fd.go @ 4177:8b75ac5e243e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Made 'staticcheck' happy with pgxutils package.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 05 Aug 2019 17:16:46 +0200
parents c489c78ed525
children 5d7ce7f926eb
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "fmt"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "io"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "time"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
24 "gemma.intevation.de/gemma/pkg/common"
4062
6c760abcff0e Move handling of PostgreSQL errors to own package
Tom Gottfried <tom@intevation.de>
parents: 4058
diff changeset
25 "gemma.intevation.de/gemma/pkg/pgxutils"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
39 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
40 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
41 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 SourceOrganization string `json:"source-organization"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
43 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
44 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
45 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
46 Password string `json:"password,omitempty"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
47 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
50
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
51 var guessFDTime = common.TimeParser([]string{
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
52 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
53 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
54 "",
4074
eb2f949ddfa2 Rename TimeGuesser to TimeParser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4071
diff changeset
55 }).Parse
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
56
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
57 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58 var s string
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
61 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
62 t, err := guessFDTime(s)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63 if err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
64 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
65 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 func init() {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
80 func (fdJobCreator) AutoAccept() bool { return false }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2171
diff changeset
82 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
84 func (fdJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
85 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
86 {"fairway_dimensions"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
87 {"level_of_service"},
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
89 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
90
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
91 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
92 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
93 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
94 id int64,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
95 ) error {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
96 // We only want to delete the features of the same LOS
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
97 // as the imported.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
98 var los int64
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
99 if err := tx.QueryRowContext(ctx, fdFindLOSSQL, id).Scan(&los); err != nil {
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
100 // Should always return a row because import will exit as unchanged
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
101 // if no new features were found while inserting.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
102 return err
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
103 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
104 // Delete the old features.
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
105 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, los); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
106 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
107 }
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
108
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
109 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
110 return err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
111 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
112
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
115
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116 type fairwayDimensionProperties struct {
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
117 HydroSorDat *fdTime `json:"hydro_sordat"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
120 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
121 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
122 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
123 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
124 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
125
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 const (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
127 fdFindLOSSQL = `
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
128 SELECT level_of_service FROM waterway.fairway_dimensions
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
129 WHERE id IN (
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
130 SELECT key FROM import.track_imports
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
131 WHERE import_id = $1 AND
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
132 relation = 'waterway.fairway_dimensions'::regclass)
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
133 LIMIT 1`
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
134
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
135 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
136 UPDATE waterway.fairway_dimensions SET staging_done = true
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
137 WHERE id IN (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
138 SELECT key FROM import.track_imports
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
139 WHERE import_id = $1 AND
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
140 relation = 'waterway.fairway_dimensions'::regclass)`
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
141
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
142 deleteFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
144 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
145 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
146 DELETE FROM waterway.fairway_dimensions
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
147 WHERE (pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
148 OR ST_Covers((SELECT a FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
149 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp))))
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
150 AND staging_done
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
151 AND level_of_service = $1::smallint`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
153 // The ST_MakeValid and ST_Buffer below are a workarround to
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
157 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
158 )
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
159 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
160 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
161 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
162 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
163 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
164 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
165 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
166 source_organization)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
167 SELECT ST_Transform(dmp.geom, 4326), $3, $4, $5, $6, $7, $8
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
168 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
169 ST_Dump(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
170 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
171 THEN ST_MakeValid(ST_Transform(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
172 new_fd, best_utm(ST_Transform(new_fd, 4326))))
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
173 ELSE ST_CollectionExtract(ST_Intersection(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
174 (SELECT ST_Buffer(a, -0.0001) FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
175 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp)))),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
176 3)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
177 END) AS dmp
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
178 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
179 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
180 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
181 `
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186 ctx context.Context,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187 importID int64,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189 feedback Feedback,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192 start := time.Now()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
194 feedback.Info("Import fairway dimensions")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4147
diff changeset
205 return nil, fmt.Errorf("unknown feature type '%s'", fd.FeatureType)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
214 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
215
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
216 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
218 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
219 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
225 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 var (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 missingProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237 badProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 features int
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
239 outside int
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
240 fds []fdSummary
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
243 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1995
diff changeset
244 feedback.Info("Get features from: '%s'", url)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
264 savepoint := Savepoint(ctx, tx, "feature")
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
265
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
266 features:
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269 missingProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
272
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
273 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
274
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
275 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941
d4249b07e719 Fairway dimension: log bad properties.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1940
diff changeset
276 feedback.Warn("bad property: %v", err)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
277 badProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
278 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
279 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
280 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
281 if props.HydroSorDat == nil || props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
282 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
283 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
284 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
285 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
286 switch feature.Geometry.Type {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
287 case "Polygon":
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
288 var p polygonSlice
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
289 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
290 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
291 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
292 var fdid int64
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
293 var lat, lon float64
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
294 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
295 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
296 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
297 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
298 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
299 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
300 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
301 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
302 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
303 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
304 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
305 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
306 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
307 })
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
308 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
309 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
310 outside++
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
311 // ignore -> filtered by responsibility_areas
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
312 continue features
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
313 case err != nil:
4147
c489c78ed525 Made 'go vet' happy with imports again.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4075
diff changeset
314 feedback.Warn(pgxutils.ReadableError{Err: err}.Error())
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
315 continue features
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
316 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
317 // Store for potential later removal.
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
318 if err = track(ctx, tx, importID, "waterway.fairway_dimensions", fdid); err != nil {
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
319 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
320 }
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
321 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
322
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
323 features++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
324 default:
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
325 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
326 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
327 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
328 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
329 }); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
330 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
331 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
332
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
333 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
334 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
335 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
336
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
337 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
338 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
339 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
340
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
341 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
342 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
343 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
344
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
345 if outside > 0 {
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
346 feedback.Info("Features outside responsibility area: %d", outside)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
347 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
348
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
349 if features == 0 {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
350 return nil, UnchangedError("No features found")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
351 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
352
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
353 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
354 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
355 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
356 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
357
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
358 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
359 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
360 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
361 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
362 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
363 }{
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
364 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
365 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
366 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
367 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
368 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
369 return &summary, err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
370 }