Mercurial > gemma

annotate pkg/imports/fd.go @ 4958:6d5d6b27c3c3 fairway-marks-import

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added pre commit hooks before commiting fairway marks to invalidate old.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 26 Feb 2020 12:54:33 +0100
parents 046a07a33b19
children 776db50c01f0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "fmt"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "io"
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
22 "strings"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23 "time"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
25 "gemma.intevation.de/gemma/pkg/common"
4062
6c760abcff0e Move handling of PostgreSQL errors to own package
Tom Gottfried <tom@intevation.de>
parents: 4058
diff changeset
26 "gemma.intevation.de/gemma/pkg/pgxutils"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
39 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
40 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
41 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43 SourceOrganization string `json:"source-organization"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
44 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
45 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
46 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
47 Password string `json:"password,omitempty"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49
4852
046a07a33b19 Fixed the golint issues of the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4832
diff changeset
50 // Description gives a short info about relevant facts of this import.
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
51 func (fd *FairwayDimension) Description() (string, error) {
4799
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
52 return strings.Join([]string{
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
53 fd.URL,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
54 fd.FeatureType,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
55 fmt.Sprintf("LOS%d", fd.LOS),
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
56 }, "|"), nil
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
57 }
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
58
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
61 var guessFDTime = common.TimeParser([]string{
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
62 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
63 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
64 "",
4074
eb2f949ddfa2 Rename TimeGuesser to TimeParser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4071
diff changeset
65 }).Parse
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
66
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 var s string
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
72 t, err := guessFDTime(s)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 if err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84 func init() {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
89
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
90 func (fdJobCreator) AutoAccept() bool { return false }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
91
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2171
diff changeset
92 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
93
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
94 func (fdJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
95 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
96 {"fairway_dimensions"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
97 {"level_of_service"},
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
98 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
99 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
100
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
101 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
102 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
103 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
104 id int64,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
105 ) error {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
106 // We only want to delete the features of the same LOS
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
107 // as the imported.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
108 var los int64
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
109 if err := tx.QueryRowContext(ctx, fdFindLOSSQL, id).Scan(&los); err != nil {
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
110 // Should always return a row because import will exit as unchanged
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
111 // if no new features were found while inserting.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
112 return err
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
113 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
114 // Delete the old features.
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
115 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, los); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
116 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
117 }
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
118
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
119 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
120 return err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 type fairwayDimensionProperties struct {
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
127 HydroSorDat *fdTime `json:"hydro_sordat"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
129
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
130 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
131 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
132 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
133 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
134 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
135
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
136 const (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
137 fdFindLOSSQL = `
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
138 SELECT level_of_service FROM waterway.fairway_dimensions
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
139 WHERE id IN (
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
140 SELECT key FROM import.track_imports
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
141 WHERE import_id = $1 AND
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
142 relation = 'waterway.fairway_dimensions'::regclass)
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
143 LIMIT 1`
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
144
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
145 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
146 UPDATE waterway.fairway_dimensions SET staging_done = true
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
147 WHERE id IN (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
148 SELECT key FROM import.track_imports
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
149 WHERE import_id = $1 AND
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
150 relation = 'waterway.fairway_dimensions'::regclass)`
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
151
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 deleteFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
154 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 DELETE FROM waterway.fairway_dimensions
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
157 WHERE (pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
158 OR ST_Covers((SELECT a FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
159 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp))))
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
160 AND staging_done
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
161 AND level_of_service = $1::smallint`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
162
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
163 // The ST_MakeValid and ST_Buffer below are a workarround to
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
164 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
165 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
166 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
167 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
168 )
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
169 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
170 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
171 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
172 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
173 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
174 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
175 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
176 source_organization)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
177 SELECT ST_Transform(dmp.geom, 4326), $3, $4, $5, $6, $7, $8
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
178 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
179 ST_Dump(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
180 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
181 THEN ST_MakeValid(ST_Transform(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
182 new_fd, best_utm(ST_Transform(new_fd, 4326))))
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
183 ELSE ST_CollectionExtract(ST_Intersection(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
184 (SELECT ST_Buffer(a, -0.0001) FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
185 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp)))),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
186 3)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
187 END) AS dmp
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
188 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
189 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
190 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
191 `
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 ctx context.Context,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 importID int64,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 feedback Feedback,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202 start := time.Now()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
204 feedback.Info("Import fairway dimensions")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
214 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4147
diff changeset
215 return nil, fmt.Errorf("unknown feature type '%s'", fd.FeatureType)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
216 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
218 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
219
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 if err != nil {
4832
5062ccb2381d Unified error message on "Unsupported CRS"
Sascha Wilde <wilde@intevation.de>
parents: 4799
diff changeset
222 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
225
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
226 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244 var (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246 missingProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 badProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 features int
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
249 outside int
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
250 fds []fdSummary
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
253 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1995
diff changeset
254 feedback.Info("Get features from: '%s'", url)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
264 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
272 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
273
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
274 savepoint := Savepoint(ctx, tx, "feature")
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
275
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
276 features:
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
277 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
278 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
279 missingProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
280 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
281 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
282
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
283 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
284
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
285 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941
d4249b07e719 Fairway dimension: log bad properties.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1940
diff changeset
286 feedback.Warn("bad property: %v", err)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
287 badProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
288 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
289 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
290 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
291 if props.HydroSorDat == nil || props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
292 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
293 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
294 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
295 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
296 switch feature.Geometry.Type {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
297 case "Polygon":
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
298 var p polygonSlice
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
299 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
300 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
301 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
302 var fdid int64
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
303 var lat, lon float64
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
304 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
305 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
306 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
307 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
308 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
309 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
310 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
311 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
312 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
313 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
314 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
315 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
316 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
317 })
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
318 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
319 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
320 outside++
4389
5e38667f740c Use stretches as areas of responsibility.
Sascha Wilde <wilde@intevation.de>
parents: 4197
diff changeset
321 // ignore -> filtered by responsibility area (stretches)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
322 continue features
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
323 case err != nil:
4197
5d7ce7f926eb Log fatal errors as errors, not warnings, in imports
Tom Gottfried <tom@intevation.de>
parents: 4177
diff changeset
324 feedback.Error(pgxutils.ReadableError{Err: err}.Error())
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
325 continue features
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
326 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
327 // Store for potential later removal.
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
328 if err = track(ctx, tx, importID, "waterway.fairway_dimensions", fdid); err != nil {
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
329 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
330 }
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
331 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
332
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
333 features++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
334 default:
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
335 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
336 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
337 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
338 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
339 }); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
340 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
341 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
342
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
343 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
344 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
345 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
346
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
347 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
348 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
349 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
350
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
351 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
352 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
353 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
354
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
355 if outside > 0 {
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
356 feedback.Info("Features outside responsibility area: %d", outside)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
357 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
358
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
359 if features == 0 {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
360 return nil, UnchangedError("No features found")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
361 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
362
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
363 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
364 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
365 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
366 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
367
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
368 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
369 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
370 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
371 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
372 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
373 }{
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
374 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
375 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
376 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
377 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
378 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
379 return &summary, err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
380 }