Mercurial > gemma

annotate pkg/imports/fd.go @ 5275:73563c4bba5b wfs-optional-sortby

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Slight improvement of log messages for fd import.
author wilde@azure1.rgb.intevation.de
date Tue, 09 Jun 2020 10:43:43 +0200
parents 41a67619c170
children 3bab0e19f08b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
5275
73563c4bba5b Slight improvement of log messages for fd import.
wilde@azure1.rgb.intevation.de
parents: 5270
diff changeset
7 // Copyright (C) 2018, 2019, 2020 by via donau
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
20 "errors"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "fmt"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "io"
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
23 "strings"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24 "time"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
26 "gemma.intevation.de/gemma/pkg/common"
4062
6c760abcff0e Move handling of PostgreSQL errors to own package
Tom Gottfried <tom@intevation.de>
parents: 4058
diff changeset
27 "gemma.intevation.de/gemma/pkg/pgxutils"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
39 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
40 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
41 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
44 SourceOrganization string `json:"source-organization"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
45 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
46 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
47 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
48 Password string `json:"password,omitempty"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
50
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
51 var errContinue = errors.New("continue")
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
52
4852
046a07a33b19 Fixed the golint issues of the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4832
diff changeset
53 // Description gives a short info about relevant facts of this import.
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
54 func (fd *FairwayDimension) Description() (string, error) {
4799
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
55 return strings.Join([]string{
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
56 fd.URL,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
57 fd.FeatureType,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
58 fmt.Sprintf("LOS%d", fd.LOS),
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
59 }, "|"), nil
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
60 }
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
61
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
62 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
64 var guessFDTime = common.TimeParser([]string{
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
65 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
66 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
67 "",
4074
eb2f949ddfa2 Rename TimeGuesser to TimeParser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4071
diff changeset
68 }).Parse
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
69
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 var s string
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
75 t, err := guessFDTime(s)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 if err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 func init() {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
89 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
90
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
91 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
92
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
93 func (fdJobCreator) AutoAccept() bool { return false }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
94
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2171
diff changeset
95 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
96
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
97 func (fdJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
98 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
99 {"fairway_dimensions"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
100 {"level_of_service"},
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
101 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
102 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
103
5035
56c589f7435d Enhance comments on implementations of StageDone()
Tom Gottfried <tom@intevation.de>
parents: 5034
diff changeset
104 // StageDone replaces fairway dimensions with those in the staging area
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
105 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
106 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
107 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
108 id int64,
5034
59a99655f34d Added feedback support for StageDone.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5032
diff changeset
109 _ Feedback,
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
110 ) error {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
111 // Delete the old features.
5026
d02f0c5c60b3 Simplify deletion of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5024
diff changeset
112 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, id); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
113 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
114 }
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
115
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
116 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
117 return err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123 type fairwayDimensionProperties struct {
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
124 HydroSorDat *fdTime `json:"hydro_sordat"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
127 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
128 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
129 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
130 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
131 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
132
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
133 const (
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
134 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
135 UPDATE waterway.fairway_dimensions SET staging_done = true
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
136 WHERE id IN (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
137 SELECT key FROM import.track_imports
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
138 WHERE import_id = $1 AND
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
139 relation = 'waterway.fairway_dimensions'::regclass)`
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
140
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141 deleteFairwayDimensionSQL = `
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
142 -- Delete entries to be replaced by those in staging area
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143 DELETE FROM waterway.fairway_dimensions
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
144 WHERE id IN (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
145 SELECT key FROM import.track_imports
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
146 WHERE import_id = $1
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
147 AND relation = 'waterway.fairway_dimensions'::regclass
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
148 AND deletion)
5026
d02f0c5c60b3 Simplify deletion of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5024
diff changeset
149 `
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
150 // Temporary table to collect IDs of unchanged entries
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
151 tmpTableSQL = `
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
152 CREATE TEMP TABLE unchanged (id int PRIMARY KEY) ON COMMIT DROP
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
153 `
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
154 // The ST_MakeValid and ST_Buffer below are a workarround to
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
157 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
158 SELECT users.current_user_area_utm() AS a
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
159 ),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
160 g AS (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
161 SELECT ST_Multi(ST_CollectionExtract(ST_MakeValid(ST_Transform(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
162 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
163 OR ST_Covers((SELECT a FROM resp),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
164 ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp)))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
165 THEN new_fd
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
166 ELSE ST_Intersection(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
167 (SELECT ST_Buffer(a, -0.0001) FROM resp),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
168 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp))))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
169 END,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
170 4326)), 3)) AS new_fd
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
171 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
172 WHERE pg_has_role('sys_admin', 'MEMBER')
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
173 OR ST_Intersects((SELECT a FROM resp),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
174 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp))))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
175 ),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
176 not_new AS (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
177 -- Collect IDs of unchanged entries in temp table
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
178 INSERT INTO unchanged
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
179 SELECT id
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
180 FROM g, waterway.fairway_dimensions
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
181 WHERE staging_done
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
182 AND validity @> current_timestamp
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
183 AND (area, level_of_service,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
184 min_width, max_width, min_depth, source_organization
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
185 ) IS NOT DISTINCT FROM (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
186 new_fd, $3, $4, $5, $6, $8)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
187 -- Return something if a duplicate in the data source is encountered
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
188 ON CONFLICT (id) DO UPDATE SET id = EXCLUDED.id
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
189 RETURNING 1
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 )
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
191 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
192 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
193 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
194 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
195 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
196 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
197 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
198 source_organization)
5031
8c590ef35280 Improve feedback if geometry cannot be stored
Tom Gottfried <tom@intevation.de>
parents: 5030
diff changeset
199 SELECT
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
200 new_fd, $3, $4, $5, $6, $7, $8
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
201 FROM g
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
202 WHERE NOT EXISTS(SELECT 1 FROM not_new)
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
203 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
204 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
205 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
206 `
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
207 // Fetch IDs of entries removed from data source
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
208 selectOldSQL = `
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
209 WITH resp AS (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
210 SELECT users.current_user_area_utm() AS a
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
211 )
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
212 SELECT id FROM waterway.fairway_dimensions
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
213 WHERE staging_done
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
214 AND validity @> current_timestamp
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
215 AND level_of_service = $1
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
216 AND (pg_has_role('sys_admin', 'MEMBER')
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
217 OR ST_Covers((SELECT a FROM resp),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
218 ST_Transform(CAST(area AS geometry), (SELECT ST_SRID(a) FROM resp))))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
219 AND id NOT IN (SELECT id FROM unchanged)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
220 `
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
221 invalidateFairwayDimensionSQL = `
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
222 WITH track AS (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
223 -- Mark entry for deletion that has been removed from the data source
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
224 INSERT INTO import.track_imports (import_id, deletion, relation, key)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
225 VALUES($1, true, 'waterway.fairway_dimensions', $2)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
226 )
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
227 -- Insert historic version with respective validity
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
228 INSERT INTO waterway.fairway_dimensions (
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
229 area,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
230 validity,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
231 level_of_service,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
232 min_width,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
233 max_width,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
234 min_depth,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
235 date_info,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
236 source_organization)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
237 SELECT
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
238 area,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
239 tstzrange(lower(validity), current_timestamp),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
240 level_of_service,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
241 min_width,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
242 max_width,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
243 min_depth,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
244 date_info,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
245 source_organization
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
246 FROM waterway.fairway_dimensions
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
247 WHERE id = $2
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
248 RETURNING id,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
249 ST_X(ST_Centroid(area::geometry)),
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
250 ST_Y(ST_Centroid(area::geometry))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
251 `
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 ctx context.Context,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 importID int64,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 feedback Feedback,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 start := time.Now()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
264 feedback.Info("Import fairway dimensions")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
272
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
273 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
274 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4147
diff changeset
275 return nil, fmt.Errorf("unknown feature type '%s'", fd.FeatureType)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
276 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
277
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
278 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
279
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
280 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
281 if err != nil {
4832
5062ccb2381d Unified error message on "Unsupported CRS"
Sascha Wilde <wilde@intevation.de>
parents: 4799
diff changeset
282 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
283 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
284 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
285
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
286 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
287 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
288 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
289 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
290 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
291
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
292 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
293 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
294 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
295 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
296 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
297
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
298 if _, err := tx.ExecContext(ctx, tmpTableSQL); err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
299 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
300 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
301
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
302 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
303 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
304 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
305 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
306 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
307
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
308 invalidateStmt, err := tx.PrepareContext(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
309 ctx, invalidateFairwayDimensionSQL)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
310 if err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
311 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
312 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
313 defer invalidateStmt.Close()
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
314
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
315 savepoint := Savepoint(ctx, tx, "feature")
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
316
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
317 var (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
318 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
319 missingProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
320 badProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
321 features int
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
322 outside int
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
323 fds []fdSummary
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
324 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
325
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
326 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1995
diff changeset
327 feedback.Info("Get features from: '%s'", url)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
328 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
329 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
330 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
331 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
332 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
333 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
334 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
335 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
336 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
337 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
338 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
339
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
340 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
341 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
342 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
343 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
344
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
345 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
346
5023
776db50c01f0 Fix repairing geometries
Tom Gottfried <tom@intevation.de>
parents: 4852
diff changeset
347 feedback.Info(
776db50c01f0 Fix repairing geometries
Tom Gottfried <tom@intevation.de>
parents: 4852
diff changeset
348 "Found %d features in data source", len(rfc.Features))
776db50c01f0 Fix repairing geometries
Tom Gottfried <tom@intevation.de>
parents: 4852
diff changeset
349
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
350 features:
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
351 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
352 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
353 missingProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
354 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
355 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
356
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
357 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
358
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
359 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941
d4249b07e719 Fairway dimension: log bad properties.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1940
diff changeset
360 feedback.Warn("bad property: %v", err)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
361 badProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
362 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
363 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
364 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
365 if props.HydroSorDat == nil || props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
366 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
367 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
368 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
369 }
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
370
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
371 // Store a feature in the database.
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
372 storeFeature := func(p polygonSlice) error {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
373
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
374 var fdid int64
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
375 var lat, lon float64
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
376 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
377 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
378 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
379 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
380 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
381 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
382 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
383 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
384 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
385 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
386 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
387 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
388 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
389 })
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
390 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
391 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
392 outside++
4389
5e38667f740c Use stretches as areas of responsibility.
Sascha Wilde <wilde@intevation.de>
parents: 4197
diff changeset
393 // ignore -> filtered by responsibility area (stretches)
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
394 return errContinue
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
395 case err != nil:
4197
5d7ce7f926eb Log fatal errors as errors, not warnings, in imports
Tom Gottfried <tom@intevation.de>
parents: 4177
diff changeset
396 feedback.Error(pgxutils.ReadableError{Err: err}.Error())
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
397 return errContinue
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
398 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
399 // Store for potential later removal.
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
400 if err = track(
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
401 ctx, tx, importID, "waterway.fairway_dimensions", fdid,
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
402 ); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
403 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
404 }
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
405 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
406
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
407 features++
5270
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
408
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
409 return nil
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
410 }
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
411
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
412 switch feature.Geometry.Type {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
413 case "Polygon":
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
414 var p polygonSlice
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
415 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
416 return err
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
417 }
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
418 switch err := storeFeature(p); {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
419 case err == errContinue:
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
420 continue features
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
421 case err != nil:
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
422 return err
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
423 }
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
424
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
425 case "MultiPolygon":
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
426 var mp multiPolygonSlice
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
427 if err := json.Unmarshal(*feature.Geometry.Coordinates, &mp); err != nil {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
428 return err
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
429 }
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
430 for i := range mp {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
431 switch err := storeFeature(mp[i]); {
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
432 case err == errContinue:
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
433 continue features
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
434 case err != nil:
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
435 return err
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
436 }
41a67619c170 Added multi geometry support for some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5262
diff changeset
437 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
438 default:
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
439 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
440 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
441 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
442 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
443 }); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
444 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
445 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
446
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
447 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
448 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
449 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
450
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
451 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
452 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
453 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
454
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
455 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
456 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
457 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
458
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
459 if outside > 0 {
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
460 feedback.Info(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
461 "Features outside responsibility area or unchanged: %d", outside)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
462 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
463
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
464 if features == 0 {
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
465 feedback.Info("No new features found")
5275
73563c4bba5b Slight improvement of log messages for fd import.
wilde@azure1.rgb.intevation.de
parents: 5270
diff changeset
466 } else {
73563c4bba5b Slight improvement of log messages for fd import.
wilde@azure1.rgb.intevation.de
parents: 5270
diff changeset
467 feedback.Info("Stored %d features", features)
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
468 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
469
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
470 // Invalidate features that have been removed from data source
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
471 res, err := tx.QueryContext(ctx, selectOldSQL, fd.LOS)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
472 if err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
473 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
474 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
475 defer res.Close()
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
476 var oldIDs []int64
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
477 for res.Next() {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
478 var oldID int64
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
479 if err := res.Scan(&oldID); err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
480 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
481 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
482 oldIDs = append(oldIDs, oldID)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
483 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
484 if err := res.Err(); err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
485 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
486 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
487
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
488 if features == 0 && len(oldIDs) == 0 {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
489 return nil, UnchangedError("Nothing changed")
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
490 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
491
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
492 if len(oldIDs) > 0 {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
493 feedback.Info(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
494 "Number of features removed from data source: %d", len(oldIDs))
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
495
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
496 var old int
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
497 for _, oldID := range oldIDs {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
498 var fdid int64
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
499 var lat, lon float64
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
500 if err := savepoint(func() error {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
501 return invalidateStmt.QueryRowContext(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
502 ctx,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
503 importID,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
504 oldID,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
505 ).Scan(&fdid, &lat, &lon)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
506 }); err != nil {
5275
73563c4bba5b Slight improvement of log messages for fd import.
wilde@azure1.rgb.intevation.de
parents: 5270
diff changeset
507 feedback.Error(pgxutils.ReadableError{Err: err}.Error() +
73563c4bba5b Slight improvement of log messages for fd import.
wilde@azure1.rgb.intevation.de
parents: 5270
diff changeset
508 "- while tracking invalidation of: %d", oldID)
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
509 continue
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
510 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
511 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
512
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
513 if err := track(
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
514 ctx, tx, importID, "waterway.fairway_dimensions", fdid,
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
515 ); err != nil {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
516 return nil, err
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
517 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
518
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
519 old++
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
520 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
521
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
522 // Do not fail if features > 0 because otherwise new features are lost
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
523 if features == 0 && old == 0 {
5262
5df1eff5e445 Made staticcheck a little happier.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 5144
diff changeset
524 return nil, fmt.Errorf("invalidating features failed")
5144
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
525 }
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
526
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
527 if old > 0 {
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
528 feedback.Info("Number of features invalidated: %d", old)
f11b9b50fcc9 Keep historic data of fairway dimensions
Tom Gottfried <tom@intevation.de>
parents: 5035
diff changeset
529 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
530 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
531
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
532 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
533 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
534 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
535 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
536
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
537 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
538 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
539 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
540 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
541 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
542 }{
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
543 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
544 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
545 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
546 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
547 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
548 return &summary, err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
549 }