Mercurial > gemma

annotate pkg/imports/fd.go @ 4827:f4abfd0ee8ad remove-octree-debris

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Renamed octree package to mesh as there is no octree any more.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 05 Nov 2019 14:30:22 +0100
parents f32d086b5dbf
children 5062ccb2381d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "fmt"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "io"
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
22 "strings"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23 "time"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
25 "gemma.intevation.de/gemma/pkg/common"
4062
6c760abcff0e Move handling of PostgreSQL errors to own package
Tom Gottfried <tom@intevation.de>
parents: 4058
diff changeset
26 "gemma.intevation.de/gemma/pkg/pgxutils"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
39 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
40 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
41 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43 SourceOrganization string `json:"source-organization"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
44 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
45 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
46 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
47 Password string `json:"password,omitempty"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
50 func (fd *FairwayDimension) Description() (string, error) {
4799
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
51 return strings.Join([]string{
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
52 fd.URL,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
53 fd.FeatureType,
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
54 fmt.Sprintf("LOS%d", fd.LOS),
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
55 }, "|"), nil
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
56 }
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4389
diff changeset
57
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59
4075
cb74aa69954e Moved TimeParser to common package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4074
diff changeset
60 var guessFDTime = common.TimeParser([]string{
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
61 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
62 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
63 "",
4074
eb2f949ddfa2 Rename TimeGuesser to TimeParser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4071
diff changeset
64 }).Parse
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
65
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67 var s string
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
71 t, err := guessFDTime(s)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72 if err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83 func init() {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
89 func (fdJobCreator) AutoAccept() bool { return false }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
90
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2171
diff changeset
91 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
92
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
93 func (fdJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
94 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
95 {"fairway_dimensions"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2923
diff changeset
96 {"level_of_service"},
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
97 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
98 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
99
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
100 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
101 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
102 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
103 id int64,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
104 ) error {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
105 // We only want to delete the features of the same LOS
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
106 // as the imported.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
107 var los int64
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
108 if err := tx.QueryRowContext(ctx, fdFindLOSSQL, id).Scan(&los); err != nil {
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
109 // Should always return a row because import will exit as unchanged
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
110 // if no new features were found while inserting.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
111 return err
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
112 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
113 // Delete the old features.
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
114 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, los); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
115 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
116 }
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
117
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
118 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
119 return err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125 type fairwayDimensionProperties struct {
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
126 HydroSorDat *fdTime `json:"hydro_sordat"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
127 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
129 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
130 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
131 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
132 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
133 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
134
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
135 const (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
136 fdFindLOSSQL = `
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
137 SELECT level_of_service FROM waterway.fairway_dimensions
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
138 WHERE id IN (
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
139 SELECT key FROM import.track_imports
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
140 WHERE import_id = $1 AND
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
141 relation = 'waterway.fairway_dimensions'::regclass)
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
142 LIMIT 1`
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
143
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
144 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
145 UPDATE waterway.fairway_dimensions SET staging_done = true
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
146 WHERE id IN (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
147 SELECT key FROM import.track_imports
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
148 WHERE import_id = $1 AND
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
149 relation = 'waterway.fairway_dimensions'::regclass)`
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
150
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
151 deleteFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
153 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 DELETE FROM waterway.fairway_dimensions
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
156 WHERE (pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
157 OR ST_Covers((SELECT a FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
158 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp))))
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
159 AND staging_done
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
160 AND level_of_service = $1::smallint`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
161
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
162 // The ST_MakeValid and ST_Buffer below are a workarround to
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
163 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
164 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
165 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
166 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
167 )
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
168 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
169 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
170 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
171 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
172 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
173 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
174 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
175 source_organization)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
176 SELECT ST_Transform(dmp.geom, 4326), $3, $4, $5, $6, $7, $8
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
177 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
178 ST_Dump(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
179 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
180 THEN ST_MakeValid(ST_Transform(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
181 new_fd, best_utm(ST_Transform(new_fd, 4326))))
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
182 ELSE ST_CollectionExtract(ST_Intersection(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
183 (SELECT ST_Buffer(a, -0.0001) FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
184 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp)))),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
185 3)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
186 END) AS dmp
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
187 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
188 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
189 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
190 `
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 ctx context.Context,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 importID int64,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 feedback Feedback,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 start := time.Now()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
203 feedback.Info("Import fairway dimensions")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4147
diff changeset
214 return nil, fmt.Errorf("unknown feature type '%s'", fd.FeatureType)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
215 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
216
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
218
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
219 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
225 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243 var (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245 missingProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246 badProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 features int
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
248 outside int
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
249 fds []fdSummary
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
252 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1995
diff changeset
253 feedback.Info("Get features from: '%s'", url)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
264 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
272
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
273 savepoint := Savepoint(ctx, tx, "feature")
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
274
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
275 features:
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
276 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
277 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
278 missingProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
279 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
280 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
281
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
282 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
283
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
284 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941
d4249b07e719 Fairway dimension: log bad properties.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1940
diff changeset
285 feedback.Warn("bad property: %v", err)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
286 badProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
287 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
288 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
289 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
290 if props.HydroSorDat == nil || props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
291 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
292 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
293 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
294 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
295 switch feature.Geometry.Type {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
296 case "Polygon":
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
297 var p polygonSlice
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
298 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
299 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
300 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
301 var fdid int64
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
302 var lat, lon float64
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
303 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
304 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
305 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
306 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
307 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
308 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
309 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
310 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
311 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
312 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
313 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
314 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
315 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
316 })
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
317 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
318 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
319 outside++
4389
5e38667f740c Use stretches as areas of responsibility.
Sascha Wilde <wilde@intevation.de>
parents: 4197
diff changeset
320 // ignore -> filtered by responsibility area (stretches)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
321 continue features
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
322 case err != nil:
4197
5d7ce7f926eb Log fatal errors as errors, not warnings, in imports
Tom Gottfried <tom@intevation.de>
parents: 4177
diff changeset
323 feedback.Error(pgxutils.ReadableError{Err: err}.Error())
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
324 continue features
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
325 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
326 // Store for potential later removal.
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
327 if err = track(ctx, tx, importID, "waterway.fairway_dimensions", fdid); err != nil {
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
328 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
329 }
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
330 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
331
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
332 features++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
333 default:
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
334 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
335 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
336 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
337 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
338 }); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
339 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
340 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
341
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
342 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
343 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
344 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
345
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
346 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
347 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
348 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
349
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
350 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
351 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
352 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
353
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
354 if outside > 0 {
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
355 feedback.Info("Features outside responsibility area: %d", outside)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
356 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
357
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
358 if features == 0 {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
359 return nil, UnchangedError("No features found")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
360 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
361
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
362 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
363 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
364 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
365 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
366
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
367 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
368 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
369 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
370 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
371 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
372 }{
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
373 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
374 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
375 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
376 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
377 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
378 return &summary, err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
379 }