Mercurial > gemma

annotate pkg/imports/dma.go @ 4852:046a07a33b19

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed the golint issues of the imports package.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 19 Nov 2019 16:08:55 +0100
parents 5062ccb2381d
children e4ab338e7ba9
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "errors"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "fmt"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "io"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23 "time"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/wfs"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27
4152
78ec61acf72e Made 'golint' happier with imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3219
diff changeset
28 // DistanceMarksAshore is an import job to import
78ec61acf72e Made 'golint' happier with imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3219
diff changeset
29 // the distance marks ashore in form of point geometries
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // and attribute data from a WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 type DistanceMarksAshore struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // URL the GetCapabilities URL of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 URL string `json:"url"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // FeatureType selects the feature type of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 FeatureType string `json:"feature-type"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 // SortBy sorts the feature by this key.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 SortBy string `json:"sort-by"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
38 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
39 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
40 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
41 Password string `json:"password,omitempty"`
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43
4852
046a07a33b19 Fixed the golint issues of the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4832
diff changeset
44 // Description gives a short info about relevant facts of this import.
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
45 func (dma *DistanceMarksAshore) Description() (string, error) {
4799
f32d086b5dbf Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4798
diff changeset
46 return dma.URL + "|" + dma.FeatureType, nil
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
47 }
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
48
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49 // DMAJobKind is the import queue type identifier.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
50 const DMAJobKind JobKind = "dma"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
51
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
52 type dmaJobCreator struct{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
53
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
54 func init() {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
55 RegisterJobCreator(DMAJobKind, dmaJobCreator{})
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
56 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
57
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58 func (dmaJobCreator) Description() string { return "distance marks" }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60 func (dmaJobCreator) AutoAccept() bool { return true }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
61
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
62 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) }
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
64 func (dmaJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
65 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
66 {"distance_marks"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
67 {},
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 // StageDone is a NOP for distance marks imports.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 // CleanUp for distance marks imports is a NOP.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 func (*DistanceMarksAshore) CleanUp() error { return nil }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 type distanceMarksAshoreProperties struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80 HydroCatdis int `json:"hydro_catdis"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83 const (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84 deleteDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
86 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 DELETE FROM waterway.distance_marks
2933
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
89 WHERE pg_has_role('sys_admin', 'MEMBER')
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
90 OR ST_Covers((SELECT a FROM resp),
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
91 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp)))
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
92 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
93 insertDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
94 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
95 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
96 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
97 INSERT INTO waterway.distance_marks (geom, catdis)
2943
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
98 SELECT ST_Transform(new_dma, 4326), $3
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
99 FROM (SELECT
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
100 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
101 THEN dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
102 ELSE ST_Intersection((SELECT a FROM resp),
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
103 ST_Transform(dma, (SELECT ST_SRID(a) FROM resp)))
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
104 END AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
105 FROM ST_GeomFromWKB($1, $2::integer) AS dma (dma)) AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
106 WHERE NOT ST_IsEmpty(new_dma)
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
107 RETURNING id
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
108 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
109 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
110
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
111 // Do executes the actual fairway dimension import.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
112 func (dma *DistanceMarksAshore) Do(
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113 ctx context.Context,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114 importID int64,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
115 conn *sql.Conn,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116 feedback Feedback,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
117 ) (interface{}, error) {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119 start := time.Now()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 feedback.Info("Import distance marks")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123 feedback.Info("Loading capabilities from %s", dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124 caps, err := wfs.GetCapabilities(dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 feedback.Error("Loading capabilities failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
127 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
129
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
130 ft := caps.FindFeatureType(dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
131 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4152
diff changeset
132 return nil, fmt.Errorf("unknown feature type '%s'", dma.FeatureType)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
133 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
134
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
135 feedback.Info("Found feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
136
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
137 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
138 if err != nil {
4832
5062ccb2381d Unified error message on "Unsupported CRS"
Sascha Wilde <wilde@intevation.de>
parents: 4799
diff changeset
139 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
140 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
142
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
143 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
144 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
145 feedback.Error("Cannot create GetFeature URLs. %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
146 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
147 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
148
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
149 tx, err := conn.BeginTx(ctx, nil)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
150 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
151 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153 defer tx.Rollback()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
157 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
158 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
159 defer insertStmt.Close()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
160
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
161 // Delete the old features.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
162 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
163 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
164 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
165
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
166 var (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
167 unsupported = stringCounter{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
168 missingProperties int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
169 badProperties int
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
170 outside int
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
171 features int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
172 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
173
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
174 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1862
diff changeset
175 feedback.Info("Get features from: '%s'", url)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
176 rfc, err := wfs.ParseRawFeatureCollection(r)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
177 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
178 return fmt.Errorf("parsing GetFeature document failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
179 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
180 if rfc.CRS != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
181 crsName := rfc.CRS.Properties.Name
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183 feedback.Error("Unsupported CRS: %d", crsName)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188 // No features -> ignore.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189 if rfc.Features == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193 feedback.Info("Using EPSG: %d", epsg)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 for _, feature := range rfc.Features {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 if feature.Geometry.Coordinates == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 missingProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 var props distanceMarksAshoreProperties
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 badProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207 switch feature.Geometry.Type {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 case "Point":
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 var p pointSlice
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 }
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
213 var dmaid int64
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
214 err := insertStmt.QueryRowContext(
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
215 ctx,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
216 p.asWKB(),
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217 epsg,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
218 props.HydroCatdis,
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
219 ).Scan(&dmaid)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
220 switch {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
221 case err == sql.ErrNoRows:
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
222 outside++
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
223 // ignore -> filtered by responsibility area
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
224 continue
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
225 case err != nil:
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 features++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 default:
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 unsupported[feature.Geometry.Type]++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 }); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 if badProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 feedback.Warn("Bad properties: %d", badProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242 if missingProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243 feedback.Warn("Missing properties: %d", missingProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246 if len(unsupported) != 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 feedback.Warn("Unsupported types found: %s", unsupported)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
250 if outside > 0 {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
251 feedback.Info("Features outside responsibility area: %d", outside)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
252 }
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
253
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254 if features == 0 {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4152
diff changeset
255 err := errors.New("no features found")
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 feedback.Error("%v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 if err = tx.Commit(); err == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261 feedback.Info("Storing %d features took %s",
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 features, time.Since(start))
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
264
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266 }