Mercurial > gemma

annotate pkg/imports/dma.go @ 3733:ec86a7155377 concave-hull

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Estimated too large triangles as triangles which have an edge which is at least 3.5 times as long as the standard dev of the longest egde per inner triangle.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 24 Jun 2019 11:39:09 +0200
parents 4acbee65275d
children 78ec61acf72e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "errors"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "fmt"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "io"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23 "time"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/wfs"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 // FairwayDimension is an import job to import
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29 // the fairway dimensions in form of polygon geometries
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // and attribute data from a WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 type DistanceMarksAshore struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // URL the GetCapabilities URL of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 URL string `json:"url"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // FeatureType selects the feature type of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 FeatureType string `json:"feature-type"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 // SortBy sorts the feature by this key.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 SortBy string `json:"sort-by"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
38 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
39 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
40 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
41 Password string `json:"password,omitempty"`
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
44 // DMAJobKind is the import queue type identifier.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
45 const DMAJobKind JobKind = "dma"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
46
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
47 type dmaJobCreator struct{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49 func init() {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
50 RegisterJobCreator(DMAJobKind, dmaJobCreator{})
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
51 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
52
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
53 func (dmaJobCreator) Description() string { return "distance marks" }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
54
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
55 func (dmaJobCreator) AutoAccept() bool { return true }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
56
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
57 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) }
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
59 func (dmaJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
60 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
61 {"distance_marks"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
62 {},
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
64 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
65
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66 // StageDone is a NOP for distance marks imports.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 // CleanUp for distance marks imports is a NOP.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72 func (*DistanceMarksAshore) CleanUp() error { return nil }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 type distanceMarksAshoreProperties struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 HydroCatdis int `json:"hydro_catdis"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 const (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 deleteDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
81 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83 DELETE FROM waterway.distance_marks
2933
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
84 WHERE pg_has_role('sys_admin', 'MEMBER')
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
85 OR ST_Covers((SELECT a FROM resp),
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
86 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp)))
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 insertDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
89 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
90 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
91 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
92 INSERT INTO waterway.distance_marks (geom, catdis)
2943
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
93 SELECT ST_Transform(new_dma, 4326), $3
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
94 FROM (SELECT
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
95 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
96 THEN dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
97 ELSE ST_Intersection((SELECT a FROM resp),
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
98 ST_Transform(dma, (SELECT ST_SRID(a) FROM resp)))
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
99 END AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
100 FROM ST_GeomFromWKB($1, $2::integer) AS dma (dma)) AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
101 WHERE NOT ST_IsEmpty(new_dma)
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
102 RETURNING id
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
103 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
104 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
105
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
106 // Do executes the actual fairway dimension import.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
107 func (dma *DistanceMarksAshore) Do(
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
108 ctx context.Context,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
109 importID int64,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
110 conn *sql.Conn,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
111 feedback Feedback,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
112 ) (interface{}, error) {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114 start := time.Now()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
115
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116 feedback.Info("Import distance marks")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
117
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118 feedback.Info("Loading capabilities from %s", dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119 caps, err := wfs.GetCapabilities(dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 feedback.Error("Loading capabilities failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125 ft := caps.FindFeatureType(dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 if ft == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
127 return nil, fmt.Errorf("Unknown feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
129
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
130 feedback.Info("Found feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
131
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
132 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
133 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
134 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
135 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
136 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
137
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
138 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
139 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
140 feedback.Error("Cannot create GetFeature URLs. %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
142 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
144 tx, err := conn.BeginTx(ctx, nil)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
145 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
146 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
147 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
148 defer tx.Rollback()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
149
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
150 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
151 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154 defer insertStmt.Close()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 // Delete the old features.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
157 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
158 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
159 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
160
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
161 var (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
162 unsupported = stringCounter{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
163 missingProperties int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
164 badProperties int
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
165 outside int
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
166 features int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
167 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
168
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
169 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1862
diff changeset
170 feedback.Info("Get features from: '%s'", url)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
171 rfc, err := wfs.ParseRawFeatureCollection(r)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
172 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
173 return fmt.Errorf("parsing GetFeature document failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
174 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
175 if rfc.CRS != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
176 crsName := rfc.CRS.Properties.Name
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
177 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
178 feedback.Error("Unsupported CRS: %d", crsName)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
179 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
180 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
181 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183 // No features -> ignore.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 if rfc.Features == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188 feedback.Info("Using EPSG: %d", epsg)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 for _, feature := range rfc.Features {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191 if feature.Geometry.Coordinates == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192 missingProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 var props distanceMarksAshoreProperties
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 badProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202 switch feature.Geometry.Type {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 case "Point":
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 var p pointSlice
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207 }
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
208 var dmaid int64
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
209 err := insertStmt.QueryRowContext(
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 ctx,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 p.asWKB(),
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 epsg,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213 props.HydroCatdis,
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
214 ).Scan(&dmaid)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
215 switch {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
216 case err == sql.ErrNoRows:
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
217 outside++
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
218 // ignore -> filtered by responsibility area
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
219 continue
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
220 case err != nil:
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 features++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224 default:
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
225 unsupported[feature.Geometry.Type]++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 }); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 if badProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 feedback.Warn("Bad properties: %d", badProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237 if missingProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 feedback.Warn("Missing properties: %d", missingProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 if len(unsupported) != 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242 feedback.Warn("Unsupported types found: %s", unsupported)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
245 if outside > 0 {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
246 feedback.Info("Features outside responsibility area: %d", outside)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
247 }
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
248
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249 if features == 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250 err := errors.New("No features found")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 feedback.Error("%v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 if err = tx.Commit(); err == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 feedback.Info("Storing %d features took %s",
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 features, time.Since(start))
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261 }