Mercurial > gemma
annotate pkg/imports/dma.go @ 2933:a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
author | Tom Gottfried <tom@intevation.de> |
---|---|
date | Thu, 04 Apr 2019 14:08:01 +0200 |
parents | 074f2bb85584 |
children | 78affd725ba5 |
rev | line source |
---|---|
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
1 // This is Free Software under GNU Affero General Public License v >= 3.0 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
2 // without warranty, see README.md and license for details. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
3 // |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
4 // SPDX-License-Identifier: AGPL-3.0-or-later |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
5 // License-Filename: LICENSES/AGPL-3.0.txt |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
6 // |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
7 // Copyright (C) 2018 by via donau |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
8 // – Österreichische Wasserstraßen-Gesellschaft mbH |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
9 // Software engineering by Intevation GmbH |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
10 // |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
11 // Author(s): |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
12 // * Raimund Renkert <raimund.renkert@intevation.de> |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
13 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
14 package imports |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
15 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
16 import ( |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
17 "context" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
18 "database/sql" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
19 "encoding/json" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
20 "errors" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
21 "fmt" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
22 "io" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
23 "time" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
24 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
25 "gemma.intevation.de/gemma/pkg/wfs" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
26 ) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
27 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
28 // FairwayDimension is an import job to import |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
29 // the fairway dimensions in form of polygon geometries |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
30 // and attribute data from a WFS service. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
31 type DistanceMarksAshore struct { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
32 // URL the GetCapabilities URL of the WFS service. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
33 URL string `json:"url"` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
34 // FeatureType selects the feature type of the WFS service. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
35 FeatureType string `json:"feature-type"` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
36 // SortBy sorts the feature by this key. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
37 SortBy string `json:"sort-by"` |
2723
a10022399e24
WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2721
diff
changeset
|
38 // User is an optional username for Basic Auth. |
a10022399e24
WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2721
diff
changeset
|
39 User string `json:"user,omitempty"` |
a10022399e24
WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2721
diff
changeset
|
40 // Password is an optional password for Basic Auth. |
a10022399e24
WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2721
diff
changeset
|
41 Password string `json:"password,omitempty"` |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
42 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
43 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
44 // DMAJobKind is the import queue type identifier. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
45 const DMAJobKind JobKind = "dma" |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
46 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
47 type dmaJobCreator struct{} |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
48 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
49 func init() { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
50 RegisterJobCreator(DMAJobKind, dmaJobCreator{}) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
51 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
52 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
53 func (dmaJobCreator) Description() string { return "distance marks" } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
54 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
55 func (dmaJobCreator) AutoAccept() bool { return true } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
56 |
2187
7c83b5277c1c
Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2168
diff
changeset
|
57 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) } |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
58 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
59 func (dmaJobCreator) Depends() []string { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
60 return []string{ |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
61 "distance_marks", |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
62 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
63 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
64 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
65 // StageDone is a NOP for distance marks imports. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
66 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64) error { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
67 return nil |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
68 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
69 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
70 // CleanUp for distance marks imports is a NOP. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
71 func (*DistanceMarksAshore) CleanUp() error { return nil } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
72 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
73 type distanceMarksAshoreProperties struct { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
74 HydroCatdis int `json:"hydro_catdis"` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
75 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
76 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
77 const ( |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
78 deleteDistanceMarksSQL = ` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
79 WITH resp AS ( |
2912
93fa55bce126
Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents:
2833
diff
changeset
|
80 SELECT users.current_user_area_utm() AS a |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
81 ) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
82 DELETE FROM waterway.distance_marks |
2933
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
83 WHERE pg_has_role('sys_admin', 'MEMBER') |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
84 OR ST_Covers((SELECT a FROM resp), |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
85 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp))) |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
86 ` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
87 insertDistanceMarksSQL = ` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
88 WITH resp AS ( |
2912
93fa55bce126
Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents:
2833
diff
changeset
|
89 SELECT users.current_user_area_utm() AS a |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
90 ) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
91 INSERT INTO waterway.distance_marks (geom, catdis) |
2933
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
92 SELECT ST_Transform( |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
93 CASE WHEN pg_has_role('sys_admin', 'MEMBER') |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
94 THEN new_dma |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
95 ELSE ST_Intersection((SELECT a FROM resp), |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
96 ST_Transform(new_dma, (SELECT ST_SRID(a) FROM resp))) |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
97 END, |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
98 4326), |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
99 $3 |
a5642ee4c6d0
Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents:
2928
diff
changeset
|
100 FROM ST_GeomFromWKB($1, $2::integer) AS new_dma (new_dma) |
2928
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
101 RETURNING id |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
102 ` |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
103 ) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
104 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
105 // Do executes the actual fairway dimension import. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
106 func (dma *DistanceMarksAshore) Do( |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
107 ctx context.Context, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
108 importID int64, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
109 conn *sql.Conn, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
110 feedback Feedback, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
111 ) (interface{}, error) { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
112 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
113 start := time.Now() |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
114 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
115 feedback.Info("Import distance marks") |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
116 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
117 feedback.Info("Loading capabilities from %s", dma.URL) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
118 caps, err := wfs.GetCapabilities(dma.URL) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
119 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
120 feedback.Error("Loading capabilities failed: %v", err) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
121 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
122 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
123 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
124 ft := caps.FindFeatureType(dma.FeatureType) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
125 if ft == nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
126 return nil, fmt.Errorf("Unknown feature type '%s'", dma.FeatureType) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
127 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
128 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
129 feedback.Info("Found feature type '%s'", dma.FeatureType) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
130 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
131 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
132 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
133 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
134 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
135 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
136 |
2721
56c8ef515d93
WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2719
diff
changeset
|
137 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy) |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
138 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
139 feedback.Error("Cannot create GetFeature URLs. %v", err) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
140 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
141 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
142 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
143 tx, err := conn.BeginTx(ctx, nil) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
144 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
145 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
146 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
147 defer tx.Rollback() |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
148 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
149 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
150 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
151 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
152 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
153 defer insertStmt.Close() |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
154 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
155 // Delete the old features. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
156 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
157 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
158 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
159 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
160 var ( |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
161 unsupported = stringCounter{} |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
162 missingProperties int |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
163 badProperties int |
2928
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
164 outside int |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
165 features int |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
166 ) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
167 |
2729
7cb027be277d
WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2723
diff
changeset
|
168 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error { |
2106
2b72f5e005aa
WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1862
diff
changeset
|
169 feedback.Info("Get features from: '%s'", url) |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
170 rfc, err := wfs.ParseRawFeatureCollection(r) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
171 if err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
172 return fmt.Errorf("parsing GetFeature document failed: %v", err) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
173 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
174 if rfc.CRS != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
175 crsName := rfc.CRS.Properties.Name |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
176 if epsg, err = wfs.CRSToEPSG(crsName); err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
177 feedback.Error("Unsupported CRS: %d", crsName) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
178 return err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
179 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
180 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
181 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
182 // No features -> ignore. |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
183 if rfc.Features == nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
184 return nil |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
185 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
186 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
187 feedback.Info("Using EPSG: %d", epsg) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
188 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
189 for _, feature := range rfc.Features { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
190 if feature.Geometry.Coordinates == nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
191 missingProperties++ |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
192 continue |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
193 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
194 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
195 var props distanceMarksAshoreProperties |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
196 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
197 if err := json.Unmarshal(*feature.Properties, &props); err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
198 badProperties++ |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
199 continue |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
200 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
201 switch feature.Geometry.Type { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
202 case "Point": |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
203 var p pointSlice |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
204 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
205 return err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
206 } |
2928
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
207 var dmaid int64 |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
208 err := insertStmt.QueryRowContext( |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
209 ctx, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
210 p.asWKB(), |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
211 epsg, |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
212 props.HydroCatdis, |
2928
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
213 ).Scan(&dmaid) |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
214 switch { |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
215 case err == sql.ErrNoRows: |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
216 outside++ |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
217 // ignore -> filtered by responsibility area |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
218 continue |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
219 case err != nil: |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
220 return err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
221 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
222 features++ |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
223 default: |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
224 unsupported[feature.Geometry.Type]++ |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
225 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
226 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
227 return nil |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
228 }); err != nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
229 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
230 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
231 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
232 if badProperties > 0 { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
233 feedback.Warn("Bad properties: %d", badProperties) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
234 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
235 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
236 if missingProperties > 0 { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
237 feedback.Warn("Missing properties: %d", missingProperties) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
238 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
239 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
240 if len(unsupported) != 0 { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
241 feedback.Warn("Unsupported types found: %s", unsupported) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
242 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
243 |
2928
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
244 if outside > 0 { |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
245 feedback.Info("Features outside responsibility area: %d", outside) |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
246 } |
074f2bb85584
Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents:
2926
diff
changeset
|
247 |
1862
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
248 if features == 0 { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
249 err := errors.New("No features found") |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
250 feedback.Error("%v", err) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
251 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
252 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
253 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
254 if err = tx.Commit(); err == nil { |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
255 feedback.Info("Storing %d features took %s", |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
256 features, time.Since(start)) |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
257 } |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
258 |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
259 return nil, err |
427f86518097
Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff
changeset
|
260 } |