Mercurial > gemma

annotate pkg/imports/dma.go @ 2943:78affd725ba5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix insert statement for dismar ashore The statement missed to handle correctly the case where the intersection is empty.
author Tom Gottfried <tom@intevation.de>
date Thu, 04 Apr 2019 18:46:17 +0200
parents a5642ee4c6d0
children 4acbee65275d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "errors"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "fmt"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "io"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23 "time"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/wfs"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 // FairwayDimension is an import job to import
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29 // the fairway dimensions in form of polygon geometries
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // and attribute data from a WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 type DistanceMarksAshore struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // URL the GetCapabilities URL of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 URL string `json:"url"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // FeatureType selects the feature type of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 FeatureType string `json:"feature-type"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 // SortBy sorts the feature by this key.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 SortBy string `json:"sort-by"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
38 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
39 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
40 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
41 Password string `json:"password,omitempty"`
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
42 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
44 // DMAJobKind is the import queue type identifier.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
45 const DMAJobKind JobKind = "dma"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
46
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
47 type dmaJobCreator struct{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49 func init() {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
50 RegisterJobCreator(DMAJobKind, dmaJobCreator{})
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
51 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
52
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
53 func (dmaJobCreator) Description() string { return "distance marks" }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
54
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
55 func (dmaJobCreator) AutoAccept() bool { return true }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
56
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
57 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) }
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59 func (dmaJobCreator) Depends() []string {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60 return []string{
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
61 "distance_marks",
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
62 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
64
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
65 // StageDone is a NOP for distance marks imports.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70 // CleanUp for distance marks imports is a NOP.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 func (*DistanceMarksAshore) CleanUp() error { return nil }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 type distanceMarksAshoreProperties struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 HydroCatdis int `json:"hydro_catdis"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 const (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 deleteDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
80 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82 DELETE FROM waterway.distance_marks
2933
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
83 WHERE pg_has_role('sys_admin', 'MEMBER')
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
84 OR ST_Covers((SELECT a FROM resp),
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
85 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp)))
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 insertDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
89 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
90 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
91 INSERT INTO waterway.distance_marks (geom, catdis)
2943
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
92 SELECT ST_Transform(new_dma, 4326), $3
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
93 FROM (SELECT
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
94 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
95 THEN dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
96 ELSE ST_Intersection((SELECT a FROM resp),
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
97 ST_Transform(dma, (SELECT ST_SRID(a) FROM resp)))
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
98 END AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
99 FROM ST_GeomFromWKB($1, $2::integer) AS dma (dma)) AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
100 WHERE NOT ST_IsEmpty(new_dma)
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
101 RETURNING id
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
102 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
103 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
104
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
105 // Do executes the actual fairway dimension import.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
106 func (dma *DistanceMarksAshore) Do(
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
107 ctx context.Context,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
108 importID int64,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
109 conn *sql.Conn,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
110 feedback Feedback,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
111 ) (interface{}, error) {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
112
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113 start := time.Now()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
115 feedback.Info("Import distance marks")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
117 feedback.Info("Loading capabilities from %s", dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118 caps, err := wfs.GetCapabilities(dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120 feedback.Error("Loading capabilities failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124 ft := caps.FindFeatureType(dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125 if ft == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 return nil, fmt.Errorf("Unknown feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
127 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
129 feedback.Info("Found feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
130
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
131 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
132 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
133 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
134 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
135 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
136
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
137 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
138 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
139 feedback.Error("Cannot create GetFeature URLs. %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
140 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
142
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143 tx, err := conn.BeginTx(ctx, nil)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
144 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
145 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
146 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
147 defer tx.Rollback()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
148
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
149 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
150 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
151 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153 defer insertStmt.Close()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 // Delete the old features.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
157 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
158 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
159
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
160 var (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
161 unsupported = stringCounter{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
162 missingProperties int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
163 badProperties int
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
164 outside int
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
165 features int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
166 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
167
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
168 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1862
diff changeset
169 feedback.Info("Get features from: '%s'", url)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
170 rfc, err := wfs.ParseRawFeatureCollection(r)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
171 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
172 return fmt.Errorf("parsing GetFeature document failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
173 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
174 if rfc.CRS != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
175 crsName := rfc.CRS.Properties.Name
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
176 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
177 feedback.Error("Unsupported CRS: %d", crsName)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
178 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
179 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
180 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
181
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182 // No features -> ignore.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183 if rfc.Features == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187 feedback.Info("Using EPSG: %d", epsg)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189 for _, feature := range rfc.Features {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 if feature.Geometry.Coordinates == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191 missingProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 var props distanceMarksAshoreProperties
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 badProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 switch feature.Geometry.Type {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202 case "Point":
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 var p pointSlice
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 }
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
207 var dmaid int64
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
208 err := insertStmt.QueryRowContext(
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 ctx,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 p.asWKB(),
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 epsg,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 props.HydroCatdis,
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
213 ).Scan(&dmaid)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
214 switch {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
215 case err == sql.ErrNoRows:
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
216 outside++
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
217 // ignore -> filtered by responsibility area
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
218 continue
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
219 case err != nil:
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 features++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 default:
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224 unsupported[feature.Geometry.Type]++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
225 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 }); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 if badProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 feedback.Warn("Bad properties: %d", badProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 if missingProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237 feedback.Warn("Missing properties: %d", missingProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240 if len(unsupported) != 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 feedback.Warn("Unsupported types found: %s", unsupported)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
244 if outside > 0 {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
245 feedback.Info("Features outside responsibility area: %d", outside)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
246 }
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
247
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 if features == 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249 err := errors.New("No features found")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250 feedback.Error("%v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254 if err = tx.Commit(); err == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 feedback.Info("Storing %d features took %s",
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 features, time.Since(start))
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 }