Mercurial > gemma

annotate pkg/imports/dma.go @ 4798:ca6a5f722471

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added Description method to most imports. Only for imports where information relevant to the administrator is relevant.
author Sascha Wilde <wilde@intevation.de>
date Fri, 25 Oct 2019 18:25:38 +0200
parents 8b75ac5e243e
children f32d086b5dbf
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "errors"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "fmt"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "io"
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
23 "strings"
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
24 "time"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 "gemma.intevation.de/gemma/pkg/wfs"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28
4152
78ec61acf72e Made 'golint' happier with imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3219
diff changeset
29 // DistanceMarksAshore is an import job to import
78ec61acf72e Made 'golint' happier with imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3219
diff changeset
30 // the distance marks ashore in form of point geometries
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 // and attribute data from a WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 type DistanceMarksAshore struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 // URL the GetCapabilities URL of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 URL string `json:"url"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 // FeatureType selects the feature type of the WFS service.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 FeatureType string `json:"feature-type"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 // SortBy sorts the feature by this key.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 SortBy string `json:"sort-by"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
39 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
40 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
41 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
42 Password string `json:"password,omitempty"`
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
43 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
44
4798
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
45 func (dma *DistanceMarksAshore) Description() (string, error) {
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
46
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
47 var descs []string
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
48
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
49 descs = append(descs, dma.URL)
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
50 descs = append(descs, dma.FeatureType)
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
51 return strings.Join(descs, "|"), nil
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
52 }
ca6a5f722471 Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents: 4177
diff changeset
53
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
54 // DMAJobKind is the import queue type identifier.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
55 const DMAJobKind JobKind = "dma"
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
56
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
57 type dmaJobCreator struct{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59 func init() {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60 RegisterJobCreator(DMAJobKind, dmaJobCreator{})
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
61 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
62
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63 func (dmaJobCreator) Description() string { return "distance marks" }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
64
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
65 func (dmaJobCreator) AutoAccept() bool { return true }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
67 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) }
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
69 func (dmaJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
70 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
71 {"distance_marks"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2943
diff changeset
72 {},
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76 // StageDone is a NOP for distance marks imports.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
79 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
81 // CleanUp for distance marks imports is a NOP.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82 func (*DistanceMarksAshore) CleanUp() error { return nil }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84 type distanceMarksAshoreProperties struct {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 HydroCatdis int `json:"hydro_catdis"`
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88 const (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
89 deleteDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
90 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
91 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
92 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
93 DELETE FROM waterway.distance_marks
2933
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
94 WHERE pg_has_role('sys_admin', 'MEMBER')
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
95 OR ST_Covers((SELECT a FROM resp),
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction
Tom Gottfried <tom@intevation.de>
parents: 2928
diff changeset
96 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp)))
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
97 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
98 insertDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
99 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
100 SELECT users.current_user_area_utm() AS a
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
101 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
102 INSERT INTO waterway.distance_marks (geom, catdis)
2943
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
103 SELECT ST_Transform(new_dma, 4326), $3
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
104 FROM (SELECT
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
105 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
106 THEN dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
107 ELSE ST_Intersection((SELECT a FROM resp),
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
108 ST_Transform(dma, (SELECT ST_SRID(a) FROM resp)))
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
109 END AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
110 FROM ST_GeomFromWKB($1, $2::integer) AS dma (dma)) AS new_dma
78affd725ba5 Fix insert statement for dismar ashore
Tom Gottfried <tom@intevation.de>
parents: 2933
diff changeset
111 WHERE NOT ST_IsEmpty(new_dma)
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
112 RETURNING id
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113 `
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
115
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116 // Do executes the actual fairway dimension import.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
117 func (dma *DistanceMarksAshore) Do(
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
118 ctx context.Context,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
119 importID int64,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
120 conn *sql.Conn,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
121 feedback Feedback,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
122 ) (interface{}, error) {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
123
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124 start := time.Now()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
125
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
126 feedback.Info("Import distance marks")
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
127
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
128 feedback.Info("Loading capabilities from %s", dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
129 caps, err := wfs.GetCapabilities(dma.URL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
130 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
131 feedback.Error("Loading capabilities failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
132 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
133 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
134
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
135 ft := caps.FindFeatureType(dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
136 if ft == nil {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4152
diff changeset
137 return nil, fmt.Errorf("unknown feature type '%s'", dma.FeatureType)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
138 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
139
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
140 feedback.Info("Found feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
142 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
144 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
145 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
146 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
147
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
148 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
149 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
150 feedback.Error("Cannot create GetFeature URLs. %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
151 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154 tx, err := conn.BeginTx(ctx, nil)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
155 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
157 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
158 defer tx.Rollback()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
159
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
160 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
161 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
162 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
163 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
164 defer insertStmt.Close()
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
165
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
166 // Delete the old features.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
167 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
168 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
169 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
170
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
171 var (
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
172 unsupported = stringCounter{}
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
173 missingProperties int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
174 badProperties int
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
175 outside int
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
176 features int
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
177 )
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
178
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
179 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1862
diff changeset
180 feedback.Info("Get features from: '%s'", url)
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
181 rfc, err := wfs.ParseRawFeatureCollection(r)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182 if err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183 return fmt.Errorf("parsing GetFeature document failed: %v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 if rfc.CRS != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186 crsName := rfc.CRS.Properties.Name
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188 feedback.Error("Unsupported CRS: %d", crsName)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
192
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193 // No features -> ignore.
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194 if rfc.Features == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 feedback.Info("Using EPSG: %d", epsg)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200 for _, feature := range rfc.Features {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 if feature.Geometry.Coordinates == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202 missingProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 var props distanceMarksAshoreProperties
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 badProperties++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 continue
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 switch feature.Geometry.Type {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213 case "Point":
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
214 var p pointSlice
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
215 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
216 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217 }
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
218 var dmaid int64
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
219 err := insertStmt.QueryRowContext(
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 ctx,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 p.asWKB(),
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 epsg,
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 props.HydroCatdis,
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
224 ).Scan(&dmaid)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
225 switch {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
226 case err == sql.ErrNoRows:
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
227 outside++
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
228 // ignore -> filtered by responsibility area
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
229 continue
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
230 case err != nil:
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231 return err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 features++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 default:
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 unsupported[feature.Geometry.Type]++
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
237 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
238 return nil
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 }); err != nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
241 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
242
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243 if badProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244 feedback.Warn("Bad properties: %d", badProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 if missingProperties > 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 feedback.Warn("Missing properties: %d", missingProperties)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 if len(unsupported) != 0 {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 feedback.Warn("Unsupported types found: %s", unsupported)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254
2928
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
255 if outside > 0 {
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
256 feedback.Info("Features outside responsibility area: %d", outside)
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
257 }
074f2bb85584 Do not count skipped features as stored
Tom Gottfried <tom@intevation.de>
parents: 2926
diff changeset
258
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259 if features == 0 {
4177
8b75ac5e243e Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 4152
diff changeset
260 err := errors.New("no features found")
1862
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261 feedback.Error("%v", err)
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
262 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
263 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
264
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265 if err = tx.Commit(); err == nil {
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266 feedback.Info("Storing %d features took %s",
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 features, time.Since(start))
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 }
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270 return nil, err
427f86518097 Added distance marks importer in backend.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271 }