Mercurial > gemma

annotate pkg/imports/wa.go @ 3681:c9e1848a516a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Handle violation of responsibility areas graceful for BN import. As many others this is an error, which should only affect the specific bottleneck failing and not the whole import.
author Sascha Wilde <wilde@intevation.de>
date Tue, 18 Jun 2019 12:43:01 +0200
parents 4acbee65275d
children 6c5c15b2fb64
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
3 //
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
6 //
2649
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping
Tom Gottfried <tom@intevation.de>
parents: 2622
diff changeset
7 // Copyright (C) 2018, 2019 by via donau
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
10 //
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
11 // Author(s):
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
2649
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping
Tom Gottfried <tom@intevation.de>
parents: 2622
diff changeset
13 // * Tom Gottfried <tom.gottfried@intevation.de>
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
14
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
15 package imports
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
16
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
17 import (
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
18 "context"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
19 "database/sql"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
20 "encoding/json"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
21 "errors"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
22 "fmt"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
23 "io"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
24 "strconv"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
25 "time"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
26
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
27 "gemma.intevation.de/gemma/pkg/wfs"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
28 )
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
29
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
30 // WaterwayArea is an import job to import
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
31 // the waterway area in form of polygon geometries
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
32 // and attribute data from a WFS service.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
33 type WaterwayArea struct {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
34 // URL the GetCapabilities URL of the WFS service.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
35 URL string `json:"url"`
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
36 // FeatureType selects the feature type of the WFS service.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
37 FeatureType string `json:"feature-type"`
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
38 // SortBy works around misconfigured services to
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
39 // establish a sort order to get the features.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
40 SortBy string `json:"sort-by"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
41 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
42 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
43 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
44 Password string `json:"password,omitempty"`
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
45 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
46
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
47 // WAJobKind is the import queue type identifier.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
48 const WAJobKind JobKind = "wa"
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
49
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
50 type waJobCreator struct{}
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
51
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
52 func init() {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
53 RegisterJobCreator(WAJobKind, waJobCreator{})
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
54 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
55
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
56 func (waJobCreator) Description() string { return "waterway area" }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
57
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
58 func (waJobCreator) AutoAccept() bool { return true }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
59
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
60 func (waJobCreator) Create() Job { return new(WaterwayArea) }
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
61
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2938
diff changeset
62 func (waJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2938
diff changeset
63 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2938
diff changeset
64 {"waterway_area"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2938
diff changeset
65 {},
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
66 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
67 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
68
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
69 // StageDone is a NOP for waterway area imports.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
70 func (waJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
71 return nil
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
72 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
73
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
74 // CleanUp for waterway area imports is a NOP.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
75 func (*WaterwayArea) CleanUp() error { return nil }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
76
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
77 type waterwayAreaProperties struct {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
78 Catccl *string `json:"ienc_catccl"`
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
79 Dirimp *string `json:"ienc_dirimp"`
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
80 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
81
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
82 const (
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
83 deleteWaterwayAreaSQL = `
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
84 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2855
diff changeset
85 SELECT users.current_user_area_utm() AS a
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
86 )
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
87 DELETE FROM waterway.waterway_area
2649
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping
Tom Gottfried <tom@intevation.de>
parents: 2622
diff changeset
88 WHERE pg_has_role('sys_admin', 'MEMBER')
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping
Tom Gottfried <tom@intevation.de>
parents: 2622
diff changeset
89 OR ST_Covers((SELECT a FROM resp),
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2855
diff changeset
90 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp)))
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
91 `
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
92 insertWaterwayAreaSQL = `
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
93 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2855
diff changeset
94 SELECT users.current_user_area_utm() AS a
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
95 )
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
96 INSERT INTO waterway.waterway_area (area, catccl, dirimp)
2855
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
97 SELECT dmp.geom, $3, $4
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
98 FROM ST_GeomFromWKB($1, $2::integer) AS new_area (new_area),
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
99 ST_Dump(ST_Transform(ST_CollectionExtract(
2649
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping
Tom Gottfried <tom@intevation.de>
parents: 2622
diff changeset
100 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
2855
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
101 THEN ST_MakeValid(ST_Transform(new_area,
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
102 best_utm(ST_Transform(new_area, 4326))))
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
103 ELSE ST_Intersection((SELECT a FROM resp),
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2855
diff changeset
104 ST_MakeValid(ST_Transform(new_area, (SELECT ST_SRID(a) FROM resp))))
2855
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
105 END,
8a8a929182f9 Enable import of axis and area independently from areas of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
106 3), 4326)) AS dmp
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
107 RETURNING id
1796
904c7e853adc Waterway area import: Clip features against responsibility area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1786
diff changeset
108 `
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
109 )
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
110
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
111 // Do executes the actual waterway axis import.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
112 func (wx *WaterwayArea) Do(
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
113 ctx context.Context,
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
114 importID int64,
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
115 conn *sql.Conn,
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
116 feedback Feedback,
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
117 ) (interface{}, error) {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
118
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
119 start := time.Now()
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
120
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
121 feedback.Info("Import waterway area")
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
122
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
123 feedback.Info("Loading capabilities from %s", wx.URL)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
124 caps, err := wfs.GetCapabilities(wx.URL)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
125 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
126 feedback.Error("Loading capabilities failed: %v", err)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
127 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
128 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
129
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
130 ft := caps.FindFeatureType(wx.FeatureType)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
131 if ft == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
132 return nil, fmt.Errorf("Unknown feature type '%s'", wx.FeatureType)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
133 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
134
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
135 feedback.Info("Found feature type '%s", wx.FeatureType)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
136
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
137 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
138 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
139 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
140 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
141 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
142
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
143 if wx.SortBy != "" {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
144 feedback.Info("Features will be sorted by '%s'", wx.SortBy)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
145 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
146
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
147 dl, err := wfs.GetFeatures(caps, wx.FeatureType, wx.SortBy)
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
148 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
149 feedback.Error("Cannot create GetFeature URLs. %v", err)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
150 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
151 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
152
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
153 tx, err := conn.BeginTx(ctx, nil)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
154 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
155 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
156 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
157 defer tx.Rollback()
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
158
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
159 insertStmt, err := tx.PrepareContext(ctx, insertWaterwayAreaSQL)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
160 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
161 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
162 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
163 defer insertStmt.Close()
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
164
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
165 // Delete the old features.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
166 if _, err := tx.ExecContext(ctx, deleteWaterwayAreaSQL); err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
167 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
168 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
169
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
170 var (
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
171 unsupported = stringCounter{}
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
172 missingProperties int
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
173 badProperties int
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
174 outside int
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
175 features int
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
176 )
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
177
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
178 if err := dl.Download(wx.User, wx.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1899
diff changeset
179 feedback.Info("Get features from: '%s'", url)
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
180 rfc, err := wfs.ParseRawFeatureCollection(r)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
181 if err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
182 return fmt.Errorf("parsing GetFeature document failed: %v", err)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
183 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
184 if rfc.CRS != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
185 crsName := rfc.CRS.Properties.Name
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
186 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
187 feedback.Error("Unsupported CRS: %d", crsName)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
188 return err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
189 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
190 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
191
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
192 // No features -> ignore.
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
193 if rfc.Features == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
194 return nil
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
195 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
196
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
197 feedback.Info("Using EPSG: %d", epsg)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
198
2758
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
199 savepoint := Savepoint(ctx, tx, "feature")
2659
00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails
Tom Gottfried <tom@intevation.de>
parents: 2649
diff changeset
200
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
201 for _, feature := range rfc.Features {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
202 if feature.Properties == nil || feature.Geometry.Coordinates == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
203 missingProperties++
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
204 continue
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
205 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
206
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
207 var props waterwayAreaProperties
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
208
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
209 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
210 badProperties++
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
211 continue
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
212 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
213
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
214 var catccl sql.NullInt64
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
215 if props.Catccl != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
216 if value, err := strconv.ParseInt(*props.Catccl, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
217 catccl = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
218 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
219 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
220 var dirimp sql.NullInt64
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
221 if props.Dirimp != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
222 if value, err := strconv.ParseInt(*props.Dirimp, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
223 dirimp = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
224 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
225 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
226
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
227 switch feature.Geometry.Type {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
228 case "Polygon":
1786
09349ca27dd7 Imports: Removed duplicated code path to store WKB polygons into database.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1785
diff changeset
229 var p polygonSlice
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
230 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
231 return err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
232 }
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
233 var waid int64
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
234 err := savepoint(func() error {
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
235 err := insertStmt.QueryRowContext(
2758
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
236 ctx,
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
237 p.asWKB(),
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
238 epsg,
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
239 catccl,
a996f2ca9fa5 Simplified savepoint handling.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2729
diff changeset
240 dirimp,
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
241 ).Scan(&waid)
2659
00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails
Tom Gottfried <tom@intevation.de>
parents: 2649
diff changeset
242 return err
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
243 })
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
244 switch {
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
245 case err == sql.ErrNoRows:
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
246 outside++
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
247 // ignore -> filtered by responsibility_areas
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
248 case err != nil:
2659
00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails
Tom Gottfried <tom@intevation.de>
parents: 2649
diff changeset
249 feedback.Warn(handleError(err).Error())
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
250 default:
2659
00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails
Tom Gottfried <tom@intevation.de>
parents: 2649
diff changeset
251 features++
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
252 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
253 default:
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
254 unsupported[feature.Geometry.Type]++
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
255 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
256 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
257 return nil
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
258 }); err != nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
259 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
260 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
261
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
262 if badProperties > 0 {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
263 feedback.Warn("Bad properties: %d", badProperties)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
264 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
265
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
266 if missingProperties > 0 {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
267 feedback.Warn("Missing properties: %d", missingProperties)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
268 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
269
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
270 if len(unsupported) != 0 {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
271 feedback.Warn("Unsupported types found: %s", unsupported)
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
272 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
273
2938
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
274 if outside > 0 {
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
275 feedback.Info("Features outside responsibility area: %d", outside)
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
276 }
ecb1e34157a3 Fix logging in area import
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
277
1828
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
278 if features == 0 {
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
279 err := errors.New("No features found")
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
280 feedback.Error("%v", err)
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
281 return nil, err
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
282 }
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1796
diff changeset
283
1785
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
284 if err = tx.Commit(); err == nil {
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
285 feedback.Info("Storing %d features took %s",
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
286 features, time.Since(start))
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
287 }
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
288
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
289 return nil, err
614c6c766691 Waterway area import: Implemented.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
diff changeset
290 }