Mercurial > gemma
annotate pkg/imports/dmv.go @ 5718:3d497077f888 uploadwg
Implemented direct file upload as alternative import method for WG.
For testing and data corrections it is useful to be able to import
waterway gauges data directly by uploading a xml file.
author | Sascha Wilde <wilde@sha-bang.de> |
---|---|
date | Thu, 18 Apr 2024 19:23:19 +0200 |
parents | 6270951dda28 |
children |
rev | line source |
---|---|
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
1 // This is Free Software under GNU Affero General Public License v >= 3.0 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
2 // without warranty, see README.md and license for details. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
3 // |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
4 // SPDX-License-Identifier: AGPL-3.0-or-later |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
5 // License-Filename: LICENSES/AGPL-3.0.txt |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
6 // |
5671
4bfcbb747768
relenc is now optional in DMV data.
Sascha Wilde <wilde@sha-bang.de>
parents:
5616
diff
changeset
|
7 // Copyright (C) 2018,2023 by via donau |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
8 // – Österreichische Wasserstraßen-Gesellschaft mbH |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
9 // Software engineering by Intevation GmbH |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
10 // |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
11 // Author(s): |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de> |
5671
4bfcbb747768
relenc is now optional in DMV data.
Sascha Wilde <wilde@sha-bang.de>
parents:
5616
diff
changeset
|
13 // * Sascha Wilde <wilde@intevation.de> |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
14 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
15 package imports |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
16 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
17 import ( |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
18 "context" |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
19 "database/sql" |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
20 "errors" |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
21 "time" |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
22 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
23 "gemma.intevation.de/gemma/pkg/models" |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
24 ) |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
25 |
4179
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
26 // DistanceMarksVirtual is a Job to import virtual distance marks |
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
27 // from an external SOAP service into the database. |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
28 type DistanceMarksVirtual struct { |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
29 // URL is the URL of the SOAP service. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
30 URL string `json:"url"` |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
31 // Username is the username used to authenticate. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
32 Username string `json:"username"` |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
33 // Passwort is the password to authenticate. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
34 Password string `json:"password"` |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
35 // Insecure indicates if HTTPS traffic |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
36 // should validate certificates or not. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
37 Insecure bool `json:"insecure"` |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
38 } |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
39 |
4852
046a07a33b19
Fixed the golint issues of the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4799
diff
changeset
|
40 // Description gives a short info about relevant facts of this import. |
5560
f2204f91d286
Join the log lines of imports to the log exports to recover data from them.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5034
diff
changeset
|
41 func (dmv *DistanceMarksVirtual) Description([]string) (string, error) { |
4799
f32d086b5dbf
Removed the mechanical touch of the last commit.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4798
diff
changeset
|
42 return dmv.URL, nil |
4798
ca6a5f722471
Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents:
4179
diff
changeset
|
43 } |
ca6a5f722471
Added Description method to most imports.
Sascha Wilde <wilde@intevation.de>
parents:
4179
diff
changeset
|
44 |
4179
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
45 // DMVJobKind is the import queue type identifier. |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
46 const DMVJobKind JobKind = "dmv" |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
47 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
48 type dmvJobCreator struct{} |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
49 |
4179
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
50 func init() { RegisterJobCreator(DMVJobKind, dmvJobCreator{}) } |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
51 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
52 func (dmvJobCreator) Description() string { return "distance marks virtual" } |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
53 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
54 func (dmvJobCreator) AutoAccept() bool { return true } |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
55 |
2187
7c83b5277c1c
Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
2168
diff
changeset
|
56 func (dmvJobCreator) Create() Job { return new(DistanceMarksVirtual) } |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
57 |
3219
4acbee65275d
Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
3163
diff
changeset
|
58 func (dmvJobCreator) Depends() [2][]string { |
5020
e4ab338e7ba9
Removed unnecessary creatings of empty slices in expressing read dependencies in imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4852
diff
changeset
|
59 return [2][]string{{"distance_marks_virtual"}} |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
60 } |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
61 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
62 // StageDone does nothing as there is no staging for distance marks virtual. |
5034
59a99655f34d
Added feedback support for StageDone.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5020
diff
changeset
|
63 func (dmvJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error { return nil } |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
64 |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
65 // CleanUp does nothing as there is nothing to cleanup with distance marks virtual. |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
66 func (*DistanceMarksVirtual) CleanUp() error { return nil } |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
67 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
68 const ( |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
69 insertDistanceMarksVirtualSQL = ` |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
70 INSERT INTO waterway.distance_marks_virtual ( |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
71 location_code, |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
72 geom, |
3778
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
73 related_enc, |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
74 wwname |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
75 ) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
76 VALUES ( |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
77 ($1::char(2), $2::char(3), $3::char(5), $4::char(5), $5::int), |
1918
62b5f6e48e82
Distance marks virtual/Waterway gauge imports: Better fix for swapped Lat/Lon fields.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1917
diff
changeset
|
78 ST_SetSRID(ST_MakePoint($6, $7), 4326)::geography, |
3778
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
79 $8, |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
80 $9 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
81 ) ON CONFLICT (location_code) DO UPDATE SET |
1918
62b5f6e48e82
Distance marks virtual/Waterway gauge imports: Better fix for swapped Lat/Lon fields.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1917
diff
changeset
|
82 geom = ST_SetSRID(ST_MakePoint($6, $7), 4326)::geography, |
3778
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
83 related_enc = $8, |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
84 wwname = $9 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
85 ` |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
86 ) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
87 |
4179
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
88 // Do executes the actual downloading and importing |
a376351d2774
Another round of doc strings to make 'golint' a little bit more happy with the imports package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
4177
diff
changeset
|
89 // of the virtual distance marks. |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
90 func (dmv *DistanceMarksVirtual) Do( |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
91 ctx context.Context, |
5711
2dd155cc95ec
Fix all revive issue (w/o machine generated stuff).
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5671
diff
changeset
|
92 _ int64, |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
93 conn *sql.Conn, |
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
94 feedback Feedback, |
5712
6270951dda28
/interface{}/any/
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5711
diff
changeset
|
95 ) (any, error) { |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
96 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
97 start := time.Now() |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
98 |
3551
a5448426e4e2
Only erase gauges that have been requested from ERDMS
Tom Gottfried <tom@intevation.de>
parents:
3219
diff
changeset
|
99 responseData, _, err := getRisData( |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
100 ctx, |
3163
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
101 conn, |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
102 feedback, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
103 dmv.Username, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
104 dmv.Password, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
105 dmv.URL, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
106 dmv.Insecure, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
107 "dismar") |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
108 if err != nil { |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
109 return nil, err |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
110 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
111 |
3163
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
112 tx, err := conn.BeginTx(ctx, nil) |
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
113 if err != nil { |
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
114 return nil, err |
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
115 } |
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
116 defer tx.Rollback() |
d9903cb34842
Handle failing INSERTs gracefully during gauges import
Tom Gottfried <tom@intevation.de>
parents:
2633
diff
changeset
|
117 |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
118 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksVirtualSQL) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
119 if err != nil { |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
120 return nil, err |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
121 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
122 defer insertStmt.Close() |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
123 |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
124 var ignored, features int |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
125 |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
126 for _, data := range responseData { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
127 for _, dr := range data.RisdataReturn { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
128 if dr.RisidxCode == nil { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
129 ignored++ |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
130 continue |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
131 } |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
132 |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
133 code, err := models.IsrsFromString(string(*dr.RisidxCode)) |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
134 if err != nil { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
135 feedback.Warn("invalid ISRS code %v", err) |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
136 ignored++ |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
137 continue |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
138 } |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
139 |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
140 if dr.Lat == nil || dr.Lon == nil { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
141 feedback.Warn("missing lat/lon: %s", code) |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
142 ignored++ |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
143 continue |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
144 } |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
145 |
5671
4bfcbb747768
relenc is now optional in DMV data.
Sascha Wilde <wilde@sha-bang.de>
parents:
5616
diff
changeset
|
146 if dr.Relenc != nil && len(*dr.Relenc) > 12 { |
5616
a4ecd66b5940
DMV: Don't accept relenc length greater 12.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5560
diff
changeset
|
147 feedback.Warn("relenc too long: %s", *dr.Relenc) |
a4ecd66b5940
DMV: Don't accept relenc length greater 12.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5560
diff
changeset
|
148 ignored++ |
a4ecd66b5940
DMV: Don't accept relenc length greater 12.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5560
diff
changeset
|
149 continue |
a4ecd66b5940
DMV: Don't accept relenc length greater 12.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5560
diff
changeset
|
150 } |
a4ecd66b5940
DMV: Don't accept relenc length greater 12.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5560
diff
changeset
|
151 |
3778
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
152 if dr.Wwname.Loc == nil { |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
153 feedback.Warn("missing wwname: %s", code) |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
154 ignored++ |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
155 continue |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
156 } |
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
157 |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
158 if _, err := insertStmt.ExecContext( |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
159 ctx, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
160 code.CountryCode, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
161 code.LoCode, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
162 code.FairwaySection, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
163 code.Orc, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
164 code.Hectometre, |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
165 float64(*dr.Lon), float64(*dr.Lat), |
5671
4bfcbb747768
relenc is now optional in DMV data.
Sascha Wilde <wilde@sha-bang.de>
parents:
5616
diff
changeset
|
166 dr.Relenc, |
3778
1b11ff97c1d1
Add wwname to import of virtual distance marks.
Sascha Wilde <wilde@intevation.de>
parents:
3551
diff
changeset
|
167 string(*dr.Wwname.Loc), |
2633
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
168 ); err != nil { |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
169 return nil, err |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
170 } |
eb1d119f253f
Fetch data from ERDMS for all allowed countries
Tom Gottfried <tom@intevation.de>
parents:
2380
diff
changeset
|
171 features++ |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
172 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
173 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
174 feedback.Info("ignored: %d", ignored) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
175 feedback.Info("features: %d", features) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
176 |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
177 if features == 0 { |
4177
8b75ac5e243e
Made 'staticcheck' happy with pgxutils package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
3778
diff
changeset
|
178 return nil, errors.New("no features found") |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
179 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
180 |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
181 if err = tx.Commit(); err == nil { |
1994
a7c4005b723f
Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents:
1918
diff
changeset
|
182 feedback.Info("Refreshing distance marks (virtual) successfully took %s.", |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
183 time.Since(start)) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
184 } else { |
1994
a7c4005b723f
Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents:
1918
diff
changeset
|
185 feedback.Error("Refreshing distance marks (virtual) failed after %s.", |
1859
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
186 time.Since(start)) |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
187 } |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
188 |
9780cb4ef6a6
Distance marks virtual import: Implemented.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1855
diff
changeset
|
189 return nil, nil |
1855
bbd653a43a6a
Distance marks virtual import: Added stub.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
190 } |