Mercurial > gemma

annotate pkg/imports/bn.go @ 3219:4acbee65275d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Import queue: Split locked dependencies in exclusively and multiple uses.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Thu, 09 May 2019 12:49:53 +0200
parents 1de0764f6af4
children bfde4f8dd323
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
3 //
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
6 //
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
7 // Copyright (C) 2018, 2019 by via donau
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
10 //
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
11 // Author(s):
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
13 // * Tom Gottfried <tom.gottfried@intevation.de>
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
14
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
15 package imports
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
16
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
17 import (
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
18 "context"
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
19 "database/sql"
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
20 "errors"
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
21 "regexp"
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
22 "strconv"
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
23 "time"
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
24
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/soap/ifbn"
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
26 )
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
27
1695
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
28 // Bottleneck is an import job to import
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
29 // bottlenecks from an IFBN SOAP service.
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
30 type Bottleneck struct {
1695
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
31 // URL is the URL of the SOAP service.
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
32 URL string `json:"url"`
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
33 // Tolerance used for axis snapping
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
34 Tolerance float64 `json:"tolerance"`
1695
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
35 // Insecure indicates if HTTPS traffic
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
36 // should validate certificates or not.
be78b429ef6e Bottleneck import: Resolved remaining golint issues.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1676
diff changeset
37 Insecure bool `json:"insecure"`
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
38 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
39
1696
ad5e1cddaa09 Imports: Resolved the remaining golint issues with this package.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1695
diff changeset
40 // BNJobKind is the import queue type identifier.
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
41 const BNJobKind JobKind = "bn"
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
42
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
43 const (
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
44 hasBottleneckSQL = `
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
45 SELECT true FROM waterway.bottlenecks WHERE bottleneck_id = $1`
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
46
1658
07f996b0e14a Bottleneck import: Cleaned up source a bit (removed outdated TODOs).
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1646
diff changeset
47 insertBottleneckSQL = `
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
48 INSERT INTO waterway.bottlenecks (
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
49 bottleneck_id,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
50 fk_g_fid,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
51 objnam,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
52 nobjnm,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
53 stretch,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
54 area,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
55 rb,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
56 lb,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
57 responsible_country,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
58 revisiting_time,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
59 limiting,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
60 date_info,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
61 source_organization
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
62 ) VALUES(
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
63 $1,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
64 isrs_fromText($2),
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
65 $3,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
66 $4,
2752
018f979f9e23 Bottlenecks import allow ISRS_from > ISRS_to.
Sascha Wilde <wilde@intevation.de>
parents: 2693
diff changeset
67 isrsrange(least(isrs_fromText($5), isrs_fromText($6)),
018f979f9e23 Bottlenecks import allow ISRS_from > ISRS_to.
Sascha Wilde <wilde@intevation.de>
parents: 2693
diff changeset
68 greatest(isrs_fromText($5), isrs_fromText($6))),
1631
6ee06eb37fa9 Use new function to calculate bn area.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1611
diff changeset
69 ISRSrange_area(
2752
018f979f9e23 Bottlenecks import allow ISRS_from > ISRS_to.
Sascha Wilde <wilde@intevation.de>
parents: 2693
diff changeset
70 ISRSrange_axis(isrsrange(least(isrs_fromText($5), isrs_fromText($6)),
018f979f9e23 Bottlenecks import allow ISRS_from > ISRS_to.
Sascha Wilde <wilde@intevation.de>
parents: 2693
diff changeset
71 greatest(isrs_fromText($5), isrs_fromText($6))),
018f979f9e23 Bottlenecks import allow ISRS_from > ISRS_to.
Sascha Wilde <wilde@intevation.de>
parents: 2693
diff changeset
72 $14),
2248
cce158db02b0 Input area as multipolygons to generate area from stretch
Tom Gottfried <tom@intevation.de>
parents: 2195
diff changeset
73 (SELECT ST_Collect(CAST(area AS geometry))
1902
c4af342be999 Use waterway area as basis for bottleneck area generation.
Sascha Wilde <wilde@intevation.de>
parents: 1754
diff changeset
74 FROM waterway.waterway_area)),
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
75 $7,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
76 $8,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
77 $9,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
78 $10,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
79 $11,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
80 $12,
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
81 $13
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
82 )
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
83 RETURNING id`
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
84
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
85 insertBottleneckMaterialSQL = `
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
86 INSERT INTO waterway.bottlenecks_riverbed_materials (
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
87 bottleneck_id,
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
88 riverbed
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
89 ) VALUES (
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
90 $1,
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
91 $2
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
92 )`
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
93 )
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
94
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
95 type bnJobCreator struct{}
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
96
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
97 func init() {
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
98 RegisterJobCreator(BNJobKind, bnJobCreator{})
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
99 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
100
1754
807569b08513 Import queue: Auto acceptance is now a property of the import kind itself and is not configurable any more.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1696
diff changeset
101 func (bnJobCreator) Description() string { return "bottlenecks" }
807569b08513 Import queue: Auto acceptance is now a property of the import kind itself and is not configurable any more.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1696
diff changeset
102
807569b08513 Import queue: Auto acceptance is now a property of the import kind itself and is not configurable any more.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1696
diff changeset
103 func (bnJobCreator) AutoAccept() bool { return false }
1646
a0982c38eac0 Import queue: Implemented email notifications.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1631
diff changeset
104
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
105 func (bnJobCreator) Create() Job { return new(Bottleneck) }
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
106
3219
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3164
diff changeset
107 func (bnJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3164
diff changeset
108 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3164
diff changeset
109 {"bottlenecks", "bottlenecks_riverbed_materials"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 3164
diff changeset
110 {"gauges"},
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
111 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
112 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
113
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
114 const (
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
115 bnStageDoneSQL = `
1533
8fc1a2298acb Pseudo fixed unstaging of bottlenecks.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1531
diff changeset
116 UPDATE waterway.bottlenecks SET staging_done = true
8fc1a2298acb Pseudo fixed unstaging of bottlenecks.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1531
diff changeset
117 WHERE id IN (
1995
59055c8301df Move import queue to its own database namespace
Tom Gottfried <tom@intevation.de>
parents: 1975
diff changeset
118 SELECT key from import.track_imports
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
119 WHERE import_id = $1 AND
1533
8fc1a2298acb Pseudo fixed unstaging of bottlenecks.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1531
diff changeset
120 relation = 'waterway.bottlenecks'::regclass)`
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
121 )
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
122
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
123 // StageDone moves the imported bottleneck out of the staging area.
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
124 func (bnJobCreator) StageDone(
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
125 ctx context.Context,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
126 tx *sql.Tx,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
127 id int64,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
128 ) error {
1594
e3abd3b3ebc6 Fixed stage done SQL for bottleneck imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1580
diff changeset
129 _, err := tx.ExecContext(ctx, bnStageDoneSQL, id)
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
130 return err
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
131 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
132
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
133 // CleanUp of a bottleneck import is a NOP.
1676
4407ecaa2192 Imports: Cosmetics.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1664
diff changeset
134 func (*Bottleneck) CleanUp() error { return nil }
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
135
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
136 var rblbRe = regexp.MustCompile(`(..)_(..)`)
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
137
2765
41f5ac76d642 BT import: allow for rb and lb to be null.
Sascha Wilde <wilde@intevation.de>
parents: 2752
diff changeset
138 func splitRBLB(s string) (*string, *string) {
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
139 m := rblbRe.FindStringSubmatch(s)
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
140 if len(m) == 0 {
2765
41f5ac76d642 BT import: allow for rb and lb to be null.
Sascha Wilde <wilde@intevation.de>
parents: 2752
diff changeset
141 return nil, nil
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
142 }
2765
41f5ac76d642 BT import: allow for rb and lb to be null.
Sascha Wilde <wilde@intevation.de>
parents: 2752
diff changeset
143 return &m[1], &m[2]
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
144 }
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
145
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
146 func revisitingTime(s string) int {
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
147 v, err := strconv.Atoi(s)
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
148 if err != nil {
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
149 v = 0
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
150 }
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
151 return v
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
152 }
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
153
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
154 // Do executes the actual bottleneck import.
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
155 func (bn *Bottleneck) Do(
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
156 ctx context.Context,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
157 importID int64,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
158 conn *sql.Conn,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
159 feedback Feedback,
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
160 ) (interface{}, error) {
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
161
2253
cfc523c70e90 Uploaded imports: Small code clean-ups.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2248
diff changeset
162 fetch := func() ([]*ifbn.BottleNeckType, error) {
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
163 client := ifbn.NewIBottleneckService(bn.URL, bn.Insecure, nil)
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
164
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
165 req := &ifbn.Export_bn_by_isrs{}
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
166
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
167 resp, err := client.Export_bn_by_isrs(req)
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
168 if err != nil {
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
169 return nil, err
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
170 }
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
171
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
172 if resp.Export_bn_by_isrsResult == nil {
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
173 return nil, errors.New("no Bottlenecks found")
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
174 }
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
175
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
176 return resp.Export_bn_by_isrsResult.BottleNeckType, nil
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
177 }
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
178
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
179 return storeBottlenecks(ctx, fetch, importID, conn, feedback, bn.Tolerance)
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
180 }
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
181
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
182 func storeBottlenecks(
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
183 ctx context.Context,
2253
cfc523c70e90 Uploaded imports: Small code clean-ups.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2248
diff changeset
184 fetch func() ([]*ifbn.BottleNeckType, error),
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
185 importID int64,
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
186 conn *sql.Conn,
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
187 feedback Feedback,
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
188 tolerance float64,
2195
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
189 ) (interface{}, error) {
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
190 start := time.Now()
3437ceee1529 Uploaded bottleneck import: Implemented in terms of normal bottleneck import.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2187
diff changeset
191
2253
cfc523c70e90 Uploaded imports: Small code clean-ups.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2248
diff changeset
192 bns, err := fetch()
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
193 if err != nil {
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
194 return nil, err
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
195 }
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
196
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
197 feedback.Info("Found %d bottlenecks for import", len(bns))
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
198
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
199 var hasStmt, insertStmt, insertMaterialStmt, trackStmt *sql.Stmt
1580
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
200
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
201 for _, x := range []struct {
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
202 sql string
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
203 stmt **sql.Stmt
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
204 }{
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
205 {hasBottleneckSQL, &hasStmt},
1658
07f996b0e14a Bottleneck import: Cleaned up source a bit (removed outdated TODOs).
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1646
diff changeset
206 {insertBottleneckSQL, &insertStmt},
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
207 {insertBottleneckMaterialSQL, &insertMaterialStmt},
1580
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
208 {trackImportSQL, &trackStmt},
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
209 } {
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
210 var err error
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
211 if *x.stmt, err = conn.PrepareContext(ctx, x.sql); err != nil {
1580
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
212 return nil, err
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
213 }
c503d2fa9776 Bottleneck import: Use a prepared statement for import tracking, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1572
diff changeset
214 defer (*x.stmt).Close()
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
215 }
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
216
1611
1835d1c9eb9b Store bottleneck_id's instead of id's in bottleneck import summary.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1595
diff changeset
217 var nids []string
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
218
2693
02d012272dd4 Log snapping tolerance
Tom Gottfried <tom@intevation.de>
parents: 2563
diff changeset
219 feedback.Info("Tolerance used to snap waterway axis: %g", tolerance)
02d012272dd4 Log snapping tolerance
Tom Gottfried <tom@intevation.de>
parents: 2563
diff changeset
220
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
221 for _, bn := range bns {
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
222 if err := storeBottleneck(
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
223 ctx, importID, conn, feedback, bn, &nids, tolerance,
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
224 hasStmt, insertStmt, insertMaterialStmt, trackStmt); err != nil {
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
225 return nil, err
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
226 }
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
227 }
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
228 if len(nids) == 0 {
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
229 return nil, UnchangedError("No new bottlenecks inserted")
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
230 }
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
231
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
232 feedback.Info("Storing %d bottlenecks took %s", len(nids), time.Since(start))
1906
32c56e6c089a Stretch import: Added forgotten source file.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1902
diff changeset
233 feedback.Info("Import of bottlenecks was successful")
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
234 summary := struct {
1611
1835d1c9eb9b Store bottleneck_id's instead of id's in bottleneck import summary.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1595
diff changeset
235 Bottlenecks []string `json:"bottlenecks"`
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
236 }{
1572
056a86b24be2 Made bottleneck primary key an int. Attention: This may break something!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1569
diff changeset
237 Bottlenecks: nids,
1569
cf0ca4a9812a Bottleneck import: Implemented backend to store bottlenecks in database.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1533
diff changeset
238 }
1906
32c56e6c089a Stretch import: Added forgotten source file.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1902
diff changeset
239 return &summary, nil
1531
24445a618513 Added stub for bottleneck importer.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
240 }
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
241
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
242 func storeBottleneck(
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
243 ctx context.Context,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
244 importID int64,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
245 conn *sql.Conn,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
246 feedback Feedback,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
247 bn *ifbn.BottleNeckType,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
248 nids *[]string,
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
249 tolerance float64,
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
250 hasStmt, insertStmt, insertMaterialStmt, trackStmt *sql.Stmt,
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
251 ) error {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
252
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
253 tx, err := conn.BeginTx(ctx, nil)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
254 if err != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
255 return err
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
256 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
257 defer tx.Rollback()
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
258
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
259 var found bool
3164
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
260 err = tx.StmtContext(ctx, hasStmt).QueryRowContext(ctx,
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
261 bn.Bottleneck_id).Scan(&found)
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
262 switch {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
263 case err == sql.ErrNoRows:
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
264 // This is good.
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
265 case err != nil:
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
266 return err
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
267 case found:
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
268 feedback.Info("'%s' already in database. Skip", bn.OBJNAM)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
269 // TODO: Deep comparison database vs. SOAP.
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
270 return nil
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
271 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
272
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
273 rb, lb := splitRBLB(bn.Rb_lb)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
274
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
275 var limiting, country string
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
276
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
277 if bn.Limiting_factor != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
278 limiting = string(*bn.Limiting_factor)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
279 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
280
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
281 if bn.Responsible_country != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
282 country = string(*bn.Responsible_country)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
283 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
284
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
285 var nid int64
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
286
3164
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
287 err = tx.StmtContext(ctx, insertStmt).QueryRowContext(
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
288 ctx,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
289 bn.Bottleneck_id,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
290 bn.Fk_g_fid,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
291 bn.OBJNAM,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
292 bn.NOBJNM,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
293 bn.From_ISRS, bn.To_ISRS,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
294 rb,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
295 lb,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
296 country,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
297 revisitingTime(bn.Revisiting_time),
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
298 limiting,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
299 bn.Date_Info,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
300 bn.Source,
2563
dc4fae4bdb8f Expose axis snapping tolerance to users
Tom Gottfried <tom@intevation.de>
parents: 2548
diff changeset
301 tolerance,
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
302 ).Scan(&nid)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
303 if err != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
304 feedback.Warn("Failed to insert '%s' into database", bn.OBJNAM)
2548
6b34d0fb4498 Improve user facing error messages for stretch area generation
Tom Gottfried <tom@intevation.de>
parents: 2532
diff changeset
305 feedback.Warn(handleError(err).Error())
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
306 return nil
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
307 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
308
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
309 if bn.Riverbed != nil {
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
310 for _, material := range bn.Riverbed.Material {
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
311 if material != nil {
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
312 mat := string(*material)
3164
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
313 if _, err := tx.StmtContext(ctx,
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
314 insertMaterialStmt).ExecContext(
2999
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
315 ctx, nid, material); err != nil {
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
316 feedback.Warn(
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
317 "Failed to insert riverbed material '%s' for bottleneck '%s'.",
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
318 mat, bn.OBJNAM)
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
319 feedback.Warn(handleError(err).Error())
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
320 }
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
321 }
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
322 }
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
323 }
b3c3c5b5b7c1 Bottleneck import: Import riverbed materials, too.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2765
diff changeset
324
3164
1de0764f6af4 Use context for preparation of transaction-specific statements
Tom Gottfried <tom@intevation.de>
parents: 2999
diff changeset
325 if _, err := tx.StmtContext(ctx, trackStmt).ExecContext(
2532
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
326 ctx, importID, "waterway.bottlenecks", nid,
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
327 ); err != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
328 return err
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
329 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
330 if err = tx.Commit(); err != nil {
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
331 return err
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
332 }
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
333 feedback.Info("Inserted '%s' into database", bn.OBJNAM)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
334 *nids = append(*nids, bn.Bottleneck_id)
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
335 return nil
452bc714bfd9 Continue importing bottlenecks if one item fails
Tom Gottfried <tom@intevation.de>
parents: 2253
diff changeset
336 }