Mercurial > gemma

comparison pkg/imports/bn.go @ 2532:452bc714bfd9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Continue importing bottlenecks if one item fails
author Tom Gottfried <tom@intevation.de>
date Thu, 07 Mar 2019 10:46:34 +0100
parents cfc523c70e90
children 6b34d0fb4498
comparison
equal deleted inserted replaced
2531:47b9a4f9a05c 2532:452bc714bfd9
2 // without warranty, see README.md and license for details. 2 // without warranty, see README.md and license for details.
3 // 3 //
4 // SPDX-License-Identifier: AGPL-3.0-or-later 4 // SPDX-License-Identifier: AGPL-3.0-or-later
5 // License-Filename: LICENSES/AGPL-3.0.txt 5 // License-Filename: LICENSES/AGPL-3.0.txt
6 // 6 //
7 // Copyright (C) 2018 by via donau 7 // Copyright (C) 2018, 2019 by via donau
8 // – Österreichische Wasserstraßen-Gesellschaft mbH 8 // – Österreichische Wasserstraßen-Gesellschaft mbH
9 // Software engineering by Intevation GmbH 9 // Software engineering by Intevation GmbH
10 // 10 //
11 // Author(s): 11 // Author(s):
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de> 12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
13 // * Tom Gottfried <tom.gottfried@intevation.de>
13 14
14 package imports 15 package imports
15 16
16 import ( 17 import (
17 "context" 18 "context"
178 return nil, err 179 return nil, err
179 } 180 }
180 181
181 feedback.Info("Found %d bottlenecks for import", len(bns)) 182 feedback.Info("Found %d bottlenecks for import", len(bns))
182 183
183 tx, err := conn.BeginTx(ctx, nil)
184 if err != nil {
185 return nil, err
186 }
187 defer tx.Rollback()
188
189 var hasStmt, insertStmt, trackStmt *sql.Stmt 184 var hasStmt, insertStmt, trackStmt *sql.Stmt
190 185
191 for _, x := range []struct { 186 for _, x := range []struct {
192 sql string 187 sql string
193 stmt **sql.Stmt 188 stmt **sql.Stmt
195 {hasBottleneckSQL, &hasStmt}, 190 {hasBottleneckSQL, &hasStmt},
196 {insertBottleneckSQL, &insertStmt}, 191 {insertBottleneckSQL, &insertStmt},
197 {trackImportSQL, &trackStmt}, 192 {trackImportSQL, &trackStmt},
198 } { 193 } {
199 var err error 194 var err error
200 if *x.stmt, err = tx.PrepareContext(ctx, x.sql); err != nil { 195 if *x.stmt, err = conn.PrepareContext(ctx, x.sql); err != nil {
201 return nil, err 196 return nil, err
202 } 197 }
203 defer (*x.stmt).Close() 198 defer (*x.stmt).Close()
204 } 199 }
205 200
206 var nids []string 201 var nids []string
207 202
208 nextBN:
209 for _, bn := range bns { 203 for _, bn := range bns {
210 204 if err := storeBottleneck(
211 var found bool 205 ctx, importID, conn, feedback, bn, &nids,
212 err := hasStmt.QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found) 206 hasStmt, insertStmt, trackStmt); err != nil {
213 switch {
214 case err == sql.ErrNoRows:
215 // This is good.
216 case err != nil:
217 return nil, err 207 return nil, err
218 case found: 208 }
219 // TODO: Deep comparison database vs. SOAP.
220 continue nextBN
221 }
222
223 rb, lb := splitRBLB(bn.Rb_lb)
224
225 var limiting, country string
226
227 if bn.Limiting_factor != nil {
228 limiting = string(*bn.Limiting_factor)
229 }
230
231 if bn.Responsible_country != nil {
232 country = string(*bn.Responsible_country)
233 }
234
235 var nid int64
236
237 err = insertStmt.QueryRowContext(
238 ctx,
239 bn.Bottleneck_id,
240 bn.Fk_g_fid,
241 bn.OBJNAM,
242 bn.NOBJNM,
243 bn.From_ISRS, bn.To_ISRS,
244 rb,
245 lb,
246 country,
247 revisitingTime(bn.Revisiting_time),
248 limiting,
249 bn.Date_Info,
250 bn.Source,
251 ).Scan(&nid)
252 if err != nil {
253 return nil, err
254 }
255 nids = append(nids, bn.Bottleneck_id)
256 if _, err := trackStmt.ExecContext(
257 ctx, importID, "waterway.bottlenecks", nid,
258 ); err != nil {
259 return nil, err
260 }
261 feedback.Info("Inserted '%s' into database", bn.OBJNAM)
262 } 209 }
263 if len(nids) == 0 { 210 if len(nids) == 0 {
264 return nil, UnchangedError("No new bottlenecks found") 211 return nil, UnchangedError("No new bottlenecks inserted")
265 } 212 }
266 213
267 feedback.Info("Storing %d bottlenecks took %s", len(nids), time.Since(start)) 214 feedback.Info("Storing %d bottlenecks took %s", len(nids), time.Since(start))
268 if err := tx.Commit(); err != nil {
269 return nil, err
270 }
271 feedback.Info("Import of bottlenecks was successful") 215 feedback.Info("Import of bottlenecks was successful")
272 summary := struct { 216 summary := struct {
273 Bottlenecks []string `json:"bottlenecks"` 217 Bottlenecks []string `json:"bottlenecks"`
274 }{ 218 }{
275 Bottlenecks: nids, 219 Bottlenecks: nids,
276 } 220 }
277 return &summary, nil 221 return &summary, nil
278 } 222 }
223
224 func storeBottleneck(
225 ctx context.Context,
226 importID int64,
227 conn *sql.Conn,
228 feedback Feedback,
229 bn *ifbn.BottleNeckType,
230 nids *[]string,
231 hasStmt, insertStmt, trackStmt *sql.Stmt,
232 ) error {
233
234 tx, err := conn.BeginTx(ctx, nil)
235 if err != nil {
236 return err
237 }
238 defer tx.Rollback()
239
240 var found bool
241 err = tx.Stmt(hasStmt).QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found)
242 switch {
243 case err == sql.ErrNoRows:
244 // This is good.
245 case err != nil:
246 return err
247 case found:
248 feedback.Info("'%s' already in database. Skip", bn.OBJNAM)
249 // TODO: Deep comparison database vs. SOAP.
250 return nil
251 }
252
253 rb, lb := splitRBLB(bn.Rb_lb)
254
255 var limiting, country string
256
257 if bn.Limiting_factor != nil {
258 limiting = string(*bn.Limiting_factor)
259 }
260
261 if bn.Responsible_country != nil {
262 country = string(*bn.Responsible_country)
263 }
264
265 var nid int64
266
267 err = tx.Stmt(insertStmt).QueryRowContext(
268 ctx,
269 bn.Bottleneck_id,
270 bn.Fk_g_fid,
271 bn.OBJNAM,
272 bn.NOBJNM,
273 bn.From_ISRS, bn.To_ISRS,
274 rb,
275 lb,
276 country,
277 revisitingTime(bn.Revisiting_time),
278 limiting,
279 bn.Date_Info,
280 bn.Source,
281 ).Scan(&nid)
282 if err != nil {
283 feedback.Warn("Failed to insert '%s' into database", bn.OBJNAM)
284 feedback.Warn("%v", err)
285 return nil
286 }
287
288 if _, err := tx.Stmt(trackStmt).ExecContext(
289 ctx, importID, "waterway.bottlenecks", nid,
290 ); err != nil {
291 return err
292 }
293 if err = tx.Commit(); err != nil {
294 return err
295 }
296 feedback.Info("Inserted '%s' into database", bn.OBJNAM)
297 *nids = append(*nids, bn.Bottleneck_id)
298 return nil
299 }