Mercurial > gemma

comparison pkg/imports/agm.go @ 3186:928cdc06ff37

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Warn only once if gauge is unknown Makes the import faster by some orders of magnitude in the worst case scenario of a large CSV file with only unknown gauges.
author Tom Gottfried <tom@intevation.de>
date Tue, 07 May 2019 18:35:11 +0200
parents adf355aaef57
children 8329c6d3cf2a
comparison
equal deleted inserted replaced
3185:505414dfe3e7 3186:928cdc06ff37
300 gaugeCheckStmt, err := tx.PrepareContext(ctx, agmGaugeCheckSQL) 300 gaugeCheckStmt, err := tx.PrepareContext(ctx, agmGaugeCheckSQL)
301 if err != nil { 301 if err != nil {
302 return nil, err 302 return nil, err
303 } 303 }
304 defer gaugeCheckStmt.Close() 304 defer gaugeCheckStmt.Close()
305
305 selectStmt, err := tx.PrepareContext(ctx, agmSelectSQL) 306 selectStmt, err := tx.PrepareContext(ctx, agmSelectSQL)
306 if err != nil { 307 if err != nil {
307 return nil, err 308 return nil, err
308 } 309 }
309 defer selectStmt.Close() 310 defer selectStmt.Close()
311
310 insertStmt, err := tx.PrepareContext(ctx, agmInsertSQL) 312 insertStmt, err := tx.PrepareContext(ctx, agmInsertSQL)
311 if err != nil { 313 if err != nil {
312 return nil, err 314 return nil, err
313 } 315 }
314 defer insertStmt.Close() 316 defer insertStmt.Close()
317
315 trackStmt, err := tx.PrepareContext(ctx, trackImportSQL) 318 trackStmt, err := tx.PrepareContext(ctx, trackImportSQL)
316 if err != nil { 319 if err != nil {
317 return nil, err 320 return nil, err
318 } 321 }
319 defer trackStmt.Close() 322 defer trackStmt.Close()
339 return nil, fmt.Errorf("Invalid ISRS code line %d: %v", line, err) 342 return nil, fmt.Errorf("Invalid ISRS code line %d: %v", line, err)
340 } 343 }
341 344
342 if exists, found := checkedGauges[*gid]; found { 345 if exists, found := checkedGauges[*gid]; found {
343 if !exists { 346 if !exists {
344 feedback.Warn("Ignoring data for unknown gauge %s", gid.String()) 347 // Just ignore the line since we have already warned
345 continue lines 348 continue lines
346 } 349 }
347 } else { // not found in gauge cache 350 } else { // not found in gauge cache
348 if err := gaugeCheckStmt.QueryRowContext( 351 if err := gaugeCheckStmt.QueryRowContext(
349 ctx, 352 ctx,