Mercurial > gemma
diff pkg/imports/bn.go @ 2532:452bc714bfd9
Continue importing bottlenecks if one item fails
author | Tom Gottfried <tom@intevation.de> |
---|---|
date | Thu, 07 Mar 2019 10:46:34 +0100 |
parents | cfc523c70e90 |
children | 6b34d0fb4498 |
line wrap: on
line diff
--- a/pkg/imports/bn.go Thu Mar 07 10:17:48 2019 +0100 +++ b/pkg/imports/bn.go Thu Mar 07 10:46:34 2019 +0100 @@ -4,12 +4,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // -// Copyright (C) 2018 by via donau +// Copyright (C) 2018, 2019 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Sascha L. Teichmann <sascha.teichmann@intevation.de> +// * Tom Gottfried <tom.gottfried@intevation.de> package imports @@ -180,12 +181,6 @@ feedback.Info("Found %d bottlenecks for import", len(bns)) - tx, err := conn.BeginTx(ctx, nil) - if err != nil { - return nil, err - } - defer tx.Rollback() - var hasStmt, insertStmt, trackStmt *sql.Stmt for _, x := range []struct { @@ -197,7 +192,7 @@ {trackImportSQL, &trackStmt}, } { var err error - if *x.stmt, err = tx.PrepareContext(ctx, x.sql); err != nil { + if *x.stmt, err = conn.PrepareContext(ctx, x.sql); err != nil { return nil, err } defer (*x.stmt).Close() @@ -205,69 +200,18 @@ var nids []string -nextBN: for _, bn := range bns { - - var found bool - err := hasStmt.QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found) - switch { - case err == sql.ErrNoRows: - // This is good. - case err != nil: - return nil, err - case found: - // TODO: Deep comparison database vs. SOAP. - continue nextBN - } - - rb, lb := splitRBLB(bn.Rb_lb) - - var limiting, country string - - if bn.Limiting_factor != nil { - limiting = string(*bn.Limiting_factor) - } - - if bn.Responsible_country != nil { - country = string(*bn.Responsible_country) - } - - var nid int64 - - err = insertStmt.QueryRowContext( - ctx, - bn.Bottleneck_id, - bn.Fk_g_fid, - bn.OBJNAM, - bn.NOBJNM, - bn.From_ISRS, bn.To_ISRS, - rb, - lb, - country, - revisitingTime(bn.Revisiting_time), - limiting, - bn.Date_Info, - bn.Source, - ).Scan(&nid) - if err != nil { + if err := storeBottleneck( + ctx, importID, conn, feedback, bn, &nids, + hasStmt, insertStmt, trackStmt); err != nil { return nil, err } - nids = append(nids, bn.Bottleneck_id) - if _, err := trackStmt.ExecContext( - ctx, importID, "waterway.bottlenecks", nid, - ); err != nil { - return nil, err - } - feedback.Info("Inserted '%s' into database", bn.OBJNAM) } if len(nids) == 0 { - return nil, UnchangedError("No new bottlenecks found") + return nil, UnchangedError("No new bottlenecks inserted") } feedback.Info("Storing %d bottlenecks took %s", len(nids), time.Since(start)) - if err := tx.Commit(); err != nil { - return nil, err - } feedback.Info("Import of bottlenecks was successful") summary := struct { Bottlenecks []string `json:"bottlenecks"` @@ -276,3 +220,80 @@ } return &summary, nil } + +func storeBottleneck( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, + bn *ifbn.BottleNeckType, + nids *[]string, + hasStmt, insertStmt, trackStmt *sql.Stmt, +) error { + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + var found bool + err = tx.Stmt(hasStmt).QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found) + switch { + case err == sql.ErrNoRows: + // This is good. + case err != nil: + return err + case found: + feedback.Info("'%s' already in database. Skip", bn.OBJNAM) + // TODO: Deep comparison database vs. SOAP. + return nil + } + + rb, lb := splitRBLB(bn.Rb_lb) + + var limiting, country string + + if bn.Limiting_factor != nil { + limiting = string(*bn.Limiting_factor) + } + + if bn.Responsible_country != nil { + country = string(*bn.Responsible_country) + } + + var nid int64 + + err = tx.Stmt(insertStmt).QueryRowContext( + ctx, + bn.Bottleneck_id, + bn.Fk_g_fid, + bn.OBJNAM, + bn.NOBJNM, + bn.From_ISRS, bn.To_ISRS, + rb, + lb, + country, + revisitingTime(bn.Revisiting_time), + limiting, + bn.Date_Info, + bn.Source, + ).Scan(&nid) + if err != nil { + feedback.Warn("Failed to insert '%s' into database", bn.OBJNAM) + feedback.Warn("%v", err) + return nil + } + + if _, err := tx.Stmt(trackStmt).ExecContext( + ctx, importID, "waterway.bottlenecks", nid, + ); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return err + } + feedback.Info("Inserted '%s' into database", bn.OBJNAM) + *nids = append(*nids, bn.Bottleneck_id) + return nil +}