changeset 2532:452bc714bfd9

Continue importing bottlenecks if one item fails
author Tom Gottfried <tom@intevation.de>
date Thu, 07 Mar 2019 10:46:34 +0100
parents 47b9a4f9a05c
children de4dc3d16647
files pkg/imports/bn.go
diffstat 1 files changed, 84 insertions(+), 63 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/bn.go	Thu Mar 07 10:17:48 2019 +0100
+++ b/pkg/imports/bn.go	Thu Mar 07 10:46:34 2019 +0100
@@ -4,12 +4,13 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // License-Filename: LICENSES/AGPL-3.0.txt
 //
-// Copyright (C) 2018 by via donau
+// Copyright (C) 2018, 2019 by via donau
 //   – Österreichische Wasserstraßen-Gesellschaft mbH
 // Software engineering by Intevation GmbH
 //
 // Author(s):
 //  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+//  * Tom Gottfried <tom.gottfried@intevation.de>
 
 package imports
 
@@ -180,12 +181,6 @@
 
 	feedback.Info("Found %d bottlenecks for import", len(bns))
 
-	tx, err := conn.BeginTx(ctx, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
-
 	var hasStmt, insertStmt, trackStmt *sql.Stmt
 
 	for _, x := range []struct {
@@ -197,7 +192,7 @@
 		{trackImportSQL, &trackStmt},
 	} {
 		var err error
-		if *x.stmt, err = tx.PrepareContext(ctx, x.sql); err != nil {
+		if *x.stmt, err = conn.PrepareContext(ctx, x.sql); err != nil {
 			return nil, err
 		}
 		defer (*x.stmt).Close()
@@ -205,69 +200,18 @@
 
 	var nids []string
 
-nextBN:
 	for _, bn := range bns {
-
-		var found bool
-		err := hasStmt.QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found)
-		switch {
-		case err == sql.ErrNoRows:
-			// This is good.
-		case err != nil:
-			return nil, err
-		case found:
-			// TODO: Deep comparison database vs. SOAP.
-			continue nextBN
-		}
-
-		rb, lb := splitRBLB(bn.Rb_lb)
-
-		var limiting, country string
-
-		if bn.Limiting_factor != nil {
-			limiting = string(*bn.Limiting_factor)
-		}
-
-		if bn.Responsible_country != nil {
-			country = string(*bn.Responsible_country)
-		}
-
-		var nid int64
-
-		err = insertStmt.QueryRowContext(
-			ctx,
-			bn.Bottleneck_id,
-			bn.Fk_g_fid,
-			bn.OBJNAM,
-			bn.NOBJNM,
-			bn.From_ISRS, bn.To_ISRS,
-			rb,
-			lb,
-			country,
-			revisitingTime(bn.Revisiting_time),
-			limiting,
-			bn.Date_Info,
-			bn.Source,
-		).Scan(&nid)
-		if err != nil {
+		if err := storeBottleneck(
+			ctx, importID, conn, feedback, bn, &nids,
+			hasStmt, insertStmt, trackStmt); err != nil {
 			return nil, err
 		}
-		nids = append(nids, bn.Bottleneck_id)
-		if _, err := trackStmt.ExecContext(
-			ctx, importID, "waterway.bottlenecks", nid,
-		); err != nil {
-			return nil, err
-		}
-		feedback.Info("Inserted '%s' into database", bn.OBJNAM)
 	}
 	if len(nids) == 0 {
-		return nil, UnchangedError("No new bottlenecks found")
+		return nil, UnchangedError("No new bottlenecks inserted")
 	}
 
 	feedback.Info("Storing %d bottlenecks took %s", len(nids), time.Since(start))
-	if err := tx.Commit(); err != nil {
-		return nil, err
-	}
 	feedback.Info("Import of bottlenecks was successful")
 	summary := struct {
 		Bottlenecks []string `json:"bottlenecks"`
@@ -276,3 +220,80 @@
 	}
 	return &summary, nil
 }
+
+func storeBottleneck(
+	ctx context.Context,
+	importID int64,
+	conn *sql.Conn,
+	feedback Feedback,
+	bn *ifbn.BottleNeckType,
+	nids *[]string,
+	hasStmt, insertStmt, trackStmt *sql.Stmt,
+) error {
+
+	tx, err := conn.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	var found bool
+	err = tx.Stmt(hasStmt).QueryRowContext(ctx, bn.Bottleneck_id).Scan(&found)
+	switch {
+	case err == sql.ErrNoRows:
+		// This is good.
+	case err != nil:
+		return err
+	case found:
+		feedback.Info("'%s' already in database. Skip", bn.OBJNAM)
+		// TODO: Deep comparison database vs. SOAP.
+		return nil
+	}
+
+	rb, lb := splitRBLB(bn.Rb_lb)
+
+	var limiting, country string
+
+	if bn.Limiting_factor != nil {
+		limiting = string(*bn.Limiting_factor)
+	}
+
+	if bn.Responsible_country != nil {
+		country = string(*bn.Responsible_country)
+	}
+
+	var nid int64
+
+	err = tx.Stmt(insertStmt).QueryRowContext(
+		ctx,
+		bn.Bottleneck_id,
+		bn.Fk_g_fid,
+		bn.OBJNAM,
+		bn.NOBJNM,
+		bn.From_ISRS, bn.To_ISRS,
+		rb,
+		lb,
+		country,
+		revisitingTime(bn.Revisiting_time),
+		limiting,
+		bn.Date_Info,
+		bn.Source,
+	).Scan(&nid)
+	if err != nil {
+		feedback.Warn("Failed to insert '%s' into database", bn.OBJNAM)
+		feedback.Warn("%v", err)
+		return nil
+	}
+
+	if _, err := tx.Stmt(trackStmt).ExecContext(
+		ctx, importID, "waterway.bottlenecks", nid,
+	); err != nil {
+		return err
+	}
+	if err = tx.Commit(); err != nil {
+		return err
+	}
+	feedback.Info("Inserted '%s' into database", bn.OBJNAM)
+	*nids = append(*nids, bn.Bottleneck_id)
+	return nil
+}