Mercurial > gemma

changeset 4931:e41d42be0e13 fairway-marks-import

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
One more callback to avoid code duplication more consequently
author Tom Gottfried <tom@intevation.de>
date Fri, 14 Feb 2020 19:34:08 +0100
parents 8b83b18a1d49
children 2b46e7c51511
files pkg/imports/fm.go pkg/imports/fm_bcnlat.go pkg/imports/fm_boycar.go pkg/imports/fm_boylat.go
diffstat 4 files changed, 252 insertions(+), 285 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/fm.go	Fri Feb 14 17:31:17 2020 +0100
+++ b/pkg/imports/fm.go	Fri Feb 14 19:34:08 2020 +0100
@@ -14,9 +14,12 @@
 package imports
 
 import (
+	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io"
+	"time"
 
 	"gemma.intevation.de/gemma/pkg/wfs"
 )
@@ -57,6 +60,8 @@
 
 // Common operation of FM imports to get features from WFS service
 func getFMFeatures(
+	ctx context.Context,
+	conn *sql.Conn,
 	feedback Feedback,
 	fm FairwayMarks,
 	// Constructor returning pointer to struct
@@ -64,12 +69,18 @@
 	newProps func() interface{},
 	// Construct pointer to featuretype from given pointSlice and properties
 	newFeat func(pointSlice, interface{}) interface{},
-) (
-	// Slice of features to be converted to featuretypes type
-	fms []interface{},
-	epsg int,
-	err error,
-) {
+	// Store features in type specific database tables
+	storeFMs func(
+		tx *sql.Tx,
+		epsg int,
+		// Slice of features to be converted to featuretypes type
+		fms []interface{},
+	) (outsideOrDup int, features int, err error),
+) (err error) {
+
+	start := time.Now()
+
+	feedback.Info("Import fairway marks")
 
 	feedback.Info("Loading capabilities from %s", fm.URL)
 	caps, err := wfs.GetCapabilities(fm.URL)
@@ -86,7 +97,7 @@
 
 	feedback.Info("Found feature type '%s", fm.FeatureType)
 
-	epsg, err = wfs.CRSToEPSG(ft.DefaultCRS)
+	epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
 	if err != nil {
 		feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
 		return
@@ -103,6 +114,7 @@
 	}
 
 	var (
+		fms               []interface{}
 		unsupported       = stringCounter{}
 		missingProperties int
 		badProperties     int
@@ -174,5 +186,31 @@
 
 	feedback.Info("Found %d usable features in data source", len(fms))
 
+	tx, err := conn.BeginTx(ctx, nil)
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	outsideOrDup, features, err := storeFMs(tx, epsg, fms)
+	if err != nil {
+		return
+	}
+
+	if outsideOrDup > 0 {
+		feedback.Info(
+			"Features outside responsibility area and duplicates: %d",
+			outsideOrDup)
+	}
+
+	if features == 0 {
+		return UnchangedError("no valid new features found")
+	}
+
+	if err = tx.Commit(); err == nil {
+		feedback.Info("Storing %d features took %s",
+			features, time.Since(start))
+	}
+
 	return
 }
--- a/pkg/imports/fm_bcnlat.go	Fri Feb 14 17:31:17 2020 +0100
+++ b/pkg/imports/fm_bcnlat.go	Fri Feb 14 19:34:08 2020 +0100
@@ -17,7 +17,6 @@
 	"context"
 	"database/sql"
 	"strings"
-	"time"
 
 	"gemma.intevation.de/gemma/pkg/pgxutils"
 )
@@ -136,124 +135,102 @@
 	feedback Feedback,
 ) (interface{}, error) {
 
-	start := time.Now()
-
-	feedback.Info("Import fairway marks of type BCNLAT/bcnlat")
-
-	fms, epsg, err := getFMFeatures(
+	err := getFMFeatures(
+		ctx,
+		conn,
 		feedback,
 		fm.FairwayMarks,
 		func() interface{} { return new(bcnlatProperties) },
 		func(p pointSlice, props interface{}) interface{} {
 			return &bcnlatFeaturetype{p, props.(*bcnlatProperties)}
 		},
-	)
-	if err != nil {
-		return nil, err
-	}
+		func(
+			tx *sql.Tx, epsg int, fms []interface{},
+		) (outsideOrDup int, features int, err error) {
+
+			feedback.Info("Store fairway marks of type BCNLAT/bcnlat")
 
-	tx, err := conn.BeginTx(ctx, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
-
-	insertStmt, err := tx.PrepareContext(ctx, insertBCNLATSQL)
-	if err != nil {
-		return nil, err
-	}
-	defer insertStmt.Close()
+			insertStmt, err := tx.PrepareContext(ctx, insertBCNLATSQL)
+			if err != nil {
+				return
+			}
+			defer insertStmt.Close()
 
-	insertDirimpStmt, err := tx.PrepareContext(ctx, insertDirimpSQL)
-	if err != nil {
-		return nil, err
-	}
-	defer insertDirimpStmt.Close()
+			insertDirimpStmt, err := tx.PrepareContext(ctx, insertDirimpSQL)
+			if err != nil {
+				return
+			}
+			defer insertDirimpStmt.Close()
 
-	savepoint := Savepoint(ctx, tx, "feature")
+			savepoint := Savepoint(ctx, tx, "feature")
+
+			for _, fm := range fms {
 
-	var (
-		outsideOrDup int
-		features     int
-	)
-	for _, fm := range fms {
+				f := fm.(*bcnlatFeaturetype)
 
-		f := fm.(*bcnlatFeaturetype)
-
-		var catlam sql.NullInt64
-		if f.props.HydroCatlam != nil {
-			catlam = sql.NullInt64{Int64: *f.props.HydroCatlam, Valid: true}
-		} else if f.props.IENCCatlam != nil {
-			catlam = sql.NullInt64{Int64: *f.props.IENCCatlam, Valid: true}
-		}
+				var catlam sql.NullInt64
+				if f.props.HydroCatlam != nil {
+					catlam = sql.NullInt64{
+						Int64: *f.props.HydroCatlam, Valid: true}
+				} else if f.props.IENCCatlam != nil {
+					catlam = sql.NullInt64{
+						Int64: *f.props.IENCCatlam, Valid: true}
+				}
 
-		var fmid int64
-		err := savepoint(func() error {
-			err := insertStmt.QueryRowContext(
-				ctx,
-				f.geom.asWKB(),
-				epsg,
-				f.props.Datsta,
-				f.props.Datend,
-				f.props.Persta,
-				f.props.Perend,
-				f.props.Objnam,
-				f.props.Nobjnm,
-				f.props.Inform,
-				f.props.Ninfom,
-				f.props.Scamin,
-				f.props.Picrep,
-				f.props.Txtdsc,
-				f.props.Sordat,
-				f.props.Sorind,
-				f.props.Colour,
-				f.props.Colpat,
-				f.props.Condtn,
-				f.props.Bcnshp,
-				catlam,
-			).Scan(&fmid)
-			return err
-		})
-		switch {
-		case err == sql.ErrNoRows:
-			outsideOrDup++
-			// ignore -> filtered by responsibility area or a duplicate
-			// TODO: handle eventual changes to dirimp
-		case err != nil:
-			feedback.Error(pgxutils.ReadableError{Err: err}.Error())
-		default:
-			features++
+				var fmid int64
+				err := savepoint(func() error {
+					err := insertStmt.QueryRowContext(
+						ctx,
+						f.geom.asWKB(),
+						epsg,
+						f.props.Datsta,
+						f.props.Datend,
+						f.props.Persta,
+						f.props.Perend,
+						f.props.Objnam,
+						f.props.Nobjnm,
+						f.props.Inform,
+						f.props.Ninfom,
+						f.props.Scamin,
+						f.props.Picrep,
+						f.props.Txtdsc,
+						f.props.Sordat,
+						f.props.Sorind,
+						f.props.Colour,
+						f.props.Colpat,
+						f.props.Condtn,
+						f.props.Bcnshp,
+						catlam,
+					).Scan(&fmid)
+					return err
+				})
+				switch {
+				case err == sql.ErrNoRows:
+					outsideOrDup++
+					// ignore -> filtered by responsibility area or a duplicate
+					// TODO: handle eventual changes to dirimp
+				case err != nil:
+					feedback.Error(pgxutils.ReadableError{Err: err}.Error())
+				default:
+					features++
 
-			if f.props.Dirimp != nil && *f.props.Dirimp != "" {
-				dirimps := strings.Split(*f.props.Dirimp, ",")
-				for _, dirimp := range dirimps {
-					if err := savepoint(func() error {
-						_, err := insertDirimpStmt.ExecContext(
-							ctx, fmid, dirimp)
-						return err
-					}); err != nil {
-						feedback.Warn(pgxutils.ReadableError{Err: err}.Error())
+					if f.props.Dirimp != nil && *f.props.Dirimp != "" {
+						dirimps := strings.Split(*f.props.Dirimp, ",")
+						for _, dirimp := range dirimps {
+							if err := savepoint(func() error {
+								_, err := insertDirimpStmt.ExecContext(
+									ctx, fmid, dirimp)
+								return err
+							}); err != nil {
+								feedback.Warn(
+									pgxutils.ReadableError{Err: err}.Error())
+							}
+						}
 					}
 				}
 			}
-		}
-	}
-
-	if outsideOrDup > 0 {
-		feedback.Info(
-			"Features outside responsibility area and duplicates: %d",
-			outsideOrDup)
-	}
-
-	if features == 0 {
-		err := UnchangedError("no valid new features found")
-		return nil, err
-	}
-
-	if err = tx.Commit(); err == nil {
-		feedback.Info("Storing %d features took %s",
-			features, time.Since(start))
-	}
+			return
+		})
 
 	return nil, err
 }
--- a/pkg/imports/fm_boycar.go	Fri Feb 14 17:31:17 2020 +0100
+++ b/pkg/imports/fm_boycar.go	Fri Feb 14 19:34:08 2020 +0100
@@ -16,7 +16,6 @@
 import (
 	"context"
 	"database/sql"
-	"time"
 
 	"gemma.intevation.de/gemma/pkg/pgxutils"
 )
@@ -131,98 +130,73 @@
 	feedback Feedback,
 ) (interface{}, error) {
 
-	start := time.Now()
-
-	feedback.Info("Import fairway marks of type BOYCAR")
-
-	fms, epsg, err := getFMFeatures(
+	err := getFMFeatures(
+		ctx,
+		conn,
 		feedback,
 		fm.FairwayMarks,
 		func() interface{} { return new(boycarProperties) },
 		func(p pointSlice, props interface{}) interface{} {
 			return &boycarFeaturetype{p, props.(*boycarProperties)}
 		},
-	)
-	if err != nil {
-		return nil, err
-	}
+		func(
+			tx *sql.Tx, epsg int, fms []interface{},
+		) (outsideOrDup int, features int, err error) {
 
-	tx, err := conn.BeginTx(ctx, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
+			feedback.Info("Store fairway marks of type BOYCAR")
 
-	insertStmt, err := tx.PrepareContext(ctx, insertBOYCARSQL)
-	if err != nil {
-		return nil, err
-	}
-	defer insertStmt.Close()
-
-	savepoint := Savepoint(ctx, tx, "feature")
+			insertStmt, err := tx.PrepareContext(ctx, insertBOYCARSQL)
+			if err != nil {
+				return
+			}
+			defer insertStmt.Close()
 
-	var (
-		outsideOrDup int
-		features     int
-	)
-	for _, fm := range fms {
+			savepoint := Savepoint(ctx, tx, "feature")
 
-		f := fm.(*boycarFeaturetype)
+			for _, fm := range fms {
+
+				f := fm.(*boycarFeaturetype)
 
-		var fmid int64
-		err := savepoint(func() error {
-			err := insertStmt.QueryRowContext(
-				ctx,
-				f.geom.asWKB(),
-				epsg,
-				f.props.Datsta,
-				f.props.Datend,
-				f.props.Persta,
-				f.props.Perend,
-				f.props.Objnam,
-				f.props.Nobjnm,
-				f.props.Inform,
-				f.props.Ninfom,
-				f.props.Scamin,
-				f.props.Picrep,
-				f.props.Txtdsc,
-				f.props.Sordat,
-				f.props.Sorind,
-				f.props.Colour,
-				f.props.Colpat,
-				f.props.Conrad,
-				f.props.Marsys,
-				f.props.Boyshp,
-				f.props.Catcam,
-			).Scan(&fmid)
-			return err
+				var fmid int64
+				err := savepoint(func() error {
+					err := insertStmt.QueryRowContext(
+						ctx,
+						f.geom.asWKB(),
+						epsg,
+						f.props.Datsta,
+						f.props.Datend,
+						f.props.Persta,
+						f.props.Perend,
+						f.props.Objnam,
+						f.props.Nobjnm,
+						f.props.Inform,
+						f.props.Ninfom,
+						f.props.Scamin,
+						f.props.Picrep,
+						f.props.Txtdsc,
+						f.props.Sordat,
+						f.props.Sorind,
+						f.props.Colour,
+						f.props.Colpat,
+						f.props.Conrad,
+						f.props.Marsys,
+						f.props.Boyshp,
+						f.props.Catcam,
+					).Scan(&fmid)
+					return err
+				})
+				switch {
+				case err == sql.ErrNoRows:
+					outsideOrDup++
+					// ignore -> filtered by responsibility_areas
+				case err != nil:
+					feedback.Error(pgxutils.ReadableError{Err: err}.Error())
+				default:
+					features++
+				}
+			}
+			return
 		})
-		switch {
-		case err == sql.ErrNoRows:
-			outsideOrDup++
-			// ignore -> filtered by responsibility_areas
-		case err != nil:
-			feedback.Error(pgxutils.ReadableError{Err: err}.Error())
-		default:
-			features++
-		}
-	}
-
-	if outsideOrDup > 0 {
-		feedback.Info(
-			"Features outside responsibility area and duplicates: %d",
-			outsideOrDup)
-	}
-
-	if features == 0 {
-		err := UnchangedError("no valid new features found")
-		return nil, err
-	}
-
-	if err = tx.Commit(); err == nil {
-		feedback.Info("Storing %d features took %s",
-			features, time.Since(start))
-	}
 
 	return nil, err
 }
--- a/pkg/imports/fm_boylat.go	Fri Feb 14 17:31:17 2020 +0100
+++ b/pkg/imports/fm_boylat.go	Fri Feb 14 19:34:08 2020 +0100
@@ -16,7 +16,6 @@
 import (
 	"context"
 	"database/sql"
-	"time"
 
 	"gemma.intevation.de/gemma/pkg/pgxutils"
 )
@@ -133,112 +132,91 @@
 	feedback Feedback,
 ) (interface{}, error) {
 
-	start := time.Now()
-
-	feedback.Info("Import fairway marks of type BOYLAT")
-
-	fms, epsg, err := getFMFeatures(
+	err := getFMFeatures(
+		ctx,
+		conn,
 		feedback,
 		fm.FairwayMarks,
 		func() interface{} { return new(boylatProperties) },
 		func(p pointSlice, props interface{}) interface{} {
 			return &boylatFeaturetype{p, props.(*boylatProperties)}
 		},
-	)
-	if err != nil {
-		return nil, err
-	}
+		func(
+			tx *sql.Tx, epsg int, fms []interface{},
+		) (outsideOrDup int, features int, err error) {
+
+			feedback.Info("Store fairway marks of type BOYLAT")
 
-	tx, err := conn.BeginTx(ctx, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
+			insertStmt, err := tx.PrepareContext(ctx, insertBOYLATSQL)
+			if err != nil {
+				return
+			}
+			defer insertStmt.Close()
 
-	insertStmt, err := tx.PrepareContext(ctx, insertBOYLATSQL)
-	if err != nil {
-		return nil, err
-	}
-	defer insertStmt.Close()
+			savepoint := Savepoint(ctx, tx, "feature")
 
-	savepoint := Savepoint(ctx, tx, "feature")
+			for _, fm := range fms {
+
+				f := fm.(*boylatFeaturetype)
 
-	var (
-		outsideOrDup int
-		features     int
-	)
-	for _, fm := range fms {
-
-		f := fm.(*boylatFeaturetype)
+				var marsys sql.NullInt64
+				if f.props.HydroMarsys != nil {
+					marsys = sql.NullInt64{
+						Int64: *f.props.HydroMarsys, Valid: true}
+				} else if f.props.IENCMarsys != nil {
+					marsys = sql.NullInt64{
+						Int64: *f.props.IENCMarsys, Valid: true}
+				}
 
-		var marsys sql.NullInt64
-		if f.props.HydroMarsys != nil {
-			marsys = sql.NullInt64{Int64: *f.props.HydroMarsys, Valid: true}
-		} else if f.props.IENCMarsys != nil {
-			marsys = sql.NullInt64{Int64: *f.props.IENCMarsys, Valid: true}
-		}
-
-		var catlam sql.NullInt64
-		if f.props.HydroCatlam != nil {
-			catlam = sql.NullInt64{Int64: *f.props.HydroCatlam, Valid: true}
-		} else if f.props.IENCCatlam != nil {
-			catlam = sql.NullInt64{Int64: *f.props.IENCCatlam, Valid: true}
-		}
+				var catlam sql.NullInt64
+				if f.props.HydroCatlam != nil {
+					catlam = sql.NullInt64{
+						Int64: *f.props.HydroCatlam, Valid: true}
+				} else if f.props.IENCCatlam != nil {
+					catlam = sql.NullInt64{
+						Int64: *f.props.IENCCatlam, Valid: true}
+				}
 
-		var fmid int64
-		err := savepoint(func() error {
-			err := insertStmt.QueryRowContext(
-				ctx,
-				f.geom.asWKB(),
-				epsg,
-				f.props.Datsta,
-				f.props.Datend,
-				f.props.Persta,
-				f.props.Perend,
-				f.props.Objnam,
-				f.props.Nobjnm,
-				f.props.Inform,
-				f.props.Ninfom,
-				f.props.Scamin,
-				f.props.Picrep,
-				f.props.Txtdsc,
-				f.props.Sordat,
-				f.props.Sorind,
-				f.props.Colour,
-				f.props.Colpat,
-				f.props.Conrad,
-				marsys,
-				f.props.Boyshp,
-				catlam,
-			).Scan(&fmid)
-			return err
+				var fmid int64
+				err := savepoint(func() error {
+					err := insertStmt.QueryRowContext(
+						ctx,
+						f.geom.asWKB(),
+						epsg,
+						f.props.Datsta,
+						f.props.Datend,
+						f.props.Persta,
+						f.props.Perend,
+						f.props.Objnam,
+						f.props.Nobjnm,
+						f.props.Inform,
+						f.props.Ninfom,
+						f.props.Scamin,
+						f.props.Picrep,
+						f.props.Txtdsc,
+						f.props.Sordat,
+						f.props.Sorind,
+						f.props.Colour,
+						f.props.Colpat,
+						f.props.Conrad,
+						marsys,
+						f.props.Boyshp,
+						catlam,
+					).Scan(&fmid)
+					return err
+				})
+				switch {
+				case err == sql.ErrNoRows:
+					outsideOrDup++
+					// ignore -> filtered by responsibility_areas
+				case err != nil:
+					feedback.Error(pgxutils.ReadableError{Err: err}.Error())
+				default:
+					features++
+				}
+			}
+			return
 		})
-		switch {
-		case err == sql.ErrNoRows:
-			outsideOrDup++
-			// ignore -> filtered by responsibility_areas
-		case err != nil:
-			feedback.Error(pgxutils.ReadableError{Err: err}.Error())
-		default:
-			features++
-		}
-	}
-
-	if outsideOrDup > 0 {
-		feedback.Info(
-			"Features outside responsibility area and duplicates: %d",
-			outsideOrDup)
-	}
-
-	if features == 0 {
-		err := UnchangedError("no valid new features found")
-		return nil, err
-	}
-
-	if err = tx.Commit(); err == nil {
-		feedback.Info("Storing %d features took %s",
-			features, time.Since(start))
-	}
 
 	return nil, err
 }