Mercurial > gemma

changeset 988:7dfd3db94e6d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
In preparation of persisting import jobs logging is done through an interface.
author Sascha L. Teichmann <teichmann@intevation.de>
date Sat, 20 Oct 2018 19:14:00 +0200
parents 3841509f6e9e
children da19c68e36ba
files pkg/imports/queue.go pkg/imports/sr.go pkg/octree/vertex.go
diffstat 3 files changed, 157 insertions(+), 127 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/queue.go	Fri Oct 19 18:09:26 2018 +0200
+++ b/pkg/imports/queue.go	Sat Oct 20 19:14:00 2018 +0200
@@ -11,9 +11,15 @@
 	"gemma.intevation.de/gemma/pkg/auth"
 )
 
+type Feedback interface {
+	Info(fmt string, args ...interface{})
+	Warn(fmt string, args ...interface{})
+	Error(fmt string, args ...interface{})
+}
+
 type Job interface {
 	User() string
-	Do(conn *sql.Conn) error
+	Do(*sql.Conn, Feedback) error
 	CleanUp() error
 }
 
@@ -42,6 +48,20 @@
 	return id
 }
 
+type logFeedback struct{}
+
+func (logFeedback) Info(fmt string, args ...interface{}) {
+	log.Printf("info: "+fmt, args...)
+}
+
+func (logFeedback) Warn(fmt string, args ...interface{}) {
+	log.Printf("warn: "+fmt, args...)
+}
+
+func (logFeedback) Error(fmt string, args ...interface{}) {
+	log.Printf("log: "+fmt, args...)
+}
+
 func importLoop() {
 	for {
 		var idj idJob
@@ -54,7 +74,11 @@
 
 		log.Printf("starting import job %d\n", idj.id)
 
-		if err := auth.RunAs(idj.job.User(), context.Background(), idj.job.Do); err != nil {
+		fn := func(conn *sql.Conn) error {
+			return idj.job.Do(conn, logFeedback{})
+		}
+
+		if err := auth.RunAs(idj.job.User(), context.Background(), fn); err != nil {
 			log.Printf("import error (job %d): %v\n", idj.id, err)
 		}
 		if err := idj.job.CleanUp(); err != nil {
--- a/pkg/imports/sr.go	Fri Oct 19 18:09:26 2018 +0200
+++ b/pkg/imports/sr.go	Sat Oct 20 19:14:00 2018 +0200
@@ -124,6 +124,127 @@
 `
 )
 
+func (sr *SoundingResult) Do(conn *sql.Conn, feedback Feedback) error {
+
+	z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip"))
+	if err != nil {
+		return err
+	}
+	defer z.Close()
+
+	feedback.Info("Looking for 'meta.json'\n")
+	mf := find("meta.json", z.File)
+	if mf == nil {
+		return errors.New("Cannot find 'meta.json'")
+	}
+
+	m, err := loadMeta(mf)
+	if err != nil {
+		return err
+	}
+
+	if err := m.validate(conn); err != nil {
+		return err
+	}
+
+	feedback.Info("Looking for '*.xyz'\n")
+	xyzf := find(".xyz", z.File)
+	if xyzf == nil {
+		return errors.New("Cannot find any *.xyz file")
+	}
+
+	xyz, err := loadXYZ(xyzf, feedback)
+	if err != nil {
+		return err
+	}
+
+	if len(xyz) == 0 {
+		return errors.New("XYZ does not contain any vertices.")
+	}
+
+	// Is there a boundary shapefile in the ZIP archive?
+	polygon, err := loadBoundary(z)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+
+	tx, err := conn.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	var id int64
+	var epsg uint32
+	start := time.Now()
+
+	err = tx.QueryRow(insertPointsSQL,
+		m.Bottleneck,
+		m.Date.Time,
+		m.DepthReference,
+		xyz.AsWKB(),
+		polygon.AsWBK(),
+		m.EPSG,
+	).Scan(&id, &epsg)
+	xyz, polygon = nil, nil // not need from now on.
+	feedback.Info("storing points took %s\n", time.Since(start))
+	if err != nil {
+		return err
+	}
+
+	feedback.Info("Best suited UTM EPSG: %d\n", epsg)
+
+	feedback.Info("Triangulate...\n")
+	start = time.Now()
+	tin, err := octree.GenerateTinByID(conn, ctx, id, epsg)
+	feedback.Info("triangulation took %s\n", time.Since(start))
+	if err != nil {
+		return err
+	}
+
+	if tin == nil {
+		return errors.New("cannot load TIN from database")
+	}
+
+	feedback.Info("Building octree...\n")
+	builder := octree.NewBuilder(tin)
+	start = time.Now()
+	builder.Build()
+	octreeIndex, err := builder.Bytes()
+	tin = nil // not needed from now on
+	feedback.Info("building octree took %s\n", time.Since(start))
+	if err != nil {
+		return err
+	}
+
+	feedback.Info("Store octree...\n")
+	start = time.Now()
+	h := sha1.New()
+	h.Write(octreeIndex)
+	checksum := hex.EncodeToString(h.Sum(nil))
+	_, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex)
+	feedback.Info("storing octree index took %s\n", time.Since(start))
+	if err != nil {
+		return err
+	}
+
+	tree := builder.Tree()
+	builder = nil // not needed from now on
+
+	feedback.Info("Generate contour lines...\n")
+	start = time.Now()
+	err = generateContours(tree, tx, id)
+	feedback.Info("generating and storing contour lines took %s\n", time.Since(start))
+	if err != nil {
+		return err
+	}
+
+	feedback.Info("Storing sounding result was successful.\n")
+	return tx.Commit()
+}
+
 func (srd *SoundingResultDate) UnmarshalJSON(data []byte) error {
 	var s string
 	if err := json.Unmarshal(data, &s); err != nil {
@@ -200,7 +321,7 @@
 	return nil
 }
 
-func loadXYZReader(r io.Reader) (octree.MultiPointZ, error) {
+func loadXYZReader(r io.Reader, feedback Feedback) (octree.MultiPointZ, error) {
 	mpz := make(octree.MultiPointZ, 0, 250000)
 	s := bufio.NewScanner(r)
 
@@ -215,21 +336,21 @@
 		}
 		var err error
 		if p.X, err = strconv.ParseFloat(text[:idx], 64); err != nil {
-			log.Printf("format error in line %d: %v\n", line, err)
+			feedback.Warn("format error in line %d: %v\n", line, err)
 			continue
 		}
 		text = text[idx+1:]
 		if idx = strings.IndexByte(text, ','); idx == -1 {
-			log.Printf("format error in line %d\n", line)
+			feedback.Warn("format error in line %d\n", line)
 			continue
 		}
 		if p.Y, err = strconv.ParseFloat(text[:idx], 64); err != nil {
-			log.Printf("format error in line %d: %v\n", line, err)
+			feedback.Warn("format error in line %d: %v\n", line, err)
 			continue
 		}
 		text = text[idx+1:]
 		if p.Z, err = strconv.ParseFloat(text, 64); err != nil {
-			log.Printf("format error in line %d: %v\n", line, err)
+			feedback.Warn("format error in line %d: %v\n", line, err)
 			continue
 		}
 		mpz = append(mpz, p)
@@ -242,13 +363,13 @@
 	return mpz, nil
 }
 
-func loadXYZ(f *zip.File) (octree.MultiPointZ, error) {
+func loadXYZ(f *zip.File, feedback Feedback) (octree.MultiPointZ, error) {
 	r, err := f.Open()
 	if err != nil {
 		return nil, err
 	}
 	defer r.Close()
-	return loadXYZReader(r)
+	return loadXYZReader(r, feedback)
 }
 
 func loadBoundary(z *zip.ReadCloser) (Polygon, error) {
@@ -287,121 +408,6 @@
 	return shapeToPolygon(s)
 }
 
-func (sr *SoundingResult) Do(conn *sql.Conn) error {
-
-	z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip"))
-	if err != nil {
-		return err
-	}
-	defer z.Close()
-
-	mf := find("meta.json", z.File)
-	if mf == nil {
-		return errors.New("Cannot find 'meta.json'")
-	}
-
-	m, err := loadMeta(mf)
-	if err != nil {
-		return err
-	}
-
-	if err := m.validate(conn); err != nil {
-		return err
-	}
-
-	xyzf := find(".xyz", z.File)
-	if xyzf == nil {
-		return errors.New("Cannot find any *.xyz file")
-	}
-
-	xyz, err := loadXYZ(xyzf)
-	if err != nil {
-		return err
-	}
-
-	if len(xyz) == 0 {
-		return errors.New("XYZ does not contain any vertices.")
-	}
-
-	// Is there a boundary shapefile in the ZIP archive?
-	polygon, err := loadBoundary(z)
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
-
-	tx, err := conn.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-
-	var id int64
-	var epsg uint32
-	start := time.Now()
-
-	err = tx.QueryRow(insertPointsSQL,
-		m.Bottleneck,
-		m.Date.Time,
-		m.DepthReference,
-		xyz.AsWKB(),
-		polygon.AsWBK(),
-		m.EPSG,
-	).Scan(&id, &epsg)
-	xyz, polygon = nil, nil // not need from now on.
-	log.Printf("storing points took %s\n", time.Since(start))
-	if err != nil {
-		return err
-	}
-
-	log.Printf("EPSG: %d\n", epsg)
-
-	start = time.Now()
-	tin, err := octree.GenerateTinByID(conn, ctx, id, epsg)
-	log.Printf("triangulation took %s\n", time.Since(start))
-	if err != nil {
-		return err
-	}
-
-	if tin == nil {
-		return errors.New("cannot load TIN from database")
-	}
-
-	builder := octree.NewBuilder(tin)
-	start = time.Now()
-	builder.Build()
-	octreeIndex, err := builder.Bytes()
-	tin = nil // not needed from now on
-	log.Printf("building octree took %s\n", time.Since(start))
-	if err != nil {
-		return err
-	}
-
-	h := sha1.New()
-	h.Write(octreeIndex)
-	checksum := hex.EncodeToString(h.Sum(nil))
-
-	start = time.Now()
-	_, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex)
-	log.Printf("storing octree index took %s\n", time.Since(start))
-	if err != nil {
-		return err
-	}
-
-	tree := builder.Tree()
-	builder = nil // not needed from now on
-
-	start = time.Now()
-	err = generateContours(tree, tx, id)
-	log.Printf("generating and storing contour lines took %s\n", time.Since(start))
-	if err != nil {
-		return err
-	}
-
-	return tx.Commit()
-}
-
 func generateContours(tree *octree.Tree, tx *sql.Tx, id int64) error {
 	stmt, err := tx.Prepare(insertContourSQL)
 	if err != nil {
--- a/pkg/octree/vertex.go	Fri Oct 19 18:09:26 2018 +0200
+++ b/pkg/octree/vertex.go	Sat Oct 20 19:14:00 2018 +0200
@@ -439,7 +439,7 @@
 		}
 	}
 
-	rings := len(out)
+	// rings := len(out)
 
 	// The rest are open line strings.
 	for _, lines := range heads {
@@ -448,8 +448,8 @@
 		}
 	}
 
-	log.Printf("segments before/after merge: %d/%d (%d rings)\n",
-		len(mls), len(out), rings)
+	// log.Printf("segments before/after merge: %d/%d (%d rings)\n",
+	// len(mls), len(out), rings)
 
 	return out
 }