# HG changeset patch # User Sascha L. Teichmann # Date 1540055640 -7200 # Node ID 7dfd3db94e6d6c6cfba64e6945ee2b2ae5421717 # Parent 3841509f6e9e6dbfeac27781006c1cc5cf9b241e In preparation of persisting import jobs logging is done through an interface. diff -r 3841509f6e9e -r 7dfd3db94e6d pkg/imports/queue.go --- a/pkg/imports/queue.go Fri Oct 19 18:09:26 2018 +0200 +++ b/pkg/imports/queue.go Sat Oct 20 19:14:00 2018 +0200 @@ -11,9 +11,15 @@ "gemma.intevation.de/gemma/pkg/auth" ) +type Feedback interface { + Info(fmt string, args ...interface{}) + Warn(fmt string, args ...interface{}) + Error(fmt string, args ...interface{}) +} + type Job interface { User() string - Do(conn *sql.Conn) error + Do(*sql.Conn, Feedback) error CleanUp() error } @@ -42,6 +48,20 @@ return id } +type logFeedback struct{} + +func (logFeedback) Info(fmt string, args ...interface{}) { + log.Printf("info: "+fmt, args...) +} + +func (logFeedback) Warn(fmt string, args ...interface{}) { + log.Printf("warn: "+fmt, args...) +} + +func (logFeedback) Error(fmt string, args ...interface{}) { + log.Printf("log: "+fmt, args...) +} + func importLoop() { for { var idj idJob @@ -54,7 +74,11 @@ log.Printf("starting import job %d\n", idj.id) - if err := auth.RunAs(idj.job.User(), context.Background(), idj.job.Do); err != nil { + fn := func(conn *sql.Conn) error { + return idj.job.Do(conn, logFeedback{}) + } + + if err := auth.RunAs(idj.job.User(), context.Background(), fn); err != nil { log.Printf("import error (job %d): %v\n", idj.id, err) } if err := idj.job.CleanUp(); err != nil { diff -r 3841509f6e9e -r 7dfd3db94e6d pkg/imports/sr.go --- a/pkg/imports/sr.go Fri Oct 19 18:09:26 2018 +0200 +++ b/pkg/imports/sr.go Sat Oct 20 19:14:00 2018 +0200 @@ -124,6 +124,127 @@ ` ) +func (sr *SoundingResult) Do(conn *sql.Conn, feedback Feedback) error { + + z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip")) + if err != nil { + return err + } + defer z.Close() + + feedback.Info("Looking for 'meta.json'\n") + mf := find("meta.json", z.File) + if mf == nil { + return errors.New("Cannot find 'meta.json'") + } + + m, err := loadMeta(mf) + if err != nil { + return err + } + + if err := m.validate(conn); err != nil { + return err + } + + feedback.Info("Looking for '*.xyz'\n") + xyzf := find(".xyz", z.File) + if xyzf == nil { + return errors.New("Cannot find any *.xyz file") + } + + xyz, err := loadXYZ(xyzf, feedback) + if err != nil { + return err + } + + if len(xyz) == 0 { + return errors.New("XYZ does not contain any vertices.") + } + + // Is there a boundary shapefile in the ZIP archive? + polygon, err := loadBoundary(z) + if err != nil { + return err + } + + ctx := context.Background() + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + var id int64 + var epsg uint32 + start := time.Now() + + err = tx.QueryRow(insertPointsSQL, + m.Bottleneck, + m.Date.Time, + m.DepthReference, + xyz.AsWKB(), + polygon.AsWBK(), + m.EPSG, + ).Scan(&id, &epsg) + xyz, polygon = nil, nil // not need from now on. + feedback.Info("storing points took %s\n", time.Since(start)) + if err != nil { + return err + } + + feedback.Info("Best suited UTM EPSG: %d\n", epsg) + + feedback.Info("Triangulate...\n") + start = time.Now() + tin, err := octree.GenerateTinByID(conn, ctx, id, epsg) + feedback.Info("triangulation took %s\n", time.Since(start)) + if err != nil { + return err + } + + if tin == nil { + return errors.New("cannot load TIN from database") + } + + feedback.Info("Building octree...\n") + builder := octree.NewBuilder(tin) + start = time.Now() + builder.Build() + octreeIndex, err := builder.Bytes() + tin = nil // not needed from now on + feedback.Info("building octree took %s\n", time.Since(start)) + if err != nil { + return err + } + + feedback.Info("Store octree...\n") + start = time.Now() + h := sha1.New() + h.Write(octreeIndex) + checksum := hex.EncodeToString(h.Sum(nil)) + _, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex) + feedback.Info("storing octree index took %s\n", time.Since(start)) + if err != nil { + return err + } + + tree := builder.Tree() + builder = nil // not needed from now on + + feedback.Info("Generate contour lines...\n") + start = time.Now() + err = generateContours(tree, tx, id) + feedback.Info("generating and storing contour lines took %s\n", time.Since(start)) + if err != nil { + return err + } + + feedback.Info("Storing sounding result was successful.\n") + return tx.Commit() +} + func (srd *SoundingResultDate) UnmarshalJSON(data []byte) error { var s string if err := json.Unmarshal(data, &s); err != nil { @@ -200,7 +321,7 @@ return nil } -func loadXYZReader(r io.Reader) (octree.MultiPointZ, error) { +func loadXYZReader(r io.Reader, feedback Feedback) (octree.MultiPointZ, error) { mpz := make(octree.MultiPointZ, 0, 250000) s := bufio.NewScanner(r) @@ -215,21 +336,21 @@ } var err error if p.X, err = strconv.ParseFloat(text[:idx], 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) + feedback.Warn("format error in line %d: %v\n", line, err) continue } text = text[idx+1:] if idx = strings.IndexByte(text, ','); idx == -1 { - log.Printf("format error in line %d\n", line) + feedback.Warn("format error in line %d\n", line) continue } if p.Y, err = strconv.ParseFloat(text[:idx], 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) + feedback.Warn("format error in line %d: %v\n", line, err) continue } text = text[idx+1:] if p.Z, err = strconv.ParseFloat(text, 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) + feedback.Warn("format error in line %d: %v\n", line, err) continue } mpz = append(mpz, p) @@ -242,13 +363,13 @@ return mpz, nil } -func loadXYZ(f *zip.File) (octree.MultiPointZ, error) { +func loadXYZ(f *zip.File, feedback Feedback) (octree.MultiPointZ, error) { r, err := f.Open() if err != nil { return nil, err } defer r.Close() - return loadXYZReader(r) + return loadXYZReader(r, feedback) } func loadBoundary(z *zip.ReadCloser) (Polygon, error) { @@ -287,121 +408,6 @@ return shapeToPolygon(s) } -func (sr *SoundingResult) Do(conn *sql.Conn) error { - - z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip")) - if err != nil { - return err - } - defer z.Close() - - mf := find("meta.json", z.File) - if mf == nil { - return errors.New("Cannot find 'meta.json'") - } - - m, err := loadMeta(mf) - if err != nil { - return err - } - - if err := m.validate(conn); err != nil { - return err - } - - xyzf := find(".xyz", z.File) - if xyzf == nil { - return errors.New("Cannot find any *.xyz file") - } - - xyz, err := loadXYZ(xyzf) - if err != nil { - return err - } - - if len(xyz) == 0 { - return errors.New("XYZ does not contain any vertices.") - } - - // Is there a boundary shapefile in the ZIP archive? - polygon, err := loadBoundary(z) - if err != nil { - return err - } - - ctx := context.Background() - - tx, err := conn.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - - var id int64 - var epsg uint32 - start := time.Now() - - err = tx.QueryRow(insertPointsSQL, - m.Bottleneck, - m.Date.Time, - m.DepthReference, - xyz.AsWKB(), - polygon.AsWBK(), - m.EPSG, - ).Scan(&id, &epsg) - xyz, polygon = nil, nil // not need from now on. - log.Printf("storing points took %s\n", time.Since(start)) - if err != nil { - return err - } - - log.Printf("EPSG: %d\n", epsg) - - start = time.Now() - tin, err := octree.GenerateTinByID(conn, ctx, id, epsg) - log.Printf("triangulation took %s\n", time.Since(start)) - if err != nil { - return err - } - - if tin == nil { - return errors.New("cannot load TIN from database") - } - - builder := octree.NewBuilder(tin) - start = time.Now() - builder.Build() - octreeIndex, err := builder.Bytes() - tin = nil // not needed from now on - log.Printf("building octree took %s\n", time.Since(start)) - if err != nil { - return err - } - - h := sha1.New() - h.Write(octreeIndex) - checksum := hex.EncodeToString(h.Sum(nil)) - - start = time.Now() - _, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex) - log.Printf("storing octree index took %s\n", time.Since(start)) - if err != nil { - return err - } - - tree := builder.Tree() - builder = nil // not needed from now on - - start = time.Now() - err = generateContours(tree, tx, id) - log.Printf("generating and storing contour lines took %s\n", time.Since(start)) - if err != nil { - return err - } - - return tx.Commit() -} - func generateContours(tree *octree.Tree, tx *sql.Tx, id int64) error { stmt, err := tx.Prepare(insertContourSQL) if err != nil { diff -r 3841509f6e9e -r 7dfd3db94e6d pkg/octree/vertex.go --- a/pkg/octree/vertex.go Fri Oct 19 18:09:26 2018 +0200 +++ b/pkg/octree/vertex.go Sat Oct 20 19:14:00 2018 +0200 @@ -439,7 +439,7 @@ } } - rings := len(out) + // rings := len(out) // The rest are open line strings. for _, lines := range heads { @@ -448,8 +448,8 @@ } } - log.Printf("segments before/after merge: %d/%d (%d rings)\n", - len(mls), len(out), rings) + // log.Printf("segments before/after merge: %d/%d (%d rings)\n", + // len(mls), len(out), rings) return out }