Mercurial > gemma
view pkg/controllers/diff.go @ 3705:7006b92c0334
Handle updates (vs. historized and new versions) separately.
We need this distinction as updated data currently can not be
reviewed. More precisely: it can not be declined after review, as the
old data is updated in place.
The current exclusion from the review is a workaround and not meant to
be the final solution. Note that there are additional minor problems,
like the fact that the updated data is not counted as changed data for
the import.
author | Sascha Wilde <wilde@intevation.de> |
---|---|
date | Wed, 19 Jun 2019 17:00:08 +0200 |
parents | 2a079d0a71c1 |
children | 0b382767b5ab |
line wrap: on
line source
// This is Free Software under GNU Affero General Public License v >= 3.0 // without warranty, see README.md and license for details. // // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // // Copyright (C) 2018, 2019 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Sascha L. Teichmann <sascha.teichmann@intevation.de> package controllers import ( "database/sql" "fmt" "log" "net/http" "sync" "time" "golang.org/x/sync/semaphore" "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/models" "gemma.intevation.de/gemma/pkg/octree" ) const ( contourTolerance = 0.1 contourStep = 0.1 ) const ( diffIDSQL = ` SELECT sd.id FROM caching.sounding_differences sd JOIN waterway.sounding_results srm ON sd.minuend = srm.id JOIN waterway.sounding_results srs ON sd.subtrahend = srs.id WHERE srm.bottleneck_id = srs.bottleneck_id AND srm.bottleneck_id = $1 AND srm.date_info = $2::date AND srs.date_info = $3::date ` insertDiffSQL = ` WITH soundings AS ( SELECT sr.id AS id, sr.date_info AS date_info FROM waterway.sounding_results sr WHERE sr.bottleneck_id = $1 ) INSERT INTO caching.sounding_differences (minuend, subtrahend) SELECT m.id, s.id FROM soundings m, soundings s WHERE m.date_info = $2::date AND s.date_info = $3::date RETURNING id ` insertDiffContourSQL = ` INSERT INTO caching.sounding_differences_contour_lines ( sounding_differences_id, height, lines ) SELECT $5, $4, ST_Transform( ST_Multi( ST_CollectionExtract( ST_SimplifyPreserveTopology( ST_Multi(ST_Collectionextract( ST_MakeValid(ST_GeomFromWKB($1, $2::integer)), 2)), $3 ), 2 ) ), 4326 ) ` ) // Only allow three diffence calculation at once. // TODO: Make this configurable? var diffCalculationSemaphore = semaphore.NewWeighted(int64(3)) func diffCalculation( input interface{}, req *http.Request, conn *sql.Conn, ) (jr JSONResult, err error) { begin := time.Now() start := begin dci := input.(*models.DiffCalculationInput) ctx := req.Context() var id int64 err = conn.QueryRowContext( ctx, diffIDSQL, dci.Bottleneck, dci.Minuend.Time, dci.Subtrahend.Time, ).Scan(&id) switch { case err == sql.ErrNoRows: // We need to calculate it. case err != nil: return default: // We already have this diff jr = JSONResult{ Result: map[string]int64{"id": id}, } return } // DoS counter measure. if err = diffCalculationSemaphore.Acquire(ctx, 1); err != nil { return } defer diffCalculationSemaphore.Release(1) minuendTree, err := octree.FromCache( ctx, conn, dci.Bottleneck, dci.Minuend.Time) log.Printf("info: loading minuend octree took %s\n", time.Since(start)) if err != nil { return } if minuendTree == nil { err = JSONError{ Code: http.StatusNotFound, Message: fmt.Sprintf("Cannot find survey for %s/%s.", dci.Bottleneck, dci.Minuend.Format(common.DateFormat)), } return } start = time.Now() subtrahendTree, err := octree.FromCache( ctx, conn, dci.Bottleneck, dci.Subtrahend.Time) log.Printf("info: loading subtrahend octree took %s\n", time.Since(start)) if err != nil { return } if subtrahendTree == nil { err = JSONError{ Code: http.StatusNotFound, Message: fmt.Sprintf("Cannot find survey for %s/%s.", dci.Bottleneck, dci.Subtrahend.Format(common.DateFormat)), } return } // We need a slow path implementation for this. if minuendTree.EPSG != subtrahendTree.EPSG { err = JSONError{ Code: http.StatusInternalServerError, Message: "Calculating differences between two different " + "EPSG code octrees are not supported, yet.", } return } start = time.Now() points := minuendTree.Diff(subtrahendTree) log.Printf("info: A - B took %v\n", time.Since(start)) // The Triangulation and the loading of the clipping // polygon can be done concurrently. jobs := make(chan func()) wg := new(sync.WaitGroup) for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() for job := range jobs { job() } }() } var ( tri *octree.Triangulation triErr error clip *octree.Polygon clipErr error ) jobs <- func() { start := time.Now() tri, triErr = points.Triangulate() log.Printf("info: triangulation took %v\n", time.Since(start)) } jobs <- func() { start := time.Now() clip, clipErr = octree.LoadClippingPolygon( ctx, conn, minuendTree.EPSG, dci.Bottleneck, dci.Minuend.Time, dci.Subtrahend.Time) log.Printf("info: loading clipping polygon took %v\n", time.Since(start)) } close(jobs) wg.Wait() switch { case triErr != nil && clipErr != nil: err = fmt.Errorf("%v %v", triErr, clipErr) return case triErr != nil: err = triErr return case clipErr != nil: err = clipErr return } start = time.Now() tin := tri.Tin() removed := tin.Clip(clip) log.Printf("info: clipping TIN took %v\n", time.Since(start)) start = time.Now() log.Printf("info: Number of triangles to clip: %d\n", len(removed)) builder := octree.NewBuilder(tin) builder.Build(removed) log.Printf("info: building octree took %v\n", time.Since(start)) tree := builder.Tree() log.Printf("info: min/max: %f %f\n", tree.Min.Z, tree.Max.Z) heights := octree.SampleDiffHeights( tree.Min.Z, tree.Max.Z, contourStep) log.Printf("info: num heights: %d\n", len(heights)) start = time.Now() // XXX: Maybe we should start this transaction earlier!? var tx *sql.Tx if tx, err = conn.BeginTx(ctx, nil); err != nil { return } defer tx.Rollback() var stmt *sql.Stmt if stmt, err = tx.PrepareContext(ctx, insertDiffContourSQL); err != nil { return } defer stmt.Close() if err = tx.QueryRowContext( ctx, insertDiffSQL, dci.Bottleneck, dci.Minuend.Time, dci.Subtrahend.Time, ).Scan(&id); err != nil { return } octree.DoContours(tree, heights, func(res *octree.ContourResult) { if err == nil && len(res.Lines) > 0 { _, err = stmt.ExecContext( ctx, res.Lines.AsWKB2D(), minuendTree.EPSG, contourTolerance, res.Height, id, ) } }) log.Printf("info: calculating and storing iso lines took %v\n", time.Since(start)) if err != nil { return } if err = tx.Commit(); err != nil { log.Printf("info: difference calculation failed after %v\n", time.Since(begin)) return } log.Printf("info: difference calculation succeed after %v\n", time.Since(begin)) jr = JSONResult{ Result: map[string]int64{"id": id}, } return }