view pkg/controllers/diff.go @ 3678:8f58851927c0

client: make layer factory only return new layer config for individual maps instead of each time it is invoked. The purpose of the factory was to support multiple maps with individual layers. But returning a new config each time it is invoked leads to bugs that rely on the layer's state. Now this factory reuses the same objects it created before, per map.
author Markus Kottlaender <markus@intevation.de>
date Mon, 17 Jun 2019 17:31:35 +0200
parents 2a079d0a71c1
children 0b382767b5ab
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2018, 2019 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package controllers

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/sync/semaphore"

	"gemma.intevation.de/gemma/pkg/common"
	"gemma.intevation.de/gemma/pkg/models"
	"gemma.intevation.de/gemma/pkg/octree"
)

const (
	contourTolerance = 0.1
	contourStep      = 0.1
)

const (
	diffIDSQL = `
SELECT sd.id FROM
  caching.sounding_differences sd JOIN
  waterway.sounding_results srm ON sd.minuend = srm.id JOIN
  waterway.sounding_results srs ON sd.subtrahend = srs.id
WHERE srm.bottleneck_id = srs.bottleneck_id AND
  srm.bottleneck_id = $1 AND
  srm.date_info = $2::date AND
  srs.date_info = $3::date
`
	insertDiffSQL = `
WITH soundings AS (
  SELECT sr.id AS id, sr.date_info AS date_info FROM
  waterway.sounding_results sr
  WHERE sr.bottleneck_id = $1
)
INSERT INTO caching.sounding_differences (minuend, subtrahend)
SELECT m.id, s.id FROM soundings m, soundings s
WHERE m.date_info = $2::date AND s.date_info = $3::date
RETURNING id
`
	insertDiffContourSQL = `
INSERT INTO caching.sounding_differences_contour_lines (
  sounding_differences_id,
  height,
  lines
)
SELECT
  $5,
  $4,
  ST_Transform(
    ST_Multi(
      ST_CollectionExtract(
        ST_SimplifyPreserveTopology(
          ST_Multi(ST_Collectionextract(
            ST_MakeValid(ST_GeomFromWKB($1, $2::integer)), 2)),
          $3
        ),
        2
      )
    ),
    4326
  )
`
)

// Only allow three diffence calculation at once.
// TODO: Make this configurable?
var diffCalculationSemaphore = semaphore.NewWeighted(int64(3))

func diffCalculation(
	input interface{},
	req *http.Request,
	conn *sql.Conn,
) (jr JSONResult, err error) {

	begin := time.Now()
	start := begin

	dci := input.(*models.DiffCalculationInput)

	ctx := req.Context()

	var id int64
	err = conn.QueryRowContext(
		ctx,
		diffIDSQL,
		dci.Bottleneck,
		dci.Minuend.Time,
		dci.Subtrahend.Time,
	).Scan(&id)

	switch {
	case err == sql.ErrNoRows:
		// We need to calculate it.
	case err != nil:
		return
	default:
		// We already have this diff
		jr = JSONResult{
			Result: map[string]int64{"id": id},
		}
		return
	}

	// DoS counter measure.
	if err = diffCalculationSemaphore.Acquire(ctx, 1); err != nil {
		return
	}
	defer diffCalculationSemaphore.Release(1)

	minuendTree, err := octree.FromCache(
		ctx, conn,
		dci.Bottleneck, dci.Minuend.Time)

	log.Printf("info: loading minuend octree took %s\n", time.Since(start))
	if err != nil {
		return
	}

	if minuendTree == nil {
		err = JSONError{
			Code: http.StatusNotFound,
			Message: fmt.Sprintf("Cannot find survey for %s/%s.",
				dci.Bottleneck,
				dci.Minuend.Format(common.DateFormat)),
		}
		return
	}

	start = time.Now()

	subtrahendTree, err := octree.FromCache(
		ctx, conn,
		dci.Bottleneck, dci.Subtrahend.Time)

	log.Printf("info: loading subtrahend octree took %s\n", time.Since(start))
	if err != nil {
		return
	}

	if subtrahendTree == nil {
		err = JSONError{
			Code: http.StatusNotFound,
			Message: fmt.Sprintf("Cannot find survey for %s/%s.",
				dci.Bottleneck,
				dci.Subtrahend.Format(common.DateFormat)),
		}
		return
	}

	// We need a slow path implementation for this.
	if minuendTree.EPSG != subtrahendTree.EPSG {
		err = JSONError{
			Code: http.StatusInternalServerError,
			Message: "Calculating differences between two different " +
				"EPSG code octrees are not supported, yet.",
		}
		return
	}

	start = time.Now()
	points := minuendTree.Diff(subtrahendTree)
	log.Printf("info: A - B took %v\n", time.Since(start))

	// The Triangulation and the loading of the clipping
	// polygon can be done concurrently.

	jobs := make(chan func())

	wg := new(sync.WaitGroup)
	for i := 0; i < 2; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for job := range jobs {
				job()
			}
		}()
	}

	var (
		tri     *octree.Triangulation
		triErr  error
		clip    *octree.Polygon
		clipErr error
	)

	jobs <- func() {
		start := time.Now()
		tri, triErr = points.Triangulate()
		log.Printf("info: triangulation took %v\n", time.Since(start))
	}

	jobs <- func() {
		start := time.Now()
		clip, clipErr = octree.LoadClippingPolygon(
			ctx, conn,
			minuendTree.EPSG,
			dci.Bottleneck,
			dci.Minuend.Time,
			dci.Subtrahend.Time)
		log.Printf("info: loading clipping polygon took %v\n", time.Since(start))
	}
	close(jobs)
	wg.Wait()

	switch {
	case triErr != nil && clipErr != nil:
		err = fmt.Errorf("%v %v", triErr, clipErr)
		return
	case triErr != nil:
		err = triErr
		return
	case clipErr != nil:
		err = clipErr
		return
	}

	start = time.Now()
	tin := tri.Tin()
	removed := tin.Clip(clip)
	log.Printf("info: clipping TIN took %v\n", time.Since(start))

	start = time.Now()
	log.Printf("info: Number of triangles to clip: %d\n", len(removed))
	builder := octree.NewBuilder(tin)
	builder.Build(removed)
	log.Printf("info: building octree took %v\n", time.Since(start))

	tree := builder.Tree()

	log.Printf("info: min/max: %f %f\n", tree.Min.Z, tree.Max.Z)

	heights := octree.SampleDiffHeights(
		tree.Min.Z, tree.Max.Z, contourStep)
	log.Printf("info: num heights: %d\n", len(heights))

	start = time.Now()

	// XXX: Maybe we should start this transaction earlier!?
	var tx *sql.Tx
	if tx, err = conn.BeginTx(ctx, nil); err != nil {
		return
	}
	defer tx.Rollback()

	var stmt *sql.Stmt
	if stmt, err = tx.PrepareContext(ctx, insertDiffContourSQL); err != nil {
		return
	}
	defer stmt.Close()

	if err = tx.QueryRowContext(
		ctx,
		insertDiffSQL,
		dci.Bottleneck,
		dci.Minuend.Time,
		dci.Subtrahend.Time,
	).Scan(&id); err != nil {
		return
	}

	octree.DoContours(tree, heights, func(res *octree.ContourResult) {
		if err == nil && len(res.Lines) > 0 {
			_, err = stmt.ExecContext(
				ctx,
				res.Lines.AsWKB2D(),
				minuendTree.EPSG,
				contourTolerance,
				res.Height,
				id,
			)
		}
	})

	log.Printf("info: calculating and storing iso lines took %v\n",
		time.Since(start))

	if err != nil {
		return
	}

	if err = tx.Commit(); err != nil {
		log.Printf("info: difference calculation failed after %v\n",
			time.Since(begin))
		return
	}
	log.Printf("info: difference calculation succeed after %v\n",
		time.Since(begin))

	jr = JSONResult{
		Result: map[string]int64{"id": id},
	}
	return
}