view pkg/imports/fm.go @ 4931:e41d42be0e13 fairway-marks-import

One more callback to avoid code duplication more consequently
author Tom Gottfried <tom@intevation.de>
date Fri, 14 Feb 2020 19:34:08 +0100
parents 644cb7c175f3
children 9f9d72a1d398
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2020 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Tom Gottfried <tom.gottfried@intevation.de>

package imports

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"time"

	"gemma.intevation.de/gemma/pkg/wfs"
)

// FairwayMarks is a struct
// to be used as the basis for imports of
// specific types for fairway marks.
type FairwayMarks struct {
	// URL the GetCapabilities URL of the WFS service.
	URL string `json:"url"`
	// FeatureType selects the feature type of the WFS service.
	FeatureType string `json:"feature-type"`
	// SortBy works around misconfigured services to
	// establish a sort order to get the features.
	SortBy string `json:"sort-by"`
	// User is an optional username for Basic Auth.
	User string `json:"user,omitempty"`
	// Password is an optional password for Basic Auth.
	Password string `json:"password,omitempty"`
}

// Properties common to all types of fairway marks
type fairwayMarksProperties struct {
	Datsta *string `json:"hydro_datsta"`
	Datend *string `json:"hydro_datend"`
	Persta *string `json:"hydro_persta"`
	Perend *string `json:"hydro_perend"`
	Objnam *string `json:"hydro_objnam"`
	Nobjnm *string `json:"hydro_nobjnm"`
	Inform *string `json:"hydro_inform"`
	Ninfom *string `json:"hydro_ninfom"`
	Scamin *int    `json:"hydro_scamin"`
	Picrep *string `json:"hydro_picrep"`
	Txtdsc *string `json:"hydro_txtdsc"`
	Sordat *string `json:"hydro_sordat"`
	Sorind *string `json:"hydro_sorind"`
}

// Common operation of FM imports to get features from WFS service
func getFMFeatures(
	ctx context.Context,
	conn *sql.Conn,
	feedback Feedback,
	fm FairwayMarks,
	// Constructor returning pointer to struct
	// representing featuretype's properties
	newProps func() interface{},
	// Construct pointer to featuretype from given pointSlice and properties
	newFeat func(pointSlice, interface{}) interface{},
	// Store features in type specific database tables
	storeFMs func(
		tx *sql.Tx,
		epsg int,
		// Slice of features to be converted to featuretypes type
		fms []interface{},
	) (outsideOrDup int, features int, err error),
) (err error) {

	start := time.Now()

	feedback.Info("Import fairway marks")

	feedback.Info("Loading capabilities from %s", fm.URL)
	caps, err := wfs.GetCapabilities(fm.URL)
	if err != nil {
		feedback.Error("Loading capabilities failed: %v", err)
		return
	}

	ft := caps.FindFeatureType(fm.FeatureType)
	if ft == nil {
		err = fmt.Errorf("unknown feature type '%s'", fm.FeatureType)
		return
	}

	feedback.Info("Found feature type '%s", fm.FeatureType)

	epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
	if err != nil {
		feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
		return
	}

	if fm.SortBy != "" {
		feedback.Info("Features will be sorted by '%s'", fm.SortBy)
	}

	dl, err := wfs.GetFeatures(caps, fm.FeatureType, fm.SortBy)
	if err != nil {
		feedback.Error("Cannot create GetFeature URLs. %v", err)
		return
	}

	var (
		fms               []interface{}
		unsupported       = stringCounter{}
		missingProperties int
		badProperties     int
	)

	err = dl.Download(fm.User, fm.Password, func(url string, r io.Reader) error {
		feedback.Info("Get features from: '%s'", url)
		rfc, err := wfs.ParseRawFeatureCollection(r)
		if err != nil {
			return fmt.Errorf("parsing GetFeature document failed: %v", err)
		}
		if rfc.CRS != nil {
			crsName := rfc.CRS.Properties.Name
			if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
				feedback.Error("Unsupported CRS: %d", crsName)
				return err
			}
		}

		// No features -> ignore.
		if rfc.Features == nil {
			return nil
		}

		feedback.Info("Using EPSG: %d", epsg)

		for _, feature := range rfc.Features {
			if feature.Properties == nil || feature.Geometry.Coordinates == nil {
				missingProperties++
				continue
			}

			props := newProps()
			if err := json.Unmarshal(*feature.Properties, props); err != nil {
				badProperties++
				continue
			}

			switch feature.Geometry.Type {
			case "Point":
				var p pointSlice
				if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
					return err
				}

				f := newFeat(p, props)
				fms = append(fms, f)
			default:
				unsupported[feature.Geometry.Type]++
			}
		}
		return nil
	})
	if err != nil {
		return
	}

	if badProperties > 0 {
		feedback.Warn("Bad properties: %d", badProperties)
	}

	if missingProperties > 0 {
		feedback.Warn("Missing properties: %d", missingProperties)
	}

	if len(unsupported) != 0 {
		feedback.Warn("Unsupported types found: %s", unsupported)
	}

	feedback.Info("Found %d usable features in data source", len(fms))

	tx, err := conn.BeginTx(ctx, nil)
	if err != nil {
		return
	}
	defer tx.Rollback()

	outsideOrDup, features, err := storeFMs(tx, epsg, fms)
	if err != nil {
		return
	}

	if outsideOrDup > 0 {
		feedback.Info(
			"Features outside responsibility area and duplicates: %d",
			outsideOrDup)
	}

	if features == 0 {
		return UnchangedError("no valid new features found")
	}

	if err = tx.Commit(); err == nil {
		feedback.Info("Storing %d features took %s",
			features, time.Since(start))
	}

	return
}