gemma: pkg/imports/dma.go annotate

annotate pkg/imports/dma.go @ 5560:f2204f91d286

Join the log lines of imports to the log exports to recover data from them. Used in SR export to extract information that where in the meta json but now are only found in the log.

author	Sascha L. Teichmann <sascha.teichmann@intevation.de>
date	Wed, 09 Feb 2022 18:34:40 +0100
parents	59a99655f34d
children	2dd155cc95ec

rev	line source
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	1 // This is Free Software under GNU Affero General Public License v >= 3.0
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	2 // without warranty, see README.md and license for details.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	3 //
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	4 // SPDX-License-Identifier: AGPL-3.0-or-later
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	5 // License-Filename: LICENSES/AGPL-3.0.txt
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	6 //
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	7 // Copyright (C) 2018 by via donau
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	8 // – Österreichische Wasserstraßen-Gesellschaft mbH
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	9 // Software engineering by Intevation GmbH
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	10 //
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	11 // Author(s):
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	12 // * Raimund Renkert <raimund.renkert@intevation.de>
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	13
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	14 package imports
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	15
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	16 import (
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	17 "context"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	18 "database/sql"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	19 "encoding/json"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	20 "errors"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	21 "fmt"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	22 "io"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	23 "time"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	24
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	25 "gemma.intevation.de/gemma/pkg/wfs"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	26 )
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	27
4152 78ec61acf72e Made 'golint' happier with imports package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 3219 diff changeset	28 // DistanceMarksAshore is an import job to import
78ec61acf72e Made 'golint' happier with imports package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 3219 diff changeset	29 // the distance marks ashore in form of point geometries
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	30 // and attribute data from a WFS service.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	31 type DistanceMarksAshore struct {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	32 // URL the GetCapabilities URL of the WFS service.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	33 URL string `json:"url"`
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	34 // FeatureType selects the feature type of the WFS service.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	35 FeatureType string `json:"feature-type"`
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	36 // SortBy sorts the feature by this key.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	37 SortBy string `json:"sort-by"`
2723 a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	38 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	39 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	40 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	41 Password string `json:"password,omitempty"`
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	42 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	43
4852 046a07a33b19 Fixed the golint issues of the imports package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4832 diff changeset	44 // Description gives a short info about relevant facts of this import.
5560 f2204f91d286 Join the log lines of imports to the log exports to recover data from them. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5034 diff changeset	45 func (dma *DistanceMarksAshore) Description([]string) (string, error) {
4799 f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	46 return dma.URL + "\|" + dma.FeatureType, nil
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4177 diff changeset	47 }
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4177 diff changeset	48
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	49 // DMAJobKind is the import queue type identifier.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	50 const DMAJobKind JobKind = "dma"
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	51
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	52 type dmaJobCreator struct{}
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	53
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	54 func init() {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	55 RegisterJobCreator(DMAJobKind, dmaJobCreator{})
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	56 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	57
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	58 func (dmaJobCreator) Description() string { return "distance marks" }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	59
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	60 func (dmaJobCreator) AutoAccept() bool { return true }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	61
2187 7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	62 func (dmaJobCreator) Create() Job { return new(DistanceMarksAshore) }
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	63
3219 4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2943 diff changeset	64 func (dmaJobCreator) Depends() [2][]string {
5020 e4ab338e7ba9 Removed unnecessary creatings of empty slices in expressing read dependencies in imports. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4852 diff changeset	65 return [2][]string{{"distance_marks"}}
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	66 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	67
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	68 // StageDone is a NOP for distance marks imports.
5034 59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5020 diff changeset	69 func (dmaJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error {
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	70 return nil
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	71 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	72
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	73 // CleanUp for distance marks imports is a NOP.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	74 func (*DistanceMarksAshore) CleanUp() error { return nil }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	75
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	76 type distanceMarksAshoreProperties struct {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	77 HydroCatdis int `json:"hydro_catdis"`
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	78 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	79
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	80 const (
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	81 deleteDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	82 WITH resp AS (
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	83 SELECT users.current_user_area_utm() AS a
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	84 )
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	85 DELETE FROM waterway.distance_marks
2933 a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction Tom Gottfried <tom@intevation.de> parents: 2928 diff changeset	86 WHERE pg_has_role('sys_admin', 'MEMBER')
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction Tom Gottfried <tom@intevation.de> parents: 2928 diff changeset	87 OR ST_Covers((SELECT a FROM resp),
a5642ee4c6d0 Let sys_admin import distance marks ashore without restriction Tom Gottfried <tom@intevation.de> parents: 2928 diff changeset	88 ST_Transform(geom::geometry, (SELECT ST_SRID(a) FROM resp)))
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	89 `
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	90 insertDistanceMarksSQL = `
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	91 WITH resp AS (
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	92 SELECT users.current_user_area_utm() AS a
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	93 )
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	94 INSERT INTO waterway.distance_marks (geom, catdis)
2943 78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	95 SELECT ST_Transform(new_dma, 4326), $3
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	96 FROM (SELECT
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	97 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	98 THEN dma
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	99 ELSE ST_Intersection((SELECT a FROM resp),
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	100 ST_Transform(dma, (SELECT ST_SRID(a) FROM resp)))
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	101 END AS new_dma
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	102 FROM ST_GeomFromWKB($1, $2::integer) AS dma (dma)) AS new_dma
78affd725ba5 Fix insert statement for dismar ashore Tom Gottfried <tom@intevation.de> parents: 2933 diff changeset	103 WHERE NOT ST_IsEmpty(new_dma)
2928 074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	104 RETURNING id
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	105 `
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	106 )
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	107
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	108 // Do executes the actual fairway dimension import.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	109 func (dma *DistanceMarksAshore) Do(
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	110 ctx context.Context,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	111 importID int64,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	112 conn *sql.Conn,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	113 feedback Feedback,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	114 ) (interface{}, error) {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	115
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	116 start := time.Now()
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	117
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	118 feedback.Info("Import distance marks")
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	119
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	120 feedback.Info("Loading capabilities from %s", dma.URL)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	121 caps, err := wfs.GetCapabilities(dma.URL)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	122 if err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	123 feedback.Error("Loading capabilities failed: %v", err)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	124 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	125 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	126
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	127 ft := caps.FindFeatureType(dma.FeatureType)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	128 if ft == nil {
4177 8b75ac5e243e Made 'staticcheck' happy with pgxutils package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4152 diff changeset	129 return nil, fmt.Errorf("unknown feature type '%s'", dma.FeatureType)
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	130 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	131
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	132 feedback.Info("Found feature type '%s'", dma.FeatureType)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	133
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	134 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	135 if err != nil {
4832 5062ccb2381d Unified error message on "Unsupported CRS" Sascha Wilde <wilde@intevation.de> parents: 4799 diff changeset	136 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	137 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	138 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	139
2721 56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2719 diff changeset	140 dl, err := wfs.GetFeatures(caps, dma.FeatureType, dma.SortBy)
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	141 if err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	142 feedback.Error("Cannot create GetFeature URLs. %v", err)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	143 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	144 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	145
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	146 tx, err := conn.BeginTx(ctx, nil)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	147 if err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	148 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	149 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	150 defer tx.Rollback()
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	151
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	152 insertStmt, err := tx.PrepareContext(ctx, insertDistanceMarksSQL)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	153 if err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	154 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	155 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	156 defer insertStmt.Close()
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	157
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	158 // Delete the old features.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	159 if _, err := tx.ExecContext(ctx, deleteDistanceMarksSQL); err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	160 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	161 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	162
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	163 var (
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	164 unsupported = stringCounter{}
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	165 missingProperties int
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	166 badProperties int
2928 074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	167 outside int
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	168 features int
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	169 )
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	170
2729 7cb027be277d WFS downloader: Really use username and password from parameters. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2723 diff changeset	171 if err := dl.Download(dma.User, dma.Password, func(url string, r io.Reader) error {
2106 2b72f5e005aa WFS imports: Write get GetFeature URLs into import log. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1862 diff changeset	172 feedback.Info("Get features from: '%s'", url)
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	173 rfc, err := wfs.ParseRawFeatureCollection(r)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	174 if err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	175 return fmt.Errorf("parsing GetFeature document failed: %v", err)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	176 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	177 if rfc.CRS != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	178 crsName := rfc.CRS.Properties.Name
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	179 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	180 feedback.Error("Unsupported CRS: %d", crsName)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	181 return err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	182 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	183 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	184
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	185 // No features -> ignore.
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	186 if rfc.Features == nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	187 return nil
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	188 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	189
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	190 feedback.Info("Using EPSG: %d", epsg)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	191
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	192 for _, feature := range rfc.Features {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	193 if feature.Geometry.Coordinates == nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	194 missingProperties++
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	195 continue
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	196 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	197
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	198 var props distanceMarksAshoreProperties
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	199
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	200 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	201 badProperties++
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	202 continue
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	203 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	204 switch feature.Geometry.Type {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	205 case "Point":
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	206 var p pointSlice
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	207 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	208 return err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	209 }
2928 074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	210 var dmaid int64
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	211 err := insertStmt.QueryRowContext(
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	212 ctx,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	213 p.asWKB(),
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	214 epsg,
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	215 props.HydroCatdis,
2928 074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	216 ).Scan(&dmaid)
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	217 switch {
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	218 case err == sql.ErrNoRows:
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	219 outside++
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	220 // ignore -> filtered by responsibility area
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	221 continue
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	222 case err != nil:
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	223 return err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	224 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	225 features++
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	226 default:
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	227 unsupported[feature.Geometry.Type]++
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	228 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	229 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	230 return nil
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	231 }); err != nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	232 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	233 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	234
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	235 if badProperties > 0 {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	236 feedback.Warn("Bad properties: %d", badProperties)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	237 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	238
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	239 if missingProperties > 0 {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	240 feedback.Warn("Missing properties: %d", missingProperties)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	241 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	242
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	243 if len(unsupported) != 0 {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	244 feedback.Warn("Unsupported types found: %s", unsupported)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	245 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	246
2928 074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	247 if outside > 0 {
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	248 feedback.Info("Features outside responsibility area: %d", outside)
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	249 }
074f2bb85584 Do not count skipped features as stored Tom Gottfried <tom@intevation.de> parents: 2926 diff changeset	250
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	251 if features == 0 {
4177 8b75ac5e243e Made 'staticcheck' happy with pgxutils package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4152 diff changeset	252 err := errors.New("no features found")
1862 427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	253 feedback.Error("%v", err)
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	254 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	255 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	256
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	257 if err = tx.Commit(); err == nil {
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	258 feedback.Info("Storing %d features took %s",
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	259 features, time.Since(start))
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	260 }
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	261
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	262 return nil, err
427f86518097 Added distance marks importer in backend. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	263 }

Mercurial > gemma

annotate pkg/imports/dma.go @ 5560:f2204f91d286