gemma: pkg/imports/wa.go annotate

annotate pkg/imports/wa.go @ 4798:ca6a5f722471

Added Description method to most imports. Only for imports where information relevant to the administrator is relevant.

author	Sascha Wilde <wilde@intevation.de>
date	Fri, 25 Oct 2019 18:25:38 +0200
parents	5d7ce7f926eb
children	f32d086b5dbf

rev	line source
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	1 // This is Free Software under GNU Affero General Public License v >= 3.0
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	2 // without warranty, see README.md and license for details.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	3 //
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	4 // SPDX-License-Identifier: AGPL-3.0-or-later
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	5 // License-Filename: LICENSES/AGPL-3.0.txt
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	6 //
2649 9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping Tom Gottfried <tom@intevation.de> parents: 2622 diff changeset	7 // Copyright (C) 2018, 2019 by via donau
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	8 // – Österreichische Wasserstraßen-Gesellschaft mbH
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	9 // Software engineering by Intevation GmbH
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	10 //
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	11 // Author(s):
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
2649 9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping Tom Gottfried <tom@intevation.de> parents: 2622 diff changeset	13 // * Tom Gottfried <tom.gottfried@intevation.de>
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	14
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	15 package imports
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	16
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	17 import (
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	18 "context"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	19 "database/sql"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	20 "encoding/json"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	21 "errors"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	22 "fmt"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	23 "io"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	24 "strconv"
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	25 "strings"
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	26 "time"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	27
4062 6c760abcff0e Move handling of PostgreSQL errors to own package Tom Gottfried <tom@intevation.de> parents: 4058 diff changeset	28 "gemma.intevation.de/gemma/pkg/pgxutils"
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	29 "gemma.intevation.de/gemma/pkg/wfs"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	30 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	31
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	32 // WaterwayArea is an import job to import
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	33 // the waterway area in form of polygon geometries
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	34 // and attribute data from a WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	35 type WaterwayArea struct {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	36 // URL the GetCapabilities URL of the WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	37 URL string `json:"url"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	38 // FeatureType selects the feature type of the WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	39 FeatureType string `json:"feature-type"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	40 // SortBy works around misconfigured services to
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	41 // establish a sort order to get the features.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	42 SortBy string `json:"sort-by"`
2723 a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	43 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	44 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	45 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	46 Password string `json:"password,omitempty"`
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	47 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	48
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	49 func (wa *WaterwayArea) Description() (string, error) {
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	50
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	51 var descs []string
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	52
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	53 descs = append(descs, wa.URL)
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	54 descs = append(descs, wa.FeatureType)
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	55 return strings.Join(descs, "\|"), nil
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	56 }
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	57
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	58 // WAJobKind is the import queue type identifier.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	59 const WAJobKind JobKind = "wa"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	60
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	61 type waJobCreator struct{}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	62
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	63 func init() {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	64 RegisterJobCreator(WAJobKind, waJobCreator{})
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	65 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	66
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	67 func (waJobCreator) Description() string { return "waterway area" }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	68
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	69 func (waJobCreator) AutoAccept() bool { return true }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	70
2187 7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	71 func (waJobCreator) Create() Job { return new(WaterwayArea) }
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	72
3219 4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2938 diff changeset	73 func (waJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2938 diff changeset	74 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2938 diff changeset	75 {"waterway_area"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2938 diff changeset	76 {},
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	77 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	78 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	79
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	80 // StageDone is a NOP for waterway area imports.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	81 func (waJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	82 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	83 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	84
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	85 // CleanUp for waterway area imports is a NOP.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	86 func (*WaterwayArea) CleanUp() error { return nil }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	87
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	88 type waterwayAreaProperties struct {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	89 Catccl *string `json:"ienc_catccl"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	90 Dirimp *string `json:"ienc_dirimp"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	91 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	92
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	93 const (
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	94 deleteWaterwayAreaSQL = `
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	95 WITH resp AS (
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2855 diff changeset	96 SELECT users.current_user_area_utm() AS a
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	97 )
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	98 DELETE FROM waterway.waterway_area
2649 9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping Tom Gottfried <tom@intevation.de> parents: 2622 diff changeset	99 WHERE pg_has_role('sys_admin', 'MEMBER')
9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping Tom Gottfried <tom@intevation.de> parents: 2622 diff changeset	100 OR ST_Covers((SELECT a FROM resp),
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2855 diff changeset	101 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp)))
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	102 `
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	103 insertWaterwayAreaSQL = `
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	104 WITH resp AS (
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2855 diff changeset	105 SELECT users.current_user_area_utm() AS a
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	106 )
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	107 INSERT INTO waterway.waterway_area (area, catccl, dirimp)
2855 8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	108 SELECT dmp.geom, $3, $4
8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	109 FROM ST_GeomFromWKB($1, $2::integer) AS new_area (new_area),
8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	110 ST_Dump(ST_Transform(ST_CollectionExtract(
2649 9b6a02923bb4 Allow sys_admin to import waterway axis and area without clipping Tom Gottfried <tom@intevation.de> parents: 2622 diff changeset	111 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
2855 8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	112 THEN ST_MakeValid(ST_Transform(new_area,
8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	113 best_utm(ST_Transform(new_area, 4326))))
8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	114 ELSE ST_Intersection((SELECT a FROM resp),
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2855 diff changeset	115 ST_MakeValid(ST_Transform(new_area, (SELECT ST_SRID(a) FROM resp))))
2855 8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	116 END,
8a8a929182f9 Enable import of axis and area independently from areas of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	117 3), 4326)) AS dmp
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	118 RETURNING id
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	119 `
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	120 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	121
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	122 // Do executes the actual waterway axis import.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	123 func (wx *WaterwayArea) Do(
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	124 ctx context.Context,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	125 importID int64,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	126 conn *sql.Conn,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	127 feedback Feedback,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	128 ) (interface{}, error) {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	129
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	130 start := time.Now()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	131
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	132 feedback.Info("Import waterway area")
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	133
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	134 feedback.Info("Loading capabilities from %s", wx.URL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	135 caps, err := wfs.GetCapabilities(wx.URL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	136 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	137 feedback.Error("Loading capabilities failed: %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	138 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	139 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	140
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	141 ft := caps.FindFeatureType(wx.FeatureType)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	142 if ft == nil {
4177 8b75ac5e243e Made 'staticcheck' happy with pgxutils package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4147 diff changeset	143 return nil, fmt.Errorf("unknown feature type '%s'", wx.FeatureType)
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	144 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	145
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	146 feedback.Info("Found feature type '%s", wx.FeatureType)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	147
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	148 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	149 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	150 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	151 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	152 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	153
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	154 if wx.SortBy != "" {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	155 feedback.Info("Features will be sorted by '%s'", wx.SortBy)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	156 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	157
2721 56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2719 diff changeset	158 dl, err := wfs.GetFeatures(caps, wx.FeatureType, wx.SortBy)
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	159 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	160 feedback.Error("Cannot create GetFeature URLs. %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	161 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	162 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	163
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	164 tx, err := conn.BeginTx(ctx, nil)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	165 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	166 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	167 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	168 defer tx.Rollback()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	169
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	170 insertStmt, err := tx.PrepareContext(ctx, insertWaterwayAreaSQL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	171 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	172 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	173 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	174 defer insertStmt.Close()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	175
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	176 // Delete the old features.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	177 if _, err := tx.ExecContext(ctx, deleteWaterwayAreaSQL); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	178 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	179 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	180
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	181 var (
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	182 unsupported = stringCounter{}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	183 missingProperties int
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	184 badProperties int
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	185 outside int
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	186 features int
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	187 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	188
2729 7cb027be277d WFS downloader: Really use username and password from parameters. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2723 diff changeset	189 if err := dl.Download(wx.User, wx.Password, func(url string, r io.Reader) error {
2106 2b72f5e005aa WFS imports: Write get GetFeature URLs into import log. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1899 diff changeset	190 feedback.Info("Get features from: '%s'", url)
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	191 rfc, err := wfs.ParseRawFeatureCollection(r)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	192 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	193 return fmt.Errorf("parsing GetFeature document failed: %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	194 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	195 if rfc.CRS != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	196 crsName := rfc.CRS.Properties.Name
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	197 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	198 feedback.Error("Unsupported CRS: %d", crsName)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	199 return err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	200 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	201 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	202
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	203 // No features -> ignore.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	204 if rfc.Features == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	205 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	206 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	207
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	208 feedback.Info("Using EPSG: %d", epsg)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	209
2758 a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	210 savepoint := Savepoint(ctx, tx, "feature")
2659 00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails Tom Gottfried <tom@intevation.de> parents: 2649 diff changeset	211
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	212 for _, feature := range rfc.Features {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	213 if feature.Properties == nil \|\| feature.Geometry.Coordinates == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	214 missingProperties++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	215 continue
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	216 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	217
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	218 var props waterwayAreaProperties
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	219
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	220 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	221 badProperties++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	222 continue
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	223 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	224
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	225 var catccl sql.NullInt64
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	226 if props.Catccl != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	227 if value, err := strconv.ParseInt(*props.Catccl, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	228 catccl = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	229 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	230 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	231 var dirimp sql.NullInt64
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	232 if props.Dirimp != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	233 if value, err := strconv.ParseInt(*props.Dirimp, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	234 dirimp = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	235 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	236 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	237
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	238 switch feature.Geometry.Type {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	239 case "Polygon":
1786 09349ca27dd7 Imports: Removed duplicated code path to store WKB polygons into database. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1785 diff changeset	240 var p polygonSlice
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	241 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	242 return err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	243 }
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	244 var waid int64
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	245 err := savepoint(func() error {
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	246 err := insertStmt.QueryRowContext(
2758 a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	247 ctx,
a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	248 p.asWKB(),
a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	249 epsg,
a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	250 catccl,
a996f2ca9fa5 Simplified savepoint handling. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2729 diff changeset	251 dirimp,
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	252 ).Scan(&waid)
2659 00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails Tom Gottfried <tom@intevation.de> parents: 2649 diff changeset	253 return err
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	254 })
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	255 switch {
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	256 case err == sql.ErrNoRows:
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	257 outside++
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	258 // ignore -> filtered by responsibility_areas
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	259 case err != nil:
4197 5d7ce7f926eb Log fatal errors as errors, not warnings, in imports Tom Gottfried <tom@intevation.de> parents: 4177 diff changeset	260 feedback.Error(pgxutils.ReadableError{Err: err}.Error())
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	261 default:
2659 00b0a7b2225a Waterway area import: do not fail if INSERT of one item fails Tom Gottfried <tom@intevation.de> parents: 2649 diff changeset	262 features++
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	263 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	264 default:
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	265 unsupported[feature.Geometry.Type]++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	266 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	267 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	268 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	269 }); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	270 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	271 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	272
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	273 if badProperties > 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	274 feedback.Warn("Bad properties: %d", badProperties)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	275 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	276
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	277 if missingProperties > 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	278 feedback.Warn("Missing properties: %d", missingProperties)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	279 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	280
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	281 if len(unsupported) != 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	282 feedback.Warn("Unsupported types found: %s", unsupported)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	283 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	284
2938 ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	285 if outside > 0 {
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	286 feedback.Info("Features outside responsibility area: %d", outside)
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	287 }
ecb1e34157a3 Fix logging in area import Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	288
1828 1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	289 if features == 0 {
4177 8b75ac5e243e Made 'staticcheck' happy with pgxutils package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4147 diff changeset	290 err := errors.New("no features found")
1828 1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	291 feedback.Error("%v", err)
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	292 return nil, err
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	293 }
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	294
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	295 if err = tx.Commit(); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	296 feedback.Info("Storing %d features took %s",
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	297 features, time.Since(start))
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	298 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	299
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	300 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	301 }

Mercurial > gemma

annotate pkg/imports/wa.go @ 4798:ca6a5f722471