gemma: pkg/imports/wfsjob.go annotate

annotate pkg/imports/wfsjob.go @ 5591:0011f50cf216 surveysperbottleneckid

Removed no longer used alternative api for surveys/ endpoint. As bottlenecks in the summary for SR imports are now identified by their id and no longer by the (not guarantied to be unique!) name, there is no longer the need to request survey data by the name+date tuple (which isn't reliable anyway). So the workaround was now reversed.

author	Sascha Wilde <wilde@sha-bang.de>
date	Wed, 06 Apr 2022 13:30:29 +0200
parents	ade07a3f2cfd
children	1222b777f51f

rev	line source
4948 821ae20b6a20 Re-added missing header lines. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4946 diff changeset	1 // This is Free Software under GNU Affero General Public License v >= 3.0
821ae20b6a20 Re-added missing header lines. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4946 diff changeset	2 // without warranty, see README.md and license for details.
821ae20b6a20 Re-added missing header lines. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4946 diff changeset	3 //
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	4 // SPDX-License-Identifier: AGPL-3.0-or-later
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	5 // License-Filename: LICENSES/AGPL-3.0.txt
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	6 //
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	7 // Copyright (C) 2020 by via donau
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	8 // – Österreichische Wasserstraßen-Gesellschaft mbH
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	9 // Software engineering by Intevation GmbH
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	10 //
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	11 // Author(s):
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	12 // * Tom Gottfried <tom.gottfried@intevation.de>
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	13 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	14
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	15 package imports
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	16
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	17 import (
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	18 "context"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	19 "database/sql"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	20 "encoding/json"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	21 "errors"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	22 "fmt"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	23 "io"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	24 "time"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	25
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	26 "gemma.intevation.de/gemma/pkg/models"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	27 "gemma.intevation.de/gemma/pkg/wfs"
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	28 )
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	29
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	30 var (
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	31 ErrFeatureIgnored = errors.New("feature ignored")
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	32 ErrFeatureDuplicated = errors.New("feature duplicated")
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	33 ErrFeaturesUnmodified = errors.New("features unmodified")
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	34 )
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	35
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	36 type (
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	37 WFSFeatureConsumer interface {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	38 Commit() error
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	39 Rollback() error
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	40
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	41 NewFeature() (kind string, properties interface{})
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	42
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	43 Consume(geom, properties interface{}, epsg int) error
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	44 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	45
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	46 WFSFeatureJobCreator struct {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	47 description string
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	48 depends [2][]string
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	49
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	50 newConsumer func(context.Context, *sql.Conn, Feedback) (WFSFeatureConsumer, error)
4980 21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	51
5034 59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	52 stageDone func(context.Context, *sql.Tx, int64, Feedback) error
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	53 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	54
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	55 WFSFeatureJob struct {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	56 models.WFSImport
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	57 creator *WFSFeatureJobCreator
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	58 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	59 )
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	60
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	61 var (
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	62 kindToGeometry = map[string]func() interface{}{
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	63 // TODO: extend me!
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	64 "Point": func() interface{} { return new(pointSlice) },
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	65 "LineString": func() interface{} { return new(lineSlice) },
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	66 "MultiLineString": func() interface{} { return new(multiLineSlice) },
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	67 }
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	68
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	69 wrapGeomKind = map[[2]string]func(interface{}) interface{}{
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	70 // TODO: extend me!
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	71 {"LineString", "MultiLineString"}: func(x interface{}) interface{} {
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	72 return &multiLineSlice{x.(lineSlice)}
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	73 },
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	74 }
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	75 )
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	76
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	77 func (wfjc *WFSFeatureJobCreator) Description() string {
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	78 return wfjc.description
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	79 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	80
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	81 func (wfjc *WFSFeatureJobCreator) Depends() [2][]string {
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	82 return wfjc.depends
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	83 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	84
4980 21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	85 func (wfjc *WFSFeatureJobCreator) AutoAccept() bool {
21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	86 return wfjc.stageDone == nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	87 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	88
5034 59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	89 func (wfjc *WFSFeatureJobCreator) StageDone(
59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	90 ctx context.Context,
59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	91 tx *sql.Tx,
59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	92 id int64,
59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	93 feedback Feedback,
59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	94 ) error {
4980 21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	95 if wfjc.stageDone == nil {
21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	96 return nil
21d2acc080c9 Add the ability to be staged to the WFS feature downloader. This will be needed for FD downloads. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4979 diff changeset	97 }
5034 59a99655f34d Added feedback support for StageDone. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5012 diff changeset	98 return wfjc.stageDone(ctx, tx, id, feedback)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	99 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	100
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	101 func (wfjc *WFSFeatureJobCreator) Create() Job {
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	102 return &WFSFeatureJob{creator: wfjc}
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	103 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	104
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	105 // Description gives a short info about relevant facts of this import.
5565 ade07a3f2cfd Forget to change signature of some imports. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5143 diff changeset	106 func (wfj *WFSFeatureJob) Description([]string) (string, error) {
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	107 return wfj.URL + "\|" + wfj.FeatureType, nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	108 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	109
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	110 // CleanUp for WFS imports is a NOP.
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	111 func (*WFSFeatureJob) CleanUp() error {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	112 return nil
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	113 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	114
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	115 func (wfj *WFSFeatureJob) Do(
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	116 ctx context.Context,
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	117 importID int64,
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	118 conn *sql.Conn,
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	119 feedback Feedback,
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	120 ) (interface{}, error) {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	121
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	122 start := time.Now()
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	123
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	124 feedback.Info("Import %s", wfj.creator.Description())
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	125
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	126 feedback.Info("Loading capabilities from %s", wfj.URL)
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	127 caps, err := wfs.GetCapabilities(wfj.URL)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	128 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	129 feedback.Error("Loading capabilities failed: %v", err)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	130 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	131 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	132
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	133 ft := caps.FindFeatureType(wfj.FeatureType)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	134 if ft == nil {
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	135 return nil, fmt.Errorf("unknown feature type '%s'", wfj.FeatureType)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	136 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	137
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	138 feedback.Info("Found feature type '%s'", wfj.FeatureType)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	139
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	140 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	141 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	142 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	143 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	144 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	145
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	146 if nilString(wfj.SortBy) != "" {
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	147 feedback.Info("Features will be sorted by '%s'", *wfj.SortBy)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	148 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	149
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	150 dl, err := wfs.GetFeatures(caps, wfj.FeatureType, nilString(wfj.SortBy))
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	151 if err != nil {
4991 6a48e5c6fe2f Re-added a import feedback line which was deleted accidently in previous commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4990 diff changeset	152 feedback.Error("Cannot create GetFeature URLs. %v", err)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	153 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	154 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	155
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	156 var (
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	157 unsupported = stringCounter{}
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	158 missingProperties int
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	159 badProperties int
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	160 dupes int
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	161 features int
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	162 )
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	163
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	164 consumer, err := wfj.creator.newConsumer(ctx, conn, feedback)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	165 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	166 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	167 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	168 defer consumer.Rollback()
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	169
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	170 if err := dl.Download(nilString(wfj.User), nilString(wfj.Password), func(url string, r io.Reader) error {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	171 feedback.Info("Get features from: '%s'", url)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	172 rfc, err := wfs.ParseRawFeatureCollection(r)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	173 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	174 return fmt.Errorf("parsing GetFeature document failed: %v", err)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	175 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	176 if rfc.CRS != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	177 crsName := rfc.CRS.Properties.Name
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	178 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	179 feedback.Error("Unsupported CRS: %d", crsName)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	180 return err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	181 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	182 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	183
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	184 // No features -> ignore.
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	185 if rfc.Features == nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	186 return nil
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	187 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	188
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	189 feedback.Info("Using EPSG: %d", epsg)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	190
4954 331a59ebaf54 Fix fairway marks import log Tom Gottfried <tom@intevation.de> parents: 4953 diff changeset	191 feedback.Info(
331a59ebaf54 Fix fairway marks import log Tom Gottfried <tom@intevation.de> parents: 4953 diff changeset	192 "Found %d features in data source", len(rfc.Features))
331a59ebaf54 Fix fairway marks import log Tom Gottfried <tom@intevation.de> parents: 4953 diff changeset	193
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	194 for _, feature := range rfc.Features {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	195 if feature.Properties == nil \|\| feature.Geometry.Coordinates == nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	196 missingProperties++
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	197 continue
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	198 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	199
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	200 kind, props := consumer.NewFeature()
799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	201
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	202 if err := json.Unmarshal(*feature.Properties, props); err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	203 badProperties++
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	204 continue
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	205 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	206
5011 2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	207 // Look if we can deserialize given type
2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	208 makeGeom := kindToGeometry[feature.Geometry.Type]
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	209 if makeGeom == nil {
5011 2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	210 unsupported[feature.Geometry.Type]++
2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	211 continue
2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	212 }
2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	213
5012 ae3a1392f9d0 Cosmetics: Tightened code a bit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5011 diff changeset	214 // Optional wrapping
ae3a1392f9d0 Cosmetics: Tightened code a bit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5011 diff changeset	215 wrap := func(x interface{}) interface{} { return x }
5011 2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	216 if feature.Geometry.Type != kind {
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	217 // Look if we can wrap it
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	218 if wrap = wrapGeomKind[[2]string{feature.Geometry.Type, kind}]; wrap == nil {
5011 2d7af72bb7ac Fix geometry type wrapping Tom Gottfried <tom@intevation.de> parents: 5010 diff changeset	219 unsupported[feature.Geometry.Type]++
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	220 continue
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	221 }
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	222 }
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	223
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	224 geom := makeGeom()
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	225 if err := json.Unmarshal(*feature.Geometry.Coordinates, geom); err != nil {
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	226 return err
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	227 }
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	228
5012 ae3a1392f9d0 Cosmetics: Tightened code a bit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5011 diff changeset	229 switch err := consumer.Consume(wrap(geom), props, epsg); {
5010 1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	230 case err == ErrFeatureDuplicated:
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	231 dupes++
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	232 case err == ErrFeatureIgnored:
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	233 // be silent
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	234 case err != nil:
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	235 return err
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	236 default:
1d1be6bd5304 Wrap geometries if needed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 5007 diff changeset	237 features++
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	238 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	239 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	240 return nil
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	241 }); err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	242 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	243 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	244
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	245 if dupes > 0 {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	246 feedback.Info(
4960 fb48bc6415c0 Restore accidentally changed import log Tom Gottfried <tom@intevation.de> parents: 4958 diff changeset	247 "Features outside responsibility area, duplicates or unchanged: %d",
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	248 dupes)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	249 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	250
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	251 if badProperties > 0 {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	252 feedback.Warn("Bad properties: %d", badProperties)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	253 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	254
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	255 if missingProperties > 0 {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	256 feedback.Warn("Missing properties: %d", missingProperties)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	257 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	258
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	259 if len(unsupported) != 0 {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	260 feedback.Warn("Unsupported types found: %s", unsupported)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	261 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	262
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	263 if err = consumer.Commit(); err == nil \|\| err == ErrFeaturesUnmodified {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	264 feedback.Info("Storing %d features took %s",
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	265 features, time.Since(start))
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	266 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	267
4958 6d5d6b27c3c3 Added pre commit hooks before commiting fairway marks to invalidate old. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4956 diff changeset	268 // Commit before eventually returning UnchangedError because we might
6d5d6b27c3c3 Added pre commit hooks before commiting fairway marks to invalidate old. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4956 diff changeset	269 // have updated last_found
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	270 if features == 0 && err == ErrFeaturesUnmodified {
4956 7cc79c65a9e5 Keep the history of fairway marks Tom Gottfried <tom@intevation.de> parents: 4954 diff changeset	271 return nil, UnchangedError("no valid new features found")
7cc79c65a9e5 Keep the history of fairway marks Tom Gottfried <tom@intevation.de> parents: 4954 diff changeset	272 }
7cc79c65a9e5 Keep the history of fairway marks Tom Gottfried <tom@intevation.de> parents: 4954 diff changeset	273
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	274 if err == ErrFeaturesUnmodified {
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	275 // It's not really an error.
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	276 err = nil
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	277 }
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	278
67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	279 return nil, err
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	280 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	281
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	282 type (
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	283 SQLGeometryConsumer struct {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	284 ctx context.Context
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	285 tx *sql.Tx
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	286 feedback Feedback
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	287 consume func(*SQLGeometryConsumer, interface{}, interface{}, int) error
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	288 newFeature func() (string, interface{})
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	289 preCommit func(*SQLGeometryConsumer) error
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	290 savepoint func(func() error) error
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	291 stmts []*sql.Stmt
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	292 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	293 )
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	294
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	295 func (sgc *SQLGeometryConsumer) Rollback() error {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	296 if tx := sgc.tx; tx != nil {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	297 sgc.releaseStmts()
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	298 sgc.tx = nil
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	299 sgc.ctx = nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	300 return tx.Rollback()
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	301 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	302 return nil
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	303 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	304
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	305 func (sgc *SQLGeometryConsumer) Commit() error {
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	306 var err error
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	307 if tx := sgc.tx; tx != nil {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	308 if sgc.preCommit != nil {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	309 err = sgc.preCommit(sgc)
4958 6d5d6b27c3c3 Added pre commit hooks before commiting fairway marks to invalidate old. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4956 diff changeset	310 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	311 sgc.releaseStmts()
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	312 sgc.tx = nil
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	313 sgc.ctx = nil
5143 733f7136a30e Do not mask error with subsequent error on commit Tom Gottfried <tom@intevation.de> parents: 5140 diff changeset	314 if err2 := tx.Commit(); err2 != nil &&
733f7136a30e Do not mask error with subsequent error on commit Tom Gottfried <tom@intevation.de> parents: 5140 diff changeset	315 (err == nil \|\| err == ErrFeaturesUnmodified) {
733f7136a30e Do not mask error with subsequent error on commit Tom Gottfried <tom@intevation.de> parents: 5140 diff changeset	316 // An error on commit that is not induced by the first
733f7136a30e Do not mask error with subsequent error on commit Tom Gottfried <tom@intevation.de> parents: 5140 diff changeset	317 // overrules the first.
5140 11c4c5b8041c Backed out changeset c9bca9a9666c Tom Gottfried <tom@intevation.de> parents: 5138 diff changeset	318 err = err2
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	319 }
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	320 }
4961 67d78b74fe43 Be more careful with unchanged errors if features were removed. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4960 diff changeset	321 return err
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	322 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	323
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	324 func (sgc *SQLGeometryConsumer) NewFeature() (string, interface{}) {
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	325 return sgc.newFeature()
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	326 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	327
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	328 func (sgc *SQLGeometryConsumer) Consume(
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	329 geom, properties interface{},
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	330 epsg int,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	331 ) error {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	332 return sgc.consume(sgc, geom, properties, epsg)
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	333 }
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	334
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	335 func (sgc *SQLGeometryConsumer) ConsumePolygon(
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	336 polygon polygonSlice,
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	337 properties interface{},
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	338 epsg int,
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	339 ) error {
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	340 return sgc.consume(sgc, polygon, properties, epsg)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	341 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	342
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	343 func newSQLConsumer(
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	344 init func(*SQLGeometryConsumer) error,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	345 consume func(*SQLGeometryConsumer, interface{}, interface{}, int) error,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	346 preCommit func(*SQLGeometryConsumer) error,
5007 799e8248de8d First step to move geometry creation for WFS features to loader. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4991 diff changeset	347 newFeature func() (string, interface{}),
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	348
4978 35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	349 ) func(context.Context, *sql.Conn, Feedback) (WFSFeatureConsumer, error) {
35a3dc12050f Give better names to generalized WFS feature downloaders. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4977 diff changeset	350 return func(ctx context.Context, conn *sql.Conn, feedback Feedback) (WFSFeatureConsumer, error) {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	351 tx, err := conn.BeginTx(ctx, nil)
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	352 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	353 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	354 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	355 sgc := &SQLGeometryConsumer{
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	356 ctx: ctx,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	357 tx: tx,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	358 feedback: feedback,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	359 consume: consume,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	360 newFeature: newFeature,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	361 preCommit: preCommit,
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	362 savepoint: Savepoint(ctx, tx, "feature"),
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	363 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	364 if err := init(sgc); err != nil {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	365 tx.Rollback()
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	366 return nil, err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	367 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	368 return sgc, nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	369 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	370 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	371
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	372 func (sgc *SQLGeometryConsumer) releaseStmts() {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	373 for i := len(sgc.stmts); i > 0; i-- {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	374 sgc.stmts[i-1].Close()
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	375 sgc.stmts[i-1] = nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	376 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	377 sgc.stmts = nil
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	378 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	379
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	380 func prepareStmnts(queries ...string) func(*SQLGeometryConsumer) error {
4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	381 return func(sgc *SQLGeometryConsumer) error {
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	382 for _, query := range queries {
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	383 stmt, err := sgc.tx.PrepareContext(sgc.ctx, query)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	384 if err != nil {
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	385 return err
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	386 }
4977 4bf6cde2d996 Generalize WFS point import job to be able to import other geometry types, too. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4961 diff changeset	387 sgc.stmts = append(sgc.stmts, stmt)
4946 b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	388 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	389 return nil
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	390 }
b0dbc0f2c748 Simplified importing of fairway marks. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: diff changeset	391 }

Mercurial > gemma

annotate pkg/imports/wfsjob.go @ 5591:0011f50cf216 surveysperbottleneckid