gemma: pkg/imports/wa.go annotate

annotate pkg/imports/wa.go @ 2168:b868cb653c4d

Import queue: The job kind parameter when creating a new job from a string was never used.

author	Sascha L. Teichmann <sascha.teichmann@intevation.de>
date	Fri, 08 Feb 2019 15:39:17 +0100
parents	86c88fc0ff5e
children	7c83b5277c1c

rev	line source
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	1 // This is Free Software under GNU Affero General Public License v >= 3.0
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	2 // without warranty, see README.md and license for details.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	3 //
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	4 // SPDX-License-Identifier: AGPL-3.0-or-later
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	5 // License-Filename: LICENSES/AGPL-3.0.txt
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	6 //
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	7 // Copyright (C) 2018 by via donau
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	8 // – Österreichische Wasserstraßen-Gesellschaft mbH
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	9 // Software engineering by Intevation GmbH
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	10 //
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	11 // Author(s):
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	13
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	14 package imports
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	15
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	16 import (
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	17 "context"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	18 "database/sql"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	19 "encoding/json"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	20 "errors"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	21 "fmt"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	22 "io"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	23 "strconv"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	24 "time"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	25
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	26 "gemma.intevation.de/gemma/pkg/common"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	27 "gemma.intevation.de/gemma/pkg/wfs"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	28 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	29
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	30 // WaterwayArea is an import job to import
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	31 // the waterway area in form of polygon geometries
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	32 // and attribute data from a WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	33 type WaterwayArea struct {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	34 // URL the GetCapabilities URL of the WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	35 URL string `json:"url"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	36 // FeatureType selects the feature type of the WFS service.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	37 FeatureType string `json:"feature-type"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	38 // SortBy works around misconfigured services to
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	39 // establish a sort order to get the features.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	40 SortBy string `json:"sort-by"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	41 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	42
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	43 // WAJobKind is the import queue type identifier.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	44 const WAJobKind JobKind = "wa"
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	45
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	46 type waJobCreator struct{}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	47
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	48 func init() {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	49 RegisterJobCreator(WAJobKind, waJobCreator{})
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	50 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	51
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	52 func (waJobCreator) Description() string { return "waterway area" }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	53
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	54 func (waJobCreator) AutoAccept() bool { return true }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	55
2168 b868cb653c4d Import queue: The job kind parameter when creating a new job from a string was never used. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2109 diff changeset	56 func (waJobCreator) Create(data string) (Job, error) {
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	57 wa := new(WaterwayArea)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	58 if err := common.FromJSONString(data, wa); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	59 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	60 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	61 return wa, nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	62 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	63
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	64 func (waJobCreator) Depends() []string {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	65 return []string{
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	66 "waterway_area",
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	67 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	68 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	69
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	70 // StageDone is a NOP for waterway area imports.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	71 func (waJobCreator) StageDone(context.Context, *sql.Tx, int64) error {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	72 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	73 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	74
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	75 // CleanUp for waterway area imports is a NOP.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	76 func (*WaterwayArea) CleanUp() error { return nil }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	77
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	78 type waterwayAreaProperties struct {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	79 Catccl *string `json:"ienc_catccl"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	80 Dirimp *string `json:"ienc_dirimp"`
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	81 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	82
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	83 const (
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	84 deleteWaterwayAreaSQL = `
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	85 WITH resp AS (
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	86 SELECT best_utm(area::geometry) AS t,
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	87 ST_Transform(area::geometry, best_utm(area::geometry)) AS a
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	88 FROM users.responsibility_areas
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	89 WHERE country = users.current_user_country()
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	90 )
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	91 DELETE FROM waterway.waterway_area
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	92 WHERE ST_Covers(
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	93 (SELECT a FROM resp),
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	94 ST_Transform(area::geometry, (SELECT t FROM resp)))
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	95 `
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	96 insertWaterwayAreaSQL = `
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	97 WITH resp AS (
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	98 SELECT best_utm(area::geometry) AS t,
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	99 ST_Transform(area::geometry, best_utm(area::geometry)) AS a
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	100 FROM users.responsibility_areas
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	101 WHERE country = users.current_user_country()
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	102 )
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	103 INSERT INTO waterway.waterway_area (area, catccl, dirimp)
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	104 SELECT ST_Transform(clipped.geom, 4326)::geography, $3, $4 FROM (
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	105 SELECT (ST_Dump(
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	106 ST_Intersection(
1899 272133cd65da Workarround for failing waterway_area imports. Raimund Renkert <raimund.renkert@intevation.de> parents: 1828 diff changeset	107 (SELECT ST_Buffer(a, 0.0001) FROM resp),
272133cd65da Workarround for failing waterway_area imports. Raimund Renkert <raimund.renkert@intevation.de> parents: 1828 diff changeset	108 ST_CollectionExtract(ST_MakeValid(ST_Transform(
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	109 ST_GeomFromWKB($1, $2::integer),
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	110 (SELECT t FROM resp)
1899 272133cd65da Workarround for failing waterway_area imports. Raimund Renkert <raimund.renkert@intevation.de> parents: 1828 diff changeset	111 )),3)
1796 904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	112 )
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	113 )).geom AS geom
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	114 ) AS clipped
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	115 WHERE clipped.geom IS NOT NULL
904c7e853adc Waterway area import: Clip features against responsibility area. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1786 diff changeset	116 `
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	117 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	118
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	119 // Do executes the actual waterway axis import.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	120 func (wx *WaterwayArea) Do(
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	121 ctx context.Context,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	122 importID int64,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	123 conn *sql.Conn,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	124 feedback Feedback,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	125 ) (interface{}, error) {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	126
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	127 start := time.Now()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	128
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	129 feedback.Info("Import waterway area")
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	130
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	131 feedback.Info("Loading capabilities from %s", wx.URL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	132 caps, err := wfs.GetCapabilities(wx.URL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	133 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	134 feedback.Error("Loading capabilities failed: %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	135 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	136 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	137
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	138 ft := caps.FindFeatureType(wx.FeatureType)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	139 if ft == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	140 return nil, fmt.Errorf("Unknown feature type '%s'", wx.FeatureType)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	141 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	142
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	143 feedback.Info("Found feature type '%s", wx.FeatureType)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	144
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	145 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	146 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	147 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	148 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	149 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	150
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	151 if wx.SortBy != "" {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	152 feedback.Info("Features will be sorted by '%s'", wx.SortBy)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	153 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	154
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	155 urls, err := wfs.GetFeaturesGET(
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	156 caps, wx.FeatureType, "application/json", wx.SortBy)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	157 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	158 feedback.Error("Cannot create GetFeature URLs. %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	159 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	160 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	161
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	162 tx, err := conn.BeginTx(ctx, nil)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	163 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	164 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	165 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	166 defer tx.Rollback()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	167
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	168 insertStmt, err := tx.PrepareContext(ctx, insertWaterwayAreaSQL)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	169 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	170 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	171 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	172 defer insertStmt.Close()
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	173
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	174 // Delete the old features.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	175 if _, err := tx.ExecContext(ctx, deleteWaterwayAreaSQL); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	176 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	177 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	178
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	179 var (
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	180 unsupported = stringCounter{}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	181 missingProperties int
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	182 badProperties int
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	183 features int
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	184 )
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	185
2106 2b72f5e005aa WFS imports: Write get GetFeature URLs into import log. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1899 diff changeset	186 if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1899 diff changeset	187 feedback.Info("Get features from: '%s'", url)
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	188 rfc, err := wfs.ParseRawFeatureCollection(r)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	189 if err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	190 return fmt.Errorf("parsing GetFeature document failed: %v", err)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	191 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	192 if rfc.CRS != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	193 crsName := rfc.CRS.Properties.Name
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	194 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	195 feedback.Error("Unsupported CRS: %d", crsName)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	196 return err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	197 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	198 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	199
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	200 // No features -> ignore.
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	201 if rfc.Features == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	202 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	203 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	204
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	205 feedback.Info("Using EPSG: %d", epsg)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	206
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	207 for _, feature := range rfc.Features {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	208 if feature.Properties == nil \|\| feature.Geometry.Coordinates == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	209 missingProperties++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	210 continue
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	211 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	212
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	213 var props waterwayAreaProperties
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	214
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	215 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	216 badProperties++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	217 continue
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	218 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	219
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	220 var catccl sql.NullInt64
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	221 if props.Catccl != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	222 if value, err := strconv.ParseInt(*props.Catccl, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	223 catccl = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	224 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	225 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	226 var dirimp sql.NullInt64
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	227 if props.Dirimp != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	228 if value, err := strconv.ParseInt(*props.Dirimp, 10, 64); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	229 dirimp = sql.NullInt64{Int64: value, Valid: true}
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	230 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	231 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	232
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	233 switch feature.Geometry.Type {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	234 case "Polygon":
1786 09349ca27dd7 Imports: Removed duplicated code path to store WKB polygons into database. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1785 diff changeset	235 var p polygonSlice
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	236 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	237 return err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	238 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	239 if _, err := insertStmt.ExecContext(
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	240 ctx,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	241 p.asWKB(),
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	242 epsg,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	243 catccl,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	244 dirimp,
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	245 ); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	246 return err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	247 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	248 features++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	249 default:
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	250 unsupported[feature.Geometry.Type]++
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	251 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	252 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	253 return nil
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	254 }); err != nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	255 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	256 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	257
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	258 if badProperties > 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	259 feedback.Warn("Bad properties: %d", badProperties)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	260 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	261
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	262 if missingProperties > 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	263 feedback.Warn("Missing properties: %d", missingProperties)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	264 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	265
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	266 if len(unsupported) != 0 {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	267 feedback.Warn("Unsupported types found: %s", unsupported)
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	268 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	269
1828 1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	270 if features == 0 {
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	271 err := errors.New("No features found")
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	272 feedback.Error("%v", err)
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	273 return nil, err
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	274 }
1ecfcf46e4da Waterway area/axis: Be more informative if features are bad. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1796 diff changeset	275
1785 614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	276 if err = tx.Commit(); err == nil {
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	277 feedback.Info("Storing %d features took %s",
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	278 features, time.Since(start))
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	279 }
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	280
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	281 return nil, err
614c6c766691 Waterway area import: Implemented. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	282 }

Mercurial > gemma

annotate pkg/imports/wa.go @ 2168:b868cb653c4d