gemma: pkg/imports/fd.go annotate

annotate pkg/imports/fd.go @ 5026:d02f0c5c60b3

Simplify deletion of fairway dimensions Remove filters that duplicate row level security policies and avoid an extra round-trip to fetch LOS.

author	Tom Gottfried <tom@intevation.de>
date	Wed, 18 Mar 2020 13:04:42 +0100
parents	36a3dce20232
children	737d7859dd86

rev	line source
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	3 //
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	6 //
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	7 // Copyright (C) 2018 by via donau
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	10 //
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	11 // Author(s):
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	13
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	14 package imports
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	15
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	16 import (
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	17 "context"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	18 "database/sql"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	19 "encoding/json"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	20 "fmt"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	21 "io"
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4389 diff changeset	22 "strings"
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	23 "time"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	24
4075 cb74aa69954e Moved TimeParser to common package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4074 diff changeset	25 "gemma.intevation.de/gemma/pkg/common"
4062 6c760abcff0e Move handling of PostgreSQL errors to own package Tom Gottfried <tom@intevation.de> parents: 4058 diff changeset	26 "gemma.intevation.de/gemma/pkg/pgxutils"
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	27 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	28 )
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	29
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	30 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	31 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	32 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	33 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	34 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	35 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	36 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	37 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	38 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	39 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	40 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	41 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	42 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	43 SourceOrganization string `json:"source-organization"`
2723 a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	44 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	45 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	46 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2721 diff changeset	47 Password string `json:"password,omitempty"`
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	48 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	49
4852 046a07a33b19 Fixed the golint issues of the imports package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4832 diff changeset	50 // Description gives a short info about relevant facts of this import.
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4389 diff changeset	51 func (fd *FairwayDimension) Description() (string, error) {
4799 f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	52 return strings.Join([]string{
f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	53 fd.URL,
f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	54 fd.FeatureType,
f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	55 fmt.Sprintf("LOS%d", fd.LOS),
f32d086b5dbf Removed the mechanical touch of the last commit. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4798 diff changeset	56 }, "\|"), nil
4798 ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4389 diff changeset	57 }
ca6a5f722471 Added Description method to most imports. Sascha Wilde <wilde@intevation.de> parents: 4389 diff changeset	58
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	59 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	60
4075 cb74aa69954e Moved TimeParser to common package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4074 diff changeset	61 var guessFDTime = common.TimeParser([]string{
1940 e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	62 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	63 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	64 "",
4074 eb2f949ddfa2 Rename TimeGuesser to TimeParser. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4071 diff changeset	65 }).Parse
1940 e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	66
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	67 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	68 var s string
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	69 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	70 return err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	71 }
1940 e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	72 t, err := guessFDTime(s)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	73 if err == nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	74 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	75 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	76 return err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	77 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	78
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	79 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	80 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	81
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	82 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	83
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	84 func init() {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	85 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	86 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	87
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	88 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	89
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	90 func (fdJobCreator) AutoAccept() bool { return false }
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	91
2187 7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2171 diff changeset	92 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	93
3219 4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2923 diff changeset	94 func (fdJobCreator) Depends() [2][]string {
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2923 diff changeset	95 return [2][]string{
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2923 diff changeset	96 {"fairway_dimensions"},
4acbee65275d Import queue: Split locked dependencies in exclusively and multiple uses. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2923 diff changeset	97 {"level_of_service"},
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	98 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	99 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	100
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	101 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	102 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	103 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	104 id int64,
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	105 ) error {
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	106 // Delete the old features.
5026 d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	107 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, id); err != nil {
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	108 return err
9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	109 }
9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	110
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	111 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	112 return err
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	113 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	114
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	115 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	116 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	117
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	118 type fairwayDimensionProperties struct {
1940 e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	119 HydroSorDat *fdTime `json:"hydro_sordat"`
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	120 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	121
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	122 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	123 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	124 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	125 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	126 }
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	127
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	128 const (
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	129 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	130 UPDATE waterway.fairway_dimensions SET staging_done = true
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	131 WHERE id IN (
2171 543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	132 SELECT key FROM import.track_imports
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	133 WHERE import_id = $1 AND
2171 543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	134 relation = 'waterway.fairway_dimensions'::regclass)`
1890 b1b0db195cc5 Put fairway dimension import to staging area. Raimund Renkert <raimund.renkert@intevation.de> parents: 1841 diff changeset	135
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	136 deleteFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	137 DELETE FROM waterway.fairway_dimensions
5026 d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	138 WHERE staging_done
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	139 AND level_of_service = (
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	140 SELECT DISTINCT level_of_service FROM waterway.fairway_dimensions
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	141 WHERE id IN (
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	142 SELECT key FROM import.track_imports
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	143 WHERE import_id = $1
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	144 AND relation = 'waterway.fairway_dimensions'::regclass))
d02f0c5c60b3 Simplify deletion of fairway dimensions Tom Gottfried <tom@intevation.de> parents: 5024 diff changeset	145 `
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	146
2171 543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	147 // The ST_MakeValid and ST_Buffer below are a workarround to
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	148 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	149 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	150 WITH resp AS (
2912 93fa55bce126 Add utility function to get users area of responsibility Tom Gottfried <tom@intevation.de> parents: 2833 diff changeset	151 SELECT users.current_user_area_utm() AS a
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	152 )
2923 9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	153 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	154 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	155 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	156 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	157 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	158 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	159 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	160 source_organization)
5023 776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	161 SELECT dmp.geom, $3, $4, $5, $6, $7, $8
2923 9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	162 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd),
5023 776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	163 ST_Dump(ST_CollectionExtract(ST_MakeValid(ST_Transform(
2923 9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	164 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
5023 776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	165 THEN new_fd
776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	166 ELSE ST_Intersection(
2923 9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions Tom Gottfried <tom@intevation.de> parents: 2922 diff changeset	167 (SELECT ST_Buffer(a, -0.0001) FROM resp),
5023 776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	168 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp))))
776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	169 END,
776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	170 4326)), 3)) AS dmp
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	171 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	172 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	173 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	174 `
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	175 )
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	176
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	177 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	178 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	179 ctx context.Context,
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	180 importID int64,
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	181 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	182 feedback Feedback,
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	183 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	184
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	185 start := time.Now()
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	186
1994 a7c4005b723f Improve some import log messages Tom Gottfried <tom@intevation.de> parents: 1941 diff changeset	187 feedback.Info("Import fairway dimensions")
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	188
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	189 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	190 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	191 if err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	192 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	193 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	194 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	195
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	196 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	197 if ft == nil {
4177 8b75ac5e243e Made 'staticcheck' happy with pgxutils package. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 4147 diff changeset	198 return nil, fmt.Errorf("unknown feature type '%s'", fd.FeatureType)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	199 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	200
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	201 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	202
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	203 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	204 if err != nil {
4832 5062ccb2381d Unified error message on "Unsupported CRS" Sascha Wilde <wilde@intevation.de> parents: 4799 diff changeset	205 feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	206 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	207 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	208
2721 56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2719 diff changeset	209 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	210 if err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	211 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	212 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	213 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	214
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	215 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	216 if err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	217 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	218 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	219 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	220
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	221 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	222 if err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	223 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	224 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	225 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	226
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	227 var (
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	228 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	229 missingProperties int
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	230 badProperties int
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	231 features int
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	232 outside int
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	233 fds []fdSummary
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	234 )
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	235
2729 7cb027be277d WFS downloader: Really use username and password from parameters. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2723 diff changeset	236 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106 2b72f5e005aa WFS imports: Write get GetFeature URLs into import log. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1995 diff changeset	237 feedback.Info("Get features from: '%s'", url)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	238 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	239 if err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	240 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	241 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	242 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	243 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	244 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	245 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	246 return err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	247 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	248 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	249
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	250 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	251 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	252 return nil
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	253 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	254
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	255 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	256
5023 776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	257 feedback.Info(
776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	258 "Found %d features in data source", len(rfc.Features))
776db50c01f0 Fix repairing geometries Tom Gottfried <tom@intevation.de> parents: 4852 diff changeset	259
2922 372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	260 savepoint := Savepoint(ctx, tx, "feature")
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	261
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	262 features:
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	263 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	264 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	265 missingProperties++
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	266 continue
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	267 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	268
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	269 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	270
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	271 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941 d4249b07e719 Fairway dimension: log bad properties. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1940 diff changeset	272 feedback.Warn("bad property: %v", err)
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	273 badProperties++
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	274 continue
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	275 }
1940 e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	276 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	277 if props.HydroSorDat == nil \|\| props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	278 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	279 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	280 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1934 diff changeset	281 }
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	282 switch feature.Geometry.Type {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	283 case "Polygon":
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	284 var p polygonSlice
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	285 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	286 return err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	287 }
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	288 var fdid int64
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	289 var lat, lon float64
2922 372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	290 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	291 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	292 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	293 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	294 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	295 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	296 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	297 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	298 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	299 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	300 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	301 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	302 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	303 })
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	304 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	305 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	306 outside++
4389 5e38667f740c Use stretches as areas of responsibility. Sascha Wilde <wilde@intevation.de> parents: 4197 diff changeset	307 // ignore -> filtered by responsibility area (stretches)
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	308 continue features
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	309 case err != nil:
4197 5d7ce7f926eb Log fatal errors as errors, not warnings, in imports Tom Gottfried <tom@intevation.de> parents: 4177 diff changeset	310 feedback.Error(pgxutils.ReadableError{Err: err}.Error())
2922 372dc8382389 Fairway dimension import: continue if a feature fails to be inserted Tom Gottfried <tom@intevation.de> parents: 2912 diff changeset	311 continue features
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	312 }
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	313 // Store for potential later removal.
9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	314 if err = track(ctx, tx, importID, "waterway.fairway_dimensions", fdid); err != nil {
9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	315 return err
9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	316 }
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	317 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893 9beff970b107 Track fairway dimension import and delete old data on staging done. Raimund Renkert <raimund.renkert@intevation.de> parents: 1890 diff changeset	318
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	319 features++
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	320 default:
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	321 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	322 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	323 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	324 return nil
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	325 }); err != nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	326 return nil, err
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	327 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	328
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	329 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	330 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	331 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	332
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	333 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	334 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	335 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	336
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	337 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	338 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	339 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	340
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	341 if outside > 0 {
5024 36a3dce20232 Amend import log Tom Gottfried <tom@intevation.de> parents: 5023 diff changeset	342 feedback.Info(
36a3dce20232 Amend import log Tom Gottfried <tom@intevation.de> parents: 5023 diff changeset	343 "Features outside responsibility area or no valid polygon: %d",
36a3dce20232 Amend import log Tom Gottfried <tom@intevation.de> parents: 5023 diff changeset	344 outside)
1934 ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	345 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 1898 diff changeset	346
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	347 if features == 0 {
2171 543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 2168 diff changeset	348 return nil, UnchangedError("No features found")
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	349 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	350
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	351 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	352 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	353 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	354 }
491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	355
1898 8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	356 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	357 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	358 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	359 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	360 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	361 }{
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	362 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	363 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	364 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	365 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	366 }
8a56a64e4410 Summarize the result of fairway dimension import for review. Raimund Renkert <raimund.renkert@intevation.de> parents: 1893 diff changeset	367 return &summary, err
1841 491f5b68da9e Implemented fairway dimension import. Raimund Renkert <raimund.renkert@intevation.de> parents: diff changeset	368 }

Mercurial > gemma

annotate pkg/imports/fd.go @ 5026:d02f0c5c60b3