Mercurial > gemma

annotate pkg/imports/fd.go @ 3055:b3c24c47931c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
client: layers legend: support multiple maps If there is more than one map, there's a layer checkbox for each of them, so layers can be toggle for each map separately.
author Markus Kottlaender <markus@intevation.de>
date Tue, 16 Apr 2019 09:26:51 +0200
parents 9bd98daf0b0f
children 4acbee65275d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
3 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
6 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2018 by via donau
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
10 //
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
11 // Author(s):
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
12 // * Raimund Renkert <raimund.renkert@intevation.de>
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
13
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
14 package imports
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
15
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
16 import (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
17 "context"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
18 "database/sql"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
19 "encoding/json"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
20 "fmt"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
21 "io"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
22 "time"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
23
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
24 "gemma.intevation.de/gemma/pkg/misc"
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/wfs"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
26 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
27
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
28 // FairwayDimension is an import job to import
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
29 // the fairway dimensions in form of polygon geometries
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
30 // and attribute data from a WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
31 type FairwayDimension struct {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
32 // URL the GetCapabilities URL of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
33 URL string `json:"url"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
34 // FeatureType selects the feature type of the WFS service.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
35 FeatureType string `json:"feature-type"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
36 SortBy string `json:"sort-by"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
37 LOS int `json:"los"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
38 MinWidth int `json:"min-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
39 MaxWidth int `json:"max-width"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
40 Depth int `json:"depth"`
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
41 SourceOrganization string `json:"source-organization"`
2723
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
42 // User is an optional username for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
43 User string `json:"user,omitempty"`
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
44 // Password is an optional password for Basic Auth.
a10022399e24 WFS downloads: Fetch user and password from config.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2721
diff changeset
45 Password string `json:"password,omitempty"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
46 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
47
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
48 type fdTime struct{ time.Time }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
49
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
50 var guessFDTime = misc.TimeGuesser([]string{
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
51 "20060102",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
52 "2006",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
53 "",
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
54 }).Guess
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
55
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
56 func (fdt *fdTime) UnmarshalJSON(data []byte) error {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
57 var s string
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
58 if err := json.Unmarshal(data, &s); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
59 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
60 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
61 t, err := guessFDTime(s)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
62 if err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
63 *fdt = fdTime{t}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
64 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
65 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
66 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
67
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
68 // FDJobKind is the import queue type identifier.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
69 const FDJobKind JobKind = "fd"
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
70
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
71 type fdJobCreator struct{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
72
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
73 func init() {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
74 RegisterJobCreator(FDJobKind, fdJobCreator{})
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
75 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
76
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
77 func (fdJobCreator) Description() string { return "fairway dimension" }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
78
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
79 func (fdJobCreator) AutoAccept() bool { return false }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
80
2187
7c83b5277c1c Import queue: Removed boilerplate code to deserialize jobs from JSON by making it part of the import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2171
diff changeset
81 func (fdJobCreator) Create() Job { return new(FairwayDimension) }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
82
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
83 func (fdJobCreator) Depends() []string {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
84 return []string{
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
85 "fairway_dimensions",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
86 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
87 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
88
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
89 func (fdJobCreator) StageDone(
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
90 ctx context.Context,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
91 tx *sql.Tx,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
92 id int64,
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
93 ) error {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
94 // We only want to delete the features of the same LOS
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
95 // as the imported.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
96 var los int64
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
97 if err := tx.QueryRowContext(ctx, fdFindLOSSQL, id).Scan(&los); err != nil {
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
98 // Should always return a row because import will exit as unchanged
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
99 // if no new features were found while inserting.
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
100 return err
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
101 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
102 // Delete the old features.
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
103 if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL, los); err != nil {
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
104 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
105 }
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
106
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
107 _, err := tx.ExecContext(ctx, fdStageDoneSQL, id)
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
108 return err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
109 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
110
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
111 // CleanUp for fairway dimension imports is a NOP.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
112 func (*FairwayDimension) CleanUp() error { return nil }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
113
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
114 type fairwayDimensionProperties struct {
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
115 HydroSorDat *fdTime `json:"hydro_sordat"`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
116 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
117
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
118 type fdSummary struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
119 Lat float64 `json:"lat"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
120 Lon float64 `json:"lon"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
121 ID int64 `json:"id"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
122 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
123
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
124 const (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
125 fdFindLOSSQL = `
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
126 SELECT level_of_service FROM waterway.fairway_dimensions
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
127 WHERE id IN (
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
128 SELECT key FROM import.track_imports
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
129 WHERE import_id = $1 AND
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
130 relation = 'waterway.fairway_dimensions'::regclass)
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
131 LIMIT 1`
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
132
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
133 fdStageDoneSQL = `
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
134 UPDATE waterway.fairway_dimensions SET staging_done = true
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
135 WHERE id IN (
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
136 SELECT key FROM import.track_imports
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
137 WHERE import_id = $1 AND
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
138 relation = 'waterway.fairway_dimensions'::regclass)`
1890
b1b0db195cc5 Put fairway dimension import to staging area.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1841
diff changeset
139
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
140 deleteFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
141 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
142 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
143 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
144 DELETE FROM waterway.fairway_dimensions
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
145 WHERE (pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
146 OR ST_Covers((SELECT a FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
147 ST_Transform(area::geometry, (SELECT ST_SRID(a) FROM resp))))
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
148 AND staging_done
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
149 AND level_of_service = $1::smallint`
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
150
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
151 // The ST_MakeValid and ST_Buffer below are a workarround to
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
152 // avoid errors due to reprojection.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
153 insertFairwayDimensionSQL = `
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
154 WITH resp AS (
2912
93fa55bce126 Add utility function to get users area of responsibility
Tom Gottfried <tom@intevation.de>
parents: 2833
diff changeset
155 SELECT users.current_user_area_utm() AS a
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
156 )
2923
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
157 INSERT INTO waterway.fairway_dimensions (
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
158 area,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
159 level_of_service,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
160 min_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
161 max_width,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
162 min_depth,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
163 date_info,
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
164 source_organization)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
165 SELECT ST_Transform(dmp.geom, 4326), $3, $4, $5, $6, $7, $8
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
166 FROM ST_GeomFromWKB($1, $2::integer) AS new_fd (new_fd),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
167 ST_Dump(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
168 CASE WHEN pg_has_role('sys_admin', 'MEMBER')
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
169 THEN ST_MakeValid(ST_Transform(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
170 new_fd, best_utm(ST_Transform(new_fd, 4326))))
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
171 ELSE ST_CollectionExtract(ST_Intersection(
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
172 (SELECT ST_Buffer(a, -0.0001) FROM resp),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
173 ST_MakeValid(ST_Transform(new_fd, (SELECT ST_SRID(a) FROM resp)))),
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
174 3)
9bd98daf0b0f Let sys_admin import fairway dimensions without restrictions
Tom Gottfried <tom@intevation.de>
parents: 2922
diff changeset
175 END) AS dmp
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
176 RETURNING id,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
177 ST_X(ST_Centroid(area::geometry)),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
178 ST_Y(ST_Centroid(area::geometry))
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
179 `
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
180 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
181
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
182 // Do executes the actual fairway dimension import.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
183 func (fd *FairwayDimension) Do(
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
184 ctx context.Context,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
185 importID int64,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
186 conn *sql.Conn,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
187 feedback Feedback,
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
188 ) (interface{}, error) {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
189
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
190 start := time.Now()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
191
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
192 feedback.Info("Import fairway dimensions")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
193
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
194 feedback.Info("Loading capabilities from %s", fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
195 caps, err := wfs.GetCapabilities(fd.URL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
196 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
197 feedback.Error("Loading capabilities failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
198 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
199 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
200
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
201 ft := caps.FindFeatureType(fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
202 if ft == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
203 return nil, fmt.Errorf("Unknown feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
204 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
205
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
206 feedback.Info("Found feature type '%s'", fd.FeatureType)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
207
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
208 epsg, err := wfs.CRSToEPSG(ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
209 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
210 feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
211 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
212 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
213
2721
56c8ef515d93 WFS downloader: Use new GML/GeoJSON factory function in gemma.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2719
diff changeset
214 dl, err := wfs.GetFeatures(caps, fd.FeatureType, fd.SortBy)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
215 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
216 feedback.Error("Cannot create GetFeature URLs. %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
217 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
218 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
219
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
220 tx, err := conn.BeginTx(ctx, nil)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
221 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
222 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
223 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
224 defer tx.Rollback()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
225
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
226 insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
227 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
228 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
229 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
230 defer insertStmt.Close()
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
231
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
232 var (
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
233 unsupported = stringCounter{}
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
234 missingProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
235 badProperties int
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
236 features int
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
237 outside int
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
238 fds []fdSummary
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
239 )
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
240
2729
7cb027be277d WFS downloader: Really use username and password from parameters.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2723
diff changeset
241 if err := dl.Download(fd.User, fd.Password, func(url string, r io.Reader) error {
2106
2b72f5e005aa WFS imports: Write get GetFeature URLs into import log.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1995
diff changeset
242 feedback.Info("Get features from: '%s'", url)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
243 rfc, err := wfs.ParseRawFeatureCollection(r)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
244 if err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
245 return fmt.Errorf("parsing GetFeature document failed: %v", err)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
246 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
247 if rfc.CRS != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
248 crsName := rfc.CRS.Properties.Name
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
249 if epsg, err = wfs.CRSToEPSG(crsName); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
250 feedback.Error("Unsupported CRS: %d", crsName)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
251 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
252 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
253 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
254
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
255 // No features -> ignore.
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
256 if rfc.Features == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
257 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
258 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
259
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
260 feedback.Info("Using EPSG: %d", epsg)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
261
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
262 savepoint := Savepoint(ctx, tx, "feature")
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
263
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
264 features:
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
265 for _, feature := range rfc.Features {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
266 if feature.Geometry.Coordinates == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
267 missingProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
268 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
269 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
270
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
271 var props fairwayDimensionProperties
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
272
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
273 if err := json.Unmarshal(*feature.Properties, &props); err != nil {
1941
d4249b07e719 Fairway dimension: log bad properties.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1940
diff changeset
274 feedback.Warn("bad property: %v", err)
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
275 badProperties++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
276 continue
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
277 }
1940
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
278 var dateInfo time.Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
279 if props.HydroSorDat == nil || props.HydroSorDat.IsZero() {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
280 dateInfo = start
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
281 } else {
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
282 dateInfo = (*props.HydroSorDat).Time
e6d851f53c99 Fairway dimension import: Made date parsing more tolerant.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1934
diff changeset
283 }
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
284 switch feature.Geometry.Type {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
285 case "Polygon":
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
286 var p polygonSlice
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
287 if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
288 return err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
289 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
290 var fdid int64
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
291 var lat, lon float64
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
292 err = savepoint(func() error {
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
293 err := insertStmt.QueryRowContext(
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
294 ctx,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
295 p.asWKB(),
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
296 epsg,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
297 fd.LOS,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
298 fd.MinWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
299 fd.MaxWidth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
300 fd.Depth,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
301 dateInfo,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
302 fd.SourceOrganization,
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
303 ).Scan(&fdid, &lat, &lon)
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
304 return err
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
305 })
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
306 switch {
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
307 case err == sql.ErrNoRows:
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
308 outside++
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
309 // ignore -> filtered by responsibility_areas
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
310 continue features
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
311 case err != nil:
2922
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
312 feedback.Warn(handleError(err).Error())
372dc8382389 Fairway dimension import: continue if a feature fails to be inserted
Tom Gottfried <tom@intevation.de>
parents: 2912
diff changeset
313 continue features
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
314 }
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
315 // Store for potential later removal.
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
316 if err = track(ctx, tx, importID, "waterway.fairway_dimensions", fdid); err != nil {
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
317 return err
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
318 }
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
319 fds = append(fds, fdSummary{ID: fdid, Lat: lat, Lon: lon})
1893
9beff970b107 Track fairway dimension import and delete old data on staging done.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1890
diff changeset
320
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
321 features++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
322 default:
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
323 unsupported[feature.Geometry.Type]++
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
324 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
325 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
326 return nil
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
327 }); err != nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
328 return nil, err
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
329 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
330
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
331 if badProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
332 feedback.Warn("Bad properties: %d", badProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
333 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
334
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
335 if missingProperties > 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
336 feedback.Warn("Missing properties: %d", missingProperties)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
337 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
338
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
339 if len(unsupported) != 0 {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
340 feedback.Warn("Unsupported types found: %s", unsupported)
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
341 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
342
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
343 if outside > 0 {
1994
a7c4005b723f Improve some import log messages
Tom Gottfried <tom@intevation.de>
parents: 1941
diff changeset
344 feedback.Info("Features outside responsibility area: %d", outside)
1934
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
345 }
ec7360afd5e1 Fairway dimension import: Dont break if there are features outside the responsibility areas.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1898
diff changeset
346
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
347 if features == 0 {
2171
543b5d071a69 Fairway dimension imports: Only delete features with the same level of services as the imported.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 2168
diff changeset
348 return nil, UnchangedError("No features found")
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
349 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
350
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
351 if err = tx.Commit(); err == nil {
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
352 feedback.Info("Storing %d features took %s",
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
353 features, time.Since(start))
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
354 }
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
355
1898
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
356 summary := struct {
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
357 Date time.Time `json:"date"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
358 LOS int `json:"los"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
359 SourceOrganization string `json:"source-organization"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
360 FdArea []fdSummary `json:"fd-area"`
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
361 }{
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
362 Date: time.Now(),
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
363 LOS: fd.LOS,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
364 SourceOrganization: fd.SourceOrganization,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
365 FdArea: fds,
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
366 }
8a56a64e4410 Summarize the result of fairway dimension import for review.
Raimund Renkert <raimund.renkert@intevation.de>
parents: 1893
diff changeset
367 return &summary, err
1841
491f5b68da9e Implemented fairway dimension import.
Raimund Renkert <raimund.renkert@intevation.de>
parents:
diff changeset
368 }