Mercurial > gemma
changeset 1841:491f5b68da9e
Implemented fairway dimension import.
author | Raimund Renkert <raimund.renkert@intevation.de> |
---|---|
date | Thu, 17 Jan 2019 09:22:17 +0100 |
parents | 00d63eb9306a |
children | 56357561938f |
files | pkg/controllers/manualimports.go pkg/controllers/routes.go pkg/imports/fd.go pkg/models/fd.go |
diffstat | 4 files changed, 352 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/controllers/manualimports.go Wed Jan 16 23:54:26 2019 +0100 +++ b/pkg/controllers/manualimports.go Thu Jan 17 09:22:17 2019 +0100 @@ -102,6 +102,22 @@ return wg, due, retries, wgi.SendEmail } +func importFairwayDimension(input interface{}) (interface{}, time.Time, int, bool) { + fdi := input.(*models.FairwayDimensionImport) + fd := &imports.FairwayDimension{ + URL: fdi.URL, + FeatureType: fdi.FeatureType, + SortBy: fdi.SortBy, + LOS: fdi.LOS, + MinWidth: fdi.MinWidth, + MaxWidth: fdi.MaxWidth, + Depth: fdi.Depth, + SourceOrganization: fdi.SourceOrganization, + } + due, retries := retry(fdi.Attributes) + return fd, due, retries, fdi.SendEmail +} + func manualImport( kind imports.JobKind, setup func(interface{}) (interface{}, time.Time, int, bool),
--- a/pkg/controllers/routes.go Wed Jan 16 23:54:26 2019 +0100 +++ b/pkg/controllers/routes.go Thu Jan 17 09:22:17 2019 +0100 @@ -211,6 +211,12 @@ NoConn: true, })).Methods(http.MethodPost) + api.Handle("/imports/fairwaydimension", waterwayAdmin(&JSONHandler{ + Input: func() interface{} { return new(models.FairwayDimensionImport) }, + Handle: manualImport(imports.FDJobKind, importFairwayDimension), + NoConn: true, + })).Methods(http.MethodPost) + // Import scheduler configuration api.Handle("/imports/config/{id:[0-9]+}/run", waterwayAdmin(&JSONHandler{
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fd.go Thu Jan 17 09:22:17 2019 +0100 @@ -0,0 +1,287 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Raimund Renkert <raimund.renkert@intevation.de> + +package imports + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "gemma.intevation.de/gemma/pkg/common" + "gemma.intevation.de/gemma/pkg/wfs" +) + +// FairwayDimension is an import job to import +// the fairway dimensions in form of polygon geometries +// and attribute data from a WFS service. +type FairwayDimension struct { + // URL the GetCapabilities URL of the WFS service. + URL string `json:"url"` + // FeatureType selects the feature type of the WFS service. + FeatureType string `json:"feature-type"` + SortBy string `json:"sort-by"` + LOS int `json:"los"` + MinWidth int `json:"min-width"` + MaxWidth int `json:"max-width"` + Depth int `json:"depth"` + SourceOrganization string `json:"source-organization"` +} + +type fdTime struct{ time.Time } + +func (fdt *fdTime) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + t, err := time.Parse("20060102", s) + if err == nil { + *fdt = fdTime{t} + } + return err +} + +// FDJobKind is the import queue type identifier. +const FDJobKind JobKind = "fd" + +type fdJobCreator struct{} + +func init() { + RegisterJobCreator(FDJobKind, fdJobCreator{}) +} + +func (fdJobCreator) Description() string { return "fairway dimension" } + +func (fdJobCreator) AutoAccept() bool { return true } + +func (fdJobCreator) Create(_ JobKind, data string) (Job, error) { + fd := new(FairwayDimension) + if err := common.FromJSONString(data, fd); err != nil { + return nil, err + } + return fd, nil +} + +func (fdJobCreator) Depends() []string { + return []string{ + "fairway_dimensions", + } +} + +// StageDone is a NOP for fairway dimensions imports. +func (fdJobCreator) StageDone(context.Context, *sql.Tx, int64) error { + return nil +} + +// CleanUp for fairway dimension imports is a NOP. +func (*FairwayDimension) CleanUp() error { return nil } + +type fairwayDimensionProperties struct { + HydroSorDat fdTime `json:"hydro_sordat"` +} + +const ( + deleteFairwayDimensionSQL = ` +WITH resp AS ( + SELECT best_utm(area::geometry) AS t, + ST_Transform(area::geometry, best_utm(area::geometry)) AS a + FROM users.responsibility_areas + WHERE country = users.current_user_country() +) +DELETE FROM waterway.fairway_dimensions +WHERE ST_Covers( + (SELECT a FROM resp), + ST_Transform(area::geometry, (SELECT t FROM resp))) +` + + // The ST_MakeValid (line125) and ST_Buffer (line124) are a workarround to + // avoid errors due to reprojection. + insertFairwayDimensionSQL = ` +WITH resp AS ( + SELECT best_utm(area::geometry) AS t, + ST_Transform(area::geometry, best_utm(area::geometry)) AS a + FROM users.responsibility_areas + WHERE country = users.current_user_country() +) +INSERT INTO waterway.fairway_dimensions (area, level_of_service, min_width, max_width, min_depth, date_info, source_organization) +SELECT ST_Transform(clipped.geom, 4326)::geography, $3, $4, $5, $6, $7, $8 FROM ( + SELECT (ST_Dump( + ST_Intersection( + (SELECT ST_Buffer(a, -0.0001) FROM resp), + ST_MakeValid(ST_Transform( + ST_GeomFromWKB($1, $2::integer), + (SELECT t FROM resp) + )) + ) + )).geom AS geom + ) AS clipped + WHERE clipped.geom IS NOT NULL +` +) + +// Do executes the actual fairway dimension import. +func (fd *FairwayDimension) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + start := time.Now() + + feedback.Info("Import fairway dimension") + + feedback.Info("Loading capabilities from %s", fd.URL) + caps, err := wfs.GetCapabilities(fd.URL) + if err != nil { + feedback.Error("Loading capabilities failed: %v", err) + return nil, err + } + + ft := caps.FindFeatureType(fd.FeatureType) + if ft == nil { + return nil, fmt.Errorf("Unknown feature type '%s'", fd.FeatureType) + } + + feedback.Info("Found feature type '%s'", fd.FeatureType) + + epsg, err := wfs.CRSToEPSG(ft.DefaultCRS) + if err != nil { + feedback.Error("Unsupported CRS name '%s'", ft.DefaultCRS) + return nil, err + } + + urls, err := wfs.GetFeaturesGET( + caps, fd.FeatureType, "application/json", fd.SortBy) + if err != nil { + feedback.Error("Cannot create GetFeature URLs. %v", err) + return nil, err + } + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + insertStmt, err := tx.PrepareContext(ctx, insertFairwayDimensionSQL) + if err != nil { + return nil, err + } + defer insertStmt.Close() + + // Delete the old features. + if _, err := tx.ExecContext(ctx, deleteFairwayDimensionSQL); err != nil { + return nil, err + } + + var ( + unsupported = stringCounter{} + missingProperties int + badProperties int + features int + ) + + if err := wfs.DownloadURLs(urls, func(r io.Reader) error { + rfc, err := wfs.ParseRawFeatureCollection(r) + if err != nil { + return fmt.Errorf("parsing GetFeature document failed: %v", err) + } + if rfc.CRS != nil { + crsName := rfc.CRS.Properties.Name + if epsg, err = wfs.CRSToEPSG(crsName); err != nil { + feedback.Error("Unsupported CRS: %d", crsName) + return err + } + } + + // No features -> ignore. + if rfc.Features == nil { + return nil + } + + feedback.Info("Using EPSG: %d", epsg) + + for _, feature := range rfc.Features { + if feature.Geometry.Coordinates == nil { + missingProperties++ + continue + } + + var props fairwayDimensionProperties + + if err := json.Unmarshal(*feature.Properties, &props); err != nil { + badProperties++ + continue + } + switch feature.Geometry.Type { + case "Polygon": + var p polygonSlice + if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil { + return err + } + if _, err := insertStmt.ExecContext( + ctx, + p.asWKB(), + epsg, + fd.LOS, + fd.MinWidth, + fd.MaxWidth, + fd.Depth, + props.HydroSorDat.Time, + fd.SourceOrganization, + ); err != nil { + feedback.Error("error: %s", err) + return err + } + features++ + default: + unsupported[feature.Geometry.Type]++ + } + } + return nil + }); err != nil { + feedback.Error("Downloading features failed: %v", err) + return nil, err + } + + if badProperties > 0 { + feedback.Warn("Bad properties: %d", badProperties) + } + + if missingProperties > 0 { + feedback.Warn("Missing properties: %d", missingProperties) + } + + if len(unsupported) != 0 { + feedback.Warn("Unsupported types found: %s", unsupported) + } + + if features == 0 { + err := errors.New("No features found") + feedback.Error("%v", err) + return nil, err + } + + if err = tx.Commit(); err == nil { + feedback.Info("Storing %d features took %s", + features, time.Since(start)) + } + + return nil, err +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/models/fd.go Thu Jan 17 09:22:17 2019 +0100 @@ -0,0 +1,43 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Raimund Renkert <raimund.renkert@intevation.de> + +package models + +import "gemma.intevation.de/gemma/pkg/common" + +type ( + // FairwayDimensionImport specifies an import of the waterway axis. + FairwayDimensionImport struct { + // URL is the capabilities URL of the WFS. + URL string `json:"url"` + // FeatureType is the layer to use. + FeatureType string `json:"feature-type"` + // SortBy sorts the feature by this key. + SortBy string `json:"sort-by"` + // SendEmail is set to true if an email should be send after + // importing the axis. + SendEmail bool `json:"send-email"` + // LOS is the level of service provided by the wfs + LOS int `json:"los"` + // MinWidth is the minimum width of the fairway for the specified LOS + MinWidth int `json:"min-width"` + // MaxWidth is the maximum width of the fairway for the specified LOS + MaxWidth int `json:"max-width"` + // Depth is the minimum depth of the fairway for the specified LOS + Depth int `json:"depth"` + // SourceOrganization specifies the source of the entry + SourceOrganization string `json:"source-organization"` + // Attributes are optional attributes. + Attributes common.Attributes `json:"attributes,omitempty"` + } +)