Mercurial > gemma
changeset 4913:8c1a3d5e3962 fairway-marks-import
Add import for fairway marks of type BOYCAR
author | Tom Gottfried <tom@intevation.de> |
---|---|
date | Mon, 10 Feb 2020 18:47:27 +0100 |
parents | bfd8ef836998 |
children | 5cdedfea740f |
files | pkg/controllers/routes.go pkg/imports/fm_boycar.go pkg/imports/modelconvert.go schema/gemma.sql |
diffstat | 4 files changed, 258 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/controllers/routes.go Mon Feb 10 18:11:52 2020 +0100 +++ b/pkg/controllers/routes.go Mon Feb 10 18:47:27 2020 +0100 @@ -244,6 +244,7 @@ "wg", "dmv", "fd", "dma", "sec", "dsec", "dst", "dsr", "fm_bcnlat", + "fm_boycar", }, "|") api.Handle("/imports/{kind:"+kinds+"}", waterwayAdmin(&mw.JSONHandler{
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fm_boycar.go Mon Feb 10 18:47:27 2020 +0100 @@ -0,0 +1,225 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2020 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried <tom.gottfried@intevation.de> + +package imports + +import ( + "context" + "database/sql" + "time" + + "gemma.intevation.de/gemma/pkg/pgxutils" +) + +// Boycar is an import job to import +// fairway marks of type BOYCAR in form of point geometries +// and attribute data from a WFS service. +type Boycar struct { + FairwayMarks +} + +// Description gives a short info about relevant facts of this import. +func (boycar *Boycar) Description() (string, error) { + return boycar.URL + "|" + boycar.FeatureType, nil +} + +// BOYCARJobKind is the import queue type identifier. +const BOYCARJobKind JobKind = "fm_boycar" + +type boycarJobCreator struct{} + +func init() { + RegisterJobCreator(BOYCARJobKind, boycarJobCreator{}) +} + +func (boycarJobCreator) Description() string { return "fairway marks boycar" } + +func (boycarJobCreator) AutoAccept() bool { return true } + +func (boycarJobCreator) Create() Job { return new(Boycar) } + +func (boycarJobCreator) Depends() [2][]string { + return [2][]string{ + {"fairway_marks_boycar"}, + {}, + } +} + +// StageDone is a NOP for fairway marks imports. +func (boycarJobCreator) StageDone(context.Context, *sql.Tx, int64) error { + return nil +} + +// CleanUp for fairway marks imports is a NOP. +func (*Boycar) CleanUp() error { return nil } + +type boycarProperties struct { + fairwayMarksProperties + Colour *string `json:"hydro_colour"` + Colpat *string `json:"hydro_colpat"` + Conrad *int `json:"hydro_conrad"` + Marsys *int `json:"hydro_marsys"` + Boyshp *int `json:"hydro_boyshp"` +} + +type boycarFeaturetype struct { + geom pointSlice + props *boycarProperties +} + +const ( + insertBOYCARSQL = ` +with a as ( + select users.current_user_area_utm() AS a +) +INSERT INTO waterway.fairway_marks_boycar ( + geom, + datsta, + datend, + persta, + perend, + objnam, + nobjnm, + inform, + ninfom, + scamin, + picrep, + txtdsc, + sordat, + sorind, + colour, + colpat, + conrad, + marsys, + boyshp +) +SELECT newfm, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20 + FROM ST_Transform(ST_GeomFromWKB($1, $2::integer), 4326) AS newfm (newfm) + WHERE pg_has_role('sys_admin', 'MEMBER') + OR ST_Intersects((select a from a), + ST_Transform(newfm, (select ST_SRID(a) from a))) +ON CONFLICT ( + CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp + ) AS waterway.fairway_marks_boycar) + ) + DO NOTHING +RETURNING id +` +) + +// Do executes the actual import. +func (fm *Boycar) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + start := time.Now() + + feedback.Info("Import fairway marks of type BOYCAR") + + fms, epsg, err := getFMFeatures( + feedback, + fm.FairwayMarks, + func() interface{} { return new(boycarProperties) }, + func(p pointSlice, props interface{}) interface{} { + return &boycarFeaturetype{p, props.(*boycarProperties)} + }, + ) + if err != nil { + return nil, err + } + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + insertStmt, err := tx.PrepareContext(ctx, insertBOYCARSQL) + if err != nil { + return nil, err + } + defer insertStmt.Close() + + savepoint := Savepoint(ctx, tx, "feature") + + var ( + outsideOrDup int + features int + ) + for _, fm := range fms { + + f := fm.(*boycarFeaturetype) + + var fmid int64 + err := savepoint(func() error { + err := insertStmt.QueryRowContext( + ctx, + f.geom.asWKB(), + epsg, + f.props.Datsta, + f.props.Datend, + f.props.Persta, + f.props.Perend, + f.props.Objnam, + f.props.Nobjnm, + f.props.Inform, + f.props.Ninfom, + f.props.Scamin, + f.props.Picrep, + f.props.Txtdsc, + f.props.Sordat, + f.props.Sorind, + f.props.Colour, + f.props.Colpat, + f.props.Conrad, + f.props.Marsys, + f.props.Boyshp, + ).Scan(&fmid) + return err + }) + switch { + case err == sql.ErrNoRows: + outsideOrDup++ + // ignore -> filtered by responsibility_areas + case err != nil: + feedback.Error(pgxutils.ReadableError{Err: err}.Error()) + default: + features++ + } + } + + if outsideOrDup > 0 { + feedback.Info( + "Features outside responsibility area and duplicates: %d", + outsideOrDup) + } + + if features == 0 { + err := UnchangedError("no valid new features found") + return nil, err + } + + if err = tx.Commit(); err == nil { + feedback.Info("Storing %d features took %s", + features, time.Since(start)) + } + + return nil, err +}
--- a/pkg/imports/modelconvert.go Mon Feb 10 18:11:52 2020 +0100 +++ b/pkg/imports/modelconvert.go Mon Feb 10 18:47:27 2020 +0100 @@ -28,6 +28,7 @@ FDJobKind: func() interface{} { return new(models.FairwayDimensionImport) }, DMAJobKind: func() interface{} { return new(models.DistanceMarksAshoreImport) }, BCNLATJobKind: func() interface{} { return new(models.FairwayMarksImport) }, + BOYCARJobKind: func() interface{} { return new(models.FairwayMarksImport) }, STJobKind: func() interface{} { return new(models.StretchImport) }, SECJobKind: func() interface{} { return new(models.SectionImport) }, DSECJobKind: func() interface{} { return new(models.SectionDelete) }, @@ -148,6 +149,17 @@ } }, + BOYCARJobKind: func(input interface{}) interface{} { + fmi := input.(*models.FairwayMarksImport) + return &FairwayMarks{ + URL: fmi.URL, + FeatureType: fmi.FeatureType, + SortBy: nilString(fmi.SortBy), + User: nilString(fmi.User), + Password: nilString(fmi.Password), + } + }, + STJobKind: func(input interface{}) interface{} { sti := input.(*models.StretchImport) return &Stretch{
--- a/schema/gemma.sql Mon Feb 10 18:11:52 2020 +0100 +++ b/schema/gemma.sql Mon Feb 10 18:47:27 2020 +0100 @@ -885,6 +885,26 @@ dirimp smallint REFERENCES dirimps, PRIMARY KEY (fm_bcnlat_id, dirimp) ) + + -- Additional attributes for IENC features BOYCAR + CREATE TABLE fairway_marks_boycar ( + id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + colour varchar, + colpat varchar, + conrad int, + marsys int, + boyshp int + ) INHERITS (fairway_marks) + -- Prevent identical entries using composite type comparison + -- (i.e. considering two NULL values in a field equal): + CREATE UNIQUE INDEX fairway_marks_boycar_distinct_rows + ON fairway_marks_boycar + ((CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp + ) AS fairway_marks_boycar) + )) ;