- Please enter a URL +

+ + Type of mark + +

+ + +

@@ -80,8 +99,19 @@ * Thomas Junk */ export default { - name: "waterwayarea", - props: ["url", "featureType", "sortBy"], + name: "fairwaymarks", + props: ["url", "featureType", "sortBy", "mark", "isUpdate"], + computed: { + selectedMark: { + get() { + return this.mark; + }, + set(value) { + this.selected = value; + this.$emit("selectedMarkChanged", value); + } + } + }, methods: { urlChanged(e) { this.$emit("urlChanged", e.target.value); @@ -92,6 +122,20 @@ sortByChanged(e) { this.$emit("sortByChanged", e.target.value); } + }, + FAIRWAYMARKS: { + BCNISD: "Beacon, isolated danger (MARITIME/Hydro feature)", + BCNLAT: "Beacon, lateral (MARITIME/Hydro feature)", + BOYCAR: "Buoy, cardinal (MARITIME/Hydro feature)", + BOYISD: "Buoy, isolated danger (MARITIME/Hydro feature)", + BOYLAT: "Buoy, lateral (MARITIME/Hydro feature)", + BOYSAW: "Buoy, safe water (MARITIME/Hydro feature)", + BOYSPP: "Buoy, special purpose/general (MARITIME/Hydro feature)", + DAYMAR: "Daymark (MARITIME/Hydro feature)", + LIGHTS: "Light (MARITIME/Hydro feature)", + RTPBCN: "Radar transponder beacon (MARITIME/Hydro feature)", + TOPMAR: "Topmark (MARITIME/Hydro feature)", + notmrk: "Notice mark (IENC feature)" } }; diff -r 0b10d3c68da0 -r 271616eff8e3 client/src/components/importoverview/ImportOverview.vue --- a/client/src/components/importoverview/ImportOverview.vue Fri Feb 14 14:31:12 2020 +0100 +++ b/client/src/components/importoverview/ImportOverview.vue Fri Feb 14 14:33:42 2020 +0100 @@ -78,10 +78,10 @@ - {{ entry.kind.toUpperCase() }} +

+ {{ entry.kind.replace("fm_", "").toUpperCase() }}

diff -r 0b10d3c68da0 -r 271616eff8e3 client/src/store/importschedule.js --- a/client/src/store/importschedule.js Fri Feb 14 14:31:12 2020 +0100 +++ b/client/src/store/importschedule.js Fri Feb 14 14:33:42 2020 +0100 @@ -60,6 +60,21 @@ distancemarksashore: "dma" }; +const FAIRWAYMARKKINDS = { + fm_bcnisd: "BCNISD", + fm_bcnlat: "BCNLAT", + fm_boycar: "BOYCAR", + fm_boyisd: "BOYISD", + fm_boylat: "BOYLAT", + fm_boysaw: "BOYSAW", + fm_boyspp: "BOYSPP", + fm_daymar: "DAYMAR", + fm_lights: "LIGHTS", + fm_rtpbcn: "RTPBCN", + fm_topmar: "TOPMAR", + fm_notmrk: "notmrk" +}; + const initializeCurrentSchedule = () => { return { id: null, @@ -91,7 +106,8 @@ depth: null, sourceOrganization: null, trys: null, - waitRetry: null + waitRetry: null, + selectedMark: null }; }; @@ -141,7 +157,14 @@ const { kind, config, id } = payload; const eMailNotification = config["send-email"]; const { cron, url } = config; - Vue.set(state.currentSchedule, "importType", KINDIMPORTTYPE[kind]); + if (FAIRWAYMARKKINDS[kind]) { + Vue.set(state.currentSchedule, "importType", "fairwaymarks"); + debugger; + Vue.set(state.currentSchedule, "selectedMark", FAIRWAYMARKKINDS[kind]); + } else { + Vue.set(state.currentSchedule, "importType", KINDIMPORTTYPE[kind]); + } + Vue.set(state.currentSchedule, "id", id); Vue.set(state.currentSchedule, "trys", config["trys"]); Vue.set(state.currentSchedule, "waitRetry", config["wait-retry"]); diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/controllers/geostyling.go --- a/pkg/controllers/geostyling.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/controllers/geostyling.go Fri Feb 14 14:33:42 2020 +0100 @@ -31,20 +31,20 @@ styleName = "style" ) -func extractStyle(req *http.Request) (string, error) { +func extractStyle(req *http.Request) ([]byte, error) { f, _, err := req.FormFile(styleName) if err != nil { - return "", err + return nil, err } defer f.Close() var buf bytes.Buffer if _, err := io.Copy(&buf, io.LimitReader(f, maxStyleSize)); err != nil { - return "", err + return nil, err } - return buf.String(), nil + return buf.Bytes(), nil } func supportedWMSFeature(name string) bool { diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/controllers/routes.go --- a/pkg/controllers/routes.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/controllers/routes.go Fri Feb 14 14:33:42 2020 +0100 @@ -4,7 +4,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // -// Copyright (C) 2018 by via donau +// Copyright (C) 2018, 2020 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // @@ -243,6 +243,9 @@ "bn", "gm", "fa", "wx", "wa", "wg", "dmv", "fd", "dma", "sec", "dsec", "dst", "dsr", + "fm_bcnlat", + "fm_boycar", + "fm_boylat", }, "|") api.Handle("/imports/{kind:"+kinds+"}", waterwayAdmin(&mw.JSONHandler{ diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/geoserver/boot.go --- a/pkg/geoserver/boot.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/geoserver/boot.go Fri Feb 14 14:33:42 2020 +0100 @@ -14,6 +14,7 @@ package geoserver import ( + "archive/zip" "bytes" "encoding/json" "encoding/xml" @@ -478,19 +479,33 @@ return stylePreprocessors[name] } +func isZip(data []byte) bool { + if len(data) == 0 { + return false + } + _, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + return err == nil +} + func updateStyle(entry *models.IntEntry, create bool) error { log.Printf("info: creating style %s\n", entry.Name) // Try to load the style data. - data, err := entry.LoadStyle() + binary, err := entry.LoadStyle() if err != nil { return err } - if processor := FindStylePreprocessor(entry.Name); processor != nil { - if data, err = processor(data); err != nil { - return err + zip := isZip(binary) + + if !zip { // We only support templating for plain XML styles. + if processor := FindStylePreprocessor(entry.Name); processor != nil { + data, err := processor(string(binary)) + if err != nil { + return err + } + binary = []byte(data) } } @@ -548,14 +563,17 @@ req, err := http.NewRequest( http.MethodPut, styleURL+"/"+url.PathEscape(entry.Name), - strings.NewReader(data)) + bytes.NewReader(binary)) if err != nil { return err } auth(req) - if isSymbologyEncoding(data) { + switch { + case zip: + asContentType(req, "application/zip") + case isSymbologyEncoding(binary): asContentType(req, "application/vnd.ogc.se+xml") - } else { + default: asContentType(req, "application/vnd.ogc.sld+xml") } resp, err := http.DefaultClient.Do(req) @@ -596,8 +614,8 @@ } // isSymbologyEncoding tries to figure out if its plain SLD or SE. -func isSymbologyEncoding(data string) bool { - decoder := xml.NewDecoder(strings.NewReader(data)) +func isSymbologyEncoding(data []byte) bool { + decoder := xml.NewDecoder(bytes.NewReader(data)) decoder.CharsetReader = charset.NewReaderLabel for { diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/imports/fm.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fm.go Fri Feb 14 14:33:42 2020 +0100 @@ -0,0 +1,178 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2020 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried + +package imports + +import ( + "encoding/json" + "fmt" + "io" + + "gemma.intevation.de/gemma/pkg/wfs" +) + +// FairwayMarks is a struct +// to be used as the basis for imports of +// specific types for fairway marks. +type FairwayMarks struct { + // URL the GetCapabilities URL of the WFS service. + URL string `json:"url"` + // FeatureType selects the feature type of the WFS service. + FeatureType string `json:"feature-type"` + // SortBy works around misconfigured services to + // establish a sort order to get the features. + SortBy string `json:"sort-by"` + // User is an optional username for Basic Auth. + User string `json:"user,omitempty"` + // Password is an optional password for Basic Auth. + Password string `json:"password,omitempty"` +} + +// Properties common to all types of fairway marks +type fairwayMarksProperties struct { + Datsta *string `json:"hydro_datsta"` + Datend *string `json:"hydro_datend"` + Persta *string `json:"hydro_persta"` + Perend *string `json:"hydro_perend"` + Objnam *string `json:"hydro_objnam"` + Nobjnm *string `json:"hydro_nobjnm"` + Inform *string `json:"hydro_inform"` + Ninfom *string `json:"hydro_ninfom"` + Scamin *int `json:"hydro_scamin"` + Picrep *string `json:"hydro_picrep"` + Txtdsc *string `json:"hydro_txtdsc"` + Sordat *string `json:"hydro_sordat"` + Sorind *string `json:"hydro_sorind"` +} + +// Common operation of FM imports to get features from WFS service +func getFMFeatures( + feedback Feedback, + fm FairwayMarks, + // Constructor returning pointer to struct + // representing featuretype's properties + newProps func() interface{}, + // Construct pointer to featuretype from given pointSlice and properties + newFeat func(pointSlice, interface{}) interface{}, +) ( + // Slice of features to be converted to featuretypes type + fms []interface{}, + epsg int, + err error, +) { + + feedback.Info("Loading capabilities from %s", fm.URL) + caps, err := wfs.GetCapabilities(fm.URL) + if err != nil { + feedback.Error("Loading capabilities failed: %v", err) + return + } + + ft := caps.FindFeatureType(fm.FeatureType) + if ft == nil { + err = fmt.Errorf("unknown feature type '%s'", fm.FeatureType) + return + } + + feedback.Info("Found feature type '%s", fm.FeatureType) + + epsg, err = wfs.CRSToEPSG(ft.DefaultCRS) + if err != nil { + feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS) + return + } + + if fm.SortBy != "" { + feedback.Info("Features will be sorted by '%s'", fm.SortBy) + } + + dl, err := wfs.GetFeatures(caps, fm.FeatureType, fm.SortBy) + if err != nil { + feedback.Error("Cannot create GetFeature URLs. %v", err) + return + } + + var ( + unsupported = stringCounter{} + missingProperties int + badProperties int + ) + + err = dl.Download(fm.User, fm.Password, func(url string, r io.Reader) error { + feedback.Info("Get features from: '%s'", url) + rfc, err := wfs.ParseRawFeatureCollection(r) + if err != nil { + return fmt.Errorf("parsing GetFeature document failed: %v", err) + } + if rfc.CRS != nil { + crsName := rfc.CRS.Properties.Name + if epsg, err = wfs.CRSToEPSG(crsName); err != nil { + feedback.Error("Unsupported CRS: %d", crsName) + return err + } + } + + // No features -> ignore. + if rfc.Features == nil { + return nil + } + + feedback.Info("Using EPSG: %d", epsg) + + for _, feature := range rfc.Features { + if feature.Properties == nil || feature.Geometry.Coordinates == nil { + missingProperties++ + continue + } + + props := newProps() + if err := json.Unmarshal(*feature.Properties, props); err != nil { + badProperties++ + continue + } + + switch feature.Geometry.Type { + case "Point": + var p pointSlice + if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil { + return err + } + + f := newFeat(p, props) + fms = append(fms, f) + default: + unsupported[feature.Geometry.Type]++ + } + } + return nil + }) + if err != nil { + return + } + + if badProperties > 0 { + feedback.Warn("Bad properties: %d", badProperties) + } + + if missingProperties > 0 { + feedback.Warn("Missing properties: %d", missingProperties) + } + + if len(unsupported) != 0 { + feedback.Warn("Unsupported types found: %s", unsupported) + } + + feedback.Info("Found %d usable features in data source", len(fms)) + + return +} diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/imports/fm_bcnlat.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fm_bcnlat.go Fri Feb 14 14:33:42 2020 +0100 @@ -0,0 +1,259 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2020 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried + +package imports + +import ( + "context" + "database/sql" + "strings" + "time" + + "gemma.intevation.de/gemma/pkg/pgxutils" +) + +// Bcnlat is an import job to import +// fairway marks of type BCNLAT in form of point geometries +// and attribute data from a WFS service. +type Bcnlat struct { + FairwayMarks +} + +// Description gives a short info about relevant facts of this import. +func (bcnlat *Bcnlat) Description() (string, error) { + return bcnlat.URL + "|" + bcnlat.FeatureType, nil +} + +// BCNLATJobKind is the import queue type identifier. +const BCNLATJobKind JobKind = "fm_bcnlat" + +type bcnlatJobCreator struct{} + +func init() { + RegisterJobCreator(BCNLATJobKind, bcnlatJobCreator{}) +} + +func (bcnlatJobCreator) Description() string { return "fairway marks bcnlat" } + +func (bcnlatJobCreator) AutoAccept() bool { return true } + +func (bcnlatJobCreator) Create() Job { return new(Bcnlat) } + +func (bcnlatJobCreator) Depends() [2][]string { + return [2][]string{ + {"fairway_marks_bcnlat"}, + {}, + } +} + +// StageDone is a NOP for fairway marks imports. +func (bcnlatJobCreator) StageDone(context.Context, *sql.Tx, int64) error { + return nil +} + +// CleanUp for fairway marks imports is a NOP. +func (*Bcnlat) CleanUp() error { return nil } + +type bcnlatProperties struct { + fairwayMarksProperties + Colour *string `json:"hydro_colour"` + Colpat *string `json:"hydro_colpat"` + Condtn *int `json:"hydro_condtn"` + Bcnshp *int `json:"hydro_bcnshp"` + HydroCatlam *int64 `json:"hydro_catlam,omitempty"` + IENCCatlam *int64 `json:"ienc_catlam,omitempty"` + Dirimp *string `json:"ienc_dirimp,omitempty"` +} + +type bcnlatFeaturetype struct { + geom pointSlice + props *bcnlatProperties +} + +const ( + insertBCNLATSQL = ` +with a as ( + select users.current_user_area_utm() AS a +) +INSERT INTO waterway.fairway_marks_bcnlat ( + geom, + datsta, + datend, + persta, + perend, + objnam, + nobjnm, + inform, + ninfom, + scamin, + picrep, + txtdsc, + sordat, + sorind, + colour, + colpat, + condtn, + bcnshp, + catlam +) +SELECT newfm, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20 + FROM ST_Transform(ST_GeomFromWKB($1, $2::integer), 4326) AS newfm (newfm) + WHERE pg_has_role('sys_admin', 'MEMBER') + OR ST_Intersects((select a from a), + ST_Transform(newfm, (select ST_SRID(a) from a))) +ON CONFLICT ( + CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, condtn, bcnshp, catlam + ) AS waterway.fairway_marks_bcnlat) + ) + DO NOTHING +RETURNING id +` + insertDirimpSQL = ` +INSERT INTO waterway.fairway_marks_bcnlat_dirimps (fm_bcnlat_id, dirimp) + VALUES ($1, $2) +` +) + +// Do executes the actual import. +func (fm *Bcnlat) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + start := time.Now() + + feedback.Info("Import fairway marks of type BCNLAT/bcnlat") + + fms, epsg, err := getFMFeatures( + feedback, + fm.FairwayMarks, + func() interface{} { return new(bcnlatProperties) }, + func(p pointSlice, props interface{}) interface{} { + return &bcnlatFeaturetype{p, props.(*bcnlatProperties)} + }, + ) + if err != nil { + return nil, err + } + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + insertStmt, err := tx.PrepareContext(ctx, insertBCNLATSQL) + if err != nil { + return nil, err + } + defer insertStmt.Close() + + insertDirimpStmt, err := tx.PrepareContext(ctx, insertDirimpSQL) + if err != nil { + return nil, err + } + defer insertDirimpStmt.Close() + + savepoint := Savepoint(ctx, tx, "feature") + + var ( + outsideOrDup int + features int + ) + for _, fm := range fms { + + f := fm.(*bcnlatFeaturetype) + + var catlam sql.NullInt64 + if f.props.HydroCatlam != nil { + catlam = sql.NullInt64{Int64: *f.props.HydroCatlam, Valid: true} + } else if f.props.IENCCatlam != nil { + catlam = sql.NullInt64{Int64: *f.props.IENCCatlam, Valid: true} + } + + var fmid int64 + err := savepoint(func() error { + err := insertStmt.QueryRowContext( + ctx, + f.geom.asWKB(), + epsg, + f.props.Datsta, + f.props.Datend, + f.props.Persta, + f.props.Perend, + f.props.Objnam, + f.props.Nobjnm, + f.props.Inform, + f.props.Ninfom, + f.props.Scamin, + f.props.Picrep, + f.props.Txtdsc, + f.props.Sordat, + f.props.Sorind, + f.props.Colour, + f.props.Colpat, + f.props.Condtn, + f.props.Bcnshp, + catlam, + ).Scan(&fmid) + return err + }) + switch { + case err == sql.ErrNoRows: + outsideOrDup++ + // ignore -> filtered by responsibility area or a duplicate + // TODO: handle eventual changes to dirimp + case err != nil: + feedback.Error(pgxutils.ReadableError{Err: err}.Error()) + default: + features++ + + if f.props.Dirimp != nil && *f.props.Dirimp != "" { + dirimps := strings.Split(*f.props.Dirimp, ",") + for _, dirimp := range dirimps { + if err := savepoint(func() error { + _, err := insertDirimpStmt.ExecContext( + ctx, fmid, dirimp) + return err + }); err != nil { + feedback.Warn(pgxutils.ReadableError{Err: err}.Error()) + } + } + } + } + } + + if outsideOrDup > 0 { + feedback.Info( + "Features outside responsibility area and duplicates: %d", + outsideOrDup) + } + + if features == 0 { + err := UnchangedError("no valid new features found") + return nil, err + } + + if err = tx.Commit(); err == nil { + feedback.Info("Storing %d features took %s", + features, time.Since(start)) + } + + return nil, err +} diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/imports/fm_boycar.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fm_boycar.go Fri Feb 14 14:33:42 2020 +0100 @@ -0,0 +1,228 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2020 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried + +package imports + +import ( + "context" + "database/sql" + "time" + + "gemma.intevation.de/gemma/pkg/pgxutils" +) + +// Boycar is an import job to import +// fairway marks of type BOYCAR in form of point geometries +// and attribute data from a WFS service. +type Boycar struct { + FairwayMarks +} + +// Description gives a short info about relevant facts of this import. +func (boycar *Boycar) Description() (string, error) { + return boycar.URL + "|" + boycar.FeatureType, nil +} + +// BOYCARJobKind is the import queue type identifier. +const BOYCARJobKind JobKind = "fm_boycar" + +type boycarJobCreator struct{} + +func init() { + RegisterJobCreator(BOYCARJobKind, boycarJobCreator{}) +} + +func (boycarJobCreator) Description() string { return "fairway marks boycar" } + +func (boycarJobCreator) AutoAccept() bool { return true } + +func (boycarJobCreator) Create() Job { return new(Boycar) } + +func (boycarJobCreator) Depends() [2][]string { + return [2][]string{ + {"fairway_marks_boycar"}, + {}, + } +} + +// StageDone is a NOP for fairway marks imports. +func (boycarJobCreator) StageDone(context.Context, *sql.Tx, int64) error { + return nil +} + +// CleanUp for fairway marks imports is a NOP. +func (*Boycar) CleanUp() error { return nil } + +type boycarProperties struct { + fairwayMarksProperties + Colour *string `json:"hydro_colour"` + Colpat *string `json:"hydro_colpat"` + Conrad *int `json:"hydro_conrad"` + Marsys *int `json:"hydro_marsys"` + Boyshp *int `json:"hydro_boyshp"` + Catcam *int `json:"hydro_catcam"` +} + +type boycarFeaturetype struct { + geom pointSlice + props *boycarProperties +} + +const ( + insertBOYCARSQL = ` +with a as ( + select users.current_user_area_utm() AS a +) +INSERT INTO waterway.fairway_marks_boycar ( + geom, + datsta, + datend, + persta, + perend, + objnam, + nobjnm, + inform, + ninfom, + scamin, + picrep, + txtdsc, + sordat, + sorind, + colour, + colpat, + conrad, + marsys, + boyshp, + catcam +) +SELECT newfm, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21 + FROM ST_Transform(ST_GeomFromWKB($1, $2::integer), 4326) AS newfm (newfm) + WHERE pg_has_role('sys_admin', 'MEMBER') + OR ST_Intersects((select a from a), + ST_Transform(newfm, (select ST_SRID(a) from a))) +ON CONFLICT ( + CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp, catcam + ) AS waterway.fairway_marks_boycar) + ) + DO NOTHING +RETURNING id +` +) + +// Do executes the actual import. +func (fm *Boycar) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + start := time.Now() + + feedback.Info("Import fairway marks of type BOYCAR") + + fms, epsg, err := getFMFeatures( + feedback, + fm.FairwayMarks, + func() interface{} { return new(boycarProperties) }, + func(p pointSlice, props interface{}) interface{} { + return &boycarFeaturetype{p, props.(*boycarProperties)} + }, + ) + if err != nil { + return nil, err + } + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + insertStmt, err := tx.PrepareContext(ctx, insertBOYCARSQL) + if err != nil { + return nil, err + } + defer insertStmt.Close() + + savepoint := Savepoint(ctx, tx, "feature") + + var ( + outsideOrDup int + features int + ) + for _, fm := range fms { + + f := fm.(*boycarFeaturetype) + + var fmid int64 + err := savepoint(func() error { + err := insertStmt.QueryRowContext( + ctx, + f.geom.asWKB(), + epsg, + f.props.Datsta, + f.props.Datend, + f.props.Persta, + f.props.Perend, + f.props.Objnam, + f.props.Nobjnm, + f.props.Inform, + f.props.Ninfom, + f.props.Scamin, + f.props.Picrep, + f.props.Txtdsc, + f.props.Sordat, + f.props.Sorind, + f.props.Colour, + f.props.Colpat, + f.props.Conrad, + f.props.Marsys, + f.props.Boyshp, + f.props.Catcam, + ).Scan(&fmid) + return err + }) + switch { + case err == sql.ErrNoRows: + outsideOrDup++ + // ignore -> filtered by responsibility_areas + case err != nil: + feedback.Error(pgxutils.ReadableError{Err: err}.Error()) + default: + features++ + } + } + + if outsideOrDup > 0 { + feedback.Info( + "Features outside responsibility area and duplicates: %d", + outsideOrDup) + } + + if features == 0 { + err := UnchangedError("no valid new features found") + return nil, err + } + + if err = tx.Commit(); err == nil { + feedback.Info("Storing %d features took %s", + features, time.Since(start)) + } + + return nil, err +} diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/imports/fm_boylat.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/fm_boylat.go Fri Feb 14 14:33:42 2020 +0100 @@ -0,0 +1,244 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2020 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Tom Gottfried + +package imports + +import ( + "context" + "database/sql" + "time" + + "gemma.intevation.de/gemma/pkg/pgxutils" +) + +// Boylat is an import job to import +// fairway marks of type BOYLAT in form of point geometries +// and attribute data from a WFS service. +type Boylat struct { + FairwayMarks +} + +// Description gives a short info about relevant facts of this import. +func (boylat *Boylat) Description() (string, error) { + return boylat.URL + "|" + boylat.FeatureType, nil +} + +// BOYLATJobKind is the import queue type identifier. +const BOYLATJobKind JobKind = "fm_boylat" + +type boylatJobCreator struct{} + +func init() { + RegisterJobCreator(BOYLATJobKind, boylatJobCreator{}) +} + +func (boylatJobCreator) Description() string { return "fairway marks boylat" } + +func (boylatJobCreator) AutoAccept() bool { return true } + +func (boylatJobCreator) Create() Job { return new(Boylat) } + +func (boylatJobCreator) Depends() [2][]string { + return [2][]string{ + {"fairway_marks_boylat"}, + {}, + } +} + +// StageDone is a NOP for fairway marks imports. +func (boylatJobCreator) StageDone(context.Context, *sql.Tx, int64) error { + return nil +} + +// CleanUp for fairway marks imports is a NOP. +func (*Boylat) CleanUp() error { return nil } + +type boylatProperties struct { + fairwayMarksProperties + Colour *string `json:"hydro_colour"` + Colpat *string `json:"hydro_colpat"` + Conrad *int `json:"hydro_conrad"` + HydroMarsys *int64 `json:"hydro_marsys,omitempty"` + IENCMarsys *int64 `json:"ienc_marsys,omitempty"` + Boyshp *int `json:"hydro_boyshp"` + HydroCatlam *int64 `json:"hydro_catlam,omitempty"` + IENCCatlam *int64 `json:"ienc_catlam,omitempty"` +} + +type boylatFeaturetype struct { + geom pointSlice + props *boylatProperties +} + +const ( + insertBOYLATSQL = ` +with a as ( + select users.current_user_area_utm() AS a +) +INSERT INTO waterway.fairway_marks_boylat ( + geom, + datsta, + datend, + persta, + perend, + objnam, + nobjnm, + inform, + ninfom, + scamin, + picrep, + txtdsc, + sordat, + sorind, + colour, + colpat, + conrad, + marsys, + boyshp, + catlam +) +SELECT newfm, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21 + FROM ST_Transform(ST_GeomFromWKB($1, $2::integer), 4326) AS newfm (newfm) + WHERE pg_has_role('sys_admin', 'MEMBER') + OR ST_Intersects((select a from a), + ST_Transform(newfm, (select ST_SRID(a) from a))) +ON CONFLICT ( + CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp, catlam + ) AS waterway.fairway_marks_boylat) + ) + DO NOTHING +RETURNING id +` +) + +// Do executes the actual import. +func (fm *Boylat) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + start := time.Now() + + feedback.Info("Import fairway marks of type BOYLAT") + + fms, epsg, err := getFMFeatures( + feedback, + fm.FairwayMarks, + func() interface{} { return new(boylatProperties) }, + func(p pointSlice, props interface{}) interface{} { + return &boylatFeaturetype{p, props.(*boylatProperties)} + }, + ) + if err != nil { + return nil, err + } + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + + insertStmt, err := tx.PrepareContext(ctx, insertBOYLATSQL) + if err != nil { + return nil, err + } + defer insertStmt.Close() + + savepoint := Savepoint(ctx, tx, "feature") + + var ( + outsideOrDup int + features int + ) + for _, fm := range fms { + + f := fm.(*boylatFeaturetype) + + var marsys sql.NullInt64 + if f.props.HydroMarsys != nil { + marsys = sql.NullInt64{Int64: *f.props.HydroMarsys, Valid: true} + } else if f.props.IENCMarsys != nil { + marsys = sql.NullInt64{Int64: *f.props.IENCMarsys, Valid: true} + } + + var catlam sql.NullInt64 + if f.props.HydroCatlam != nil { + catlam = sql.NullInt64{Int64: *f.props.HydroCatlam, Valid: true} + } else if f.props.IENCCatlam != nil { + catlam = sql.NullInt64{Int64: *f.props.IENCCatlam, Valid: true} + } + + var fmid int64 + err := savepoint(func() error { + err := insertStmt.QueryRowContext( + ctx, + f.geom.asWKB(), + epsg, + f.props.Datsta, + f.props.Datend, + f.props.Persta, + f.props.Perend, + f.props.Objnam, + f.props.Nobjnm, + f.props.Inform, + f.props.Ninfom, + f.props.Scamin, + f.props.Picrep, + f.props.Txtdsc, + f.props.Sordat, + f.props.Sorind, + f.props.Colour, + f.props.Colpat, + f.props.Conrad, + marsys, + f.props.Boyshp, + catlam, + ).Scan(&fmid) + return err + }) + switch { + case err == sql.ErrNoRows: + outsideOrDup++ + // ignore -> filtered by responsibility_areas + case err != nil: + feedback.Error(pgxutils.ReadableError{Err: err}.Error()) + default: + features++ + } + } + + if outsideOrDup > 0 { + feedback.Info( + "Features outside responsibility area and duplicates: %d", + outsideOrDup) + } + + if features == 0 { + err := UnchangedError("no valid new features found") + return nil, err + } + + if err = tx.Commit(); err == nil { + feedback.Info("Storing %d features took %s", + features, time.Since(start)) + } + + return nil, err +} diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/imports/modelconvert.go --- a/pkg/imports/modelconvert.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/imports/modelconvert.go Fri Feb 14 14:33:42 2020 +0100 @@ -4,7 +4,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // -// Copyright (C) 2018 by via donau +// Copyright (C) 2018, 2020 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // @@ -18,20 +18,23 @@ ) var kindToImportModel = map[JobKind]func() interface{}{ - BNJobKind: func() interface{} { return new(models.BottleneckImport) }, - GMJobKind: func() interface{} { return new(models.GaugeMeasurementImport) }, - FAJobKind: func() interface{} { return new(models.FairwayAvailabilityImport) }, - WXJobKind: func() interface{} { return new(models.WaterwayAxisImport) }, - WAJobKind: func() interface{} { return new(models.WaterwayAreaImport) }, - WGJobKind: func() interface{} { return new(models.WaterwayGaugeImport) }, - DMVJobKind: func() interface{} { return new(models.DistanceMarksVirtualImport) }, - FDJobKind: func() interface{} { return new(models.FairwayDimensionImport) }, - DMAJobKind: func() interface{} { return new(models.DistanceMarksAshoreImport) }, - STJobKind: func() interface{} { return new(models.StretchImport) }, - SECJobKind: func() interface{} { return new(models.SectionImport) }, - DSECJobKind: func() interface{} { return new(models.SectionDelete) }, - DSTJobKind: func() interface{} { return new(models.StretchDelete) }, - DSRJobKind: func() interface{} { return new(models.SoundingResultDelete) }, + BNJobKind: func() interface{} { return new(models.BottleneckImport) }, + GMJobKind: func() interface{} { return new(models.GaugeMeasurementImport) }, + FAJobKind: func() interface{} { return new(models.FairwayAvailabilityImport) }, + WXJobKind: func() interface{} { return new(models.WaterwayAxisImport) }, + WAJobKind: func() interface{} { return new(models.WaterwayAreaImport) }, + WGJobKind: func() interface{} { return new(models.WaterwayGaugeImport) }, + DMVJobKind: func() interface{} { return new(models.DistanceMarksVirtualImport) }, + FDJobKind: func() interface{} { return new(models.FairwayDimensionImport) }, + DMAJobKind: func() interface{} { return new(models.DistanceMarksAshoreImport) }, + BCNLATJobKind: func() interface{} { return new(models.FairwayMarksImport) }, + BOYCARJobKind: func() interface{} { return new(models.FairwayMarksImport) }, + BOYLATJobKind: func() interface{} { return new(models.FairwayMarksImport) }, + STJobKind: func() interface{} { return new(models.StretchImport) }, + SECJobKind: func() interface{} { return new(models.SectionImport) }, + DSECJobKind: func() interface{} { return new(models.SectionDelete) }, + DSTJobKind: func() interface{} { return new(models.StretchDelete) }, + DSRJobKind: func() interface{} { return new(models.SoundingResultDelete) }, } // ImportModelForJobKind returns the constructor function to @@ -136,6 +139,39 @@ } }, + BCNLATJobKind: func(input interface{}) interface{} { + fmi := input.(*models.FairwayMarksImport) + return &FairwayMarks{ + URL: fmi.URL, + FeatureType: fmi.FeatureType, + SortBy: nilString(fmi.SortBy), + User: nilString(fmi.User), + Password: nilString(fmi.Password), + } + }, + + BOYCARJobKind: func(input interface{}) interface{} { + fmi := input.(*models.FairwayMarksImport) + return &FairwayMarks{ + URL: fmi.URL, + FeatureType: fmi.FeatureType, + SortBy: nilString(fmi.SortBy), + User: nilString(fmi.User), + Password: nilString(fmi.Password), + } + }, + + BOYLATJobKind: func(input interface{}) interface{} { + fmi := input.(*models.FairwayMarksImport) + return &FairwayMarks{ + URL: fmi.URL, + FeatureType: fmi.FeatureType, + SortBy: nilString(fmi.SortBy), + User: nilString(fmi.User), + Password: nilString(fmi.Password), + } + }, + STJobKind: func(input interface{}) interface{} { sti := input.(*models.StretchImport) return &Stretch{ diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/models/imports.go --- a/pkg/models/imports.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/models/imports.go Fri Feb 14 14:33:42 2020 +0100 @@ -4,7 +4,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // -// Copyright (C) 2018, 2019 by via donau +// Copyright (C) 2018, 2019, 2020 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // @@ -78,6 +78,11 @@ WFSImport } + // FairwayMarksImport specifies an import of fairway marks. + FairwayMarksImport struct { + WFSImport + } + // FairwayDimensionImport specifies an import of the waterway axis. FairwayDimensionImport struct { WFSImport diff -r 0b10d3c68da0 -r 271616eff8e3 pkg/models/intservices.go --- a/pkg/models/intservices.go Fri Feb 14 14:31:12 2020 +0100 +++ b/pkg/models/intservices.go Fri Feb 14 14:33:42 2020 +0100 @@ -54,20 +54,20 @@ ORDER by name` selectStyleSQL = ` -SELECT XMLSERIALIZE(DOCUMENT style AS text) +SELECT style FROM sys_admin.published_services WHERE name = $1 AND schema = $2` updateStyleSQL = ` UPDATE sys_admin.published_services -SET style = XMLPARSE(DOCUMENT $1) +SET style = $1 WHERE name = $2 AND schema = $3` ) var InternalServices = &IntServices{} -func (e *IntEntry) LoadStyle() (string, error) { - var style string +func (e *IntEntry) LoadStyle() ([]byte, error) { + var style []byte ctx := context.Background() err := auth.RunAs(ctx, "sys_admin", func(conn *sql.Conn) error { @@ -79,7 +79,7 @@ return style, err } -func UpdateInternalStyle(req *http.Request, name, style string) error { +func UpdateInternalStyle(req *http.Request, name string, style []byte) error { return auth.RunAsSessionUser(req, func(conn *sql.Conn) error { _, err := conn.ExecContext( req.Context(), updateStyleSQL, diff -r 0b10d3c68da0 -r 271616eff8e3 schema/gemma.sql --- a/schema/gemma.sql Fri Feb 14 14:31:12 2020 +0100 +++ b/schema/gemma.sql Fri Feb 14 14:33:42 2020 +0100 @@ -406,8 +406,8 @@ key_column varchar, -- SRID to be used with SQL view: srid int REFERENCES spatial_ref_sys, - -- SLD style document: - style xml CHECK(style IS DOCUMENT), + -- SLD style document or ZIP blob: + style bytea, as_wms boolean NOT NULL DEFAULT TRUE, as_wfs boolean NOT NULL DEFAULT TRUE, -- Either give a valid relation or a SQL statement: @@ -840,6 +840,93 @@ CHECK(measure_type = 'minimum guaranteed' OR value_lifetime IS NOT NULL) ) + + -- Attributes common to all fairway marks + -- according to IENC feature catalogue + CREATE TABLE fairway_marks ( + geom geography(POINT, 4326) NOT NULL, + datsta varchar, + datend varchar, + persta varchar, + perend varchar, + objnam varchar, + nobjnm varchar, + inform varchar, + ninfom varchar, + scamin int, + picrep varchar, + txtdsc varchar, + sordat varchar, + sorind varchar + ) + + -- Additional attributes for IENC features BCNLAT/bcnlat + CREATE TABLE fairway_marks_bcnlat ( + id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + colour varchar, + colpat varchar, + condtn int, + bcnshp int, + catlam int + ) INHERITS (fairway_marks) + -- Prevent identical entries using composite type comparison + -- (i.e. considering two NULL values in a field equal): + CREATE UNIQUE INDEX fairway_marks_bcnlat_distinct_rows + ON fairway_marks_bcnlat + ((CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, condtn, bcnshp, catlam + ) AS fairway_marks_bcnlat) + )) + + CREATE TABLE fairway_marks_bcnlat_dirimps ( + fm_bcnlat_id int REFERENCES fairway_marks_bcnlat, + dirimp smallint REFERENCES dirimps, + PRIMARY KEY (fm_bcnlat_id, dirimp) + ) + + -- Additional attributes for IENC feature BOYCAR + CREATE TABLE fairway_marks_boycar ( + id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + colour varchar, + colpat varchar, + conrad int, + marsys int, + boyshp int, + catcam int + ) INHERITS (fairway_marks) + -- Prevent identical entries using composite type comparison + -- (i.e. considering two NULL values in a field equal): + CREATE UNIQUE INDEX fairway_marks_boycar_distinct_rows + ON fairway_marks_boycar + ((CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp, catcam + ) AS fairway_marks_boycar) + )) + + -- Additional attributes for IENC feature BOYLAT + CREATE TABLE fairway_marks_boylat ( + id int PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + colour varchar, + colpat varchar, + conrad int, + marsys int, + boyshp int, + catlam int + ) INHERITS (fairway_marks) + -- Prevent identical entries using composite type comparison + -- (i.e. considering two NULL values in a field equal): + CREATE UNIQUE INDEX fairway_marks_boylat_distinct_rows + ON fairway_marks_boylat + ((CAST((geom, + datsta, datend, persta, perend, objnam, nobjnm, inform, ninfom, + scamin, picrep, txtdsc, sordat, sorind, + 0, colour, colpat, conrad, marsys, boyshp, catlam + ) AS fairway_marks_boylat) + )) ; diff -r 0b10d3c68da0 -r 271616eff8e3 schema/updates/1317/01.migrate_styles.sql --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/schema/updates/1317/01.migrate_styles.sql Fri Feb 14 14:33:42 2020 +0100 @@ -0,0 +1,4 @@ +ALTER TABLE sys_admin.published_services ADD COLUMN bin_blob bytea; +UPDATE sys_admin.published_services SET bin_blob = style::text::bytea WHERE style is NOT NULL; +ALTER TABLE sys_admin.published_services DROP COLUMN style; +ALTER TABLE sys_admin.published_services RENAME COLUMN bin_blob TO style; diff -r 0b10d3c68da0 -r 271616eff8e3 schema/version.sql --- a/schema/version.sql Fri Feb 14 14:31:12 2020 +0100 +++ b/schema/version.sql Fri Feb 14 14:33:42 2020 +0100 @@ -1,1 +1,1 @@ -INSERT INTO gemma_schema_version(version) VALUES (1316); +INSERT INTO gemma_schema_version(version) VALUES (1317); diff -r 0b10d3c68da0 -r 271616eff8e3 style-templates/upload-styles.sh --- a/style-templates/upload-styles.sh Fri Feb 14 14:31:12 2020 +0100 +++ b/style-templates/upload-styles.sh Fri Feb 14 14:33:42 2020 +0100 @@ -19,9 +19,11 @@ usage() { cat < /dev/null then echo "== Configuring geoserver styles" >&2 - for style in $(basename -s .sld-template $(ls $datadir/*.sld-template)) + if [ $# -gt 0 ]; then + files=("$@") + else + files=($(find . -name "*.sld-template" -or -name "*.zip")) + fi + for file in ${files[@]} do - echo "uploading $style ..." + style=$(basename $(basename "$file" .zip) .sld-template) + echo "uploading ${style} ..." curl -f -s -S -H "X-Gemma-Auth:${token}" -X POST \ - -F style=@"${datadir}/${style}.sld-template" \ + -F style=@"${file}" \ "http://${g_host}:${g_port}/api/geo/style/${style}" done echo 'done.'