Mercurial > gemma
view pkg/imports/fm.go @ 4933:45ef039cf4a8 fairway-marks-import
grant select rights to sys_admin to new group layers tables.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Sat, 15 Feb 2020 19:08:02 +0100 |
parents | e41d42be0e13 |
children | 9f9d72a1d398 |
line wrap: on
line source
// This is Free Software under GNU Affero General Public License v >= 3.0 // without warranty, see README.md and license for details. // // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // // Copyright (C) 2020 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Tom Gottfried <tom.gottfried@intevation.de> package imports import ( "context" "database/sql" "encoding/json" "fmt" "io" "time" "gemma.intevation.de/gemma/pkg/wfs" ) // FairwayMarks is a struct // to be used as the basis for imports of // specific types for fairway marks. type FairwayMarks struct { // URL the GetCapabilities URL of the WFS service. URL string `json:"url"` // FeatureType selects the feature type of the WFS service. FeatureType string `json:"feature-type"` // SortBy works around misconfigured services to // establish a sort order to get the features. SortBy string `json:"sort-by"` // User is an optional username for Basic Auth. User string `json:"user,omitempty"` // Password is an optional password for Basic Auth. Password string `json:"password,omitempty"` } // Properties common to all types of fairway marks type fairwayMarksProperties struct { Datsta *string `json:"hydro_datsta"` Datend *string `json:"hydro_datend"` Persta *string `json:"hydro_persta"` Perend *string `json:"hydro_perend"` Objnam *string `json:"hydro_objnam"` Nobjnm *string `json:"hydro_nobjnm"` Inform *string `json:"hydro_inform"` Ninfom *string `json:"hydro_ninfom"` Scamin *int `json:"hydro_scamin"` Picrep *string `json:"hydro_picrep"` Txtdsc *string `json:"hydro_txtdsc"` Sordat *string `json:"hydro_sordat"` Sorind *string `json:"hydro_sorind"` } // Common operation of FM imports to get features from WFS service func getFMFeatures( ctx context.Context, conn *sql.Conn, feedback Feedback, fm FairwayMarks, // Constructor returning pointer to struct // representing featuretype's properties newProps func() interface{}, // Construct pointer to featuretype from given pointSlice and properties newFeat func(pointSlice, interface{}) interface{}, // Store features in type specific database tables storeFMs func( tx *sql.Tx, epsg int, // Slice of features to be converted to featuretypes type fms []interface{}, ) (outsideOrDup int, features int, err error), ) (err error) { start := time.Now() feedback.Info("Import fairway marks") feedback.Info("Loading capabilities from %s", fm.URL) caps, err := wfs.GetCapabilities(fm.URL) if err != nil { feedback.Error("Loading capabilities failed: %v", err) return } ft := caps.FindFeatureType(fm.FeatureType) if ft == nil { err = fmt.Errorf("unknown feature type '%s'", fm.FeatureType) return } feedback.Info("Found feature type '%s", fm.FeatureType) epsg, err := wfs.CRSToEPSG(ft.DefaultCRS) if err != nil { feedback.Error("Unsupported CRS: '%s'", ft.DefaultCRS) return } if fm.SortBy != "" { feedback.Info("Features will be sorted by '%s'", fm.SortBy) } dl, err := wfs.GetFeatures(caps, fm.FeatureType, fm.SortBy) if err != nil { feedback.Error("Cannot create GetFeature URLs. %v", err) return } var ( fms []interface{} unsupported = stringCounter{} missingProperties int badProperties int ) err = dl.Download(fm.User, fm.Password, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil { return fmt.Errorf("parsing GetFeature document failed: %v", err) } if rfc.CRS != nil { crsName := rfc.CRS.Properties.Name if epsg, err = wfs.CRSToEPSG(crsName); err != nil { feedback.Error("Unsupported CRS: %d", crsName) return err } } // No features -> ignore. if rfc.Features == nil { return nil } feedback.Info("Using EPSG: %d", epsg) for _, feature := range rfc.Features { if feature.Properties == nil || feature.Geometry.Coordinates == nil { missingProperties++ continue } props := newProps() if err := json.Unmarshal(*feature.Properties, props); err != nil { badProperties++ continue } switch feature.Geometry.Type { case "Point": var p pointSlice if err := json.Unmarshal(*feature.Geometry.Coordinates, &p); err != nil { return err } f := newFeat(p, props) fms = append(fms, f) default: unsupported[feature.Geometry.Type]++ } } return nil }) if err != nil { return } if badProperties > 0 { feedback.Warn("Bad properties: %d", badProperties) } if missingProperties > 0 { feedback.Warn("Missing properties: %d", missingProperties) } if len(unsupported) != 0 { feedback.Warn("Unsupported types found: %s", unsupported) } feedback.Info("Found %d usable features in data source", len(fms)) tx, err := conn.BeginTx(ctx, nil) if err != nil { return } defer tx.Rollback() outsideOrDup, features, err := storeFMs(tx, epsg, fms) if err != nil { return } if outsideOrDup > 0 { feedback.Info( "Features outside responsibility area and duplicates: %d", outsideOrDup) } if features == 0 { return UnchangedError("no valid new features found") } if err = tx.Commit(); err == nil { feedback.Info("Storing %d features took %s", features, time.Since(start)) } return }