Mercurial > gemma
view pkg/wfs/download.go @ 1665:da0d1a19ebe6
Fairway availability import: Made schedulable, too.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Sun, 23 Dec 2018 13:30:33 +0100 |
parents | 92da44ba610c |
children | 6caf5cd6249e |
line wrap: on
line source
// This is Free Software under GNU Affero General Public License v >= 3.0 // without warranty, see README.md and license for details. // // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // // Copyright (C) 2018 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Sascha L. Teichmann <sascha.teichmann@intevation.de> package wfs import ( "bufio" "encoding/xml" "errors" "fmt" "io" "log" "net/http" "net/url" "strconv" "golang.org/x/net/html/charset" ) var ( ErrNoSuchFeatureType = errors.New("No such feature type") ErrGetFeatureNotSupported = errors.New("GetFeature not supported") ErrMethodGetNotSupported = errors.New("GET not supported") ErrNoNumberMatchedFound = errors.New("No numberMatched attribute found") ErrOutputFormatNotSupported = errors.New("Output format not supported") ) func GetCapabilities(capURL string) (*Capabilities, error) { base, err := url.Parse(capURL) if err != nil { return nil, err } v := url.Values{} v.Set("SERVICE", "WFS") v.Set("REQUEST", "GetCapabilities") v.Set("ACCEPTVERSIONS", "2.0.0,1.1.0,1.0.0") base.RawQuery = v.Encode() baseURL := base.String() resp, err := http.Get(baseURL) if err != nil { return nil, err } defer resp.Body.Close() caps, err := ParseCapabilities(bufio.NewReader(resp.Body)) if err == nil { caps.BaseURL = baseURL } return caps, err } func numberFeaturesGET(u *url.URL, featureType, version string) (int, error) { v := url.Values{} v.Set("SERVICE", "WFS") v.Set("REQUEST", "GetFeature") v.Set("resultType", "hits") v.Set("VERSION", version) v.Set("TYPENAMES", featureType) q := *u q.RawQuery = v.Encode() resp, err := http.Get(q.String()) if err != nil { return 0, err } defer resp.Body.Close() dec := xml.NewDecoder(resp.Body) dec.CharsetReader = charset.NewReaderLabel var result struct { NumberMatched *int `xml:"numberMatched,attr"` } if err := dec.Decode(&result); err != nil { return 0, err } if result.NumberMatched == nil { return 0, ErrNoNumberMatchedFound } return *result.NumberMatched, nil } func GetFeaturesGET( caps *Capabilities, featureTypeName, outputFormat string, sortBy string, ) ([]string, error) { feature := caps.FindFeatureType(featureTypeName) if feature == nil { return nil, ErrNoSuchFeatureType } op := caps.FindOperation("GetFeature") if op == nil { return nil, ErrGetFeatureNotSupported } if op.DCP.HTTP.Get == nil { return nil, ErrMethodGetNotSupported } getRaw := op.DCP.HTTP.Get.HRef getU, err := url.Parse(getRaw) if err != nil { return nil, err } // The URL could be relative so resolve against Capabilities URL. if !getU.IsAbs() { base, err := url.Parse(caps.BaseURL) if err != nil { return nil, err } getU = getU.ResolveReference(base) } if !op.SupportsOutputFormat(outputFormat) { return nil, ErrOutputFormatNotSupported } wfsVersion := caps.HighestWFSVersion(WFS2_0_0) featuresPerPage, supportsPaging := op.FeaturesPerPage() var numFeatures int if supportsPaging { log.Printf("Paging supported with %d feature per page.\n", featuresPerPage) if !op.SupportsHits() { supportsPaging = false } else { numFeatures, err = numberFeaturesGET(getU, featureTypeName, wfsVersion) if err != nil { log.Printf("error: %v\n", err) supportsPaging = false } else { log.Printf("Number of features: %d\n", numFeatures) } } } var downloadURLs []string wfs2 := !versionIsLess(wfsVersion, WFS2_0_0) addNS := func(v url.Values) { if len(feature.Namespaces) == 0 { return } // Only use first namespace ns := feature.Namespaces[0] if wfs2 { v.Set("NAMESPACES", fmt.Sprintf("(%s,%s)", ns.Space, ns.Local)) } else { v.Set("NAMESPACE", fmt.Sprintf("(%s:%s)", ns.Space, ns.Local)) } } addOutputFormat := func(v url.Values) { if outputFormat != "" { v.Set("outputFormat", outputFormat) } } addSortBy := func(v url.Values) { if sortBy != "" { v.Set("sortBy", sortBy) } } if supportsPaging { pagedURL := func(ofs, count int) string { v := url.Values{} v.Set("SERVICE", "WFS") v.Set("REQUEST", "GetFeature") v.Set("VERSION", wfsVersion) v.Set("startIndex", strconv.Itoa(ofs)) if wfs2 { v.Set("count", strconv.Itoa(count)) } else { v.Set("maxFeatures", strconv.Itoa(count)) } v.Set("TYPENAMES", featureTypeName) addNS(v) addOutputFormat(v) addSortBy(v) q := *getU q.RawQuery = v.Encode() return q.String() } if numFeatures <= featuresPerPage { log.Println("All features can be fetched in one page") downloadURLs = []string{pagedURL(0, numFeatures)} } else { log.Println("Features need to be downloaded in pages.") for pos := 0; pos < numFeatures; { var count int if rest := numFeatures - pos; rest >= numFeatures { count = numFeatures } else { count = rest } downloadURLs = append(downloadURLs, pagedURL(pos, count)) pos += count } } } else { // No paging support. v := url.Values{} v.Set("SERVICE", "WFS") v.Set("REQUEST", "GetFeature") v.Set("VERSION", wfsVersion) v.Set("TYPENAMES", featureTypeName) addNS(v) addOutputFormat(v) addSortBy(v) q := *getU q.RawQuery = v.Encode() downloadURLs = []string{q.String()} } return downloadURLs, nil } func downloadURL(url string, handler func(io.Reader) error) error { resp, err := http.Get(url) if err != nil { return err } if resp.StatusCode < 200 || resp.StatusCode > 299 { return fmt.Errorf("Invalid HTTP status code: %d (%s)", resp.StatusCode, resp.Status) } defer resp.Body.Close() return handler(resp.Body) } func DownloadURLs(urls []string, handler func(io.Reader) error) error { for _, url := range urls { if err := downloadURL(url, handler); err != nil { return nil } } return nil }