diff pkg/wfs/download.go @ 1608:427f9010b4a9

WFS download: Started with GET downloader (paged and unpaged).
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 17 Dec 2018 18:27:57 +0100
parents
children efc409e330a6
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/wfs/download.go	Mon Dec 17 18:27:57 2018 +0100
@@ -0,0 +1,198 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2018 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+
+package wfs
+
+import (
+	"bufio"
+	"encoding/xml"
+	"errors"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"golang.org/x/net/html/charset"
+)
+
+var (
+	ErrNoSuchFeatureType      = errors.New("No such feature type")
+	ErrGetFeatureNotSupported = errors.New("GetFeature not supported")
+	ErrMethodGetNotSupported  = errors.New("GET not supported")
+	ErrNoNumberMatchedFound   = errors.New("No numberMatched attribute found")
+)
+
+func GetCapabilities(capURL string) (*Capabilities, error) {
+
+	base, err := url.Parse(capURL)
+	if err != nil {
+		return nil, err
+	}
+	v := url.Values{}
+	v.Set("SERVICE", "WFS")
+	v.Set("REQUEST", "GetCapabilities")
+	v.Set("ACCEPTVERSIONS", "2.0.0,1.1.0,1.0.0")
+	base.RawQuery = v.Encode()
+
+	baseURL := base.String()
+	resp, err := http.Get(baseURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	caps, err := ParseCapabilities(bufio.NewReader(resp.Body))
+	if err == nil {
+		caps.BaseURL = baseURL
+	}
+	return caps, err
+}
+
+func numberFeaturesGET(u *url.URL, featureType, version string) (int, error) {
+
+	v := url.Values{}
+	v.Set("SERVICE", "WFS")
+	v.Set("REQUEST", "GetFeature")
+	v.Set("resultType", "hits")
+	v.Set("VERSION", version)
+	v.Set("TYPENAMES", featureType)
+
+	q := *u
+	q.RawQuery = v.Encode()
+
+	resp, err := http.Get(q.String())
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+	dec := xml.NewDecoder(resp.Body)
+	dec.CharsetReader = charset.NewReaderLabel
+
+	var result struct {
+		NumberMatched *int `xml:"numberMatched,attr"`
+	}
+
+	if err := dec.Decode(&result); err != nil {
+		return 0, err
+	}
+
+	if result.NumberMatched == nil {
+		return 0, ErrNoNumberMatchedFound
+	}
+
+	return *result.NumberMatched, nil
+}
+
+func GetFeaturesGET(caps *Capabilities, featureTypeName string) error {
+
+	if caps.FindFeatureType(featureTypeName) == nil {
+		return ErrNoSuchFeatureType
+	}
+	op := caps.FindOperation("GetFeature")
+	if op == nil {
+		return ErrGetFeatureNotSupported
+	}
+
+	if op.DCP.HTTP.Get == nil {
+		return ErrMethodGetNotSupported
+	}
+
+	getRaw := op.DCP.HTTP.Get.HRef
+	getU, err := url.Parse(getRaw)
+	if err != nil {
+		return err
+	}
+	// The URL could be relative so resolve against Capabilities URL.
+	if !getU.IsAbs() {
+		base, err := url.Parse(caps.BaseURL)
+		if err != nil {
+			return err
+		}
+		getU = getU.ResolveReference(base)
+	}
+
+	wfsVersion := caps.HighestWFSVersion(WFS2_0_0)
+
+	featuresPerPage, supportsPaging := op.FeaturesPerPage()
+
+	var numFeatures int
+
+	if supportsPaging {
+		log.Printf("Paging supported with %d feature per page.\n",
+			featuresPerPage)
+
+		if !op.SupportsHits() {
+			supportsPaging = false
+		} else {
+			numFeatures, err = numberFeaturesGET(getU, featureTypeName, wfsVersion)
+			if err != nil {
+				log.Printf("error: %v\n", err)
+				supportsPaging = false
+			} else {
+				log.Printf("Number of features: %d\n", numFeatures)
+			}
+		}
+	}
+
+	var downloadURLs []string
+
+	if supportsPaging {
+		wfs2 := !versionIsLess(wfsVersion, WFS2_0_0)
+		pagedURL := func(ofs, count int) string {
+			v := url.Values{}
+			v.Set("SERVICE", "WFS")
+			v.Set("REQUEST", "GetFeature")
+			v.Set("VERSION", wfsVersion)
+			v.Set("startIndex", strconv.Itoa(ofs))
+			if wfs2 {
+				v.Set("count", strconv.Itoa(count))
+			} else {
+				v.Set("maxFeatures", strconv.Itoa(count))
+			}
+			v.Set("TYPENAMES", featureTypeName)
+			q := *getU
+			q.RawQuery = v.Encode()
+			return q.String()
+		}
+		if numFeatures <= featuresPerPage {
+			log.Println("All features can be fetched in one page")
+			downloadURLs = []string{pagedURL(0, numFeatures)}
+		} else {
+			log.Println("Features need to be downloaded in pages.")
+			for pos := 0; pos < numFeatures; {
+				var count int
+				if rest := numFeatures - pos; rest >= numFeatures {
+					count = numFeatures
+				} else {
+					count = rest
+				}
+				downloadURLs = append(downloadURLs, pagedURL(pos, count))
+				pos += count
+			}
+		}
+	} else { // No paging support.
+		v := url.Values{}
+		v.Set("SERVICE", "WFS")
+		v.Set("REQUEST", "GetFeature")
+		v.Set("VERSION", wfsVersion)
+		v.Set("TYPENAMES", featureTypeName)
+		q := *getU
+		q.RawQuery = v.Encode()
+		downloadURLs = []string{q.String()}
+	}
+
+	// TODO: Implement me!
+
+	log.Printf("%v\n", downloadURLs)
+
+	return nil
+}