view pkg/wfs/download.go @ 1614:efc409e330a6

WFS downloader: Propagate namespaces of feature type into GetFeature request.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 18 Dec 2018 12:42:43 +0100
parents 427f9010b4a9
children f59550310143
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2018 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package wfs

import (
	"bufio"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strconv"

	"golang.org/x/net/html/charset"
)

var (
	ErrNoSuchFeatureType      = errors.New("No such feature type")
	ErrGetFeatureNotSupported = errors.New("GetFeature not supported")
	ErrMethodGetNotSupported  = errors.New("GET not supported")
	ErrNoNumberMatchedFound   = errors.New("No numberMatched attribute found")
)

func GetCapabilities(capURL string) (*Capabilities, error) {

	base, err := url.Parse(capURL)
	if err != nil {
		return nil, err
	}
	v := url.Values{}
	v.Set("SERVICE", "WFS")
	v.Set("REQUEST", "GetCapabilities")
	v.Set("ACCEPTVERSIONS", "2.0.0,1.1.0,1.0.0")
	base.RawQuery = v.Encode()

	baseURL := base.String()
	resp, err := http.Get(baseURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	caps, err := ParseCapabilities(bufio.NewReader(resp.Body))
	if err == nil {
		caps.BaseURL = baseURL
	}
	return caps, err
}

func numberFeaturesGET(u *url.URL, featureType, version string) (int, error) {

	v := url.Values{}
	v.Set("SERVICE", "WFS")
	v.Set("REQUEST", "GetFeature")
	v.Set("resultType", "hits")
	v.Set("VERSION", version)
	v.Set("TYPENAMES", featureType)

	q := *u
	q.RawQuery = v.Encode()

	resp, err := http.Get(q.String())
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()
	dec := xml.NewDecoder(resp.Body)
	dec.CharsetReader = charset.NewReaderLabel

	var result struct {
		NumberMatched *int `xml:"numberMatched,attr"`
	}

	if err := dec.Decode(&result); err != nil {
		return 0, err
	}

	if result.NumberMatched == nil {
		return 0, ErrNoNumberMatchedFound
	}

	return *result.NumberMatched, nil
}

func GetFeaturesGET(caps *Capabilities, featureTypeName string) error {

	feature := caps.FindFeatureType(featureTypeName)
	if feature == nil {
		return ErrNoSuchFeatureType
	}
	op := caps.FindOperation("GetFeature")
	if op == nil {
		return ErrGetFeatureNotSupported
	}

	if op.DCP.HTTP.Get == nil {
		return ErrMethodGetNotSupported
	}

	getRaw := op.DCP.HTTP.Get.HRef
	getU, err := url.Parse(getRaw)
	if err != nil {
		return err
	}
	// The URL could be relative so resolve against Capabilities URL.
	if !getU.IsAbs() {
		base, err := url.Parse(caps.BaseURL)
		if err != nil {
			return err
		}
		getU = getU.ResolveReference(base)
	}

	wfsVersion := caps.HighestWFSVersion(WFS2_0_0)

	featuresPerPage, supportsPaging := op.FeaturesPerPage()

	var numFeatures int

	if supportsPaging {
		log.Printf("Paging supported with %d feature per page.\n",
			featuresPerPage)

		if !op.SupportsHits() {
			supportsPaging = false
		} else {
			numFeatures, err = numberFeaturesGET(getU, featureTypeName, wfsVersion)
			if err != nil {
				log.Printf("error: %v\n", err)
				supportsPaging = false
			} else {
				log.Printf("Number of features: %d\n", numFeatures)
			}
		}
	}

	var downloadURLs []string
	wfs2 := !versionIsLess(wfsVersion, WFS2_0_0)

	addNS := func(v url.Values) {
		if len(feature.Namespaces) == 0 {
			return
		}
		// Only use first namespace
		ns := feature.Namespaces[0]
		if wfs2 {
			v.Set("NAMESPACES", fmt.Sprintf("(%s,%s)", ns.Space, ns.Local))
		} else {
			v.Set("NAMESPACE", fmt.Sprintf("(%s:%s)", ns.Space, ns.Local))
		}
	}

	if supportsPaging {
		pagedURL := func(ofs, count int) string {
			v := url.Values{}
			v.Set("SERVICE", "WFS")
			v.Set("REQUEST", "GetFeature")
			v.Set("VERSION", wfsVersion)
			v.Set("startIndex", strconv.Itoa(ofs))
			if wfs2 {
				v.Set("count", strconv.Itoa(count))
			} else {
				v.Set("maxFeatures", strconv.Itoa(count))
			}
			v.Set("TYPENAMES", featureTypeName)
			addNS(v)
			q := *getU
			q.RawQuery = v.Encode()
			return q.String()
		}
		if numFeatures <= featuresPerPage {
			log.Println("All features can be fetched in one page")
			downloadURLs = []string{pagedURL(0, numFeatures)}
		} else {
			log.Println("Features need to be downloaded in pages.")
			for pos := 0; pos < numFeatures; {
				var count int
				if rest := numFeatures - pos; rest >= numFeatures {
					count = numFeatures
				} else {
					count = rest
				}
				downloadURLs = append(downloadURLs, pagedURL(pos, count))
				pos += count
			}
		}
	} else { // No paging support.
		v := url.Values{}
		v.Set("SERVICE", "WFS")
		v.Set("REQUEST", "GetFeature")
		v.Set("VERSION", wfsVersion)
		v.Set("TYPENAMES", featureTypeName)
		addNS(v)
		q := *getU
		q.RawQuery = v.Encode()
		downloadURLs = []string{q.String()}
	}

	// TODO: Implement me!

	log.Printf("%v\n", downloadURLs)

	return nil
}