view pkg/wfs/global.go @ 3681:c9e1848a516a

Handle violation of responsibility areas graceful for BN import. As many others this is an error, which should only affect the specific bottleneck failing and not the whole import.
author Sascha Wilde <wilde@intevation.de>
date Tue, 18 Jun 2019 12:43:01 +0200
parents 2b6f73c30016
children 04876d865528
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2019 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package wfs

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"

	"gemma.intevation.de/gemma/pkg/config"
)

var (
	FormatGeoJSON = []string{"geojson", "application/json"}
	FormatGML     = []string{
		"application/gml+xml; version=3.2",
		"gml2", "gml3", "gml32",
		"text/xml; subtype=gml/2.1.2",
		"text/xml; subtype=gml/3.1.1",
		"text/xml; subtype=gml/3.2",
	}
)

type (
	Downloader interface {
		Download(user, password string, handler func(string, io.Reader) error) error
	}

	GeoJSONDownloader []string
	GMLDownloader     []string
)

var GetFeatures = setup()

func setup() func(*Capabilities, string, string) (Downloader, error) {
	path, err := exec.LookPath("ogr2ogr")
	if err != nil {
		log.Println("info: ogr2ogr not installed. Using direct GeoJSON WFS download.")
		return GetFeaturesGeoJSON
	}
	log.Printf("info: ogr2ogr found at %s. Using GML WFS download.\n", path)
	return GetFeaturesGML
}

func GetFeaturesGeoJSON(
	caps *Capabilities,
	featureTypeName string,
	sortBy string,
) (Downloader, error) {
	urls, err := GetFeaturesGET(
		caps,
		featureTypeName,
		FormatGeoJSON,
		sortBy)
	return GeoJSONDownloader(urls), err
}

func GetFeaturesGML(
	caps *Capabilities,
	featureTypeName string,
	sortBy string,
) (Downloader, error) {
	urls, err := GetFeaturesGET(
		caps,
		featureTypeName,
		FormatGML,
		sortBy)
	return GMLDownloader(urls), err
}

func (gjd GeoJSONDownloader) Download(user, password string, handler func(string, io.Reader) error) error {
	return DownloadURLs(user, password, []string(gjd), handler)
}

func places(n int) int {
	places := 1
	if n < 0 {
		places++
		n = -n
	}
	for value := 10; n >= value; value *= 10 {
		places++
	}
	return places
}

func (gmd GMLDownloader) Download(user, password string, handler func(string, io.Reader) error) error {

	if len(gmd) == 0 {
		return errors.New("Nothing to download")
	}

	tmpDir := config.TmpDir()
	dlDir, err := ioutil.TempDir(tmpDir, "wfs-downloads")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dlDir)
	digits := places(len(gmd))

	var files []string

	for i, url := range gmd {
		fname := filepath.Join(dlDir, fmt.Sprintf("%0*d.gml", digits, i))
		if err := downloadURL(user, password, url, func(_ string, r io.Reader) error {
			f, err := os.Create(fname)
			if err != nil {
				return err
			}
			if _, err = io.Copy(f, r); err != nil {
				f.Close()
				return err
			}
			return f.Close()
		}); err != nil {
			return err
		}
		files = append(files, fname)
	}

	out := filepath.Join(dlDir, "out.geojson")

	cmd := exec.Command("ogr2ogr", append([]string{"-f", "GeoJSON", out}, files...)...)

	msg, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("ogr2ogr failed: %s", msg)
	}

	in, err := os.Open(out)
	if err != nil {
		return err
	}
	defer in.Close()

	return handler(gmd[0], bufio.NewReader(in))
}