Mercurial > gemma

changeset 2719:2b6f73c30016

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
WFS downloader: Add BasicAuth support for downloading. TODO: Use it.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 19 Mar 2019 12:20:54 +0100
parents eae8ec732101
children 305af1e2975d 56c8ef515d93
files cmd/wfs/dump.go cmd/wfs/main.go pkg/imports/dma.go pkg/imports/fd.go pkg/imports/wa.go pkg/imports/wp.go pkg/imports/wx.go pkg/wfs/download.go pkg/wfs/global.go
diffstat 9 files changed, 44 insertions(+), 31 deletions(-) [+]
line wrap: on
line diff
--- a/cmd/wfs/dump.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/cmd/wfs/dump.go	Tue Mar 19 12:20:54 2019 +0100
@@ -14,19 +14,17 @@
 package main
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"log"
-	"os"
 
 	"gemma.intevation.de/gemma/pkg/wfs"
 )
 
-func parseFeatures(downloader wfs.Downloader, defaultCRS string) error {
+func parseFeatures(user, password string, downloader wfs.Downloader, defaultCRS string) error {
 
-	return downloader.Download(func(url string, r io.Reader) error {
+	return downloader.Download(user, password, func(url string, r io.Reader) error {
 
 		log.Printf("Get features from: '%s'\n", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
@@ -61,18 +59,6 @@
 	})
 }
 
-func dumpURLs(urls []string) error {
-	out := bufio.NewWriter(os.Stdout)
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
-		log.Printf("Get features from: '%s'\n", url)
-		_, err := io.Copy(out, r)
-		return err
-	}); err != nil {
-		return err
-	}
-	return out.Flush()
-}
-
 func dump(caps *wfs.Capabilities) {
 	fmt.Println("service identification")
 	fmt.Println("----------------------")
--- a/cmd/wfs/main.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/cmd/wfs/main.go	Tue Mar 19 12:20:54 2019 +0100
@@ -31,6 +31,8 @@
 		dumpCaps    = flag.Bool("dump-caps", false, "Dump capabilities document")
 		featureType = flag.String("features", "", "feature to get")
 		sortBy      = flag.String("sortby", "", "Sort features by this property")
+		user        = flag.String("user", "", "user name")
+		password    = flag.String("password", "", "user password")
 	)
 	flag.Parse()
 
@@ -53,6 +55,6 @@
 		dl, err := wfs.GetFeatures(caps, *featureType, *sortBy)
 		check(err)
 
-		parseFeatures(dl, feature.DefaultCRS)
+		parseFeatures(*user, *password, dl, feature.DefaultCRS)
 	}
 }
--- a/pkg/imports/dma.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/imports/dma.go	Tue Mar 19 12:20:54 2019 +0100
@@ -169,7 +169,10 @@
 		features          int
 	)
 
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
+	// TODO: Fetch from parameters.
+	user, password := "", ""
+
+	if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error {
 		feedback.Info("Get features from: '%s'", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
 		if err != nil {
--- a/pkg/imports/fd.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/imports/fd.go	Tue Mar 19 12:20:54 2019 +0100
@@ -235,7 +235,10 @@
 		fds               []fdSummary
 	)
 
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
+	// TODO: Fetch from parameters.
+	user, password := "", ""
+
+	if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error {
 		feedback.Info("Get features from: '%s'", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
 		if err != nil {
--- a/pkg/imports/wa.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/imports/wa.go	Tue Mar 19 12:20:54 2019 +0100
@@ -176,7 +176,10 @@
 		features          int
 	)
 
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
+	// TODO: Fetch from parameters.
+	user, password := "", ""
+
+	if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error {
 		feedback.Info("Get features from: '%s'", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
 		if err != nil {
--- a/pkg/imports/wp.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/imports/wp.go	Tue Mar 19 12:20:54 2019 +0100
@@ -244,7 +244,10 @@
 		features          int
 	)
 
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
+	// TODO: Fetch from parameters.
+	user, password := "", ""
+
+	if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error {
 		feedback.Info("Get features from: '%s'", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
 		if err != nil {
--- a/pkg/imports/wx.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/imports/wx.go	Tue Mar 19 12:20:54 2019 +0100
@@ -176,7 +176,10 @@
 		features          int
 	)
 
-	if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error {
+	// TODO: Fetch from parameters.
+	user, password := "", ""
+
+	if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error {
 		feedback.Info("Get features from: '%s'", url)
 		rfc, err := wfs.ParseRawFeatureCollection(r)
 		if err != nil {
--- a/pkg/wfs/download.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/wfs/download.go	Tue Mar 19 12:20:54 2019 +0100
@@ -252,8 +252,18 @@
 	return downloadURLs, nil
 }
 
-func downloadURL(url string, handler func(string, io.Reader) error) error {
-	resp, err := http.Get(url)
+func downloadURL(user, password, url string, handler func(string, io.Reader) error) error {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	if user != "" || password != "" {
+		req.SetBasicAuth(user, password)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+
 	if err != nil {
 		return err
 	}
@@ -385,9 +395,9 @@
 
 // DownloadURLs does the actual GetFeature requests downloads
 // and hands the resulting io.Readers over to the given handler.
-func DownloadURLs(urls []string, handler func(string, io.Reader) error) error {
+func DownloadURLs(user, password string, urls []string, handler func(string, io.Reader) error) error {
 	for _, url := range urls {
-		if err := downloadURL(url, handler); err != nil {
+		if err := downloadURL(user, password, url, handler); err != nil {
 			return err
 		}
 	}
--- a/pkg/wfs/global.go	Tue Mar 19 10:06:22 2019 +0100
+++ b/pkg/wfs/global.go	Tue Mar 19 12:20:54 2019 +0100
@@ -40,7 +40,7 @@
 
 type (
 	Downloader interface {
-		Download(handler func(string, io.Reader) error) error
+		Download(user, password string, handler func(string, io.Reader) error) error
 	}
 
 	GeoJSONDownloader []string
@@ -85,8 +85,8 @@
 	return GMLDownloader(urls), err
 }
 
-func (gjd GeoJSONDownloader) Download(handler func(string, io.Reader) error) error {
-	return DownloadURLs([]string(gjd), handler)
+func (gjd GeoJSONDownloader) Download(user, password string, handler func(string, io.Reader) error) error {
+	return DownloadURLs(user, password, []string(gjd), handler)
 }
 
 func places(n int) int {
@@ -101,7 +101,7 @@
 	return places
 }
 
-func (gmd GMLDownloader) Download(handler func(string, io.Reader) error) error {
+func (gmd GMLDownloader) Download(user, password string, handler func(string, io.Reader) error) error {
 
 	if len(gmd) == 0 {
 		return errors.New("Nothing to download")
@@ -119,7 +119,7 @@
 
 	for i, url := range gmd {
 		fname := filepath.Join(dlDir, fmt.Sprintf("%0*d.gml", digits, i))
-		if err := downloadURL(url, func(_ string, r io.Reader) error {
+		if err := downloadURL(user, password, url, func(_ string, r io.Reader) error {
 			f, err := os.Create(fname)
 			if err != nil {
 				return err