Mercurial > gemma
changeset 2719:2b6f73c30016
WFS downloader: Add BasicAuth support for downloading. TODO: Use it.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Tue, 19 Mar 2019 12:20:54 +0100 |
parents | eae8ec732101 |
children | 305af1e2975d 56c8ef515d93 |
files | cmd/wfs/dump.go cmd/wfs/main.go pkg/imports/dma.go pkg/imports/fd.go pkg/imports/wa.go pkg/imports/wp.go pkg/imports/wx.go pkg/wfs/download.go pkg/wfs/global.go |
diffstat | 9 files changed, 44 insertions(+), 31 deletions(-) [+] |
line wrap: on
line diff
--- a/cmd/wfs/dump.go Tue Mar 19 10:06:22 2019 +0100 +++ b/cmd/wfs/dump.go Tue Mar 19 12:20:54 2019 +0100 @@ -14,19 +14,17 @@ package main import ( - "bufio" "errors" "fmt" "io" "log" - "os" "gemma.intevation.de/gemma/pkg/wfs" ) -func parseFeatures(downloader wfs.Downloader, defaultCRS string) error { +func parseFeatures(user, password string, downloader wfs.Downloader, defaultCRS string) error { - return downloader.Download(func(url string, r io.Reader) error { + return downloader.Download(user, password, func(url string, r io.Reader) error { log.Printf("Get features from: '%s'\n", url) rfc, err := wfs.ParseRawFeatureCollection(r) @@ -61,18 +59,6 @@ }) } -func dumpURLs(urls []string) error { - out := bufio.NewWriter(os.Stdout) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { - log.Printf("Get features from: '%s'\n", url) - _, err := io.Copy(out, r) - return err - }); err != nil { - return err - } - return out.Flush() -} - func dump(caps *wfs.Capabilities) { fmt.Println("service identification") fmt.Println("----------------------")
--- a/cmd/wfs/main.go Tue Mar 19 10:06:22 2019 +0100 +++ b/cmd/wfs/main.go Tue Mar 19 12:20:54 2019 +0100 @@ -31,6 +31,8 @@ dumpCaps = flag.Bool("dump-caps", false, "Dump capabilities document") featureType = flag.String("features", "", "feature to get") sortBy = flag.String("sortby", "", "Sort features by this property") + user = flag.String("user", "", "user name") + password = flag.String("password", "", "user password") ) flag.Parse() @@ -53,6 +55,6 @@ dl, err := wfs.GetFeatures(caps, *featureType, *sortBy) check(err) - parseFeatures(dl, feature.DefaultCRS) + parseFeatures(*user, *password, dl, feature.DefaultCRS) } }
--- a/pkg/imports/dma.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/imports/dma.go Tue Mar 19 12:20:54 2019 +0100 @@ -169,7 +169,10 @@ features int ) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { + // TODO: Fetch from parameters. + user, password := "", "" + + if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil {
--- a/pkg/imports/fd.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/imports/fd.go Tue Mar 19 12:20:54 2019 +0100 @@ -235,7 +235,10 @@ fds []fdSummary ) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { + // TODO: Fetch from parameters. + user, password := "", "" + + if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil {
--- a/pkg/imports/wa.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/imports/wa.go Tue Mar 19 12:20:54 2019 +0100 @@ -176,7 +176,10 @@ features int ) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { + // TODO: Fetch from parameters. + user, password := "", "" + + if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil {
--- a/pkg/imports/wp.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/imports/wp.go Tue Mar 19 12:20:54 2019 +0100 @@ -244,7 +244,10 @@ features int ) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { + // TODO: Fetch from parameters. + user, password := "", "" + + if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil {
--- a/pkg/imports/wx.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/imports/wx.go Tue Mar 19 12:20:54 2019 +0100 @@ -176,7 +176,10 @@ features int ) - if err := wfs.DownloadURLs(urls, func(url string, r io.Reader) error { + // TODO: Fetch from parameters. + user, password := "", "" + + if err := wfs.DownloadURLs(user, password, urls, func(url string, r io.Reader) error { feedback.Info("Get features from: '%s'", url) rfc, err := wfs.ParseRawFeatureCollection(r) if err != nil {
--- a/pkg/wfs/download.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/wfs/download.go Tue Mar 19 12:20:54 2019 +0100 @@ -252,8 +252,18 @@ return downloadURLs, nil } -func downloadURL(url string, handler func(string, io.Reader) error) error { - resp, err := http.Get(url) +func downloadURL(user, password, url string, handler func(string, io.Reader) error) error { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + + if user != "" || password != "" { + req.SetBasicAuth(user, password) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { return err } @@ -385,9 +395,9 @@ // DownloadURLs does the actual GetFeature requests downloads // and hands the resulting io.Readers over to the given handler. -func DownloadURLs(urls []string, handler func(string, io.Reader) error) error { +func DownloadURLs(user, password string, urls []string, handler func(string, io.Reader) error) error { for _, url := range urls { - if err := downloadURL(url, handler); err != nil { + if err := downloadURL(user, password, url, handler); err != nil { return err } }
--- a/pkg/wfs/global.go Tue Mar 19 10:06:22 2019 +0100 +++ b/pkg/wfs/global.go Tue Mar 19 12:20:54 2019 +0100 @@ -40,7 +40,7 @@ type ( Downloader interface { - Download(handler func(string, io.Reader) error) error + Download(user, password string, handler func(string, io.Reader) error) error } GeoJSONDownloader []string @@ -85,8 +85,8 @@ return GMLDownloader(urls), err } -func (gjd GeoJSONDownloader) Download(handler func(string, io.Reader) error) error { - return DownloadURLs([]string(gjd), handler) +func (gjd GeoJSONDownloader) Download(user, password string, handler func(string, io.Reader) error) error { + return DownloadURLs(user, password, []string(gjd), handler) } func places(n int) int { @@ -101,7 +101,7 @@ return places } -func (gmd GMLDownloader) Download(handler func(string, io.Reader) error) error { +func (gmd GMLDownloader) Download(user, password string, handler func(string, io.Reader) error) error { if len(gmd) == 0 { return errors.New("Nothing to download") @@ -119,7 +119,7 @@ for i, url := range gmd { fname := filepath.Join(dlDir, fmt.Sprintf("%0*d.gml", digits, i)) - if err := downloadURL(url, func(_ string, r io.Reader) error { + if err := downloadURL(user, password, url, func(_ string, r io.Reader) error { f, err := os.Create(fname) if err != nil { return err