Mercurial > gemma
changeset 2110:fc4ebe7eaeff
WFS downloader: Don't use temp files. TODO: Use a streaming scanner for the detection of the exception documents.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Mon, 04 Feb 2019 18:31:55 +0100 |
parents | 86c88fc0ff5e |
children | dd4e1f5f66b8 |
files | pkg/wfs/download.go |
diffstat | 1 files changed, 10 insertions(+), 26 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/wfs/download.go Mon Feb 04 17:48:36 2019 +0100 +++ b/pkg/wfs/download.go Mon Feb 04 18:31:55 2019 +0100 @@ -15,15 +15,14 @@ import ( "bufio" + "bytes" "encoding/xml" "errors" "fmt" "io" - "io/ioutil" "log" "net/http" "net/url" - "os" "strconv" "gemma.intevation.de/gemma/pkg/config" @@ -259,34 +258,19 @@ resp.StatusCode, resp.Status) } defer resp.Body.Close() - tmp, err := ioutil.TempFile(config.TmpDir(), "wfs*.tmp") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) + + var already bytes.Buffer - _, err = io.Copy(tmp, resp.Body) - err2 := tmp.Close() - if err != nil { - return err - } - if err2 != nil { - return err2 - } - f, err := os.Open(tmp.Name()) - if err != nil { - return err - } - defer f.Close() + // Prevent the XML reader from consuming everything. + limit := io.LimitReader(resp.Body, 16*1024) + in := io.TeeReader(limit, &already) - if err := scanExceptionReport(f); err != nil { + if err := scanExceptionReport(in); err != nil { return err } - if _, err := f.Seek(0, 0); err != nil { - return err - } - return handler(url, bufio.NewReader(f)) + multi := io.MultiReader(bytes.NewReader(already.Bytes()), resp.Body) + return handler(url, multi) } type ExceptionReport struct { @@ -319,7 +303,7 @@ var er ExceptionReport - if err := decoder.Decode(&er); err != nil { + if decoder.Decode(&er) != nil { // Fine because of other kind of document return nil }