changeset 2110:fc4ebe7eaeff

WFS downloader: Don't use temp files. TODO: Use a streaming scanner for the detection of the exception documents.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 04 Feb 2019 18:31:55 +0100
parents 86c88fc0ff5e
children dd4e1f5f66b8
files pkg/wfs/download.go
diffstat 1 files changed, 10 insertions(+), 26 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/wfs/download.go	Mon Feb 04 17:48:36 2019 +0100
+++ b/pkg/wfs/download.go	Mon Feb 04 18:31:55 2019 +0100
@@ -15,15 +15,14 @@
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
-	"os"
 	"strconv"
 
 	"gemma.intevation.de/gemma/pkg/config"
@@ -259,34 +258,19 @@
 			resp.StatusCode, resp.Status)
 	}
 	defer resp.Body.Close()
-	tmp, err := ioutil.TempFile(config.TmpDir(), "wfs*.tmp")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tmp.Name())
+
+	var already bytes.Buffer
 
-	_, err = io.Copy(tmp, resp.Body)
-	err2 := tmp.Close()
-	if err != nil {
-		return err
-	}
-	if err2 != nil {
-		return err2
-	}
-	f, err := os.Open(tmp.Name())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
+	// Prevent the XML reader from consuming everything.
+	limit := io.LimitReader(resp.Body, 16*1024)
+	in := io.TeeReader(limit, &already)
 
-	if err := scanExceptionReport(f); err != nil {
+	if err := scanExceptionReport(in); err != nil {
 		return err
 	}
 
-	if _, err := f.Seek(0, 0); err != nil {
-		return err
-	}
-	return handler(url, bufio.NewReader(f))
+	multi := io.MultiReader(bytes.NewReader(already.Bytes()), resp.Body)
+	return handler(url, multi)
 }
 
 type ExceptionReport struct {
@@ -319,7 +303,7 @@
 
 	var er ExceptionReport
 
-	if err := decoder.Decode(&er); err != nil {
+	if decoder.Decode(&er) != nil {
 		// Fine because of other kind of document
 		return nil
 	}