changeset 1221:c193649d4f11

Add an area for temp uploads on the server to be addressed by tokens. If unused they will me thrown away after 45 minutes. There could be max 100 of them. If there are more to upload the oldest are removed first.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 19 Nov 2018 16:15:30 +0100
parents d11d5e39c8d0
children a8b682f3d13d
files pkg/controllers/srimports.go pkg/misc/tmpfiles.go
diffstat 2 files changed, 215 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/controllers/srimports.go	Mon Nov 19 16:02:34 2018 +0100
+++ b/pkg/controllers/srimports.go	Mon Nov 19 16:15:30 2018 +0100
@@ -15,16 +15,19 @@
 
 import (
 	"bufio"
+	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"gemma.intevation.de/gemma/pkg/auth"
 	"gemma.intevation.de/gemma/pkg/config"
 	"gemma.intevation.de/gemma/pkg/imports"
+	"gemma.intevation.de/gemma/pkg/misc"
 )
 
 const (
@@ -34,6 +37,24 @@
 
 func downloadSoundingResult(req *http.Request) (string, error) {
 
+	// Check first if we have a token.
+	if token := req.FormValue("token"); token != "" {
+		if _, err := hex.DecodeString(token); err != nil {
+			return "", err
+		}
+		dir := config.TmpDir()
+		if dir == "" {
+			dir = os.TempDir()
+		}
+		// XXX: This should hopefully be race-free enough.
+		now := time.Now().Format("2006-15-04-05")
+		dst := filepath.Join(dir, soundingResultName+"-"+token+"-"+now)
+		if err := misc.UnmakeTempFile(token, dst); err != nil {
+			return "", err
+		}
+	}
+
+	// Check for direct upload.
 	f, _, err := req.FormFile(soundingResultName)
 	if err != nil {
 		return "", err
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pkg/misc/tmpfiles.go	Mon Nov 19 16:15:30 2018 +0100
@@ -0,0 +1,194 @@
+// This is Free Software under GNU Affero General Public License v >= 3.0
+// without warranty, see README.md and license for details.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// License-Filename: LICENSES/AGPL-3.0.txt
+//
+// Copyright (C) 2018 by via donau
+//   – Österreichische Wasserstraßen-Gesellschaft mbH
+// Software engineering by Intevation GmbH
+//
+// Author(s):
+//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
+
+package misc
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gemma.intevation.de/gemma/pkg/common"
+	"gemma.intevation.de/gemma/pkg/config"
+)
+
+const (
+	tmpFilesDir     = "tmpfiles"
+	maxTmpFiles     = 100
+	maxTmpFileAge   = 45 * time.Minute
+	tmpfilesCleanUp = 15 * time.Minute
+)
+
+var tmpfilesMu sync.Mutex
+
+func init() {
+	go func() {
+		config.WaitReady()
+		for {
+			if err := cleanupTmpFiles(); err != nil {
+				log.Printf("error: %v\n", err)
+			}
+			time.Sleep(tmpfilesCleanUp)
+		}
+	}()
+}
+
+func tmpfilesDir() string {
+	dir := config.TmpDir()
+	if dir == "" {
+		dir = os.TempDir()
+	}
+	return filepath.Join(dir, tmpFilesDir)
+}
+
+func toError(errs []error) error {
+	var b strings.Builder
+	for i, err := range errs {
+		if i > 0 {
+			fmt.Fprintf(&b, ", ")
+		}
+		fmt.Fprintf(&b, "%v", err)
+	}
+	return errors.New(b.String())
+}
+
+func cleanupTmpFiles() error {
+	tmpfilesMu.Lock()
+	defer tmpfilesMu.Unlock()
+
+	tmp := tmpfilesDir()
+
+	_, err := os.Stat(tmp)
+	switch {
+	case os.IsNotExist(err):
+		return nil
+	case err != nil:
+		return err
+	}
+
+	f, err := os.Open(tmp)
+	if err != nil {
+		return err
+	}
+
+	files, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return err
+	}
+
+	bestBefore := time.Now().Add(-maxTmpFileAge)
+
+	var errs []error
+	var deleted int
+
+	for _, fi := range files {
+		if fi.ModTime().Before(bestBefore) {
+			fname := filepath.Join(tmp, fi.Name())
+			if err := os.RemoveAll(fname); err != nil {
+				errs = append(errs, err)
+			} else {
+				deleted++
+			}
+		}
+	}
+
+	// If empty remove temp folder.
+	if deleted == len(files) {
+		if err := os.RemoveAll(tmp); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return toError(errs)
+	}
+
+	return nil
+}
+
+func UnmakeTempFile(token, path string) error {
+	tmpfilesMu.Lock()
+	defer tmpfilesMu.Unlock()
+	tmp := tmpfilesDir()
+	return os.Rename(filepath.Join(tmp, token), path)
+}
+
+func MakeTempFile(fname string) (string, error) {
+	tmpfilesMu.Lock()
+	defer tmpfilesMu.Unlock()
+
+	tmp := tmpfilesDir()
+
+	_, err := os.Stat(tmp)
+	switch {
+	case os.IsNotExist(err):
+		if err := os.MkdirAll(tmp, 0700); err != nil {
+			return "", err
+		}
+	case err != nil:
+		return "", err
+	}
+
+	f, err := os.Open(tmp)
+	if err != nil {
+		return "", err
+	}
+
+	files, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return "", err
+	}
+
+	// If there are too many throw away old.
+	if len(files) >= maxTmpFiles {
+		sort.Slice(files, func(i, j int) bool {
+			return files[i].ModTime().Before(files[j].ModTime())
+		})
+
+		var errs []error
+		for len(files) >= maxTmpFiles {
+			fname := filepath.Join(tmp, files[0].Name())
+			if err := os.RemoveAll(fname); err != nil {
+				errs = append(errs, err)
+			}
+			files = files[1:]
+		}
+		if len(errs) > 0 {
+			return "", toError(errs)
+		}
+	}
+
+	var token string
+again:
+	token = generateToken()
+	for _, f := range files {
+		if f.Name() == token {
+			goto again
+		}
+	}
+	err = os.Rename(fname, filepath.Join(tmp, token))
+	return token, err
+}
+
+func generateToken() string {
+	return hex.EncodeToString(common.GenerateRandomKey(20))
+}