# HG changeset patch # User Sascha L. Teichmann # Date 1542640530 -3600 # Node ID c193649d4f1187a9a07af1b60d9398f735c5c263 # Parent d11d5e39c8d09365a1a3f788c70673f08166a0ae Add an area for temp uploads on the server to be addressed by tokens. If unused they will me thrown away after 45 minutes. There could be max 100 of them. If there are more to upload the oldest are removed first. diff -r d11d5e39c8d0 -r c193649d4f11 pkg/controllers/srimports.go --- a/pkg/controllers/srimports.go Mon Nov 19 16:02:34 2018 +0100 +++ b/pkg/controllers/srimports.go Mon Nov 19 16:15:30 2018 +0100 @@ -15,16 +15,19 @@ import ( "bufio" + "encoding/hex" "io" "io/ioutil" "log" "net/http" "os" "path/filepath" + "time" "gemma.intevation.de/gemma/pkg/auth" "gemma.intevation.de/gemma/pkg/config" "gemma.intevation.de/gemma/pkg/imports" + "gemma.intevation.de/gemma/pkg/misc" ) const ( @@ -34,6 +37,24 @@ func downloadSoundingResult(req *http.Request) (string, error) { + // Check first if we have a token. + if token := req.FormValue("token"); token != "" { + if _, err := hex.DecodeString(token); err != nil { + return "", err + } + dir := config.TmpDir() + if dir == "" { + dir = os.TempDir() + } + // XXX: This should hopefully be race-free enough. + now := time.Now().Format("2006-15-04-05") + dst := filepath.Join(dir, soundingResultName+"-"+token+"-"+now) + if err := misc.UnmakeTempFile(token, dst); err != nil { + return "", err + } + } + + // Check for direct upload. f, _, err := req.FormFile(soundingResultName) if err != nil { return "", err diff -r d11d5e39c8d0 -r c193649d4f11 pkg/misc/tmpfiles.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/misc/tmpfiles.go Mon Nov 19 16:15:30 2018 +0100 @@ -0,0 +1,194 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann + +package misc + +import ( + "encoding/hex" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "gemma.intevation.de/gemma/pkg/common" + "gemma.intevation.de/gemma/pkg/config" +) + +const ( + tmpFilesDir = "tmpfiles" + maxTmpFiles = 100 + maxTmpFileAge = 45 * time.Minute + tmpfilesCleanUp = 15 * time.Minute +) + +var tmpfilesMu sync.Mutex + +func init() { + go func() { + config.WaitReady() + for { + if err := cleanupTmpFiles(); err != nil { + log.Printf("error: %v\n", err) + } + time.Sleep(tmpfilesCleanUp) + } + }() +} + +func tmpfilesDir() string { + dir := config.TmpDir() + if dir == "" { + dir = os.TempDir() + } + return filepath.Join(dir, tmpFilesDir) +} + +func toError(errs []error) error { + var b strings.Builder + for i, err := range errs { + if i > 0 { + fmt.Fprintf(&b, ", ") + } + fmt.Fprintf(&b, "%v", err) + } + return errors.New(b.String()) +} + +func cleanupTmpFiles() error { + tmpfilesMu.Lock() + defer tmpfilesMu.Unlock() + + tmp := tmpfilesDir() + + _, err := os.Stat(tmp) + switch { + case os.IsNotExist(err): + return nil + case err != nil: + return err + } + + f, err := os.Open(tmp) + if err != nil { + return err + } + + files, err := f.Readdir(-1) + f.Close() + if err != nil { + return err + } + + bestBefore := time.Now().Add(-maxTmpFileAge) + + var errs []error + var deleted int + + for _, fi := range files { + if fi.ModTime().Before(bestBefore) { + fname := filepath.Join(tmp, fi.Name()) + if err := os.RemoveAll(fname); err != nil { + errs = append(errs, err) + } else { + deleted++ + } + } + } + + // If empty remove temp folder. + if deleted == len(files) { + if err := os.RemoveAll(tmp); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return toError(errs) + } + + return nil +} + +func UnmakeTempFile(token, path string) error { + tmpfilesMu.Lock() + defer tmpfilesMu.Unlock() + tmp := tmpfilesDir() + return os.Rename(filepath.Join(tmp, token), path) +} + +func MakeTempFile(fname string) (string, error) { + tmpfilesMu.Lock() + defer tmpfilesMu.Unlock() + + tmp := tmpfilesDir() + + _, err := os.Stat(tmp) + switch { + case os.IsNotExist(err): + if err := os.MkdirAll(tmp, 0700); err != nil { + return "", err + } + case err != nil: + return "", err + } + + f, err := os.Open(tmp) + if err != nil { + return "", err + } + + files, err := f.Readdir(-1) + f.Close() + if err != nil { + return "", err + } + + // If there are too many throw away old. + if len(files) >= maxTmpFiles { + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime().Before(files[j].ModTime()) + }) + + var errs []error + for len(files) >= maxTmpFiles { + fname := filepath.Join(tmp, files[0].Name()) + if err := os.RemoveAll(fname); err != nil { + errs = append(errs, err) + } + files = files[1:] + } + if len(errs) > 0 { + return "", toError(errs) + } + } + + var token string +again: + token = generateToken() + for _, f := range files { + if f.Name() == token { + goto again + } + } + err = os.Rename(fname, filepath.Join(tmp, token)) + return token, err +} + +func generateToken() string { + return hex.EncodeToString(common.GenerateRandomKey(20)) +}