Mercurial > gemma
view pkg/controllers/srimports.go @ 1741:44398a8bdf94
Approved gauge measurements: Added a stub to upload a CSV file for parsing.
TODO: Implement the parsing and store the values in the DB.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Wed, 09 Jan 2019 18:26:52 +0100 |
parents | 49e047c2106e |
children | 807569b08513 |
line wrap: on
line source
// This is Free Software under GNU Affero General Public License v >= 3.0 // without warranty, see README.md and license for details. // // SPDX-License-Identifier: AGPL-3.0-or-later // License-Filename: LICENSES/AGPL-3.0.txt // // Copyright (C) 2018 by via donau // – Österreichische Wasserstraßen-Gesellschaft mbH // Software engineering by Intevation GmbH // // Author(s): // * Sascha L. Teichmann <sascha.teichmann@intevation.de> package controllers import ( "archive/zip" "bufio" "database/sql" "encoding/hex" "fmt" "io" "io/ioutil" "log" "net/http" "os" "path/filepath" "strconv" "sync" "time" "github.com/gorilla/mux" "gemma.intevation.de/gemma/pkg/auth" "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/config" "gemma.intevation.de/gemma/pkg/imports" "gemma.intevation.de/gemma/pkg/misc" "gemma.intevation.de/gemma/pkg/models" ) const ( maxSoundingResultSize = 25 * 1024 * 1024 soundingResultName = "soundingresult" ) func fetchSoundingResult(req *http.Request) (string, error) { // Check first if we have a token. if token := req.FormValue("token"); token != "" { if _, err := hex.DecodeString(token); err != nil { return "", err } dir := config.TmpDir() if dir == "" { dir = os.TempDir() } // XXX: This should hopefully be race-free enough. now := time.Now().Format("2006-15-04-05") dst := filepath.Join(dir, soundingResultName+"-"+token+"-"+now) if err := misc.UnmakeTempFile(token, dst); err != nil { return "", err } return dst, nil } return storeSoundingResult(req) } func storeSoundingResult(req *http.Request) (string, error) { // Check for direct upload. f, _, err := req.FormFile(soundingResultName) if err != nil { return "", err } defer f.Close() dir, err := ioutil.TempDir(config.TmpDir(), soundingResultName) if err != nil { return "", err } o, err := os.Create(filepath.Join(dir, "sr.zip")) if err != nil { os.RemoveAll(dir) return "", err } out := bufio.NewWriter(o) if _, err = io.Copy(out, io.LimitReader(f, maxSoundingResultSize)); err != nil { o.Close() os.RemoveAll(dir) return "", err } if err = out.Flush(); err != nil { o.Close() os.RemoveAll(dir) return "", err } return dir, nil } func fetchSoundingResultMetaOverrides(sr *imports.SoundingResult, req *http.Request) error { if v := req.FormValue("epsg"); v != "" { epsg, err := strconv.ParseUint(v, 10, 32) if err != nil { return err } srid := uint(epsg) sr.EPSG = &srid } if v := req.FormValue("date"); v != "" { date, err := time.Parse(models.SoundingResultDateFormat, v) if err != nil { return err } sr.Date = &models.SoundingResultDate{Time: date} } if v := req.FormValue("depth-reference"); v != "" { sr.DepthReference = &v } if v := req.FormValue("bottleneck"); v != "" { sr.Bottleneck = &v } return nil } func importSoundingResult(rw http.ResponseWriter, req *http.Request) { sr := new(imports.SoundingResult) if err := fetchSoundingResultMetaOverrides(sr, req); err != nil { log.Printf("error: %v\n", err) http.Error(rw, "error: "+err.Error(), http.StatusBadRequest) return } dir, err := fetchSoundingResult(req) if err != nil { log.Printf("error: %v\n", err) http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) return } sr.Dir = dir serialized, err := common.ToJSONString(sr) if err != nil { log.Printf("error: %v\n", err) http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) return } session, _ := auth.GetSession(req) sendEmail := req.FormValue("bottleneck") != "" var due time.Time if d := req.FormValue("due"); d != "" { var err error if due, err = time.Parse("2006-01-02T15:04:05", d); err != nil { log.Printf("error: %v\n", err) } } retries := -1 if r := req.FormValue("retries"); r != "" { var err error if retries, err = strconv.Atoi(r); err != nil { log.Printf("error: %v\n", err) retries = -1 } } jobID, err := imports.AddJob( imports.SRJobKind, due, retries, session.User, sendEmail, false, serialized) if err != nil { log.Printf("error: %v\n", err) http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) return } log.Printf("info: added import #%d to queue\n", jobID) result := struct { ID int64 `json:"id"` }{ ID: jobID, } SendJSON(rw, http.StatusCreated, &result) } func loadMeta(f *zip.File) (*models.SoundingResultMeta, error) { r, err := f.Open() if err != nil { return nil, err } defer r.Close() var m models.SoundingResultMeta return &m, m.Decode(r) } func uploadSoundingResult( _ interface{}, req *http.Request, conn *sql.Conn, ) (jr JSONResult, err error) { var dir string if dir, err = storeSoundingResult(req); err != nil { return } srFile := filepath.Join(dir, "sr.zip") var zr *zip.ReadCloser if zr, err = zip.OpenReader(srFile); err != nil { return } var once sync.Once closeOnce := func() { zr.Close() } defer once.Do(closeOnce) var messages []string var result struct { Token string `json:"token,omitempty"` Meta interface{} `json:"meta,omitempty"` Messages []string `json:"messages,omitempty"` } find := func(ext string) *zip.File { return common.FindInZIP(zr, ext) } noXYZ := find(".xyz") == nil && find(".txt") == nil if noXYZ { messages = append(messages, "no .xyz or .txt file found.") } if mj := find("meta.json"); mj == nil { messages = append(messages, "no 'meta.json' file found.") } else { if meta, err := loadMeta(mj); err != nil { messages = append(messages, fmt.Sprintf("'meta.json' found but invalid: %v", err)) } else { errs := meta.Validate(req.Context(), conn) for _, err := range errs { messages = append(messages, fmt.Sprintf("invalid 'meta.json': %v", err)) } result.Meta = meta } } once.Do(closeOnce) code := http.StatusCreated // If there are no XYZ data we cant help the user anyway. if noXYZ { code = http.StatusBadRequest if err2 := os.RemoveAll(dir); err2 != nil { log.Printf("error: %v\n", err2) } } else if result.Token, err = misc.MakeTempFile(dir); err != nil { if err2 := os.RemoveAll(dir); err2 != nil { log.Printf("error: %v\n", err2) } return } result.Messages = messages jr = JSONResult{ Code: code, Result: &result, } return } func deleteSoundingUpload(rw http.ResponseWriter, req *http.Request) { token := mux.Vars(req)["token"] if _, err := hex.DecodeString(token); err != nil { http.Error(rw, "Invalid token", http.StatusBadRequest) return } if err := misc.DeleteTempFile(token); err != nil { http.Error(rw, fmt.Sprintf("error: %v", err), http.StatusInternalServerError) return } result := struct { Message string `json:"message"` }{ Message: fmt.Sprintf("Token %s deleted.", token), } SendJSON(rw, http.StatusOK, &result) }