Mercurial > gemma
changeset 1741:44398a8bdf94
Approved gauge measurements: Added a stub to upload a CSV file for parsing.
TODO: Implement the parsing and store the values in the DB.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Wed, 09 Jan 2019 18:26:52 +0100 |
parents | 82a36eaf2366 |
children | 8bbdad3f9af7 |
files | pkg/controllers/agmimports.go pkg/controllers/routes.go pkg/imports/agm.go |
diffstat | 3 files changed, 231 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/controllers/agmimports.go Wed Jan 09 18:26:52 2019 +0100 @@ -0,0 +1,135 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann <sascha.teichmann@intevation.de> + +package controllers + +import ( + "bufio" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "gemma.intevation.de/gemma/pkg/auth" + "gemma.intevation.de/gemma/pkg/common" + "gemma.intevation.de/gemma/pkg/config" + "gemma.intevation.de/gemma/pkg/imports" +) + +const ( + maxApprovedGaugeMeasurementSize = 25 * 1024 * 1024 + approvedGaugeMeasurementsName = "approvedgm" +) + +func storeApprovedGaugeMeasurements(req *http.Request) (string, error) { + + // Check for direct upload. + f, _, err := req.FormFile(approvedGaugeMeasurementsName) + if err != nil { + return "", err + } + defer f.Close() + + dir, err := ioutil.TempDir(config.TmpDir(), approvedGaugeMeasurementsName) + if err != nil { + return "", err + } + + o, err := os.Create(filepath.Join(dir, "agm.csv")) + if err != nil { + os.RemoveAll(dir) + return "", err + } + + out := bufio.NewWriter(o) + + if _, err = io.Copy(out, io.LimitReader(f, maxApprovedGaugeMeasurementSize)); err != nil { + o.Close() + os.RemoveAll(dir) + return "", err + } + + if err = out.Flush(); err != nil { + o.Close() + os.RemoveAll(dir) + return "", err + } + + return dir, nil +} + +func importApprovedGaugeMeasurements(rw http.ResponseWriter, req *http.Request) { + + dir, err := storeApprovedGaugeMeasurements(req) + if err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) + return + } + + agm := &imports.ApprovedGaugeMeasurements{Dir: dir} + + serialized, err := common.ToJSONString(agm) + if err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) + return + } + + session, _ := auth.GetSession(req) + + sendEmail := req.FormValue("bottleneck") != "" + + var due time.Time + if d := req.FormValue("due"); d != "" { + var err error + if due, err = time.Parse("2006-01-02T15:04:05", d); err != nil { + log.Printf("error: %v\n", err) + } + } + + retries := -1 + if r := req.FormValue("retries"); r != "" { + var err error + if retries, err = strconv.Atoi(r); err != nil { + log.Printf("error: %v\n", err) + retries = -1 + } + } + + jobID, err := imports.AddJob( + imports.AGMJobKind, + due, retries, + session.User, + sendEmail, false, + serialized) + + if err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) + return + } + + log.Printf("info: added import #%d to queue\n", jobID) + + result := struct { + ID int64 `json:"id"` + }{ + ID: jobID, + } + SendJSON(rw, http.StatusCreated, &result) +}
--- a/pkg/controllers/routes.go Wed Jan 09 17:28:41 2019 +0100 +++ b/pkg/controllers/routes.go Wed Jan 09 18:26:52 2019 +0100 @@ -172,6 +172,9 @@ api.Handle("/imports/soundingresult", waterwayAdmin( http.HandlerFunc(importSoundingResult))).Methods(http.MethodPost) + api.Handle("/imports/approvedgm", waterwayAdmin( + http.HandlerFunc(importApprovedGaugeMeasurements))).Methods(http.MethodPost) + api.Handle("/imports/bottleneck", waterwayAdmin(&JSONHandler{ Input: func() interface{} { return new(models.BottleneckImport) }, Handle: manualImport(imports.BNJobKind, importBottleneck),
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/agm.go Wed Jan 09 18:26:52 2019 +0100 @@ -0,0 +1,93 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann <sascha.teichmann@intevation.de> + +package imports + +import ( + "context" + "database/sql" + "os" + + "gemma.intevation.de/gemma/pkg/common" +) + +type ApprovedGaugeMeasurements struct { + Dir string `json:"dir"` +} + +// GMAPJobKind is the unique name of an approved gauge measurements import job. +const AGMJobKind JobKind = "agm" + +type agmJobCreator struct{} + +func init() { + RegisterJobCreator(AGMJobKind, agmJobCreator{}) +} + +func (agmJobCreator) Description() string { + return "approved gauge measurements" +} + +func (agmJobCreator) Create(_ JobKind, data string) (Job, error) { + agm := new(ApprovedGaugeMeasurements) + if err := common.FromJSONString(data, agm); err != nil { + return nil, err + } + return agm, nil +} + +func (agmJobCreator) Depends() []string { + return []string{ + "gauges", + "gauge_measurements", + } +} + +const ( + // TODO: re-add staging_done field in table and fix RLS policy + // issue for raw import. + agmStageDoneSQL = ` +UPDATE waterway.gauge_measurements SET staging_done = true +WHERE id = ( + SELECT key from waterway.track_imports + WHERE import_id = $1 AND + relation = 'waterway.gauge_measurements'::regclass)` +) + +func (agmJobCreator) StageDone( + ctx context.Context, + tx *sql.Tx, + id int64, +) error { + _, err := tx.ExecContext(ctx, agmStageDoneSQL, id) + return err +} + +// CleanUp removes the folder containing the CSV file with the +// the approved gauge measurements. +func (agm *ApprovedGaugeMeasurements) CleanUp() error { + return os.RemoveAll(agm.Dir) +} + +// Do executes the actual approved gauge measurements import. +func (agm *ApprovedGaugeMeasurements) Do( + ctx context.Context, + importID int64, + conn *sql.Conn, + feedback Feedback, +) (interface{}, error) { + + // TODO: Implement me! + + return nil, nil +}