Mercurial > gemma
changeset 2078:dacf79a0658e
Waterway profile import: Parse header line. TODO: Parse the body and store the data into the database.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Wed, 30 Jan 2019 20:27:07 +0100 |
parents | e9aaf6808682 |
children | 9318973487a1 |
files | pkg/controllers/wpimports.go pkg/imports/wp.go |
diffstat | 2 files changed, 84 insertions(+), 1 deletions(-) [+] |
line wrap: on
line diff
--- a/pkg/controllers/wpimports.go Wed Jan 30 17:28:50 2019 +0100 +++ b/pkg/controllers/wpimports.go Wed Jan 30 20:27:07 2019 +0100 @@ -93,7 +93,7 @@ sendEmail := req.FormValue("email") != "" jobID, err := imports.AddJob( - imports.AGMJobKind, + imports.WPJobKind, time.Time{}, // due nil, // trys nil, // wait retry
--- a/pkg/imports/wp.go Wed Jan 30 17:28:50 2019 +0100 +++ b/pkg/imports/wp.go Wed Jan 30 20:27:07 2019 +0100 @@ -14,10 +14,15 @@ package imports import ( + "bufio" "context" "database/sql" + "encoding/csv" "errors" + "fmt" "os" + "path/filepath" + "strings" "gemma.intevation.de/gemma/pkg/common" ) @@ -73,6 +78,84 @@ conn *sql.Conn, feedback Feedback, ) (interface{}, error) { + + //start := time.Now() + + f, err := os.Open(filepath.Join(wp.Dir, "wp.csv")) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(bufio.NewReader(f)) + r.Comma = ';' + r.ReuseRecord = true + + headers, err := r.Read() + if err != nil { + return nil, err + } + + var ( + locationIdx = -1 + validFromIdx = -1 + validToIdx = -1 + lnwlIdx = -1 + mwlIdx = -1 + hnwlIdx = -1 + fe30Idx = -1 + fe100Idx = -1 + dateInfoIdx = -1 + sourceIdx = -1 + ) + + type headerField struct { + idx *int + substr string + } + + fields := []headerField{ + {&locationIdx, "location"}, + {&validFromIdx, "valid_from"}, + {&validToIdx, "valid_to"}, + {&lnwlIdx, "lnwl"}, + {&mwlIdx, "mwl"}, + {&hnwlIdx, "hnwl"}, + {&fe30Idx, "fe30"}, + {&fe100Idx, "fe100"}, + {&dateInfoIdx, "date_info"}, + {&sourceIdx, "source"}, + } + +nextHeader: + for i, h := range headers { + h = strings.ToLower(h) + for j := range fields { + if strings.Contains(h, fields[j].substr) { + if *fields[j].idx != -1 { + return nil, fmt.Errorf( + "CSV has more than one column with name containing '%s'", + fields[j].substr) + } + *fields[j].idx = i + continue nextHeader + } + } + } + + var missing []string + for i := range fields { + if *fields[i].idx == -1 { + missing = append(missing, fields[i].substr) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf( + "CSV is missing columns: %s", + strings.Join(missing, ", ")) + } + // TODO: Implement me! + return nil, errors.New("Not implemented, yet!") }