Mercurial > gemma
view pkg/imports/sr.go @ 959:6ab012d0f0c2
Started with writing an importer job for sounding results.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Tue, 16 Oct 2018 18:20:50 +0200 |
parents | |
children | e23ae2c83427 |
line wrap: on
line source
package imports import ( "archive/zip" "bufio" "context" "database/sql" "encoding/json" "errors" "fmt" "io" "log" "os" "path/filepath" "strconv" "strings" "time" "gemma.intevation.de/gemma/pkg/octree" ) type SoundingResult struct { who string dir string } const SoundingResultDateFormat = "2006-01-02" type SoundingResultDate struct{ time.Time } type SoundingResultMeta struct { Date SoundingResultDate `json:"date"` Bottleneck string `json:"bottleneck"` EPSG uint `json:"epsg"` DepthReference string `json:"depth-reference"` } func (srd *SoundingResultDate) UnmarshalJSON(data []byte) error { var s string if err := json.Unmarshal(data, &s); err != nil { return err } d, err := time.Parse(SoundingResultDateFormat, s) if err == nil { *srd = SoundingResultDate{d} } return err } func (sr *SoundingResult) Who() string { return sr.who } func (sr *SoundingResult) CleanUp() error { return os.RemoveAll(sr.dir) } func find(needle string, haystack []*zip.File) *zip.File { needle = strings.ToLower(needle) for _, straw := range haystack { if strings.HasSuffix(strings.ToLower(straw.Name), needle) { return straw } } return nil } func loadMeta(f *zip.File) (*SoundingResultMeta, error) { r, err := f.Open() if err != nil { return nil, err } defer r.Close() var m SoundingResultMeta err = json.NewDecoder(r).Decode(&m) return &m, err } func (m *SoundingResultMeta) validate(conn *sql.Conn) error { var b bool err := conn.QueryRowContext(context.Background(), `SELECT true FROM internal.depth_references WHERE depth_reference = $1`, m.DepthReference).Scan(&b) switch { case err == sql.ErrNoRows: return fmt.Errorf("Unknown depth reference '%s'\n", m.DepthReference) case err != nil: return err case !b: return errors.New("Unexpected depth reference") } err = conn.QueryRowContext(context.Background(), `SELECT true FROM waterway.bottlenecks WHERE bottleneck_id = $1`, m.Bottleneck).Scan(&b) switch { case err == sql.ErrNoRows: return fmt.Errorf("Unknown bottleneck '%s'\n", m.Bottleneck) case err != nil: return err case !b: return errors.New("Unexpected bottleneck") } return nil } func loadXYZReader(r io.Reader) (octree.MultiPointZ, error) { mpz := make(octree.MultiPointZ, 0, 250000) s := bufio.NewScanner(r) for line := 1; s.Scan(); line++ { text := s.Text() var p octree.Vertex // fmt.Sscanf(text, "%f,%f,%f") is 4 times slower. idx := strings.IndexByte(text, ',') if idx == -1 { log.Printf("format error in line %d\n", line) continue } var err error if p.X, err = strconv.ParseFloat(text[:idx], 64); err != nil { log.Printf("format error in line %d: %v\n", line, err) continue } text = text[idx+1:] if idx = strings.IndexByte(text, ','); idx == -1 { log.Printf("format error in line %d\n", line) continue } if p.Y, err = strconv.ParseFloat(text[:idx], 64); err != nil { log.Printf("format error in line %d: %v\n", line, err) continue } text = text[idx+1:] if p.Z, err = strconv.ParseFloat(text, 64); err != nil { log.Printf("format error in line %d: %v\n", line, err) continue } mpz = append(mpz, p) } if err := s.Err(); err != nil { return nil, err } return mpz, nil } func loadXYZ(f *zip.File) (octree.MultiPointZ, error) { r, err := f.Open() if err != nil { return nil, err } defer r.Close() return loadXYZReader(r) } func (sr *SoundingResult) Do(conn *sql.Conn) error { z, err := zip.OpenReader(filepath.Join(sr.dir, "upload.zip")) if err != nil { return err } defer z.Close() mf := find("meta.json", z.File) if mf == nil { return errors.New("Cannot find 'meta.json'") } m, err := loadMeta(mf) if err != nil { return err } if err := m.validate(conn); err != nil { return err } xyzf := find(".xyz", z.File) if xyzf == nil { return errors.New("Cannot find any *.xyz file") } xyz, err := loadXYZ(xyzf) if err != nil { return err } if len(xyz) == 0 { return errors.New("XYZ does not contain any vertices.") } // TODO: Implement more. return nil }