Mercurial > gemma

comparison pkg/imports/sr.go @ 959:6ab012d0f0c2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Started with writing an importer job for sounding results.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 16 Oct 2018 18:20:50 +0200
parents
children e23ae2c83427
comparison
equal deleted inserted replaced
958:2818ad6c7d32 959:6ab012d0f0c2
1 package imports
2
3 import (
4 "archive/zip"
5 "bufio"
6 "context"
7 "database/sql"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "log"
13 "os"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "time"
18
19 "gemma.intevation.de/gemma/pkg/octree"
20 )
21
22 type SoundingResult struct {
23 who string
24 dir string
25 }
26
27 const SoundingResultDateFormat = "2006-01-02"
28
29 type SoundingResultDate struct{ time.Time }
30
31 type SoundingResultMeta struct {
32 Date SoundingResultDate `json:"date"`
33 Bottleneck string `json:"bottleneck"`
34 EPSG uint `json:"epsg"`
35 DepthReference string `json:"depth-reference"`
36 }
37
38 func (srd *SoundingResultDate) UnmarshalJSON(data []byte) error {
39 var s string
40 if err := json.Unmarshal(data, &s); err != nil {
41 return err
42 }
43 d, err := time.Parse(SoundingResultDateFormat, s)
44 if err == nil {
45 *srd = SoundingResultDate{d}
46 }
47 return err
48 }
49
50 func (sr *SoundingResult) Who() string {
51 return sr.who
52 }
53
54 func (sr *SoundingResult) CleanUp() error {
55 return os.RemoveAll(sr.dir)
56 }
57
58 func find(needle string, haystack []*zip.File) *zip.File {
59 needle = strings.ToLower(needle)
60 for _, straw := range haystack {
61 if strings.HasSuffix(strings.ToLower(straw.Name), needle) {
62 return straw
63 }
64 }
65 return nil
66 }
67
68 func loadMeta(f *zip.File) (*SoundingResultMeta, error) {
69 r, err := f.Open()
70 if err != nil {
71 return nil, err
72 }
73 defer r.Close()
74 var m SoundingResultMeta
75 err = json.NewDecoder(r).Decode(&m)
76 return &m, err
77 }
78
79 func (m *SoundingResultMeta) validate(conn *sql.Conn) error {
80
81 var b bool
82 err := conn.QueryRowContext(context.Background(),
83 `SELECT true FROM internal.depth_references WHERE depth_reference = $1`,
84 m.DepthReference).Scan(&b)
85 switch {
86 case err == sql.ErrNoRows:
87 return fmt.Errorf("Unknown depth reference '%s'\n", m.DepthReference)
88 case err != nil:
89 return err
90 case !b:
91 return errors.New("Unexpected depth reference")
92 }
93
94 err = conn.QueryRowContext(context.Background(),
95 `SELECT true FROM waterway.bottlenecks WHERE bottleneck_id = $1`,
96 m.Bottleneck).Scan(&b)
97 switch {
98 case err == sql.ErrNoRows:
99 return fmt.Errorf("Unknown bottleneck '%s'\n", m.Bottleneck)
100 case err != nil:
101 return err
102 case !b:
103 return errors.New("Unexpected bottleneck")
104 }
105
106 return nil
107 }
108
109 func loadXYZReader(r io.Reader) (octree.MultiPointZ, error) {
110 mpz := make(octree.MultiPointZ, 0, 250000)
111 s := bufio.NewScanner(r)
112
113 for line := 1; s.Scan(); line++ {
114 text := s.Text()
115 var p octree.Vertex
116 // fmt.Sscanf(text, "%f,%f,%f") is 4 times slower.
117 idx := strings.IndexByte(text, ',')
118 if idx == -1 {
119 log.Printf("format error in line %d\n", line)
120 continue
121 }
122 var err error
123 if p.X, err = strconv.ParseFloat(text[:idx], 64); err != nil {
124 log.Printf("format error in line %d: %v\n", line, err)
125 continue
126 }
127 text = text[idx+1:]
128 if idx = strings.IndexByte(text, ','); idx == -1 {
129 log.Printf("format error in line %d\n", line)
130 continue
131 }
132 if p.Y, err = strconv.ParseFloat(text[:idx], 64); err != nil {
133 log.Printf("format error in line %d: %v\n", line, err)
134 continue
135 }
136 text = text[idx+1:]
137 if p.Z, err = strconv.ParseFloat(text, 64); err != nil {
138 log.Printf("format error in line %d: %v\n", line, err)
139 continue
140 }
141 mpz = append(mpz, p)
142 }
143
144 if err := s.Err(); err != nil {
145 return nil, err
146 }
147
148 return mpz, nil
149 }
150
151 func loadXYZ(f *zip.File) (octree.MultiPointZ, error) {
152 r, err := f.Open()
153 if err != nil {
154 return nil, err
155 }
156 defer r.Close()
157 return loadXYZReader(r)
158 }
159
160 func (sr *SoundingResult) Do(conn *sql.Conn) error {
161
162 z, err := zip.OpenReader(filepath.Join(sr.dir, "upload.zip"))
163 if err != nil {
164 return err
165 }
166 defer z.Close()
167
168 mf := find("meta.json", z.File)
169 if mf == nil {
170 return errors.New("Cannot find 'meta.json'")
171 }
172
173 m, err := loadMeta(mf)
174 if err != nil {
175 return err
176 }
177
178 if err := m.validate(conn); err != nil {
179 return err
180 }
181
182 xyzf := find(".xyz", z.File)
183 if xyzf == nil {
184 return errors.New("Cannot find any *.xyz file")
185 }
186
187 xyz, err := loadXYZ(xyzf)
188 if err != nil {
189 return err
190 }
191
192 if len(xyz) == 0 {
193 return errors.New("XYZ does not contain any vertices.")
194 }
195
196 // TODO: Implement more.
197
198 return nil
199 }