comparison pkg/imports/sr.go @ 1392:0e1d89241cda

Imports: An Import (e.g. a sounding result import) can now write a 'summary' of a successful import. This is done if the import switches to to state 'pending'.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 28 Nov 2018 09:52:34 +0100
parents 8d59457a5651
children eb82076fdffe
comparison
equal deleted inserted replaced
1391:801ae5f4bc5b 1392:0e1d89241cda
113 ST_Transform(ST_GeomFromWKB($5, $6::integer), 4326)::geography 113 ST_Transform(ST_GeomFromWKB($5, $6::integer), 4326)::geography
114 END) 114 END)
115 ) 115 )
116 RETURNING 116 RETURNING
117 id, 117 id,
118 ST_X(ST_Centroid(point_cloud::geometry)),
119 ST_Y(ST_Centroid(point_cloud::geometry)),
118 CASE WHEN ST_Y(ST_Centroid(point_cloud::geometry)) > 0 THEN 120 CASE WHEN ST_Y(ST_Centroid(point_cloud::geometry)) > 0 THEN
119 32600 121 32600
120 ELSE 122 ELSE
121 32700 123 32700
122 END + floor((ST_X(ST_Centroid(point_cloud::geometry))+180)/6)::int + 1` 124 END + floor((ST_X(ST_Centroid(point_cloud::geometry))+180)/6)::int + 1`
167 func (sr *SoundingResult) Do( 169 func (sr *SoundingResult) Do(
168 ctx context.Context, 170 ctx context.Context,
169 importID int64, 171 importID int64,
170 conn *sql.Conn, 172 conn *sql.Conn,
171 feedback Feedback, 173 feedback Feedback,
172 ) error { 174 ) (interface{}, error) {
173 175
174 z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip")) 176 z, err := zip.OpenReader(filepath.Join(sr.Dir, "sr.zip"))
175 if err != nil { 177 if err != nil {
176 return err 178 return nil, err
177 } 179 }
178 defer z.Close() 180 defer z.Close()
179 181
180 feedback.Info("Looking for 'meta.json'") 182 feedback.Info("Looking for 'meta.json'")
181 mf := common.FindInZIP(z, "meta.json") 183 mf := common.FindInZIP(z, "meta.json")
182 if mf == nil && !sr.completeOverride() { 184 if mf == nil && !sr.completeOverride() {
183 return errors.New("Cannot find 'meta.json'") 185 return nil, errors.New("Cannot find 'meta.json'")
184 } 186 }
185 187
186 m, err := sr.loadMeta(mf) 188 m, err := sr.loadMeta(mf)
187 if err != nil { 189 if err != nil {
188 return err 190 return nil, err
189 } 191 }
190 192
191 if err := m.Validate(ctx, conn); err != nil { 193 if err := m.Validate(ctx, conn); err != nil {
192 return common.ToError(err) 194 return nil, common.ToError(err)
193 } 195 }
194 196
195 feedback.Info("Looking for '*.xyz'") 197 feedback.Info("Looking for '*.xyz'")
196 xyzf := common.FindInZIP(z, ".xyz") 198 xyzf := common.FindInZIP(z, ".xyz")
197 if xyzf == nil { 199 if xyzf == nil {
198 return errors.New("Cannot find any *.xyz file") 200 return nil, errors.New("Cannot find any *.xyz file")
199 } 201 }
200 202
201 xyz, err := loadXYZ(xyzf, feedback) 203 xyz, err := loadXYZ(xyzf, feedback)
202 if err != nil { 204 if err != nil {
203 return err 205 return nil, err
204 } 206 }
205 207
206 if len(xyz) == 0 { 208 if len(xyz) == 0 {
207 return errors.New("XYZ does not contain any vertices") 209 return nil, errors.New("XYZ does not contain any vertices")
208 } 210 }
209 211
210 // Is there a boundary shapefile in the ZIP archive? 212 // Is there a boundary shapefile in the ZIP archive?
211 polygon, err := loadBoundary(z) 213 polygon, err := loadBoundary(z)
212 if err != nil { 214 if err != nil {
213 return err 215 return nil, err
214 } 216 }
215 217
216 tx, err := conn.BeginTx(ctx, nil) 218 tx, err := conn.BeginTx(ctx, nil)
217 if err != nil { 219 if err != nil {
218 return err 220 return nil, err
219 } 221 }
220 defer tx.Rollback() 222 defer tx.Rollback()
221 223
222 var id int64 224 var (
223 var epsg uint32 225 id int64
226 epsg uint32
227 lat, lon float64
228 )
224 start := time.Now() 229 start := time.Now()
225 230
226 err = tx.QueryRow(insertPointsSQL, 231 err = tx.QueryRow(insertPointsSQL,
227 m.Bottleneck, 232 m.Bottleneck,
228 m.Date.Time, 233 m.Date.Time,
229 m.DepthReference, 234 m.DepthReference,
230 xyz.AsWKB(), 235 xyz.AsWKB(),
231 polygon.AsWBK(), 236 polygon.AsWBK(),
232 m.EPSG, 237 m.EPSG,
233 ).Scan(&id, &epsg) 238 ).Scan(&id, &lat, &lon, &epsg)
234 xyz, polygon = nil, nil // not need from now on. 239 xyz, polygon = nil, nil // not need from now on.
235 feedback.Info("storing points took %s", time.Since(start)) 240 feedback.Info("storing points took %s", time.Since(start))
236 if err != nil { 241 if err != nil {
237 return err 242 return nil, err
238 } 243 }
239 244
240 feedback.Info("Best suited UTM EPSG: %d", epsg) 245 feedback.Info("Best suited UTM EPSG: %d", epsg)
241 246
242 feedback.Info("Triangulate...") 247 feedback.Info("Triangulate...")
243 start = time.Now() 248 start = time.Now()
244 tin, err := octree.GenerateTinByID(ctx, conn, id, epsg) 249 tin, err := octree.GenerateTinByID(ctx, conn, id, epsg)
245 feedback.Info("triangulation took %s", time.Since(start)) 250 feedback.Info("triangulation took %s", time.Since(start))
246 if err != nil { 251 if err != nil {
247 return err 252 return nil, err
248 } 253 }
249 254
250 if tin == nil { 255 if tin == nil {
251 return errors.New("cannot load TIN from database") 256 return nil, errors.New("cannot load TIN from database")
252 } 257 }
253 258
254 feedback.Info("Building octree...") 259 feedback.Info("Building octree...")
255 builder := octree.NewBuilder(tin) 260 builder := octree.NewBuilder(tin)
256 start = time.Now() 261 start = time.Now()
257 builder.Build() 262 builder.Build()
258 octreeIndex, err := builder.Bytes() 263 octreeIndex, err := builder.Bytes()
259 tin = nil // not needed from now on 264 tin = nil // not needed from now on
260 feedback.Info("building octree took %s", time.Since(start)) 265 feedback.Info("building octree took %s", time.Since(start))
261 if err != nil { 266 if err != nil {
262 return err 267 return nil, err
263 } 268 }
264 269
265 feedback.Info("Store octree...") 270 feedback.Info("Store octree...")
266 start = time.Now() 271 start = time.Now()
267 h := sha1.New() 272 h := sha1.New()
268 h.Write(octreeIndex) 273 h.Write(octreeIndex)
269 checksum := hex.EncodeToString(h.Sum(nil)) 274 checksum := hex.EncodeToString(h.Sum(nil))
270 _, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex) 275 _, err = tx.Exec(insertOctreeSQL, id, checksum, octreeIndex)
271 feedback.Info("storing octree index took %s", time.Since(start)) 276 feedback.Info("storing octree index took %s", time.Since(start))
272 if err != nil { 277 if err != nil {
273 return err 278 return nil, err
274 } 279 }
275 280
276 tree := builder.Tree() 281 tree := builder.Tree()
277 builder = nil // not needed from now on 282 builder = nil // not needed from now on
278 283
279 feedback.Info("Generate contour lines...") 284 feedback.Info("Generate contour lines...")
280 start = time.Now() 285 start = time.Now()
281 err = generateContours(tree, tx, id) 286 err = generateContours(tree, tx, id)
282 feedback.Info("generating and storing contour lines took %s", time.Since(start)) 287 feedback.Info("generating and storing contour lines took %s", time.Since(start))
283 if err != nil { 288 if err != nil {
284 return err 289 return nil, err
285 } 290 }
286 291
287 // Store for potential later removal. 292 // Store for potential later removal.
288 if err = track(ctx, tx, importID, "waterway.sounding_results", id); err != nil { 293 if err = track(ctx, tx, importID, "waterway.sounding_results", id); err != nil {
289 return err 294 return nil, err
290 } 295 }
291 296
292 if err = tx.Commit(); err == nil { 297 if err = tx.Commit(); err == nil {
293 feedback.Info("Storing sounding result was successful.") 298 feedback.Info("Storing sounding result was successful.")
294 } 299 }
295 return err 300
301 summary := struct {
302 Bottleneck string `json:"bottleneck"`
303 Date models.SoundingResultDate `json:"date"`
304 Lat float64 `json:"lat"`
305 Lon float64 `json:"lon"`
306 }{
307 Bottleneck: m.Bottleneck,
308 Date: m.Date,
309 Lat: lat,
310 Lon: lon,
311 }
312 return &summary, err
296 } 313 }
297 314
298 func (sr *SoundingResult) CleanUp() error { 315 func (sr *SoundingResult) CleanUp() error {
299 return os.RemoveAll(sr.Dir) 316 return os.RemoveAll(sr.Dir)
300 } 317 }