Mercurial > gemma

comparison pkg/imports/stsh.go @ 4332:8080007d3c06

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
shape upload stretch import: Finished implementation (staging, summary).
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Thu, 05 Sep 2019 10:32:45 +0200
parents 3d6a2c6b436c
children eca3afe766d7 5e38667f740c
comparison
equal deleted inserted replaced
4331:98497ac4af3c 4332:8080007d3c06
17 "archive/zip" 17 "archive/zip"
18 "context" 18 "context"
19 "database/sql" 19 "database/sql"
20 "errors" 20 "errors"
21 "fmt" 21 "fmt"
22 "log"
23 "os" 22 "os"
24 "path" 23 "path"
25 "path/filepath" 24 "path/filepath"
26 "strings" 25 "strings"
27 "time" 26 "time"
28 27
29 shp "github.com/jonas-p/go-shp" 28 shp "github.com/jonas-p/go-shp"
30 29
31 "gemma.intevation.de/gemma/pkg/common" 30 "gemma.intevation.de/gemma/pkg/common"
31 "gemma.intevation.de/gemma/pkg/models"
32 ) 32 )
33 33
34 type StretchShape struct { 34 type StretchShape struct {
35 Dir string `json:"dir"` 35 Dir string `json:"dir"`
36 } 36 }
73 $7, 73 $7,
74 $8) 74 $8)
75 RETURNING id` 75 RETURNING id`
76 ) 76 )
77 77
78 // StageDone is merely the same as for the normal stretch import.
78 func (stshJobCreator) StageDone( 79 func (stshJobCreator) StageDone(
79 ctx context.Context, 80 ctx context.Context,
80 tx *sql.Tx, 81 tx *sql.Tx,
81 id int64, 82 id int64,
82 ) error { 83 ) error {
83 // TODO: Implement me! 84 return stJobCreator{}.StageDone(ctx, tx, id)
84 return nil
85 } 85 }
86 86
87 func (stsh *StretchShape) CleanUp() error { 87 func (stsh *StretchShape) CleanUp() error {
88 return os.RemoveAll(stsh.Dir) 88 return os.RemoveAll(stsh.Dir)
89 } 89 }
90 90
91 func fixAttribute(s string) string { 91 func fixAttribute(s string) string {
92 return strings.TrimRight(s, "\x00") 92 return strings.TrimRight(s, "\x00")
93 }
94
95 type stretchSummary struct {
96 ID int64 `json:"id"`
97 Name string `json:"name"`
98 From string `json:"from"`
99 To string `json:"to"`
100 ObjNam string `json:"objnam"`
101 NObjNam *string `json:"nobjnam"`
102 Date models.Date `json:"date-info"`
103 Countries models.UniqueCountries `json:"countries"`
104 }
105
106 func parseUniqueCountries(s string) (models.UniqueCountries, error) {
107 unique := map[models.Country]struct{}{}
108 var countries models.UniqueCountries
109 for _, c := range strings.Split(s, ",") {
110 if c = strings.ToUpper(strings.TrimSpace(c)); c == "" {
111 continue
112 }
113 n := models.Country(c)
114 if !n.Valid() {
115 return nil, fmt.Errorf("'%s' is not a valid country code", c)
116 }
117 if _, found := unique[n]; found {
118 return nil, fmt.Errorf("'%s' is not a unique country code", c)
119 }
120 countries = append(countries, n)
121 unique[n] = struct{}{}
122 }
123 return countries, nil
93 } 124 }
94 125
95 func (stsh *StretchShape) Do( 126 func (stsh *StretchShape) Do(
96 ctx context.Context, 127 ctx context.Context,
97 importID int64, 128 importID int64,
211 if err != nil { 242 if err != nil {
212 return nil, err 243 return nil, err
213 } 244 }
214 defer insCountryStmt.Close() 245 defer insCountryStmt.Close()
215 246
247 trackStmt, err := tx.PrepareContext(ctx, trackImportSQL)
248 if err != nil {
249 return nil, err
250 }
251 defer trackStmt.Close()
252
253 var stretches []*stretchSummary
254
216 for sr.Next() { 255 for sr.Next() {
217 256
218 _, p := sr.Shape() 257 _, p := sr.Shape()
219 if p == nil { 258 if p == nil {
220 feedback.Warn("Invalid NULL geometry found.") 259 feedback.Warn("Invalid NULL geometry found.")
225 feedback.Warn("Invalid geometry found: %v.", err) 264 feedback.Warn("Invalid geometry found: %v.", err)
226 continue 265 continue
227 } 266 }
228 267
229 var ( 268 var (
230 name = fixAttribute(sr.Attribute(nameIdx)) 269 name = fixAttribute(sr.Attribute(nameIdx))
231 objnam = fixAttribute(sr.Attribute(objnamIdx)) 270 objnam = fixAttribute(sr.Attribute(objnamIdx))
232 nobjnam = fixAttribute(sr.Attribute(nobjnamIdx)) 271 nobjnam = fixAttribute(sr.Attribute(nobjnamIdx))
233 lower = fixAttribute(sr.Attribute(lowerIdx)) 272 lower = fixAttribute(sr.Attribute(lowerIdx))
234 upper = fixAttribute(sr.Attribute(upperIdx)) 273 upper = fixAttribute(sr.Attribute(upperIdx))
235 dateInfo = fixAttribute(sr.Attribute(dateInfoIdx)) 274 dateInfo = fixAttribute(sr.Attribute(dateInfoIdx))
236 source = fixAttribute(sr.Attribute(sourceIdx)) 275 source = fixAttribute(sr.Attribute(sourceIdx))
237 countries = fixAttribute(sr.Attribute(countriesIdx)) 276 cnts = fixAttribute(sr.Attribute(countriesIdx))
238 ) 277 )
239 278
240 log.Printf("name : %+q\n", name) 279 feedback.Info("Importing stretch %s (%s - %s).",
241 log.Printf("objnam : %+q\n", objnam) 280 name, lower, upper)
242 log.Printf("nobjnam : %+q\n", nobjnam)
243 log.Printf("lower : %+q\n", lower)
244 log.Printf("upper : %+q\n", upper)
245 log.Printf("dateinfo : %+q\n", dateInfo)
246 log.Printf("source : %+q\n", source)
247 log.Printf("countries: %+q\n", countries)
248 281
249 date, err := common.ParseTime(dateInfo) 282 date, err := common.ParseTime(dateInfo)
250 if err != nil { 283 if err != nil {
251 feedback.Warn("Invalid time value: %v.", err) 284 feedback.Warn("Invalid time value: %v.", err)
285 continue
286 }
287
288 countries, err := parseUniqueCountries(cnts)
289 if err != nil {
290 feedback.Warn("Countries: %v.", err)
252 continue 291 continue
253 } 292 }
254 293
255 var nobjnamNull sql.NullString 294 var nobjnamNull sql.NullString
256 if nobjnam != "" { 295 if nobjnam != "" {
260 } 299 }
261 } 300 }
262 301
263 // Convert to a multi polygon. 302 // Convert to a multi polygon.
264 area := poly.MultiPolygonGeom().AsWKB() 303 area := poly.MultiPolygonGeom().AsWKB()
265
266 log.Printf("len geom: %d\n", len(area))
267 304
268 var id int64 305 var id int64
269 306
270 if err := insStmt.QueryRowContext( 307 if err := insStmt.QueryRowContext(
271 ctx, 308 ctx,
278 source, 315 source,
279 ).Scan(&id); err != nil { 316 ).Scan(&id); err != nil {
280 return nil, err 317 return nil, err
281 } 318 }
282 319
283 log.Println("after insert") 320 // Store the countries
284 321 for _, country := range countries {
285 for _, country := range strings.Split(countries, ",") {
286 if country = strings.TrimSpace(country); country == "" {
287 continue
288 }
289 if _, err := insCountryStmt.ExecContext(ctx, country); err != nil { 322 if _, err := insCountryStmt.ExecContext(ctx, country); err != nil {
290 return nil, err 323 return nil, err
291 } 324 }
292 } 325 }
293 326
294 // TODO: Implement me! 327 // Finally track the stretch
328
329 if _, err := trackStmt.ExecContext(
330 ctx,
331 importID,
332 "waterway.stretches",
333 id,
334 ); err != nil {
335 return nil, err
336 }
337
338 stretch := &stretchSummary{
339 ID: id,
340 Name: name,
341 From: lower,
342 To: upper,
343 ObjNam: objnam,
344 Date: models.Date{date},
345 Countries: countries,
346 }
347
348 if nobjnamNull.Valid {
349 stretch.NObjNam = &nobjnamNull.String
350 }
351
352 stretches = append(stretches, stretch)
295 } 353 }
296 354
297 if err := sr.Err(); err != nil { 355 if err := sr.Err(); err != nil {
298 return nil, err 356 return nil, err
299 } 357 }
300 358
301 return nil, errors.New("Not implemented, yet!") 359 if len(stretches) == 0 {
302 } 360 return nil, UnchangedError("No stretches written.")
361 }
362
363 if err := tx.Commit(); err != nil {
364 return nil, err
365 }
366
367 return stretches, nil
368 }