Mercurial > gemma

comparison pkg/imports/bn.go @ 1569:cf0ca4a9812a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Bottleneck import: Implemented backend to store bottlenecks in database.
author Raimund Renkert <raimund.renkert@intevation.de>
date Wed, 12 Dec 2018 15:48:33 +0100
parents 8fc1a2298acb
children 056a86b24be2
comparison
equal deleted inserted replaced
1568:ce59e73a0cef 1569:cf0ca4a9812a
15 15
16 import ( 16 import (
17 "context" 17 "context"
18 "database/sql" 18 "database/sql"
19 "errors" 19 "errors"
20 "regexp"
21 "strconv"
22 "time"
20 23
21 "gemma.intevation.de/gemma/pkg/common" 24 "gemma.intevation.de/gemma/pkg/common"
22 "gemma.intevation.de/gemma/pkg/soap/ifbn" 25 "gemma.intevation.de/gemma/pkg/soap/ifbn"
23 ) 26 )
24 27
26 URL string `json:"url"` 29 URL string `json:"url"`
27 Insecure bool `json:"insecure"` 30 Insecure bool `json:"insecure"`
28 } 31 }
29 32
30 const BNJobKind JobKind = "bn" 33 const BNJobKind JobKind = "bn"
34
35 const insertSQL = `INSERT INTO waterway.bottlenecks (
36 bottleneck_id,
37 fk_g_fid,
38 objnam,
39 nobjnm,
40 stretch,
41 area,
42 rb,
43 lb,
44 responsible_country,
45 revisiting_time,
46 limiting,
47 date_info,
48 source_organization
49 ) VALUES(
50 $1,
51 isrs_fromText($2),
52 $3,
53 $4,
54 isrsrange(isrs_fromText($5), isrs_fromText($6)),
55 ST_MakePolygon(ST_ExteriorRing(ST_Buffer(ST_SetSRID(ST_Makepoint(13.05501, 47.80949), 4326), 0.01)))::Geography,
56 $7,
57 $8,
58 $9,
59 $10,
60 $11,
61 $12,
62 $13
63 ) ON CONFLICT (bottleneck_id) DO NOTHING`
31 64
32 type bnJobCreator struct{} 65 type bnJobCreator struct{}
33 66
34 func init() { 67 func init() {
35 RegisterJobCreator(BNJobKind, bnJobCreator{}) 68 RegisterJobCreator(BNJobKind, bnJobCreator{})
71 } 104 }
72 105
73 // CleanUp of a bottleneck import is a NOP. 106 // CleanUp of a bottleneck import is a NOP.
74 func (bn *Bottleneck) CleanUp() error { return nil } 107 func (bn *Bottleneck) CleanUp() error { return nil }
75 108
109 var rblbRe = regexp.MustCompile(`(..)_(..)`)
110
111 func splitRBLB(s string) (string, string) {
112 m := rblbRe.FindStringSubmatch(s)
113 if len(m) == 0 {
114 return "", ""
115 }
116 return m[1], m[2]
117 }
118
119 func revisitingTime(s string) int {
120 v, err := strconv.Atoi(s)
121 if err != nil {
122 v = 0
123 }
124 return v
125 }
126
76 // Do executes the actual bottleneck import. 127 // Do executes the actual bottleneck import.
77 func (bn *Bottleneck) Do( 128 func (bn *Bottleneck) Do(
78 ctx context.Context, 129 ctx context.Context,
79 importID int64, 130 importID int64,
80 conn *sql.Conn, 131 conn *sql.Conn,
94 err := errors.New("no Bottlenecks found") 145 err := errors.New("no Bottlenecks found")
95 feedback.Error("%v", err) 146 feedback.Error("%v", err)
96 return nil, err 147 return nil, err
97 } 148 }
98 149
99 // TODO: Implement me!
100 bns := resp.Export_bn_by_isrsResult.BottleNeckType 150 bns := resp.Export_bn_by_isrsResult.BottleNeckType
101 151 feedback.Info("Found %d bottlenecks for import", len(bns))
102 _ = bns 152
103 153 tx, err := conn.BeginTx(ctx, nil)
104 return nil, nil 154 if err != nil {
105 } 155 return nil, err
156 }
157 defer tx.Rollback()
158
159 start := time.Now()
160 for i := range bns {
161 bn := bns[i]
162 rb, lb := splitRBLB(bn.Rb_lb)
163
164 var limiting, country string
165
166 if bn.Limiting_factor != nil {
167 limiting = string(*bn.Limiting_factor)
168 }
169
170 if bn.Responsible_country != nil {
171 country = string(*bn.Responsible_country)
172 }
173
174 tx.Exec(insertSQL,
175 bn.Bottleneck_id,
176 bn.Fk_g_fid,
177 bn.OBJNAM,
178 bn.NOBJNM,
179 bn.From_ISRS, bn.To_ISRS,
180 rb,
181 lb,
182 country,
183 revisitingTime(bn.Revisiting_time),
184 limiting,
185 bn.Date_Info,
186 bn.Source,
187 )
188 feedback.Info("Insert \"%s\" into database", bn.OBJNAM)
189 //TODO: Track for potential later removal? Bottlenecks have an string PK, track wants int64.
190 // if err = track(ctx, tx, importID, "waterway.bottlenecks", bn.Bottleneck_id); err != nil {
191 // return nil, err
192 // }
193 }
194
195 feedback.Info("Storing %d bottlenecks took %s", len(bns), time.Since(start))
196 if err = tx.Commit(); err == nil {
197 feedback.Info("Import of bottlenecks was successful")
198 }
199
200 // TODO: needs to be filled.
201 summary := struct {
202 BottleneckCount int `json:"bottleneckCount"`
203 }{
204 BottleneckCount: len(bns),
205 }
206 return &summary, err
207 }