Mercurial > gemma

comparison pkg/controllers/manualimports.go @ 1985:8eeb0b5eb340

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Imports: Made retries and the waiting between the attempts configurable.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 23 Jan 2019 17:58:57 +0100
parents 71b722809b2b
children 9c65153bb56e
comparison
equal deleted inserted replaced
1983:f9f1babe52ae 1985:8eeb0b5eb340
24 "gemma.intevation.de/gemma/pkg/common" 24 "gemma.intevation.de/gemma/pkg/common"
25 "gemma.intevation.de/gemma/pkg/imports" 25 "gemma.intevation.de/gemma/pkg/imports"
26 "gemma.intevation.de/gemma/pkg/models" 26 "gemma.intevation.de/gemma/pkg/models"
27 ) 27 )
28 28
29 func retry(a common.Attributes) (time.Time, int) { 29 func retry(a common.Attributes) (time.Time, *int, *time.Duration) {
30 due, _ := a.Time("due") 30 due, _ := a.Time("due")
31 retries, ok := a.Int("retries") 31 ret, ok := a.Int("retries")
32 if !ok { 32 var retries *int
33 retries = -1 33 if ok {
34 } 34 retries = &ret
35 return due, retries 35 }
36 } 36 dur, ok := a.Duration("wait-retry")
37 37 var duration *time.Duration
38 func importBottleneck(input interface{}) (interface{}, time.Time, int, bool) { 38 if ok {
39 duration = &dur
40 }
41 return due, retries, duration
42 }
43
44 func importBottleneck(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
39 bi := input.(*models.BottleneckImport) 45 bi := input.(*models.BottleneckImport)
40 bn := &imports.Bottleneck{ 46 bn := &imports.Bottleneck{
41 URL: bi.URL, 47 URL: bi.URL,
42 Insecure: bi.Insecure, 48 Insecure: bi.Insecure,
43 } 49 }
44 due, retries := retry(bi.Attributes) 50 due, retries, duration := retry(bi.Attributes)
45 return bn, due, retries, bi.SendEmail 51 return bn, due, retries, duration, bi.SendEmail
46 } 52 }
47 53
48 func importGaugeMeasurement(input interface{}) (interface{}, time.Time, int, bool) { 54 func importGaugeMeasurement(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
49 gi := input.(*models.GaugeMeasurementImport) 55 gi := input.(*models.GaugeMeasurementImport)
50 gm := &imports.GaugeMeasurement{ 56 gm := &imports.GaugeMeasurement{
51 URL: gi.URL, 57 URL: gi.URL,
52 Insecure: gi.Insecure, 58 Insecure: gi.Insecure,
53 } 59 }
54 due, retries := retry(gi.Attributes) 60 due, retries, duration := retry(gi.Attributes)
55 return gm, due, retries, gi.SendEmail 61 return gm, due, retries, duration, gi.SendEmail
56 } 62 }
57 63
58 func importFairwayAvailability(input interface{}) (interface{}, time.Time, int, bool) { 64 func importFairwayAvailability(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
59 fai := input.(*models.FairwayAvailabilityImport) 65 fai := input.(*models.FairwayAvailabilityImport)
60 fa := &imports.FairwayAvailability{ 66 fa := &imports.FairwayAvailability{
61 URL: fai.URL, 67 URL: fai.URL,
62 Insecure: fai.Insecure, 68 Insecure: fai.Insecure,
63 } 69 }
64 due, retries := retry(fai.Attributes) 70 due, retries, duration := retry(fai.Attributes)
65 return fa, due, retries, fai.SendEmail 71 return fa, due, retries, duration, fai.SendEmail
66 } 72 }
67 73
68 func importWaterwayAxis(input interface{}) (interface{}, time.Time, int, bool) { 74 func importWaterwayAxis(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
69 wxi := input.(*models.WaterwayAxisImport) 75 wxi := input.(*models.WaterwayAxisImport)
70 wx := &imports.WaterwayAxis{ 76 wx := &imports.WaterwayAxis{
71 URL: wxi.URL, 77 URL: wxi.URL,
72 FeatureType: wxi.FeatureType, 78 FeatureType: wxi.FeatureType,
73 SortBy: wxi.SortBy, 79 SortBy: wxi.SortBy,
74 } 80 }
75 due, retries := retry(wxi.Attributes) 81 due, retries, duration := retry(wxi.Attributes)
76 return wx, due, retries, wxi.SendEmail 82 return wx, due, retries, duration, wxi.SendEmail
77 } 83 }
78 84
79 func importWaterwayArea(input interface{}) (interface{}, time.Time, int, bool) { 85 func importWaterwayArea(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
80 wai := input.(*models.WaterwayAreaImport) 86 wai := input.(*models.WaterwayAreaImport)
81 wa := &imports.WaterwayArea{ 87 wa := &imports.WaterwayArea{
82 URL: wai.URL, 88 URL: wai.URL,
83 FeatureType: wai.FeatureType, 89 FeatureType: wai.FeatureType,
84 SortBy: wai.SortBy, 90 SortBy: wai.SortBy,
85 } 91 }
86 due, retries := retry(wai.Attributes) 92 due, retries, duration := retry(wai.Attributes)
87 return wa, due, retries, wai.SendEmail 93 return wa, due, retries, duration, wai.SendEmail
88 } 94 }
89 95
90 func importWaterwayGauge(input interface{}) (interface{}, time.Time, int, bool) { 96 func importWaterwayGauge(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
91 wgi := input.(*models.WaterwayGaugeImport) 97 wgi := input.(*models.WaterwayGaugeImport)
92 username, _ := wgi.Attributes.Get("username") 98 username, _ := wgi.Attributes.Get("username")
93 password, _ := wgi.Attributes.Get("password") 99 password, _ := wgi.Attributes.Get("password")
94 insecure := wgi.Attributes.Bool("insecure") 100 insecure := wgi.Attributes.Bool("insecure")
95 wg := &imports.WaterwayGauge{ 101 wg := &imports.WaterwayGauge{
96 URL: wgi.URL, 102 URL: wgi.URL,
97 Username: username, 103 Username: username,
98 Password: password, 104 Password: password,
99 Insecure: insecure, 105 Insecure: insecure,
100 } 106 }
101 due, retries := retry(wgi.Attributes) 107 due, retries, duration := retry(wgi.Attributes)
102 return wg, due, retries, wgi.SendEmail 108 return wg, due, retries, duration, wgi.SendEmail
103 } 109 }
104 110
105 func importDistancemarksVirtual(input interface{}) (interface{}, time.Time, int, bool) { 111 func importDistancemarksVirtual(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
106 dmvi := input.(*models.DistanceMarksVirtualImport) 112 dmvi := input.(*models.DistanceMarksVirtualImport)
107 username, _ := dmvi.Attributes.Get("username") 113 username, _ := dmvi.Attributes.Get("username")
108 password, _ := dmvi.Attributes.Get("password") 114 password, _ := dmvi.Attributes.Get("password")
109 insecure := dmvi.Attributes.Bool("insecure") 115 insecure := dmvi.Attributes.Bool("insecure")
110 wg := &imports.DistanceMarksVirtual{ 116 wg := &imports.DistanceMarksVirtual{
111 URL: dmvi.URL, 117 URL: dmvi.URL,
112 Username: username, 118 Username: username,
113 Password: password, 119 Password: password,
114 Insecure: insecure, 120 Insecure: insecure,
115 } 121 }
116 due, retries := retry(dmvi.Attributes) 122 due, retries, duration := retry(dmvi.Attributes)
117 return wg, due, retries, dmvi.SendEmail 123 return wg, due, retries, duration, dmvi.SendEmail
118 } 124 }
119 125
120 func importFairwayDimension(input interface{}) (interface{}, time.Time, int, bool) { 126 func importFairwayDimension(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
121 fdi := input.(*models.FairwayDimensionImport) 127 fdi := input.(*models.FairwayDimensionImport)
122 fd := &imports.FairwayDimension{ 128 fd := &imports.FairwayDimension{
123 URL: fdi.URL, 129 URL: fdi.URL,
124 FeatureType: fdi.FeatureType, 130 FeatureType: fdi.FeatureType,
125 SortBy: fdi.SortBy, 131 SortBy: fdi.SortBy,
127 MinWidth: fdi.MinWidth, 133 MinWidth: fdi.MinWidth,
128 MaxWidth: fdi.MaxWidth, 134 MaxWidth: fdi.MaxWidth,
129 Depth: fdi.Depth, 135 Depth: fdi.Depth,
130 SourceOrganization: fdi.SourceOrganization, 136 SourceOrganization: fdi.SourceOrganization,
131 } 137 }
132 due, retries := retry(fdi.Attributes) 138 due, retries, duration := retry(fdi.Attributes)
133 return fd, due, retries, fdi.SendEmail 139 return fd, due, retries, duration, fdi.SendEmail
134 } 140 }
135 141
136 func importDistanceMarksAshore(input interface{}) (interface{}, time.Time, int, bool) { 142 func importDistanceMarksAshore(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
137 dmai := input.(*models.DistanceMarksAshoreImport) 143 dmai := input.(*models.DistanceMarksAshoreImport)
138 dma := &imports.DistanceMarksAshore{ 144 dma := &imports.DistanceMarksAshore{
139 URL: dmai.URL, 145 URL: dmai.URL,
140 FeatureType: dmai.FeatureType, 146 FeatureType: dmai.FeatureType,
141 SortBy: dmai.SortBy, 147 SortBy: dmai.SortBy,
142 } 148 }
143 due, retries := retry(dmai.Attributes) 149 due, retries, duration := retry(dmai.Attributes)
144 return dma, due, retries, dmai.SendEmail 150 return dma, due, retries, duration, dmai.SendEmail
145 } 151 }
146 152
147 func importStretch(input interface{}) (interface{}, time.Time, int, bool) { 153 func importStretch(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
148 sti := input.(*models.StretchImport) 154 sti := input.(*models.StretchImport)
149 st := &imports.Stretch{ 155 st := &imports.Stretch{
150 Name: sti.Name, 156 Name: sti.Name,
151 From: sti.From, 157 From: sti.From,
152 To: sti.To, 158 To: sti.To,
154 NObjNam: sti.NObjNam, 160 NObjNam: sti.NObjNam,
155 Source: sti.Source, 161 Source: sti.Source,
156 Date: sti.Date, 162 Date: sti.Date,
157 Countries: sti.Countries, 163 Countries: sti.Countries,
158 } 164 }
159 due, retries := retry(sti.Attributes) 165 due, retries, duration := retry(sti.Attributes)
160 return st, due, retries, sti.SendEmail 166 return st, due, retries, duration, sti.SendEmail
161 } 167 }
162 168
163 func manualImport( 169 func manualImport(
164 kind imports.JobKind, 170 kind imports.JobKind,
165 setup func(interface{}) (interface{}, time.Time, int, bool), 171 setup func(interface{}) (interface{}, time.Time, *int, *time.Duration, bool),
166 ) func(interface{}, *http.Request, *sql.Conn) (JSONResult, error) { 172 ) func(interface{}, *http.Request, *sql.Conn) (JSONResult, error) {
167 173
168 return func(input interface{}, req *http.Request, _ *sql.Conn) ( 174 return func(input interface{}, req *http.Request, _ *sql.Conn) (
169 jr JSONResult, err error) { 175 jr JSONResult, err error) {
170 176
171 what, due, retries, sendEmail := setup(input) 177 what, due, retries, waitDuration, sendEmail := setup(input)
172 178
173 var serialized string 179 var serialized string
174 if serialized, err = common.ToJSONString(what); err != nil { 180 if serialized, err = common.ToJSONString(what); err != nil {
175 return 181 return
176 } 182 }
178 session, _ := auth.GetSession(req) 184 session, _ := auth.GetSession(req)
179 185
180 var jobID int64 186 var jobID int64
181 if jobID, err = imports.AddJob( 187 if jobID, err = imports.AddJob(
182 kind, 188 kind,
183 due, retries, 189 due,
190 retries,
191 waitDuration,
184 session.User, 192 session.User,
185 sendEmail, 193 sendEmail,
186 serialized, 194 serialized,
187 ); err != nil { 195 ); err != nil {
188 return 196 return