Mercurial > gemma

comparison pkg/controllers/importqueue.go @ 4791:1fef4679b07a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added an endpoint GET api/imports/export to export imports.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Fri, 25 Oct 2019 13:03:50 +0200
parents 47922c1a088d
children d727641911a5
comparison
equal deleted inserted replaced
4790:6f3730196ebb 4791:1fef4679b07a
14 package controllers 14 package controllers
15 15
16 import ( 16 import (
17 "context" 17 "context"
18 "database/sql" 18 "database/sql"
19 "encoding/csv"
19 "encoding/json" 20 "encoding/json"
20 "fmt" 21 "fmt"
21 "log" 22 "log"
22 "net/http" 23 "net/http"
23 "strconv" 24 "strconv"
55 EXISTS(SELECT 1 FROM import.import_logs 56 EXISTS(SELECT 1 FROM import.import_logs
56 WHERE kind = 'warn'::log_type and import_id = imports.id) AS has_warnings 57 WHERE kind = 'warn'::log_type and import_id = imports.id) AS has_warnings
57 FROM import.imports 58 FROM import.imports
58 WHERE 59 WHERE
59 ` 60 `
61 selectExportSQL = `
62 SELECT
63 imports.id AS id,
64 state::varchar,
65 enqueued,
66 changed,
67 kind,
68 username,
69 (SELECT country FROM users.list_users lu
70 WHERE lu.username = import.imports.username) AS country,
71 signer,
72 EXISTS(SELECT 1 FROM import.import_logs
73 WHERE kind = 'warn'::log_type and import_id = imports.id) AS has_warnings,
74 data
75 FROM import.imports
76 WHERE
77 `
60 selectEnqueuedSQL = ` 78 selectEnqueuedSQL = `
61 SELECT enqueued FROM import.imports 79 SELECT enqueued FROM import.imports
62 WHERE 80 WHERE
63 ` 81 `
64 selectImportSummarySQL = ` 82 selectImportSummarySQL = `
87 type filledStmt struct { 105 type filledStmt struct {
88 stmt strings.Builder 106 stmt strings.Builder
89 args []interface{} 107 args []interface{}
90 } 108 }
91 109
92 func buildFilters(req *http.Request) (*filledStmt, *filledStmt, *filledStmt, error) { 110 func buildFilters(projection string, req *http.Request) (*filledStmt, *filledStmt, *filledStmt, error) {
93 111
94 var l, a, b filterAnd 112 var l, a, b filterAnd
95 113
96 var noBefore, noAfter bool 114 var noBefore, noAfter bool
97 115
173 fa.stmt.WriteString(selectEnqueuedSQL) 191 fa.stmt.WriteString(selectEnqueuedSQL)
174 fb.stmt.WriteString(selectEnqueuedSQL) 192 fb.stmt.WriteString(selectEnqueuedSQL)
175 193
176 var counting bool 194 var counting bool
177 195
178 switch count := strings.ToLower(req.FormValue("count")); count { 196 if projection != "" {
179 case "1", "t", "true": 197 fl.stmt.WriteString(projection)
180 counting = true 198 } else {
181 fl.stmt.WriteString(selectImportsCountSQL) 199 switch count := strings.ToLower(req.FormValue("count")); count {
182 default: 200 case "1", "t", "true":
183 fl.stmt.WriteString(selectImportsSQL) 201 counting = true
202 fl.stmt.WriteString(selectImportsCountSQL)
203 default:
204 fl.stmt.WriteString(selectImportsSQL)
205 }
184 } 206 }
185 207
186 if len(l) == 0 { 208 if len(l) == 0 {
187 fl.stmt.WriteString(" TRUE ") 209 fl.stmt.WriteString(" TRUE ")
188 } else { 210 } else {
229 return nil 251 return nil
230 } 252 }
231 return &models.ImportTime{Time: when.UTC()} 253 return &models.ImportTime{Time: when.UTC()}
232 } 254 }
233 255
256 func exportImports(rw http.ResponseWriter, req *http.Request) {
257
258 list, _, _, err := buildFilters(selectExportSQL, req)
259 if err != nil {
260 http.Error(rw, "error: "+err.Error(), http.StatusBadRequest)
261 return
262 }
263
264 rw.Header().Add("Content-Type", "text/csv")
265 out := csv.NewWriter(rw)
266
267 record := []string{
268 "#id",
269 "#kind",
270 "#enqueued",
271 "#changed",
272 "#user",
273 "#country",
274 "#signer",
275 "#state",
276 "#warnings",
277 "#source",
278 }
279
280 if err := out.Write(record); err != nil {
281 // Too late for HTTP status message.
282 log.Printf("error: %v\n", err)
283 return
284 }
285
286 conn := mw.GetDBConn(req)
287 ctx := req.Context()
288 var rows *sql.Rows
289 if rows, err = conn.QueryContext(ctx, list.stmt.String(), list.args...); err != nil {
290 log.Printf("error: %v\n", err)
291 return
292 }
293 defer rows.Close()
294
295 stringString := func(s sql.NullString) string {
296 if s.Valid {
297 return s.String
298 }
299 return ""
300 }
301
302 // Extract some meta infos from the import.
303 type Description interface {
304 Description() (string, error)
305 }
306
307 for rows.Next() {
308 var (
309 id int64
310 state string
311 enqueued time.Time
312 changed time.Time
313 kind string
314 user string
315 country string
316 signer sql.NullString
317 warnings bool
318 data string
319 description string
320 )
321 if err = rows.Scan(
322 &id,
323 &state,
324 &enqueued,
325 &changed,
326 &kind,
327 &user,
328 &country,
329 &signer,
330 &warnings,
331 &data,
332 ); err != nil {
333 return
334 }
335
336 // Do some introspection on the job to be more verbose.
337 if jc := imports.FindJobCreator(imports.JobKind(kind)); jc != nil {
338 job := jc.Create()
339 if err := common.FromJSONString(data, job); err != nil {
340 log.Printf("error: %v\n", err)
341 } else if desc, ok := job.(Description); ok {
342 if description, err = desc.Description(); err != nil {
343 log.Printf("error: %v\n", err)
344 }
345 }
346 }
347
348 record[0] = strconv.FormatInt(id, 10)
349 record[1] = kind
350 record[2] = enqueued.UTC().Format(common.TimeFormat)
351 record[3] = changed.UTC().Format(common.TimeFormat)
352 record[4] = user
353 record[5] = country
354 record[6] = stringString(signer)
355 record[7] = state
356 record[8] = strconv.FormatBool(warnings)
357 record[9] = strings.Replace(description, ",", "|", -1)
358
359 if err := out.Write(record); err != nil {
360 log.Printf("error: %v\n", err)
361 return
362 }
363 }
364
365 out.Flush()
366 if err := out.Error(); err != nil {
367 log.Printf("error: %v\n", err)
368 }
369
370 if err = rows.Err(); err != nil {
371 log.Printf("error: %v\n", err)
372 return
373 }
374 }
375
234 func listImports(req *http.Request) (jr mw.JSONResult, err error) { 376 func listImports(req *http.Request) (jr mw.JSONResult, err error) {
235 377
236 var list, before, after *filledStmt 378 var list, before, after *filledStmt
237 379
238 if list, before, after, err = buildFilters(req); err != nil { 380 if list, before, after, err = buildFilters("", req); err != nil {
239 return 381 return
240 } 382 }
241 383
242 ctx := req.Context() 384 ctx := req.Context()
243 385