Mercurial > gemma

changeset 2675:43e1abaf9ee3 import-overview-rework

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged default into import-overview-rework branch.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Fri, 15 Mar 2019 10:18:49 +0100
parents df2a31e8f257 (diff) b219ca1514f4 (current diff)
children fe93c48f76c9
files
diffstat 2 files changed, 161 insertions(+), 97 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/controllers/importqueue.go	Fri Mar 15 09:34:06 2019 +0100
+++ b/pkg/controllers/importqueue.go	Fri Mar 15 10:18:49 2019 +0100
@@ -14,6 +14,7 @@
 package controllers
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -21,6 +22,7 @@
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/jackc/pgx/pgtype"
@@ -32,13 +34,12 @@
 
 const (
 	warningSQLPrefix = `
-`
-	selectImportsSQL = `
 WITH warned AS (
   SELECT distinct(import_id) AS id
   FROM import.import_logs
   WHERE kind = 'warn'::log_type
-)
+)`
+	selectImportsSQL = warningSQLPrefix + `
 SELECT
   imports.id AS id,
   state::varchar,
@@ -46,21 +47,21 @@
   kind,
   username,
   signer,
-  summary,
-  EXISTS (
-    SELECT true FROM warned
-	WHERE warned.id = imports.id
-  ) AS has_warnings
-`
-	withoutWarningsSQL = `
+  summary IS NOT NULL AS has_summary,
+  imports.id IN (SELECT id FROM warned) AS has_warnings
 FROM import.imports
+WHERE
 `
-	withWarningsSQL = `
-FROM import.imports JOIN warned ON imports.id = warned.id
+	selectBeforeSQL = warningSQLPrefix + `
+SELECT enqueued FROM import.imports
+WHERE
 `
-
-	selectHasImportSQL = `
-SELECT true FROM import.imports WHERE id = $1`
+	selectAfterSQL = warningSQLPrefix + `
+SELECT enqueued FROM import.imports
+WHERE
+`
+	selectImportSummaySQL = `
+SELECT summary FROM import.imports WHERE id = $1`
 
 	selectHasNoRunningImportSQL = `
 SELECT true FROM import.imports
@@ -126,93 +127,127 @@
 	return &ta
 }
 
-func queryImportListStmt(
-	conn *sql.Conn,
-	req *http.Request,
-) (*sql.Rows, error) {
+type filterBuilder struct {
+	stmt    strings.Builder
+	args    []interface{}
+	hasCond bool
+}
+
+func (fb *filterBuilder) arg(format string, v ...interface{}) {
+	indices := make([]interface{}, len(v))
+	for i := range indices {
+		indices[i] = len(fb.args) + i + 1
+	}
+	fmt.Fprintf(&fb.stmt, format, indices...)
+	fb.args = append(fb.args, v...)
+}
 
-	var (
-		stmt   strings.Builder
-		args   []interface{}
-		states *pgtype.TextArray
-		kinds  *pgtype.TextArray
-		ids    *pgtype.Int8Array
-	)
+func (fb *filterBuilder) cond(format string, v ...interface{}) {
+	if fb.hasCond {
+		fb.stmt.WriteString(" AND ")
+	} else {
+		fb.hasCond = true
+	}
+	fb.arg(format, v...)
+}
 
-	arg := func(format string, v interface{}) {
-		fmt.Fprintf(&stmt, format, len(args)+1)
-		args = append(args, v)
-	}
+func buildFilters(req *http.Request) (l, b, a *filterBuilder, err error) {
+
+	l = new(filterBuilder)
+	a = new(filterBuilder)
+	b = new(filterBuilder)
 
-	var warnings bool
+	var noBefore, noAfter bool
 
-	switch warn := strings.ToLower(req.FormValue("warnings")); warn {
-	case "1", "t", "true":
-		warnings = true
-	}
+	l.stmt.WriteString(selectImportsSQL)
+	a.stmt.WriteString(selectAfterSQL)
+	b.stmt.WriteString(selectBeforeSQL)
 
 	if st := req.FormValue("states"); st != "" {
-		states = toTextArray(st, imports.ImportStateNames)
+		states := toTextArray(st, imports.ImportStateNames)
+		l.cond(" state = ANY($%d) ", states)
+		a.cond(" state = ANY($%d) ", states)
+		b.cond(" state = ANY($%d) ", states)
 	}
 
 	if ks := req.FormValue("kinds"); ks != "" {
-		kinds = toTextArray(ks, imports.ImportKindNames())
+		kinds := toTextArray(ks, imports.ImportKindNames())
+		l.cond(" kind = ANY($%d) ", kinds)
+		a.cond(" kind = ANY($%d) ", kinds)
+		b.cond(" kind = ANY($%d) ", kinds)
 	}
 
 	if idss := req.FormValue("ids"); idss != "" {
-		ids = toInt8Array(idss)
+		ids := toInt8Array(idss)
+		l.cond(" id = ANY($%d) ", ids)
+		a.cond(" id = ANY($%d) ", ids)
+		b.cond(" id = ANY($%d) ", ids)
 	}
 
-	stmt.WriteString(selectImportsSQL)
-	if warnings {
-		stmt.WriteString(withWarningsSQL)
+	if from := req.FormValue("from"); from != "" {
+		var fromTime time.Time
+		if fromTime, err = time.Parse(models.ImportTimeFormat, from); err != nil {
+			return
+		}
+		l.cond(" enqueued >= $%d ", fromTime)
+		b.cond(" enqueued < $%d", fromTime)
 	} else {
-		stmt.WriteString(withoutWarningsSQL)
+		noBefore = true
 	}
 
-	if states != nil || kinds != nil || ids != nil {
-		stmt.WriteString(" WHERE ")
+	if to := req.FormValue("to"); to != "" {
+		var toTime time.Time
+		if toTime, err = time.Parse(models.ImportTimeFormat, to); err != nil {
+			return
+		}
+		l.cond(" enqueued <= $%d ", toTime)
+		a.cond(" enqueued > $%d", toTime)
+	} else {
+		noAfter = true
 	}
 
-	if states != nil {
-		arg(" state = ANY($%d) ", states)
-	}
-
-	if states != nil && (kinds != nil || ids != nil) {
-		stmt.WriteString("AND")
-	}
-
-	if kinds != nil {
-		arg(" kind = ANY($%d) ", kinds)
+	switch warn := strings.ToLower(req.FormValue("warnings")); warn {
+	case "1", "t", "true":
+		l.cond(" id IN (SELECT id FROM warned) ")
+		a.cond(" id IN (SELECT id FROM warned) ")
+		b.cond(" id IN (SELECT id FROM warned) ")
 	}
 
-	if (states != nil || kinds != nil) && ids != nil {
-		stmt.WriteString("AND")
+	if !l.hasCond {
+		l.stmt.WriteString(" TRUE ")
 	}
-
-	if ids != nil {
-		arg(" id = ANY($%d) ", ids)
+	if !b.hasCond {
+		b.stmt.WriteString(" TRUE ")
+	}
+	if !a.hasCond {
+		a.stmt.WriteString(" TRUE ")
 	}
 
-	stmt.WriteString(" ORDER BY enqueued DESC ")
+	l.stmt.WriteString(" ORDER BY enqueued DESC ")
+	a.stmt.WriteString(" ORDER BY enqueued LIMIT 1")
+	b.stmt.WriteString(" ORDER BY enqueued LIMIT 1")
 
-	if lim := req.FormValue("limit"); lim != "" {
-		limit, err := strconv.ParseInt(lim, 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		arg(" LIMIT $%d ", limit)
+	if noBefore {
+		b = nil
 	}
+	if noAfter {
+		a = nil
+	}
+	return
+}
 
-	if ofs := req.FormValue("offset"); ofs != "" {
-		offset, err := strconv.ParseInt(ofs, 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		arg(" OFFSET $%d ", offset)
+func neighbored(ctx context.Context, conn *sql.Conn, fb *filterBuilder) *models.ImportTime {
+
+	var when time.Time
+	err := conn.QueryRowContext(ctx, fb.stmt.String(), fb.args...).Scan(&when)
+	switch {
+	case err == sql.ErrNoRows:
+		return nil
+	case err != nil:
+		log.Printf("warn: %v\n", err)
+		return nil
 	}
-
-	return conn.QueryContext(req.Context(), stmt.String(), args...)
+	return &models.ImportTime{when}
 }
 
 func listImports(
@@ -221,27 +256,35 @@
 	conn *sql.Conn,
 ) (jr JSONResult, err error) {
 
+	var list, before, after *filterBuilder
+
+	if list, before, after, err = buildFilters(req); err != nil {
+		return
+	}
+
+	ctx := req.Context()
+
 	var rows *sql.Rows
-	rows, err = queryImportListStmt(conn, req)
-	if err != nil {
+	if rows, err = conn.QueryContext(ctx, list.stmt.String(), list.args...); err != nil {
 		return
 	}
 	defer rows.Close()
 
 	imports := make([]*models.Import, 0, 20)
 
-	var signer, summary sql.NullString
+	var signer sql.NullString
 
 	for rows.Next() {
 		var it models.Import
+		var enqueued time.Time
 		if err = rows.Scan(
 			&it.ID,
 			&it.State,
-			&it.Enqueued,
+			&enqueued,
 			&it.Kind,
 			&it.User,
 			&signer,
-			&summary,
+			&it.Summary,
 			&it.Warnings,
 		); err != nil {
 			return
@@ -249,12 +292,7 @@
 		if signer.Valid {
 			it.Signer = signer.String
 		}
-		if summary.Valid {
-			if err = json.NewDecoder(
-				strings.NewReader(summary.String)).Decode(&it.Summary); err != nil {
-				return
-			}
-		}
+		it.Enqueued = models.ImportTime{enqueued}
 		imports = append(imports, &it)
 	}
 
@@ -262,11 +300,25 @@
 		return
 	}
 
+	var prev, next *models.ImportTime
+
+	if before != nil {
+		prev = neighbored(ctx, conn, before)
+	}
+
+	if after != nil {
+		next = neighbored(ctx, conn, after)
+	}
+
 	jr = JSONResult{
 		Result: struct {
-			Imports []*models.Import `json:"imports"`
+			Prev    *models.ImportTime `json:"prev,omitempty"`
+			Next    *models.ImportTime `json:"next,omitempty"`
+			Imports []*models.Import   `json:"imports"`
 		}{
 			Imports: imports,
+			Prev:    prev,
+			Next:    next,
 		},
 	}
 	return
@@ -283,8 +335,8 @@
 	id, _ := strconv.ParseInt(mux.Vars(req)["id"], 10, 64)
 
 	// Check if he have such a import job first.
-	var dummy bool
-	err = conn.QueryRowContext(ctx, selectHasImportSQL, id).Scan(&dummy)
+	var summary sql.NullString
+	err = conn.QueryRowContext(ctx, selectImportSummaySQL, id).Scan(&summary)
 	switch {
 	case err == sql.ErrNoRows:
 		err = JSONError{
@@ -296,6 +348,14 @@
 		return
 	}
 
+	var sum interface{}
+	if summary.Valid {
+		if err = json.NewDecoder(
+			strings.NewReader(summary.String)).Decode(&sum); err != nil {
+			return
+		}
+	}
+
 	// We have it -> generate log entries.
 	var rows *sql.Rows
 	rows, err = conn.QueryContext(ctx, selectImportLogsSQL, id)
@@ -320,8 +380,10 @@
 
 	jr = JSONResult{
 		Result: struct {
+			Summary interface{}              `json:"summary,omitempty"`
 			Entries []*models.ImportLogEntry `json:"entries"`
 		}{
+			Summary: sum,
 			Entries: entries,
 		},
 	}
--- a/pkg/models/import.go	Fri Mar 15 09:34:06 2019 +0100
+++ b/pkg/models/import.go	Fri Mar 15 10:18:49 2019 +0100
@@ -19,18 +19,20 @@
 	"time"
 )
 
+const ImportTimeFormat = "2006-01-02T15:04:05.000"
+
 type (
 	ImportTime struct{ time.Time }
 
 	Import struct {
-		ID       int64       `json:"id"`
-		State    string      `json:"state"`
-		Enqueued ImportTime  `json:"enqueued"`
-		Kind     string      `json:"kind"`
-		User     string      `json:"user"`
-		Signer   string      `json:"signer,omitempty"`
-		Summary  interface{} `json:"summary,omitempty"`
-		Warnings bool        `json:"warnings,omitempty"`
+		ID       int64      `json:"id"`
+		State    string     `json:"state"`
+		Enqueued ImportTime `json:"enqueued"`
+		Kind     string     `json:"kind"`
+		User     string     `json:"user"`
+		Signer   string     `json:"signer,omitempty"`
+		Summary  bool       `json:"summary,omitempty"`
+		Warnings bool       `json:"warnings,omitempty"`
 	}
 
 	ImportLogEntry struct {
@@ -62,7 +64,7 @@
 }
 
 func (it ImportTime) MarshalJSON() ([]byte, error) {
-	return json.Marshal(it.Format("2006-01-02T15:04:05.000"))
+	return json.Marshal(it.Format(ImportTimeFormat))
 }
 
 func (it *ImportTime) Scan(x interface{}) error {