# HG changeset patch # User Sascha L. Teichmann # Date 1552661980 -3600 # Node ID 3b98de34de90f1671eedf42899fd7c3ea91e5ee7 # Parent c4da269238a44fcc7ffb48520720981cdcd22f37# Parent 47b789a276185be326bc4f9f003a4e504848d30f Merged import-overview-rework back into default. diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/AdditionalDetail.vue --- a/client/src/components/importoverview/AdditionalDetail.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/AdditionalDetail.vue Fri Mar 15 15:59:40 2019 +0100 @@ -2,13 +2,19 @@

- +

@@ -29,7 +35,7 @@ export default { name: "additionaldetail", - props: ["entry"], + props: ["entry", "details"], components: { BottleneckDetail: () => import("./BottleneckDetail.vue"), ApprovedGaugeMeasurementDetail: () => diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/AdditionalLog.vue --- a/client/src/components/importoverview/AdditionalLog.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/AdditionalLog.vue Fri Mar 15 15:59:40 2019 +0100 @@ -49,37 +49,13 @@ * Author(s): * Thomas Junk */ -import { displayError } from "@/lib/errors.js"; -import { HTTP } from "@/lib/http.js"; - export default { name: "additionallogs", - props: ["entry"], + props: ["details"], data() { return { - logLines: [] + logLines: this.details.entries }; - }, - methods: { - loadEntries() { - HTTP.get("/imports/" + this.entry.id, { - headers: { "X-Gemma-Auth": localStorage.getItem("token") } - }) - .then(response => { - const { entries } = response.data; - this.logLines = entries; - }) - .catch(error => { - const { status, data } = error.response; - displayError({ - title: this.$gettext("Backend Error"), - message: `${status}: ${data.message || data}` - }); - }); - } - }, - mounted() { - this.loadEntries(); } }; diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/ApprovedGaugeMeasurementDetail.vue --- a/client/src/components/importoverview/ApprovedGaugeMeasurementDetail.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/ApprovedGaugeMeasurementDetail.vue Fri Mar 15 15:59:40 2019 +0100 @@ -1,6 +1,6 @@ @@ -84,7 +90,7 @@ export default { name: "logdetail", - props: ["entry"], + props: ["entry", "details"], components: { SoundingResultDetail: () => import("./SoundingResultDetail.vue"), StretchDetail: () => import("./StretchDetails.vue"), diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/LogEntry.vue --- a/client/src/components/importoverview/LogEntry.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/LogEntry.vue Fri Mar 15 15:59:40 2019 +0100 @@ -58,7 +58,11 @@

- +

@@ -79,10 +83,32 @@ */ import { mapState } from "vuex"; import { STATES } from "@/store/imports.js"; +import { displayError } from "@/lib/errors.js"; +import { HTTP } from "@/lib/http.js"; export default { name: "importlogentry", props: ["entry"], + data() { + return { + details: null + }; + }, + mounted() { + HTTP.get("/imports/" + this.entry.id, { + headers: { "X-Gemma-Auth": localStorage.getItem("token") } + }) + .then(response => { + this.details = response.data; + }) + .catch(error => { + const { status, data } = error.response; + displayError({ + title: this.$gettext("Backend Error"), + message: `${status}: ${data.message || data}` + }); + }); + }, components: { LogDetail: () => import("./LogDetail.vue") }, diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/SoundingResultDetail.vue --- a/client/src/components/importoverview/SoundingResultDetail.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/SoundingResultDetail.vue Fri Mar 15 15:59:40 2019 +0100 @@ -2,7 +2,7 @@

- {{ entry.summary.bottleneck }} + {{ details.summary.bottleneck }}

@@ -23,7 +23,7 @@ */ export default { name: "soundingresultdetails", - props: ["entry"], + props: ["entry", "details"], methods: { moveMap(coordinates) { this.$store.commit("map/moveMap", { @@ -33,7 +33,7 @@ }); }, zoomTo() { - const { lat, lon, bottleneck, date } = this.entry.summary; + const { lat, lon, bottleneck, date } = this.details.summary; const coordinates = [lat, lon]; this.moveMap(coordinates); this.$store diff -r c4da269238a4 -r 3b98de34de90 client/src/components/importoverview/StretchDetails.vue --- a/client/src/components/importoverview/StretchDetails.vue Fri Mar 15 12:43:30 2019 +0100 +++ b/client/src/components/importoverview/StretchDetails.vue Fri Mar 15 15:59:40 2019 +0100 @@ -2,7 +2,7 @@

@@ -26,7 +26,7 @@ export default { name: "stretchdetails", - props: ["entry"], + props: ["entry", "details"], methods: { moveToExtent(feature) { this.$store.commit("map/moveToExtent", { @@ -36,7 +36,7 @@ }); }, zoomToStretch() { - const name = this.entry.summary.stretch; + const name = this.details.summary.stretch; this.$store.commit("map/setLayerVisible", LAYERS.STRETCHES); this.$store .dispatch("imports/loadStretch", name) diff -r c4da269238a4 -r 3b98de34de90 pkg/controllers/importqueue.go --- a/pkg/controllers/importqueue.go Fri Mar 15 12:43:30 2019 +0100 +++ b/pkg/controllers/importqueue.go Fri Mar 15 15:59:40 2019 +0100 @@ -14,6 +14,7 @@ package controllers import ( + "context" "database/sql" "encoding/json" "fmt" @@ -21,6 +22,7 @@ "net/http" "strconv" "strings" + "time" "github.com/gorilla/mux" "github.com/jackc/pgx/pgtype" @@ -32,13 +34,17 @@ const ( warningSQLPrefix = ` -` - selectImportsSQL = ` WITH warned AS ( SELECT distinct(import_id) AS id FROM import.import_logs WHERE kind = 'warn'::log_type -) +)` + selectImportsCountSQL = warningSQLPrefix + ` +SELECT count(*) +FROM import.imports +WHERE +` + selectImportsSQL = warningSQLPrefix + ` SELECT imports.id AS id, state::varchar, @@ -46,21 +52,21 @@ kind, username, signer, - summary, - EXISTS ( - SELECT true FROM warned - WHERE warned.id = imports.id - ) AS has_warnings -` - withoutWarningsSQL = ` + summary IS NOT NULL AS has_summary, + imports.id IN (SELECT id FROM warned) AS has_warnings FROM import.imports +WHERE ` - withWarningsSQL = ` -FROM import.imports JOIN warned ON imports.id = warned.id + selectBeforeSQL = warningSQLPrefix + ` +SELECT enqueued FROM import.imports +WHERE ` - - selectHasImportSQL = ` -SELECT true FROM import.imports WHERE id = $1` + selectAfterSQL = warningSQLPrefix + ` +SELECT enqueued FROM import.imports +WHERE +` + selectImportSummaySQL = ` +SELECT summary FROM import.imports WHERE id = $1` selectHasNoRunningImportSQL = ` SELECT true FROM import.imports @@ -126,93 +132,142 @@ return &ta } -func queryImportListStmt( - conn *sql.Conn, - req *http.Request, -) (*sql.Rows, error) { +type filterBuilder struct { + stmt strings.Builder + args []interface{} + hasCond bool +} + +func (fb *filterBuilder) arg(format string, v ...interface{}) { + indices := make([]interface{}, len(v)) + for i := range indices { + indices[i] = len(fb.args) + i + 1 + } + fmt.Fprintf(&fb.stmt, format, indices...) + fb.args = append(fb.args, v...) +} + +func (fb *filterBuilder) cond(format string, v ...interface{}) { + if fb.hasCond { + fb.stmt.WriteString(" AND ") + } else { + fb.hasCond = true + } + fb.arg(format, v...) +} - var ( - stmt strings.Builder - args []interface{} - states *pgtype.TextArray - kinds *pgtype.TextArray - ids *pgtype.Int8Array - ) +func buildFilters(req *http.Request) (l, b, a *filterBuilder, err error) { + + l = new(filterBuilder) + a = new(filterBuilder) + b = new(filterBuilder) + + var noBefore, noAfter bool + + var counting bool - arg := func(format string, v interface{}) { - fmt.Fprintf(&stmt, format, len(args)+1) - args = append(args, v) + switch count := strings.ToLower(req.FormValue("count")); count { + case "1", "t", "true": + counting = true + l.stmt.WriteString(selectImportsCountSQL) + default: + l.stmt.WriteString(selectImportsSQL) + } + a.stmt.WriteString(selectAfterSQL) + b.stmt.WriteString(selectBeforeSQL) + + cond := func(format string, v ...interface{}) { + l.cond(format, v...) + a.cond(format, v...) + b.cond(format, v...) } - var warnings bool - - switch warn := strings.ToLower(req.FormValue("warnings")); warn { - case "1", "t", "true": - warnings = true + if query := req.FormValue("query"); query != "" { + query = "%" + query + "%" + cond(` (kind ILIKE $%d OR username ILIKE $%d OR signer ILIKE $%d OR `+ + `id IN (SELECT import_id FROM import.import_logs WHERE msg ILIKE $%d)) `, + query, query, query, query) } if st := req.FormValue("states"); st != "" { - states = toTextArray(st, imports.ImportStateNames) + states := toTextArray(st, imports.ImportStateNames) + cond(" state = ANY($%d) ", states) } if ks := req.FormValue("kinds"); ks != "" { - kinds = toTextArray(ks, imports.ImportKindNames()) + kinds := toTextArray(ks, imports.ImportKindNames()) + cond(" kind = ANY($%d) ", kinds) } if idss := req.FormValue("ids"); idss != "" { - ids = toInt8Array(idss) + ids := toInt8Array(idss) + cond(" id = ANY($%d) ", ids) } - stmt.WriteString(selectImportsSQL) - if warnings { - stmt.WriteString(withWarningsSQL) + if from := req.FormValue("from"); from != "" { + var fromTime time.Time + if fromTime, err = time.Parse(models.ImportTimeFormat, from); err != nil { + return + } + l.cond(" enqueued >= $%d ", fromTime) + b.cond(" enqueued < $%d", fromTime) } else { - stmt.WriteString(withoutWarningsSQL) + noBefore = true } - if states != nil || kinds != nil || ids != nil { - stmt.WriteString(" WHERE ") + if to := req.FormValue("to"); to != "" { + var toTime time.Time + if toTime, err = time.Parse(models.ImportTimeFormat, to); err != nil { + return + } + l.cond(" enqueued <= $%d ", toTime) + a.cond(" enqueued > $%d", toTime) + } else { + noAfter = true } - if states != nil { - arg(" state = ANY($%d) ", states) - } - - if states != nil && (kinds != nil || ids != nil) { - stmt.WriteString("AND") - } - - if kinds != nil { - arg(" kind = ANY($%d) ", kinds) + switch warn := strings.ToLower(req.FormValue("warnings")); warn { + case "1", "t", "true": + cond(" id IN (SELECT id FROM warned) ") } - if (states != nil || kinds != nil) && ids != nil { - stmt.WriteString("AND") + if !l.hasCond { + l.stmt.WriteString(" TRUE ") + } + if !b.hasCond { + b.stmt.WriteString(" TRUE ") + } + if !a.hasCond { + a.stmt.WriteString(" TRUE ") } - if ids != nil { - arg(" id = ANY($%d) ", ids) + if !counting { + l.stmt.WriteString(" ORDER BY enqueued DESC ") + a.stmt.WriteString(" ORDER BY enqueued LIMIT 1") + b.stmt.WriteString(" ORDER BY enqueued LIMIT 1") } - stmt.WriteString(" ORDER BY enqueued DESC ") + if noBefore { + b = nil + } + if noAfter { + a = nil + } + return +} - if lim := req.FormValue("limit"); lim != "" { - limit, err := strconv.ParseInt(lim, 10, 64) - if err != nil { - return nil, err - } - arg(" LIMIT $%d ", limit) - } +func neighbored(ctx context.Context, conn *sql.Conn, fb *filterBuilder) *models.ImportTime { - if ofs := req.FormValue("offset"); ofs != "" { - offset, err := strconv.ParseInt(ofs, 10, 64) - if err != nil { - return nil, err - } - arg(" OFFSET $%d ", offset) + var when time.Time + err := conn.QueryRowContext(ctx, fb.stmt.String(), fb.args...).Scan(&when) + switch { + case err == sql.ErrNoRows: + return nil + case err != nil: + log.Printf("warn: %v\n", err) + return nil } - - return conn.QueryContext(req.Context(), stmt.String(), args...) + return &models.ImportTime{when} } func listImports( @@ -221,27 +276,53 @@ conn *sql.Conn, ) (jr JSONResult, err error) { + var list, before, after *filterBuilder + + if list, before, after, err = buildFilters(req); err != nil { + return + } + + ctx := req.Context() + + // Fast path for counting + + switch count := strings.ToLower(req.FormValue("count")); count { + case "1", "t", "true": + var count int64 + err = conn.QueryRowContext(ctx, list.stmt.String(), list.args...).Scan(&count) + switch { + case err == sql.ErrNoRows: + count, err = 0, nil + case err != nil: + return + } + jr = JSONResult{Result: count} + return + } + + // Generate the list + var rows *sql.Rows - rows, err = queryImportListStmt(conn, req) - if err != nil { + if rows, err = conn.QueryContext(ctx, list.stmt.String(), list.args...); err != nil { return } defer rows.Close() imports := make([]*models.Import, 0, 20) - var signer, summary sql.NullString + var signer sql.NullString for rows.Next() { var it models.Import + var enqueued time.Time if err = rows.Scan( &it.ID, &it.State, - &it.Enqueued, + &enqueued, &it.Kind, &it.User, &signer, - &summary, + &it.Summary, &it.Warnings, ); err != nil { return @@ -249,12 +330,7 @@ if signer.Valid { it.Signer = signer.String } - if summary.Valid { - if err = json.NewDecoder( - strings.NewReader(summary.String)).Decode(&it.Summary); err != nil { - return - } - } + it.Enqueued = models.ImportTime{enqueued} imports = append(imports, &it) } @@ -262,11 +338,25 @@ return } + var prev, next *models.ImportTime + + if before != nil { + prev = neighbored(ctx, conn, before) + } + + if after != nil { + next = neighbored(ctx, conn, after) + } + jr = JSONResult{ Result: struct { - Imports []*models.Import `json:"imports"` + Prev *models.ImportTime `json:"prev,omitempty"` + Next *models.ImportTime `json:"next,omitempty"` + Imports []*models.Import `json:"imports"` }{ Imports: imports, + Prev: prev, + Next: next, }, } return @@ -283,8 +373,8 @@ id, _ := strconv.ParseInt(mux.Vars(req)["id"], 10, 64) // Check if he have such a import job first. - var dummy bool - err = conn.QueryRowContext(ctx, selectHasImportSQL, id).Scan(&dummy) + var summary sql.NullString + err = conn.QueryRowContext(ctx, selectImportSummaySQL, id).Scan(&summary) switch { case err == sql.ErrNoRows: err = JSONError{ @@ -296,6 +386,14 @@ return } + var sum interface{} + if summary.Valid { + if err = json.NewDecoder( + strings.NewReader(summary.String)).Decode(&sum); err != nil { + return + } + } + // We have it -> generate log entries. var rows *sql.Rows rows, err = conn.QueryContext(ctx, selectImportLogsSQL, id) @@ -320,8 +418,10 @@ jr = JSONResult{ Result: struct { + Summary interface{} `json:"summary,omitempty"` Entries []*models.ImportLogEntry `json:"entries"` }{ + Summary: sum, Entries: entries, }, } diff -r c4da269238a4 -r 3b98de34de90 pkg/models/import.go --- a/pkg/models/import.go Fri Mar 15 12:43:30 2019 +0100 +++ b/pkg/models/import.go Fri Mar 15 15:59:40 2019 +0100 @@ -19,18 +19,20 @@ "time" ) +const ImportTimeFormat = "2006-01-02T15:04:05.000" + type ( ImportTime struct{ time.Time } Import struct { - ID int64 `json:"id"` - State string `json:"state"` - Enqueued ImportTime `json:"enqueued"` - Kind string `json:"kind"` - User string `json:"user"` - Signer string `json:"signer,omitempty"` - Summary interface{} `json:"summary,omitempty"` - Warnings bool `json:"warnings,omitempty"` + ID int64 `json:"id"` + State string `json:"state"` + Enqueued ImportTime `json:"enqueued"` + Kind string `json:"kind"` + User string `json:"user"` + Signer string `json:"signer,omitempty"` + Summary bool `json:"summary,omitempty"` + Warnings bool `json:"warnings,omitempty"` } ImportLogEntry struct { @@ -62,7 +64,7 @@ } func (it ImportTime) MarshalJSON() ([]byte, error) { - return json.Marshal(it.Format("2006-01-02T15:04:05.000")) + return json.Marshal(it.Format(ImportTimeFormat)) } func (it *ImportTime) Scan(x interface{}) error {