Mercurial > gemma

changeset 2687:3b98de34de90

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged import-overview-rework back into default.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Fri, 15 Mar 2019 15:59:40 +0100
parents c4da269238a4 (current diff) 47b789a27618 (diff)
children d316a6e41f54
files
diffstat 10 files changed, 258 insertions(+), 142 deletions(-) [+]
line wrap: on
line diff
--- a/client/src/components/importoverview/AdditionalDetail.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/AdditionalDetail.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -2,13 +2,19 @@
   <div>
     <FairwayDimensionDetail
       :entry="entry"
+      :details="details"
       v-if="isFairwayDimension"
     ></FairwayDimensionDetail>
     <ApprovedGaugeMeasurementDetail
       :entry="entry"
+      :details="details"
       v-if="isApprovedGaugeMeasurement"
     ></ApprovedGaugeMeasurementDetail>
-    <BottleneckDetail :entry="entry" v-if="isBottleneck"></BottleneckDetail>
+    <BottleneckDetail
+      :details="details"
+      :entry="entry"
+      v-if="isBottleneck"
+    ></BottleneckDetail>
   </div>
 </template>
 
@@ -29,7 +35,7 @@
 
 export default {
   name: "additionaldetail",
-  props: ["entry"],
+  props: ["entry", "details"],
   components: {
     BottleneckDetail: () => import("./BottleneckDetail.vue"),
     ApprovedGaugeMeasurementDetail: () =>
--- a/client/src/components/importoverview/AdditionalLog.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/AdditionalLog.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -49,37 +49,13 @@
  * Author(s):
  * Thomas Junk <thomas.junk@intevation.de>
  */
-import { displayError } from "@/lib/errors.js";
-import { HTTP } from "@/lib/http.js";
-
 export default {
   name: "additionallogs",
-  props: ["entry"],
+  props: ["details"],
   data() {
     return {
-      logLines: []
+      logLines: this.details.entries
     };
-  },
-  methods: {
-    loadEntries() {
-      HTTP.get("/imports/" + this.entry.id, {
-        headers: { "X-Gemma-Auth": localStorage.getItem("token") }
-      })
-        .then(response => {
-          const { entries } = response.data;
-          this.logLines = entries;
-        })
-        .catch(error => {
-          const { status, data } = error.response;
-          displayError({
-            title: this.$gettext("Backend Error"),
-            message: `${status}: ${data.message || data}`
-          });
-        });
-    }
-  },
-  mounted() {
-    this.loadEntries();
   }
 };
 </script>
--- a/client/src/components/importoverview/ApprovedGaugeMeasurementDetail.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/ApprovedGaugeMeasurementDetail.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -1,6 +1,6 @@
 <template>
   <div class="diffs">
-    <div v-for="(result, index) in entry.summary" :key="index">
+    <div v-for="(result, index) in details.summary" :key="index">
       <div class="pl-2 d-flex flex-row">
         <div
           @click="toggleDiff(index)"
@@ -105,7 +105,7 @@
 
 export default {
   name: "agmdetails",
-  props: ["entry"],
+  props: ["entry", "details"],
   data() {
     return {
       showDiff: NODIFF
--- a/client/src/components/importoverview/BottleneckDetail.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/BottleneckDetail.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -73,7 +73,7 @@
 
 export default {
   name: "bottleneckdetails",
-  props: ["entry"],
+  props: ["entry", "details"],
   data() {
     return {
       bottlenecks: [],
@@ -86,7 +86,7 @@
   methods: {
     loadBottlenecks() {
       const generateFilter = () => {
-        const { bottlenecks } = this.entry.summary;
+        const { bottlenecks } = this.details.summary;
         if (bottlenecks.length === 1)
           return equalToFilter("bottleneck_id", bottlenecks[0]);
         const orExpressions = bottlenecks.map(x => {
--- a/client/src/components/importoverview/LogDetail.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/LogDetail.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -18,19 +18,24 @@
         ></font-awesome-icon>
         <span class="text-info"><translate>Additional Info</translate></span>
         <span class="text-info" v-if="isApprovedGaugeMeasurement">
-          ({{ entry.summary.length }})</span
+          ({{ details.summary.length }})</span
         >
         <span v-if="isBottleneck" class="text-info text-left">
-          ({{ entry.summary.bottlenecks.length }})</span
+          ({{ details.summary.bottlenecks.length }})</span
         >
         <span class="text-left" v-if="isFairwayDimension"
-          >{{ entry.summary["source-organization"] }} (LOS:
-          {{ entry.summary.los }})</span
+          >{{ details.summary["source-organization"] }} (LOS:
+          {{ details.summary.los }})</span
         >
       </div>
-      <StretchDetail v-if="isStretch" :entry="entry"></StretchDetail>
+      <StretchDetail
+        v-if="isStretch"
+        :entry="entry"
+        :details="details"
+      ></StretchDetail>
       <SoundingResultDetail
         :entry="entry"
+        :details="details"
         v-if="isSoundingResult"
       ></SoundingResultDetail>
     </div>
@@ -38,6 +43,7 @@
       v-if="entry.id === showAdditional"
       class="ml-2 d-flex flex-row"
       :entry="entry"
+      :details="details"
     ></AdditionalDetail>
     <div class="d-flex fex-row">
       <font-awesome-icon
@@ -60,7 +66,7 @@
     <AdditionalLog
       v-if="entry.id === showLogs"
       class="ml-4 d-flex flex-row mr-1"
-      :entry="entry"
+      :details="details"
     ></AdditionalLog>
   </div>
 </template>
@@ -84,7 +90,7 @@
 
 export default {
   name: "logdetail",
-  props: ["entry"],
+  props: ["entry", "details"],
   components: {
     SoundingResultDetail: () => import("./SoundingResultDetail.vue"),
     StretchDetail: () => import("./StretchDetails.vue"),
--- a/client/src/components/importoverview/LogEntry.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/LogEntry.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -58,7 +58,11 @@
       </div>
     </div>
     <div class="ml-1 d-flex flex-row">
-      <LogDetail :entry="entry" v-if="show === entry.id"></LogDetail>
+      <LogDetail
+        :entry="entry"
+        :details="details"
+        v-if="show === entry.id"
+      ></LogDetail>
     </div>
   </div>
 </template>
@@ -79,10 +83,32 @@
  */
 import { mapState } from "vuex";
 import { STATES } from "@/store/imports.js";
+import { displayError } from "@/lib/errors.js";
+import { HTTP } from "@/lib/http.js";
 
 export default {
   name: "importlogentry",
   props: ["entry"],
+  data() {
+    return {
+      details: null
+    };
+  },
+  mounted() {
+    HTTP.get("/imports/" + this.entry.id, {
+      headers: { "X-Gemma-Auth": localStorage.getItem("token") }
+    })
+      .then(response => {
+        this.details = response.data;
+      })
+      .catch(error => {
+        const { status, data } = error.response;
+        displayError({
+          title: this.$gettext("Backend Error"),
+          message: `${status}: ${data.message || data}`
+        });
+      });
+  },
   components: {
     LogDetail: () => import("./LogDetail.vue")
   },
--- a/client/src/components/importoverview/SoundingResultDetail.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/SoundingResultDetail.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -2,7 +2,7 @@
   <div>
     <span class="empty"></span>
     <a @click="zoomTo()" class="text-info pointer">
-      {{ entry.summary.bottleneck }}
+      {{ details.summary.bottleneck }}
     </a>
   </div>
 </template>
@@ -23,7 +23,7 @@
  */
 export default {
   name: "soundingresultdetails",
-  props: ["entry"],
+  props: ["entry", "details"],
   methods: {
     moveMap(coordinates) {
       this.$store.commit("map/moveMap", {
@@ -33,7 +33,7 @@
       });
     },
     zoomTo() {
-      const { lat, lon, bottleneck, date } = this.entry.summary;
+      const { lat, lon, bottleneck, date } = this.details.summary;
       const coordinates = [lat, lon];
       this.moveMap(coordinates);
       this.$store
--- a/client/src/components/importoverview/StretchDetails.vue	Fri Mar 15 12:43:30 2019 +0100
+++ b/client/src/components/importoverview/StretchDetails.vue	Fri Mar 15 15:59:40 2019 +0100
@@ -2,7 +2,7 @@
   <div>
     <span class="empty">&nbsp;</span>
     <a @click="zoomToStretch()" class="text-info pointer">{{
-      entry.summary.stretch
+      details.summary.stretch
     }}</a>
   </div>
 </template>
@@ -26,7 +26,7 @@
 
 export default {
   name: "stretchdetails",
-  props: ["entry"],
+  props: ["entry", "details"],
   methods: {
     moveToExtent(feature) {
       this.$store.commit("map/moveToExtent", {
@@ -36,7 +36,7 @@
       });
     },
     zoomToStretch() {
-      const name = this.entry.summary.stretch;
+      const name = this.details.summary.stretch;
       this.$store.commit("map/setLayerVisible", LAYERS.STRETCHES);
       this.$store
         .dispatch("imports/loadStretch", name)
--- a/pkg/controllers/importqueue.go	Fri Mar 15 12:43:30 2019 +0100
+++ b/pkg/controllers/importqueue.go	Fri Mar 15 15:59:40 2019 +0100
@@ -14,6 +14,7 @@
 package controllers
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -21,6 +22,7 @@
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/jackc/pgx/pgtype"
@@ -32,13 +34,17 @@
 
 const (
 	warningSQLPrefix = `
-`
-	selectImportsSQL = `
 WITH warned AS (
   SELECT distinct(import_id) AS id
   FROM import.import_logs
   WHERE kind = 'warn'::log_type
-)
+)`
+	selectImportsCountSQL = warningSQLPrefix + `
+SELECT count(*)
+FROM import.imports
+WHERE
+`
+	selectImportsSQL = warningSQLPrefix + `
 SELECT
   imports.id AS id,
   state::varchar,
@@ -46,21 +52,21 @@
   kind,
   username,
   signer,
-  summary,
-  EXISTS (
-    SELECT true FROM warned
-	WHERE warned.id = imports.id
-  ) AS has_warnings
-`
-	withoutWarningsSQL = `
+  summary IS NOT NULL AS has_summary,
+  imports.id IN (SELECT id FROM warned) AS has_warnings
 FROM import.imports
+WHERE
 `
-	withWarningsSQL = `
-FROM import.imports JOIN warned ON imports.id = warned.id
+	selectBeforeSQL = warningSQLPrefix + `
+SELECT enqueued FROM import.imports
+WHERE
 `
-
-	selectHasImportSQL = `
-SELECT true FROM import.imports WHERE id = $1`
+	selectAfterSQL = warningSQLPrefix + `
+SELECT enqueued FROM import.imports
+WHERE
+`
+	selectImportSummaySQL = `
+SELECT summary FROM import.imports WHERE id = $1`
 
 	selectHasNoRunningImportSQL = `
 SELECT true FROM import.imports
@@ -126,93 +132,142 @@
 	return &ta
 }
 
-func queryImportListStmt(
-	conn *sql.Conn,
-	req *http.Request,
-) (*sql.Rows, error) {
+type filterBuilder struct {
+	stmt    strings.Builder
+	args    []interface{}
+	hasCond bool
+}
+
+func (fb *filterBuilder) arg(format string, v ...interface{}) {
+	indices := make([]interface{}, len(v))
+	for i := range indices {
+		indices[i] = len(fb.args) + i + 1
+	}
+	fmt.Fprintf(&fb.stmt, format, indices...)
+	fb.args = append(fb.args, v...)
+}
+
+func (fb *filterBuilder) cond(format string, v ...interface{}) {
+	if fb.hasCond {
+		fb.stmt.WriteString(" AND ")
+	} else {
+		fb.hasCond = true
+	}
+	fb.arg(format, v...)
+}
 
-	var (
-		stmt   strings.Builder
-		args   []interface{}
-		states *pgtype.TextArray
-		kinds  *pgtype.TextArray
-		ids    *pgtype.Int8Array
-	)
+func buildFilters(req *http.Request) (l, b, a *filterBuilder, err error) {
+
+	l = new(filterBuilder)
+	a = new(filterBuilder)
+	b = new(filterBuilder)
+
+	var noBefore, noAfter bool
+
+	var counting bool
 
-	arg := func(format string, v interface{}) {
-		fmt.Fprintf(&stmt, format, len(args)+1)
-		args = append(args, v)
+	switch count := strings.ToLower(req.FormValue("count")); count {
+	case "1", "t", "true":
+		counting = true
+		l.stmt.WriteString(selectImportsCountSQL)
+	default:
+		l.stmt.WriteString(selectImportsSQL)
+	}
+	a.stmt.WriteString(selectAfterSQL)
+	b.stmt.WriteString(selectBeforeSQL)
+
+	cond := func(format string, v ...interface{}) {
+		l.cond(format, v...)
+		a.cond(format, v...)
+		b.cond(format, v...)
 	}
 
-	var warnings bool
-
-	switch warn := strings.ToLower(req.FormValue("warnings")); warn {
-	case "1", "t", "true":
-		warnings = true
+	if query := req.FormValue("query"); query != "" {
+		query = "%" + query + "%"
+		cond(` (kind ILIKE $%d OR username ILIKE $%d OR signer ILIKE $%d OR `+
+			`id IN (SELECT import_id FROM import.import_logs WHERE msg ILIKE $%d)) `,
+			query, query, query, query)
 	}
 
 	if st := req.FormValue("states"); st != "" {
-		states = toTextArray(st, imports.ImportStateNames)
+		states := toTextArray(st, imports.ImportStateNames)
+		cond(" state = ANY($%d) ", states)
 	}
 
 	if ks := req.FormValue("kinds"); ks != "" {
-		kinds = toTextArray(ks, imports.ImportKindNames())
+		kinds := toTextArray(ks, imports.ImportKindNames())
+		cond(" kind = ANY($%d) ", kinds)
 	}
 
 	if idss := req.FormValue("ids"); idss != "" {
-		ids = toInt8Array(idss)
+		ids := toInt8Array(idss)
+		cond(" id = ANY($%d) ", ids)
 	}
 
-	stmt.WriteString(selectImportsSQL)
-	if warnings {
-		stmt.WriteString(withWarningsSQL)
+	if from := req.FormValue("from"); from != "" {
+		var fromTime time.Time
+		if fromTime, err = time.Parse(models.ImportTimeFormat, from); err != nil {
+			return
+		}
+		l.cond(" enqueued >= $%d ", fromTime)
+		b.cond(" enqueued < $%d", fromTime)
 	} else {
-		stmt.WriteString(withoutWarningsSQL)
+		noBefore = true
 	}
 
-	if states != nil || kinds != nil || ids != nil {
-		stmt.WriteString(" WHERE ")
+	if to := req.FormValue("to"); to != "" {
+		var toTime time.Time
+		if toTime, err = time.Parse(models.ImportTimeFormat, to); err != nil {
+			return
+		}
+		l.cond(" enqueued <= $%d ", toTime)
+		a.cond(" enqueued > $%d", toTime)
+	} else {
+		noAfter = true
 	}
 
-	if states != nil {
-		arg(" state = ANY($%d) ", states)
-	}
-
-	if states != nil && (kinds != nil || ids != nil) {
-		stmt.WriteString("AND")
-	}
-
-	if kinds != nil {
-		arg(" kind = ANY($%d) ", kinds)
+	switch warn := strings.ToLower(req.FormValue("warnings")); warn {
+	case "1", "t", "true":
+		cond(" id IN (SELECT id FROM warned) ")
 	}
 
-	if (states != nil || kinds != nil) && ids != nil {
-		stmt.WriteString("AND")
+	if !l.hasCond {
+		l.stmt.WriteString(" TRUE ")
+	}
+	if !b.hasCond {
+		b.stmt.WriteString(" TRUE ")
+	}
+	if !a.hasCond {
+		a.stmt.WriteString(" TRUE ")
 	}
 
-	if ids != nil {
-		arg(" id = ANY($%d) ", ids)
+	if !counting {
+		l.stmt.WriteString(" ORDER BY enqueued DESC ")
+		a.stmt.WriteString(" ORDER BY enqueued LIMIT 1")
+		b.stmt.WriteString(" ORDER BY enqueued LIMIT 1")
 	}
 
-	stmt.WriteString(" ORDER BY enqueued DESC ")
+	if noBefore {
+		b = nil
+	}
+	if noAfter {
+		a = nil
+	}
+	return
+}
 
-	if lim := req.FormValue("limit"); lim != "" {
-		limit, err := strconv.ParseInt(lim, 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		arg(" LIMIT $%d ", limit)
-	}
+func neighbored(ctx context.Context, conn *sql.Conn, fb *filterBuilder) *models.ImportTime {
 
-	if ofs := req.FormValue("offset"); ofs != "" {
-		offset, err := strconv.ParseInt(ofs, 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		arg(" OFFSET $%d ", offset)
+	var when time.Time
+	err := conn.QueryRowContext(ctx, fb.stmt.String(), fb.args...).Scan(&when)
+	switch {
+	case err == sql.ErrNoRows:
+		return nil
+	case err != nil:
+		log.Printf("warn: %v\n", err)
+		return nil
 	}
-
-	return conn.QueryContext(req.Context(), stmt.String(), args...)
+	return &models.ImportTime{when}
 }
 
 func listImports(
@@ -221,27 +276,53 @@
 	conn *sql.Conn,
 ) (jr JSONResult, err error) {
 
+	var list, before, after *filterBuilder
+
+	if list, before, after, err = buildFilters(req); err != nil {
+		return
+	}
+
+	ctx := req.Context()
+
+	// Fast path for counting
+
+	switch count := strings.ToLower(req.FormValue("count")); count {
+	case "1", "t", "true":
+		var count int64
+		err = conn.QueryRowContext(ctx, list.stmt.String(), list.args...).Scan(&count)
+		switch {
+		case err == sql.ErrNoRows:
+			count, err = 0, nil
+		case err != nil:
+			return
+		}
+		jr = JSONResult{Result: count}
+		return
+	}
+
+	// Generate the list
+
 	var rows *sql.Rows
-	rows, err = queryImportListStmt(conn, req)
-	if err != nil {
+	if rows, err = conn.QueryContext(ctx, list.stmt.String(), list.args...); err != nil {
 		return
 	}
 	defer rows.Close()
 
 	imports := make([]*models.Import, 0, 20)
 
-	var signer, summary sql.NullString
+	var signer sql.NullString
 
 	for rows.Next() {
 		var it models.Import
+		var enqueued time.Time
 		if err = rows.Scan(
 			&it.ID,
 			&it.State,
-			&it.Enqueued,
+			&enqueued,
 			&it.Kind,
 			&it.User,
 			&signer,
-			&summary,
+			&it.Summary,
 			&it.Warnings,
 		); err != nil {
 			return
@@ -249,12 +330,7 @@
 		if signer.Valid {
 			it.Signer = signer.String
 		}
-		if summary.Valid {
-			if err = json.NewDecoder(
-				strings.NewReader(summary.String)).Decode(&it.Summary); err != nil {
-				return
-			}
-		}
+		it.Enqueued = models.ImportTime{enqueued}
 		imports = append(imports, &it)
 	}
 
@@ -262,11 +338,25 @@
 		return
 	}
 
+	var prev, next *models.ImportTime
+
+	if before != nil {
+		prev = neighbored(ctx, conn, before)
+	}
+
+	if after != nil {
+		next = neighbored(ctx, conn, after)
+	}
+
 	jr = JSONResult{
 		Result: struct {
-			Imports []*models.Import `json:"imports"`
+			Prev    *models.ImportTime `json:"prev,omitempty"`
+			Next    *models.ImportTime `json:"next,omitempty"`
+			Imports []*models.Import   `json:"imports"`
 		}{
 			Imports: imports,
+			Prev:    prev,
+			Next:    next,
 		},
 	}
 	return
@@ -283,8 +373,8 @@
 	id, _ := strconv.ParseInt(mux.Vars(req)["id"], 10, 64)
 
 	// Check if he have such a import job first.
-	var dummy bool
-	err = conn.QueryRowContext(ctx, selectHasImportSQL, id).Scan(&dummy)
+	var summary sql.NullString
+	err = conn.QueryRowContext(ctx, selectImportSummaySQL, id).Scan(&summary)
 	switch {
 	case err == sql.ErrNoRows:
 		err = JSONError{
@@ -296,6 +386,14 @@
 		return
 	}
 
+	var sum interface{}
+	if summary.Valid {
+		if err = json.NewDecoder(
+			strings.NewReader(summary.String)).Decode(&sum); err != nil {
+			return
+		}
+	}
+
 	// We have it -> generate log entries.
 	var rows *sql.Rows
 	rows, err = conn.QueryContext(ctx, selectImportLogsSQL, id)
@@ -320,8 +418,10 @@
 
 	jr = JSONResult{
 		Result: struct {
+			Summary interface{}              `json:"summary,omitempty"`
 			Entries []*models.ImportLogEntry `json:"entries"`
 		}{
+			Summary: sum,
 			Entries: entries,
 		},
 	}
--- a/pkg/models/import.go	Fri Mar 15 12:43:30 2019 +0100
+++ b/pkg/models/import.go	Fri Mar 15 15:59:40 2019 +0100
@@ -19,18 +19,20 @@
 	"time"
 )
 
+const ImportTimeFormat = "2006-01-02T15:04:05.000"
+
 type (
 	ImportTime struct{ time.Time }
 
 	Import struct {
-		ID       int64       `json:"id"`
-		State    string      `json:"state"`
-		Enqueued ImportTime  `json:"enqueued"`
-		Kind     string      `json:"kind"`
-		User     string      `json:"user"`
-		Signer   string      `json:"signer,omitempty"`
-		Summary  interface{} `json:"summary,omitempty"`
-		Warnings bool        `json:"warnings,omitempty"`
+		ID       int64      `json:"id"`
+		State    string     `json:"state"`
+		Enqueued ImportTime `json:"enqueued"`
+		Kind     string     `json:"kind"`
+		User     string     `json:"user"`
+		Signer   string     `json:"signer,omitempty"`
+		Summary  bool       `json:"summary,omitempty"`
+		Warnings bool       `json:"warnings,omitempty"`
 	}
 
 	ImportLogEntry struct {
@@ -62,7 +64,7 @@
 }
 
 func (it ImportTime) MarshalJSON() ([]byte, error) {
-	return json.Marshal(it.Format("2006-01-02T15:04:05.000"))
+	return json.Marshal(it.Format(ImportTimeFormat))
 }
 
 func (it *ImportTime) Scan(x interface{}) error {