Mercurial > gemma

changeset 5560:f2204f91d286

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Join the log lines of imports to the log exports to recover data from them. Used in SR export to extract information that where in the meta json but now are only found in the log.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 09 Feb 2022 18:34:40 +0100
parents ce9a9a1bf92f
children b91716d2acc6
files pkg/controllers/importqueue.go pkg/imports/bn.go pkg/imports/dma.go pkg/imports/dmv.go pkg/imports/dsr.go pkg/imports/fa.go pkg/imports/fd.go pkg/imports/gm.go pkg/imports/report.go pkg/imports/sec.go pkg/imports/sr.go
diffstat 11 files changed, 167 insertions(+), 60 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/controllers/importqueue.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/controllers/importqueue.go	Wed Feb 09 18:34:40 2022 +0100
@@ -65,15 +65,17 @@
   state::varchar,
   enqueued,
   changed,
-  kind,
+  imports.kind,
   username,
   (SELECT country FROM users.list_users lu
     WHERE lu.username = import.imports.username) AS country,
   signer,
   EXISTS(SELECT 1 FROM import.import_logs
     WHERE kind = 'warn'::log_type and import_id = imports.id) AS has_warnings,
-  data
-FROM import.imports
+  data,
+  il.msg
+FROM import.imports RIGHT JOIN import.import_logs il
+  ON import.imports.id = il.import_id
 WHERE
 `
 	selectEnqueuedSQL = `
@@ -122,11 +124,11 @@
 	}
 
 	// Always filter review jobs. They are only for internal use.
-	cond(` NOT kind LIKE '%%` + imports.ReviewJobSuffix + `'`)
+	cond(` NOT imports.kind LIKE '%%` + imports.ReviewJobSuffix + `'`)
 
 	if query := req.FormValue("query"); query != "" {
 		query = "%" + query + "%"
-		cond(` (kind ILIKE $%d OR username ILIKE $%d OR signer ILIKE $%d OR `+
+		cond(` (imports.kind ILIKE $%d OR username ILIKE $%d OR signer ILIKE $%d OR `+
 			`id IN (SELECT import_id FROM import.import_logs WHERE msg ILIKE $%d)) `,
 			query, query, query, query)
 	}
@@ -146,7 +148,7 @@
 
 	if ks := req.FormValue("kinds"); ks != "" {
 		kinds := toTextArray(ks, imports.ImportKindNames())
-		cond(" kind = ANY($%d) ", kinds)
+		cond(" imports.kind = ANY($%d) ", kinds)
 	}
 
 	if idss := req.FormValue("ids"); idss != "" {
@@ -305,65 +307,109 @@
 
 	// Extract some meta infos from the import.
 	type Description interface {
-		Description() (string, error)
+		Description([]string) (string, error)
+	}
+
+	type dataset struct {
+		id       int64
+		state    string
+		enqueued time.Time
+		changed  time.Time
+		kind     string
+		user     string
+		country  string
+		signer   sql.NullString
+		warnings bool
+		data     string
+		msgs     []string
 	}
 
+	// Log unsupported description interfaces per kind only once.
+	unsupported := make(map[string]bool)
+
+	store := func(ds *dataset) error {
+		if ds == nil {
+			return nil
+		}
+
+		var description string
+
+		// Do some introspection on the job to be more verbose.
+		if jc := imports.FindJobCreator(imports.JobKind(ds.kind)); jc != nil {
+			job := jc.Create()
+			if err := common.FromJSONString(ds.data, job); err != nil {
+				log.Errorf("%v\n", err)
+			} else if desc, ok := job.(Description); ok {
+				description, err = desc.Description(ds.msgs)
+				if err != nil {
+					log.Errorf("%v\n", err)
+				}
+				description = strings.Replace(description, ",", "|", -1)
+			} else {
+				if !unsupported[ds.kind] {
+					unsupported[ds.kind] = true
+					log.Debugf("%s: description not supported\n", ds.kind)
+				}
+			}
+		}
+
+		record[0] = strconv.FormatInt(ds.id, 10)
+		record[1] = ds.kind
+		record[2] = ds.enqueued.UTC().Format(common.TimeFormat)
+		record[3] = ds.changed.UTC().Format(common.TimeFormat)
+		record[4] = ds.user
+		record[5] = ds.country
+		record[6] = stringString(ds.signer)
+		record[7] = ds.state
+		record[8] = strconv.FormatBool(ds.warnings)
+		record[9] = description
+
+		return out.Write(record)
+	}
+
+	var last *dataset
+
 	for rows.Next() {
 		var (
-			id          int64
-			state       string
-			enqueued    time.Time
-			changed     time.Time
-			kind        string
-			user        string
-			country     string
-			signer      sql.NullString
-			warnings    bool
-			data        string
-			description string
+			curr dataset
+			msg  sql.NullString
 		)
 		if err = rows.Scan(
-			&id,
-			&state,
-			&enqueued,
-			&changed,
-			&kind,
-			&user,
-			&country,
-			&signer,
-			&warnings,
-			&data,
+			&curr.id,
+			&curr.state,
+			&curr.enqueued,
+			&curr.changed,
+			&curr.kind,
+			&curr.user,
+			&curr.country,
+			&curr.signer,
+			&curr.warnings,
+			&curr.data,
+			&msg,
 		); err != nil {
 			return
 		}
 
-		// Do some introspection on the job to be more verbose.
-		if jc := imports.FindJobCreator(imports.JobKind(kind)); jc != nil {
-			job := jc.Create()
-			if err := common.FromJSONString(data, job); err != nil {
-				log.Errorf("%v\n", err)
-			} else if desc, ok := job.(Description); ok {
-				if description, err = desc.Description(); err != nil {
-					log.Errorf("%v\n", err)
-				}
+		if last != nil && last.id == curr.id {
+			if msg.Valid {
+				last.msgs = append(last.msgs, msg.String)
 			}
+			continue
+		}
+		if msg.Valid {
+			curr.msgs = append(curr.msgs, msg.String)
 		}
 
-		record[0] = strconv.FormatInt(id, 10)
-		record[1] = kind
-		record[2] = enqueued.UTC().Format(common.TimeFormat)
-		record[3] = changed.UTC().Format(common.TimeFormat)
-		record[4] = user
-		record[5] = country
-		record[6] = stringString(signer)
-		record[7] = state
-		record[8] = strconv.FormatBool(warnings)
-		record[9] = strings.Replace(description, ",", "|", -1)
-
-		if err := out.Write(record); err != nil {
+		if err := store(last); err != nil {
 			log.Errorf("%v\n", err)
 			return
 		}
+		last = &curr
+	}
+
+	if err := store(last); err != nil {
+		log.Errorf("%v\n", err)
+		return
 	}
 
 	out.Flush()
--- a/pkg/imports/bn.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/bn.go	Wed Feb 09 18:34:40 2022 +0100
@@ -160,7 +160,7 @@
 )
 
 // Description gives a short info about relevant facts of this import.
-func (bn *Bottleneck) Description() (string, error) {
+func (bn *Bottleneck) Description([]string) (string, error) {
 	return bn.URL, nil
 }
 
--- a/pkg/imports/dma.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/dma.go	Wed Feb 09 18:34:40 2022 +0100
@@ -42,7 +42,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (dma *DistanceMarksAshore) Description() (string, error) {
+func (dma *DistanceMarksAshore) Description([]string) (string, error) {
 	return dma.URL + "|" + dma.FeatureType, nil
 }
 
--- a/pkg/imports/dmv.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/dmv.go	Wed Feb 09 18:34:40 2022 +0100
@@ -37,7 +37,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (dmv *DistanceMarksVirtual) Description() (string, error) {
+func (dmv *DistanceMarksVirtual) Description([]string) (string, error) {
 	return dmv.URL, nil
 }
 
--- a/pkg/imports/dsr.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/dsr.go	Wed Feb 09 18:34:40 2022 +0100
@@ -30,7 +30,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (dsr *DeleteSoundingResult) Description() (string, error) {
+func (dsr *DeleteSoundingResult) Description([]string) (string, error) {
 	return dsr.BottleneckID + "|" + dsr.Date.Format(common.DateFormat), nil
 }
 
--- a/pkg/imports/fa.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/fa.go	Wed Feb 09 18:34:40 2022 +0100
@@ -160,7 +160,7 @@
 )
 
 // Description gives a short info about relevant facts of this import.
-func (fa *FairwayAvailability) Description() (string, error) {
+func (fa *FairwayAvailability) Description([]string) (string, error) {
 	return fa.URL, nil
 }
 
--- a/pkg/imports/fd.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/fd.go	Wed Feb 09 18:34:40 2022 +0100
@@ -48,7 +48,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (fd *FairwayDimension) Description() (string, error) {
+func (fd *FairwayDimension) Description([]string) (string, error) {
 	return strings.Join([]string{
 		fd.URL,
 		fd.FeatureType,
@@ -486,7 +486,7 @@
 					oldID,
 				).Scan(&fdid, &lat, &lon)
 			}); err != nil {
-				feedback.Error(pgxutils.ReadableError{Err: err}.Error() +
+				feedback.Error(pgxutils.ReadableError{Err: err}.Error()+
 					"- while tracking invalidation of: %d", oldID)
 				continue
 			}
--- a/pkg/imports/gm.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/gm.go	Wed Feb 09 18:34:40 2022 +0100
@@ -43,7 +43,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (gm *GaugeMeasurement) Description() (string, error) {
+func (gm *GaugeMeasurement) Description([]string) (string, error) {
 	return gm.URL, nil
 }
 
--- a/pkg/imports/report.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/report.go	Wed Feb 09 18:34:40 2022 +0100
@@ -87,7 +87,7 @@
 // RequiresRoles enforces to be a sys_admin to run this .
 func (*Report) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} }
 
-func (r *Report) Description() (string, error) { return string(r.Name), nil }
+func (r *Report) Description([]string) (string, error) { return string(r.Name), nil }
 
 func (*Report) CleanUp() error { return nil }
 
--- a/pkg/imports/sec.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/sec.go	Wed Feb 09 18:34:40 2022 +0100
@@ -35,7 +35,7 @@
 }
 
 // Description gives a short info about relevant facts of this import.
-func (sec *Section) Description() (string, error) {
+func (sec *Section) Description([]string) (string, error) {
 	return strings.Join([]string{
 		sec.Name,
 		sec.ObjNam,
--- a/pkg/imports/sr.go	Thu Dec 02 12:37:33 2021 +0100
+++ b/pkg/imports/sr.go	Wed Feb 09 18:34:40 2022 +0100
@@ -28,13 +28,16 @@
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	shp "github.com/jonas-p/go-shp"
 
 	"gemma.intevation.de/gemma/pkg/common"
+	"gemma.intevation.de/gemma/pkg/log"
 	"gemma.intevation.de/gemma/pkg/mesh"
 	"gemma.intevation.de/gemma/pkg/models"
 	"gemma.intevation.de/gemma/pkg/wkb"
@@ -239,23 +242,81 @@
 `
 )
 
+var (
+	bottleneckRe,
+	surveyTypeRe,
+	dateRe,
+	negateZRe *regexp.Regexp
+	recoverRegsOnce sync.Once
+)
+
+func compileRecoverRegs() {
+	bottleneckRe = regexp.MustCompile(`Bottleneck:\s*(.+)\s*$`)
+	surveyTypeRe = regexp.MustCompile(`Processing as\s+([^\s]+)\s+beam scan.`)
+	dateRe = regexp.MustCompile(`Survey date:\s*(\d{4}-\d{2}-\d{2}).`)
+	negateZRe = regexp.MustCompile(`Z values will be negated.`)
+}
+
 // Description gives a short info about relevant facts of this import.
-func (sr *SoundingResult) Description() (string, error) {
+func (sr *SoundingResult) Description(msgs []string) (string, error) {
+
+	recoverRegsOnce.Do(compileRecoverRegs)
+
+	//log.Debugln(strings.Join(msgs, "\n") + "\n\n")
+
+	var (
+		bottleneck, st, date string
+		negZ                 *bool
+	)
+
+	for _, msg := range msgs {
+		if m := bottleneckRe.FindStringSubmatch(msg); m != nil {
+			bottleneck = m[1]
+			continue
+		}
+		if m := surveyTypeRe.FindStringSubmatch(msg); m != nil {
+			st = m[1]
+			continue
+		}
+		if m := dateRe.FindStringSubmatch(msg); m != nil {
+			date = m[1]
+			continue
+		}
+		if negateZRe.MatchString(msg) {
+			t := true
+			negZ = &t
+		}
+	}
 
 	var descs []string
 
 	if sr.Bottleneck != nil {
 		descs = append(descs, *sr.Bottleneck)
+	} else if bottleneck != "" {
+		log.Debugf("bottleneck recovered: %s\n", bottleneck)
+		descs = append(descs, bottleneck)
 	}
+
 	if sr.Date != nil {
 		descs = append(descs, (*sr).Date.Format(common.DateFormat))
+	} else if date != "" {
+		log.Debugf("date recovered: %s\n", date)
+		descs = append(descs, date)
 	}
+
 	if sr.NegateZ != nil && *sr.NegateZ {
+	} else if negZ != nil && *negZ {
+		log.Debugln("negateZ recovered")
 		descs = append(descs, "negateZ")
 	}
+
 	if sr.SurveyType != nil {
 		descs = append(descs, string(*sr.SurveyType))
+	} else if st != "" {
+		log.Debugf("survey type recovered: %s\n", st)
+		descs = append(descs, st)
 	}
+
 	return strings.Join(descs, "|"), nil
 }