Mercurial > gemma

changeset 2025:070ac9dd61a1 unify_imports

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Bring manual imports to new import modeling.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Thu, 24 Jan 2019 18:19:43 +0100
parents 6d9fbc62c5a6
children ec92398b3af7
files pkg/controllers/manualimports.go pkg/models/importbase.go pkg/models/imports.go
diffstat 3 files changed, 79 insertions(+), 111 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/controllers/manualimports.go	Thu Jan 24 17:38:31 2019 +0100
+++ b/pkg/controllers/manualimports.go	Thu Jan 24 18:19:43 2019 +0100
@@ -26,84 +26,70 @@
 	"gemma.intevation.de/gemma/pkg/models"
 )
 
-func importBottleneck(input interface{}) (interface{}, common.Attributes, bool) {
+func importBottleneck(input interface{}) interface{} {
 	bi := input.(*models.BottleneckImport)
-	bn := &imports.Bottleneck{
+	return &imports.Bottleneck{
 		URL:      bi.URL,
 		Insecure: bi.Insecure,
 	}
-	return bn, bi.Attributes, bi.SendEmail
 }
 
-func importGaugeMeasurement(input interface{}) (interface{}, common.Attributes, bool) {
+func importGaugeMeasurement(input interface{}) interface{} {
 	gi := input.(*models.GaugeMeasurementImport)
-	gm := &imports.GaugeMeasurement{
+	return &imports.GaugeMeasurement{
 		URL:      gi.URL,
 		Insecure: gi.Insecure,
 	}
-	return gm, gi.Attributes, gi.SendEmail
 }
 
-func importFairwayAvailability(input interface{}) (interface{}, common.Attributes, bool) {
+func importFairwayAvailability(input interface{}) interface{} {
 	fai := input.(*models.FairwayAvailabilityImport)
-	fa := &imports.FairwayAvailability{
+	return &imports.FairwayAvailability{
 		URL:      fai.URL,
 		Insecure: fai.Insecure,
 	}
-	return fa, fai.Attributes, fai.SendEmail
 }
 
-func importWaterwayAxis(input interface{}) (interface{}, common.Attributes, bool) {
+func importWaterwayAxis(input interface{}) interface{} {
 	wxi := input.(*models.WaterwayAxisImport)
-	wx := &imports.WaterwayAxis{
+	return &imports.WaterwayAxis{
 		URL:         wxi.URL,
 		FeatureType: wxi.FeatureType,
 		SortBy:      wxi.SortBy,
 	}
-	return wx, wxi.Attributes, wxi.SendEmail
 }
 
-func importWaterwayArea(input interface{}) (interface{}, common.Attributes, bool) {
+func importWaterwayArea(input interface{}) interface{} {
 	wai := input.(*models.WaterwayAreaImport)
-	wa := &imports.WaterwayArea{
+	return &imports.WaterwayArea{
 		URL:         wai.URL,
 		FeatureType: wai.FeatureType,
 		SortBy:      wai.SortBy,
 	}
-	return wa, wai.Attributes, wai.SendEmail
 }
 
-func importWaterwayGauge(input interface{}) (interface{}, common.Attributes, bool) {
+func importWaterwayGauge(input interface{}) interface{} {
 	wgi := input.(*models.WaterwayGaugeImport)
-	username, _ := wgi.Attributes.Get("username")
-	password, _ := wgi.Attributes.Get("password")
-	insecure := wgi.Attributes.Bool("insecure")
-	wg := &imports.WaterwayGauge{
-		URL:      wgi.URL,
-		Username: username,
-		Password: password,
-		Insecure: insecure,
+	return &imports.WaterwayGauge{
+		Username: nilString(wgi.User),
+		Password: nilString(wgi.Password),
+		Insecure: wgi.Insecure,
 	}
-	return wg, wgi.Attributes, wgi.SendEmail
 }
 
-func importDistancemarksVirtual(input interface{}) (interface{}, common.Attributes, bool) {
+func importDistancemarksVirtual(input interface{}) interface{} {
 	dmvi := input.(*models.DistanceMarksVirtualImport)
-	username, _ := dmvi.Attributes.Get("username")
-	password, _ := dmvi.Attributes.Get("password")
-	insecure := dmvi.Attributes.Bool("insecure")
-	wg := &imports.DistanceMarksVirtual{
+	return &imports.DistanceMarksVirtual{
 		URL:      dmvi.URL,
-		Username: username,
-		Password: password,
-		Insecure: insecure,
+		Username: nilString(dmvi.User),
+		Password: nilString(dmvi.Password),
+		Insecure: dmvi.Insecure,
 	}
-	return wg, dmvi.Attributes, dmvi.SendEmail
 }
 
-func importFairwayDimension(input interface{}) (interface{}, common.Attributes, bool) {
+func importFairwayDimension(input interface{}) interface{} {
 	fdi := input.(*models.FairwayDimensionImport)
-	fd := &imports.FairwayDimension{
+	return &imports.FairwayDimension{
 		URL:                fdi.URL,
 		FeatureType:        fdi.FeatureType,
 		SortBy:             fdi.SortBy,
@@ -113,22 +99,20 @@
 		Depth:              fdi.Depth,
 		SourceOrganization: fdi.SourceOrganization,
 	}
-	return fd, fdi.Attributes, fdi.SendEmail
 }
 
-func importDistanceMarksAshore(input interface{}) (interface{}, common.Attributes, bool) {
+func importDistanceMarksAshore(input interface{}) interface{} {
 	dmai := input.(*models.DistanceMarksAshoreImport)
-	dma := &imports.DistanceMarksAshore{
+	return &imports.DistanceMarksAshore{
 		URL:         dmai.URL,
 		FeatureType: dmai.FeatureType,
 		SortBy:      dmai.SortBy,
 	}
-	return dma, dmai.Attributes, dmai.SendEmail
 }
 
-func importStretch(input interface{}) (interface{}, common.Attributes, bool) {
+func importStretch(input interface{}) interface{} {
 	sti := input.(*models.StretchImport)
-	st := &imports.Stretch{
+	return &imports.Stretch{
 		Name:      sti.Name,
 		From:      sti.From,
 		To:        sti.To,
@@ -138,7 +122,6 @@
 		Date:      sti.Date,
 		Countries: sti.Countries,
 	}
-	return st, sti.Attributes, sti.SendEmail
 }
 
 func retry(a common.Attributes) (time.Time, *int, *time.Duration) {
@@ -156,33 +139,54 @@
 	return due, retries, duration
 }
 
+func nilString(s *string) string {
+	if s != nil {
+		return *s
+	}
+	return ""
+}
+
 func manualImport(
 	kind imports.JobKind,
-	setup func(interface{}) (interface{}, common.Attributes, bool),
+	setup func(interface{}) interface{},
 ) func(interface{}, *http.Request, *sql.Conn) (JSONResult, error) {
 
 	return func(input interface{}, req *http.Request, _ *sql.Conn) (
 		jr JSONResult, err error) {
 
-		what, attrs, sendEmail := setup(input)
-
-		due, retries, waitRetry := retry(attrs)
-
+		what := setup(input)
 		var serialized string
 		if serialized, err = common.ToJSONString(what); err != nil {
 			return
 		}
 
+		var (
+			due       time.Time
+			trys      *int
+			waitRetry *time.Duration
+			email     bool
+		)
+
+		if qctg, ok := input.(models.QueueConfigurationGetter); ok {
+			qct := qctg.GetQueueConfiguration()
+			if qct.Due != nil {
+				due = *qct.Due
+			}
+			trys = qct.Trys
+			waitRetry = qct.WaitRetry
+			email = qct.EMail
+		}
+
 		session, _ := auth.GetSession(req)
 
 		var jobID int64
 		if jobID, err = imports.AddJob(
 			kind,
 			due,
-			retries,
+			trys,
 			waitRetry,
 			session.User,
-			sendEmail,
+			email,
 			serialized,
 		); err != nil {
 			return
--- a/pkg/models/importbase.go	Thu Jan 24 17:38:31 2019 +0100
+++ b/pkg/models/importbase.go	Thu Jan 24 18:19:43 2019 +0100
@@ -27,6 +27,7 @@
 	QueueConfigurationType struct {
 		Trys      *int           `json:"trys,omitempty"`
 		WaitRetry *time.Duration `json:"wait-retry,omitempty"`
+		Due       *time.Time     `json:"due,omitempty"`
 		Cron      *CronSpec      `json:"cron,omitempty"`
 		EMail     bool           `json:"send-email,omitempty"`
 	}
--- a/pkg/models/imports.go	Thu Jan 24 17:38:31 2019 +0100
+++ b/pkg/models/imports.go	Thu Jan 24 18:19:43 2019 +0100
@@ -12,97 +12,67 @@
 //  * Sascha L. Teichmann <sascha.teichmann@intevation.de>
 package models
 
-import "gemma.intevation.de/gemma/pkg/common"
-
 type (
 	BottleneckImport struct {
-		URL        string            `json:"url"`
-		Insecure   bool              `json:"insecure"`
-		SendEmail  bool              `json:"send-email"`
-		Attributes common.Attributes `json:"attributes,omitempty"`
+		URLType
+		QueueConfigurationType
 	}
 
 	// GaugeMeasurementImport contains data used to define the endpoint
 	GaugeMeasurementImport struct {
-		URL       string `json:"url"`
-		Insecure  bool   `json:"insecure"`
-		SendEmail bool   `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
+		URLType
+		QueueConfigurationType
 	}
 
 	// FairwayAvailabilityImport contains data used to define the endpoint
 	FairwayAvailabilityImport struct {
-		URL       string `json:"url"`
-		Insecure  bool   `json:"insecure"`
-		SendEmail bool   `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
+		URLType
+		QueueConfigurationType
 	}
 
 	// WaterwayAxisImport specifies an import of the waterway axis.
 	WaterwayAxisImport struct {
-		// URL is the capabilities URL of the WFS.
-		URL string `json:"url"`
+		URLType
+		QueueConfigurationType
+
 		// FeatureType is the layer to use.
 		FeatureType string `json:"feature-type"`
 		// SortBy sorts the feature by this key.
 		SortBy string `json:"sort-by"`
-		// SendEmail is set to true if an email should be send after
-		// importing the axis.
-		SendEmail bool `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
 	}
 
 	// WaterwayAreaImport specifies an import of the waterway area.
 	WaterwayAreaImport struct {
-		// URL is the capabilities URL of the WFS.
-		URL string `json:"url"`
+		URLType
+		QueueConfigurationType
+
 		// FeatureType is the layer to use.
 		FeatureType string `json:"feature-type"`
 		// SortBy sorts the feature by this key.
 		SortBy string `json:"sort-by"`
-		// SendEmail is set to true if an email should be send after
-		// importing the axis.
-		SendEmail bool `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
 	}
 
 	// WaterwayAxisImport specifies an import of waterway gauges.
 	WaterwayGaugeImport struct {
-		// URL is the SOAP service URL.
-		URL string `json:"url"`
-		// SendEmail is set to true if an email should be send after
-		// importing the waterway gauges.
-		SendEmail bool `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
+		URLType
+		QueueConfigurationType
 	}
 
 	// DistanceMarksVirtualImport specifies an import of distance marks virtual.
 	DistanceMarksVirtualImport struct {
-		// URL is the SOAP service URL.
-		URL string `json:"url"`
-		// SendEmail is set to true if an email should be send after
-		// importing the waterway gauges.
-		SendEmail bool `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
+		URLType
+		QueueConfigurationType
 	}
 
 	// FairwayDimensionImport specifies an import of the waterway axis.
 	FairwayDimensionImport struct {
-		// URL is the capabilities URL of the WFS.
-		URL string `json:"url"`
+		URLType
+		QueueConfigurationType
+
 		// FeatureType is the layer to use.
 		FeatureType string `json:"feature-type"`
 		// SortBy sorts the feature by this key.
 		SortBy string `json:"sort-by"`
-		// SendEmail is set to true if an email should be send after
-		// importing the axis.
-		SendEmail bool `json:"send-email"`
 		// LOS is the level of service provided by the wfs
 		LOS int `json:"los"`
 		// MinWidth is the minimum width of the fairway for the specified LOS
@@ -113,26 +83,22 @@
 		Depth int `json:"depth"`
 		// SourceOrganization specifies the source of the entry
 		SourceOrganization string `json:"source-organization"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
 	}
 
 	// DistanceMarksAshoreImport specifies an import of the distance marks.
 	DistanceMarksAshoreImport struct {
-		// URL is the capabilities URL of the WFS.
-		URL string `json:"url"`
+		URLType
+		QueueConfigurationType
+
 		// FeatureType is the layer to use.
 		FeatureType string `json:"feature-type"`
 		// SortBy sorts the feature by this key.
 		SortBy string `json:"sort-by"`
-		// SendEmail is set to true if an email should be send after
-		// importing the axis.
-		SendEmail bool `json:"send-email"`
-		// Attributes are optional attributes.
-		Attributes common.Attributes `json:"attributes,omitempty"`
 	}
 
 	StretchImport struct {
+		QueueConfigurationType
+
 		Name      string          `json:"name"`
 		From      Isrs            `json:"from"`
 		To        Isrs            `json:"to"`
@@ -141,8 +107,5 @@
 		Source    string          `json:"source-organization"`
 		Date      Date            `json:"date-info"`
 		Countries UniqueCountries `json:"countries"`
-
-		SendEmail  bool              `json:"send-email"`
-		Attributes common.Attributes `json:"attributes,omitempty"`
 	}
 )