diff pkg/controllers/manualimports.go @ 1985:8eeb0b5eb340

Imports: Made retries and the waiting between the attempts configurable.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 23 Jan 2019 17:58:57 +0100
parents 71b722809b2b
children 9c65153bb56e
line wrap: on
line diff
--- a/pkg/controllers/manualimports.go	Wed Jan 23 16:25:43 2019 +0100
+++ b/pkg/controllers/manualimports.go	Wed Jan 23 17:58:57 2019 +0100
@@ -26,68 +26,74 @@
 	"gemma.intevation.de/gemma/pkg/models"
 )
 
-func retry(a common.Attributes) (time.Time, int) {
+func retry(a common.Attributes) (time.Time, *int, *time.Duration) {
 	due, _ := a.Time("due")
-	retries, ok := a.Int("retries")
-	if !ok {
-		retries = -1
+	ret, ok := a.Int("retries")
+	var retries *int
+	if ok {
+		retries = &ret
 	}
-	return due, retries
+	dur, ok := a.Duration("wait-retry")
+	var duration *time.Duration
+	if ok {
+		duration = &dur
+	}
+	return due, retries, duration
 }
 
-func importBottleneck(input interface{}) (interface{}, time.Time, int, bool) {
+func importBottleneck(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	bi := input.(*models.BottleneckImport)
 	bn := &imports.Bottleneck{
 		URL:      bi.URL,
 		Insecure: bi.Insecure,
 	}
-	due, retries := retry(bi.Attributes)
-	return bn, due, retries, bi.SendEmail
+	due, retries, duration := retry(bi.Attributes)
+	return bn, due, retries, duration, bi.SendEmail
 }
 
-func importGaugeMeasurement(input interface{}) (interface{}, time.Time, int, bool) {
+func importGaugeMeasurement(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	gi := input.(*models.GaugeMeasurementImport)
 	gm := &imports.GaugeMeasurement{
 		URL:      gi.URL,
 		Insecure: gi.Insecure,
 	}
-	due, retries := retry(gi.Attributes)
-	return gm, due, retries, gi.SendEmail
+	due, retries, duration := retry(gi.Attributes)
+	return gm, due, retries, duration, gi.SendEmail
 }
 
-func importFairwayAvailability(input interface{}) (interface{}, time.Time, int, bool) {
+func importFairwayAvailability(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	fai := input.(*models.FairwayAvailabilityImport)
 	fa := &imports.FairwayAvailability{
 		URL:      fai.URL,
 		Insecure: fai.Insecure,
 	}
-	due, retries := retry(fai.Attributes)
-	return fa, due, retries, fai.SendEmail
+	due, retries, duration := retry(fai.Attributes)
+	return fa, due, retries, duration, fai.SendEmail
 }
 
-func importWaterwayAxis(input interface{}) (interface{}, time.Time, int, bool) {
+func importWaterwayAxis(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	wxi := input.(*models.WaterwayAxisImport)
 	wx := &imports.WaterwayAxis{
 		URL:         wxi.URL,
 		FeatureType: wxi.FeatureType,
 		SortBy:      wxi.SortBy,
 	}
-	due, retries := retry(wxi.Attributes)
-	return wx, due, retries, wxi.SendEmail
+	due, retries, duration := retry(wxi.Attributes)
+	return wx, due, retries, duration, wxi.SendEmail
 }
 
-func importWaterwayArea(input interface{}) (interface{}, time.Time, int, bool) {
+func importWaterwayArea(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	wai := input.(*models.WaterwayAreaImport)
 	wa := &imports.WaterwayArea{
 		URL:         wai.URL,
 		FeatureType: wai.FeatureType,
 		SortBy:      wai.SortBy,
 	}
-	due, retries := retry(wai.Attributes)
-	return wa, due, retries, wai.SendEmail
+	due, retries, duration := retry(wai.Attributes)
+	return wa, due, retries, duration, wai.SendEmail
 }
 
-func importWaterwayGauge(input interface{}) (interface{}, time.Time, int, bool) {
+func importWaterwayGauge(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	wgi := input.(*models.WaterwayGaugeImport)
 	username, _ := wgi.Attributes.Get("username")
 	password, _ := wgi.Attributes.Get("password")
@@ -98,11 +104,11 @@
 		Password: password,
 		Insecure: insecure,
 	}
-	due, retries := retry(wgi.Attributes)
-	return wg, due, retries, wgi.SendEmail
+	due, retries, duration := retry(wgi.Attributes)
+	return wg, due, retries, duration, wgi.SendEmail
 }
 
-func importDistancemarksVirtual(input interface{}) (interface{}, time.Time, int, bool) {
+func importDistancemarksVirtual(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	dmvi := input.(*models.DistanceMarksVirtualImport)
 	username, _ := dmvi.Attributes.Get("username")
 	password, _ := dmvi.Attributes.Get("password")
@@ -113,11 +119,11 @@
 		Password: password,
 		Insecure: insecure,
 	}
-	due, retries := retry(dmvi.Attributes)
-	return wg, due, retries, dmvi.SendEmail
+	due, retries, duration := retry(dmvi.Attributes)
+	return wg, due, retries, duration, dmvi.SendEmail
 }
 
-func importFairwayDimension(input interface{}) (interface{}, time.Time, int, bool) {
+func importFairwayDimension(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	fdi := input.(*models.FairwayDimensionImport)
 	fd := &imports.FairwayDimension{
 		URL:                fdi.URL,
@@ -129,22 +135,22 @@
 		Depth:              fdi.Depth,
 		SourceOrganization: fdi.SourceOrganization,
 	}
-	due, retries := retry(fdi.Attributes)
-	return fd, due, retries, fdi.SendEmail
+	due, retries, duration := retry(fdi.Attributes)
+	return fd, due, retries, duration, fdi.SendEmail
 }
 
-func importDistanceMarksAshore(input interface{}) (interface{}, time.Time, int, bool) {
+func importDistanceMarksAshore(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	dmai := input.(*models.DistanceMarksAshoreImport)
 	dma := &imports.DistanceMarksAshore{
 		URL:         dmai.URL,
 		FeatureType: dmai.FeatureType,
 		SortBy:      dmai.SortBy,
 	}
-	due, retries := retry(dmai.Attributes)
-	return dma, due, retries, dmai.SendEmail
+	due, retries, duration := retry(dmai.Attributes)
+	return dma, due, retries, duration, dmai.SendEmail
 }
 
-func importStretch(input interface{}) (interface{}, time.Time, int, bool) {
+func importStretch(input interface{}) (interface{}, time.Time, *int, *time.Duration, bool) {
 	sti := input.(*models.StretchImport)
 	st := &imports.Stretch{
 		Name:      sti.Name,
@@ -156,19 +162,19 @@
 		Date:      sti.Date,
 		Countries: sti.Countries,
 	}
-	due, retries := retry(sti.Attributes)
-	return st, due, retries, sti.SendEmail
+	due, retries, duration := retry(sti.Attributes)
+	return st, due, retries, duration, sti.SendEmail
 }
 
 func manualImport(
 	kind imports.JobKind,
-	setup func(interface{}) (interface{}, time.Time, int, bool),
+	setup func(interface{}) (interface{}, time.Time, *int, *time.Duration, bool),
 ) func(interface{}, *http.Request, *sql.Conn) (JSONResult, error) {
 
 	return func(input interface{}, req *http.Request, _ *sql.Conn) (
 		jr JSONResult, err error) {
 
-		what, due, retries, sendEmail := setup(input)
+		what, due, retries, waitDuration, sendEmail := setup(input)
 
 		var serialized string
 		if serialized, err = common.ToJSONString(what); err != nil {
@@ -180,7 +186,9 @@
 		var jobID int64
 		if jobID, err = imports.AddJob(
 			kind,
-			due, retries,
+			due,
+			retries,
+			waitDuration,
 			session.User,
 			sendEmail,
 			serialized,