changeset 992:a978b2b26a88

Run do and cleanup of import jobs in own go routines with crash handler. This way the import job may die badly w/o killing the job queue, too.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 22 Oct 2018 11:24:25 +0200
parents a301d240905f
children 6421c51309eb d9cc1e906469
files pkg/imports/queue.go
diffstat 1 files changed, 24 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/queue.go	Mon Oct 22 10:45:17 2018 +0200
+++ b/pkg/imports/queue.go	Mon Oct 22 11:24:25 2018 +0200
@@ -4,7 +4,9 @@
 	"container/list"
 	"context"
 	"database/sql"
+	"fmt"
 	"log"
+	"runtime/debug"
 	"sync"
 	"sync/atomic"
 
@@ -90,6 +92,22 @@
 	log.Printf("error: "+fmt, args...)
 }
 
+func survive(fn func() error) func() error {
+	return func() error {
+		errCh := make(chan error)
+		go func() {
+			defer func() {
+				if err := recover(); err != nil {
+					errCh <- fmt.Errorf("%v: %s",
+						err, string(debug.Stack()))
+				}
+			}()
+			errCh <- fn()
+		}()
+		return <-errCh
+	}
+}
+
 func importLoop() {
 	for {
 		var idj idJob
@@ -114,14 +132,14 @@
 			continue
 		}
 
-		fn := func(conn *sql.Conn) error {
-			return job.Do(conn, logFeedback{})
-		}
-
-		if err := auth.RunAs(idj.user, context.Background(), fn); err != nil {
+		do := survive(func() error {
+			return auth.RunAs(idj.user, context.Background(),
+				func(conn *sql.Conn) error { return job.Do(conn, logFeedback{}) })
+		})
+		if err := do(); err != nil {
 			log.Printf("import error (job %d): %v\n", idj.id, err)
 		}
-		if err := job.CleanUp(); err != nil {
+		if err := survive(job.CleanUp)(); err != nil {
 			log.Printf("cleanup error (job %d): %v\n", idj.id, err)
 		}
 	}