Mercurial > gemma

annotate pkg/imports/queue.go @ 984:3c9ea8ab856a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed small glitches in sounding result importer.
author Sascha L. Teichmann <teichmann@intevation.de>
date Fri, 19 Oct 2018 16:05:46 +0200
parents 7934b5c1a910
children 3841509f6e9e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
1 package imports
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
2
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
3 import (
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
4 "container/list"
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
5 "context"
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
6 "database/sql"
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
7 "log"
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
8 "sync"
978
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
9 "sync/atomic"
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
10
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
11 "gemma.intevation.de/gemma/pkg/auth"
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
12 )
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
13
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
14 type Job interface {
979
7934b5c1a910 Finally enqueue sounding result import job to import jobs.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 978
diff changeset
15 User() string
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
16 Do(conn *sql.Conn) error
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
17 CleanUp() error
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
18 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
19
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
20 var (
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
21 queueCond = sync.NewCond(new(sync.Mutex))
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
22 queue = list.New()
978
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
23
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
24 jobID int64
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
25 )
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
26
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
27 func init() {
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
28 go importLoop()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
29 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
30
978
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
31 func AddJob(job Job) int64 {
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
32 id := atomic.AddInt64(&jobID, 1)
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
33 queueCond.L.Lock()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
34 defer queueCond.L.Unlock()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
35 queue.PushBack(job)
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
36 queueCond.Signal()
978
544a5cfe07cd Started with endpoint for uploading sounding result and trigger respective import job.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 958
diff changeset
37 return id
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
38 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
39
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
40 func importLoop() {
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
41 for {
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
42 var job Job
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
43 queueCond.L.Lock()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
44 for queue.Len() == 0 {
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
45 queueCond.Wait()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
46 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
47 job = queue.Remove(queue.Front()).(Job)
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
48 queueCond.L.Unlock()
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
49
979
7934b5c1a910 Finally enqueue sounding result import job to import jobs.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 978
diff changeset
50 if err := auth.RunAs(job.User(), context.Background(), job.Do); err != nil {
958
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
51 log.Printf("import error: %v\n", err)
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
52 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
53 if err := job.CleanUp(); err != nil {
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
54 log.Printf("cleanup error: %v\n", err)
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
55 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
56 }
2818ad6c7d32 Started with import queue.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
57 }