Mercurial > gemma
annotate pkg/imports/statsupdate.go @ 5591:0011f50cf216 surveysperbottleneckid
Removed no longer used alternative api for surveys/ endpoint.
As bottlenecks in the summary for SR imports are now identified by
their id and no longer by the (not guarantied to be unique!) name,
there is no longer the need to request survey data by the name+date
tuple (which isn't reliable anyway). So the workaround was now
reversed.
author | Sascha Wilde <wilde@sha-bang.de> |
---|---|
date | Wed, 06 Apr 2022 13:30:29 +0200 |
parents | ade07a3f2cfd |
children | 1222b777f51f |
rev | line source |
---|---|
5387
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
1 // This is Free Software under GNU Affero General Public License v >= 3.0 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
2 // without warranty, see README.md and license for details. |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
3 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
4 // SPDX-License-Identifier: AGPL-3.0-or-later |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
5 // License-Filename: LICENSES/AGPL-3.0.txt |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
6 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
7 // Copyright (C) 2021 by via donau |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
8 // – Österreichische Wasserstraßen-Gesellschaft mbH |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
9 // Software engineering by Intevation GmbH |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
10 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
11 // Author(s): |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de> |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
13 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
14 package imports |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
15 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
16 import ( |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
17 "context" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
18 "database/sql" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
19 "errors" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
20 "fmt" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
21 "time" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
22 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
23 "gemma.intevation.de/gemma/pkg/auth" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
24 "gemma.intevation.de/gemma/pkg/common" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
25 "gemma.intevation.de/gemma/pkg/models" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
26 ) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
27 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
28 type StatsUpdate struct { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
29 models.QueueConfigurationType |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
30 Name string `json:"name"` |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
31 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
32 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
33 const StatsUpdateJobKind JobKind = "statsupdate" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
34 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
35 type statsUpdateJobCreator struct{} |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
36 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
37 func init() { RegisterJobCreator(StatsUpdateJobKind, statsUpdateJobCreator{}) } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
38 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
39 func (statsUpdateJobCreator) Description() string { return "statsupdate" } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
40 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
41 func (statsUpdateJobCreator) AutoAccept() bool { return true } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
42 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
43 func (statsUpdateJobCreator) Create() Job { return new(StatsUpdate) } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
44 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
45 func (statsUpdateJobCreator) Depends() [2][]string { return [2][]string{{}, {}} } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
46 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
47 func (statsUpdateJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
48 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
49 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
50 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
51 // RequiresRoles enforces to be a sys_admin to run this . |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
52 func (*StatsUpdate) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
53 |
5565
ade07a3f2cfd
Forget to change signature of some imports.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
5387
diff
changeset
|
54 func (su *StatsUpdate) Description([]string) (string, error) { return su.Name, nil } |
5387
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
55 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
56 func (*StatsUpdate) CleanUp() error { return nil } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
57 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
58 func (su *StatsUpdate) MarshalAttributes(attrs common.Attributes) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
59 if err := su.QueueConfigurationType.MarshalAttributes(attrs); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
60 return err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
61 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
62 attrs.Set("name", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
63 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
64 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
65 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
66 func (su *StatsUpdate) UnmarshalAttributes(attrs common.Attributes) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
67 if err := su.QueueConfigurationType.UnmarshalAttributes(attrs); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
68 return err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
69 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
70 name, found := attrs.Get("name") |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
71 if !found { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
72 return errors.New("missing 'name' attribute") |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
73 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
74 su.Name = name |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
75 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
76 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
77 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
78 const loadUpdateStatsScriptSQL = `SELECT script FROM sys_admin.stats_updates WHERE name = $1` |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
79 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
80 func (su *StatsUpdate) Do( |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
81 ctx context.Context, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
82 importID int64, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
83 conn *sql.Conn, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
84 feedback Feedback, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
85 ) (interface{}, error) { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
86 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
87 start := time.Now() |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
88 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
89 feedback.Info("Running stats update %s.", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
90 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
91 tx, err := conn.BeginTx(ctx, nil) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
92 if err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
93 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
94 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
95 defer tx.Rollback() |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
96 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
97 var script string |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
98 switch err := tx.QueryRowContext(ctx, loadUpdateStatsScriptSQL, su.Name).Scan(&script); { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
99 case err == sql.ErrNoRows: |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
100 return nil, fmt.Errorf("no update script found for '%s'", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
101 case err != nil: |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
102 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
103 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
104 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
105 if _, err := tx.ExecContext(ctx, script); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
106 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
107 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
108 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
109 if err := tx.Commit(); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
110 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
111 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
112 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
113 feedback.Info("Running stats update took %v.", time.Since(start)) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
114 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
115 return nil, nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
116 } |