Mercurial > gemma

annotate pkg/imports/statsupdate.go @ 5560:f2204f91d286

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Join the log lines of imports to the log exports to recover data from them. Used in SR export to extract information that where in the meta json but now are only found in the log.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Wed, 09 Feb 2022 18:34:40 +0100
parents 8e30b926b94d
children ade07a3f2cfd
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5387
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
1 // This is Free Software under GNU Affero General Public License v >= 3.0
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
2 // without warranty, see README.md and license for details.
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
3 //
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
4 // SPDX-License-Identifier: AGPL-3.0-or-later
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
5 // License-Filename: LICENSES/AGPL-3.0.txt
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
6 //
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
7 // Copyright (C) 2021 by via donau
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
8 // – Österreichische Wasserstraßen-Gesellschaft mbH
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
9 // Software engineering by Intevation GmbH
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
10 //
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
11 // Author(s):
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de>
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
13
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
14 package imports
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
15
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
16 import (
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
17 "context"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
18 "database/sql"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
19 "errors"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
20 "fmt"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
21 "time"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
22
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
23 "gemma.intevation.de/gemma/pkg/auth"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
24 "gemma.intevation.de/gemma/pkg/common"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
25 "gemma.intevation.de/gemma/pkg/models"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
26 )
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
27
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
28 type StatsUpdate struct {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
29 models.QueueConfigurationType
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
30 Name string `json:"name"`
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
31 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
32
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
33 const StatsUpdateJobKind JobKind = "statsupdate"
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
34
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
35 type statsUpdateJobCreator struct{}
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
36
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
37 func init() { RegisterJobCreator(StatsUpdateJobKind, statsUpdateJobCreator{}) }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
38
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
39 func (statsUpdateJobCreator) Description() string { return "statsupdate" }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
40
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
41 func (statsUpdateJobCreator) AutoAccept() bool { return true }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
42
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
43 func (statsUpdateJobCreator) Create() Job { return new(StatsUpdate) }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
44
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
45 func (statsUpdateJobCreator) Depends() [2][]string { return [2][]string{{}, {}} }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
46
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
47 func (statsUpdateJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
48 return nil
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
49 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
50
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
51 // RequiresRoles enforces to be a sys_admin to run this .
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
52 func (*StatsUpdate) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
53
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
54 func (su *StatsUpdate) Description() (string, error) { return su.Name, nil }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
55
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
56 func (*StatsUpdate) CleanUp() error { return nil }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
57
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
58 func (su *StatsUpdate) MarshalAttributes(attrs common.Attributes) error {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
59 if err := su.QueueConfigurationType.MarshalAttributes(attrs); err != nil {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
60 return err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
61 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
62 attrs.Set("name", su.Name)
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
63 return nil
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
64 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
65
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
66 func (su *StatsUpdate) UnmarshalAttributes(attrs common.Attributes) error {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
67 if err := su.QueueConfigurationType.UnmarshalAttributes(attrs); err != nil {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
68 return err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
69 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
70 name, found := attrs.Get("name")
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
71 if !found {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
72 return errors.New("missing 'name' attribute")
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
73 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
74 su.Name = name
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
75 return nil
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
76 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
77
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
78 const loadUpdateStatsScriptSQL = `SELECT script FROM sys_admin.stats_updates WHERE name = $1`
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
79
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
80 func (su *StatsUpdate) Do(
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
81 ctx context.Context,
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
82 importID int64,
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
83 conn *sql.Conn,
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
84 feedback Feedback,
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
85 ) (interface{}, error) {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
86
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
87 start := time.Now()
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
88
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
89 feedback.Info("Running stats update %s.", su.Name)
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
90
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
91 tx, err := conn.BeginTx(ctx, nil)
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
92 if err != nil {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
93 return nil, err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
94 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
95 defer tx.Rollback()
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
96
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
97 var script string
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
98 switch err := tx.QueryRowContext(ctx, loadUpdateStatsScriptSQL, su.Name).Scan(&script); {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
99 case err == sql.ErrNoRows:
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
100 return nil, fmt.Errorf("no update script found for '%s'", su.Name)
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
101 case err != nil:
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
102 return nil, err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
103 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
104
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
105 if _, err := tx.ExecContext(ctx, script); err != nil {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
106 return nil, err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
107 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
108
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
109 if err := tx.Commit(); err != nil {
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
110 return nil, err
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
111 }
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
112
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
113 feedback.Info("Running stats update took %v.", time.Since(start))
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
114
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
115 return nil, nil
8e30b926b94d Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
116 }