Mercurial > gemma
annotate pkg/imports/statsupdate.go @ 5520:05db984d3db1
Improve performance of bottleneck area calculation
Avoid buffer calculations by replacing them with simple distance comparisons
and calculate the boundary of the result geometry only once per iteration.
In some edge cases with very large numbers of iterations, this reduced
the runtime of a bottleneck import by a factor of more than twenty.
author | Tom Gottfried <tom@intevation.de> |
---|---|
date | Thu, 21 Oct 2021 19:50:39 +0200 |
parents | 8e30b926b94d |
children | ade07a3f2cfd |
rev | line source |
---|---|
5387
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
1 // This is Free Software under GNU Affero General Public License v >= 3.0 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
2 // without warranty, see README.md and license for details. |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
3 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
4 // SPDX-License-Identifier: AGPL-3.0-or-later |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
5 // License-Filename: LICENSES/AGPL-3.0.txt |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
6 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
7 // Copyright (C) 2021 by via donau |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
8 // – Österreichische Wasserstraßen-Gesellschaft mbH |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
9 // Software engineering by Intevation GmbH |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
10 // |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
11 // Author(s): |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
12 // * Sascha L. Teichmann <sascha.teichmann@intevation.de> |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
13 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
14 package imports |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
15 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
16 import ( |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
17 "context" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
18 "database/sql" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
19 "errors" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
20 "fmt" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
21 "time" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
22 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
23 "gemma.intevation.de/gemma/pkg/auth" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
24 "gemma.intevation.de/gemma/pkg/common" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
25 "gemma.intevation.de/gemma/pkg/models" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
26 ) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
27 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
28 type StatsUpdate struct { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
29 models.QueueConfigurationType |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
30 Name string `json:"name"` |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
31 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
32 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
33 const StatsUpdateJobKind JobKind = "statsupdate" |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
34 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
35 type statsUpdateJobCreator struct{} |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
36 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
37 func init() { RegisterJobCreator(StatsUpdateJobKind, statsUpdateJobCreator{}) } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
38 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
39 func (statsUpdateJobCreator) Description() string { return "statsupdate" } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
40 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
41 func (statsUpdateJobCreator) AutoAccept() bool { return true } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
42 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
43 func (statsUpdateJobCreator) Create() Job { return new(StatsUpdate) } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
44 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
45 func (statsUpdateJobCreator) Depends() [2][]string { return [2][]string{{}, {}} } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
46 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
47 func (statsUpdateJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
48 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
49 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
50 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
51 // RequiresRoles enforces to be a sys_admin to run this . |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
52 func (*StatsUpdate) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
53 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
54 func (su *StatsUpdate) Description() (string, error) { return su.Name, nil } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
55 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
56 func (*StatsUpdate) CleanUp() error { return nil } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
57 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
58 func (su *StatsUpdate) MarshalAttributes(attrs common.Attributes) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
59 if err := su.QueueConfigurationType.MarshalAttributes(attrs); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
60 return err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
61 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
62 attrs.Set("name", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
63 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
64 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
65 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
66 func (su *StatsUpdate) UnmarshalAttributes(attrs common.Attributes) error { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
67 if err := su.QueueConfigurationType.UnmarshalAttributes(attrs); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
68 return err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
69 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
70 name, found := attrs.Get("name") |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
71 if !found { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
72 return errors.New("missing 'name' attribute") |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
73 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
74 su.Name = name |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
75 return nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
76 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
77 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
78 const loadUpdateStatsScriptSQL = `SELECT script FROM sys_admin.stats_updates WHERE name = $1` |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
79 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
80 func (su *StatsUpdate) Do( |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
81 ctx context.Context, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
82 importID int64, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
83 conn *sql.Conn, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
84 feedback Feedback, |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
85 ) (interface{}, error) { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
86 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
87 start := time.Now() |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
88 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
89 feedback.Info("Running stats update %s.", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
90 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
91 tx, err := conn.BeginTx(ctx, nil) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
92 if err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
93 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
94 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
95 defer tx.Rollback() |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
96 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
97 var script string |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
98 switch err := tx.QueryRowContext(ctx, loadUpdateStatsScriptSQL, su.Name).Scan(&script); { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
99 case err == sql.ErrNoRows: |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
100 return nil, fmt.Errorf("no update script found for '%s'", su.Name) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
101 case err != nil: |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
102 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
103 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
104 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
105 if _, err := tx.ExecContext(ctx, script); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
106 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
107 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
108 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
109 if err := tx.Commit(); err != nil { |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
110 return nil, err |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
111 } |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
112 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
113 feedback.Info("Running stats update took %v.", time.Since(start)) |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
114 |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
115 return nil, nil |
8e30b926b94d
Added an import to run update stats scripts.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
116 } |