view pkg/imports/report.go @ 5520:05db984d3db1

Improve performance of bottleneck area calculation Avoid buffer calculations by replacing them with simple distance comparisons and calculate the boundary of the result geometry only once per iteration. In some edge cases with very large numbers of iterations, this reduced the runtime of a bottleneck import by a factor of more than twenty.
author Tom Gottfried <tom@intevation.de>
date Thu, 21 Oct 2021 19:50:39 +0200
parents 5f47eeea988d
children f2204f91d286
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2021 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package imports

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"gemma.intevation.de/gemma/pkg/auth"
	"gemma.intevation.de/gemma/pkg/common"
	"gemma.intevation.de/gemma/pkg/config"
	"gemma.intevation.de/gemma/pkg/log"
	"gemma.intevation.de/gemma/pkg/misc"
	"gemma.intevation.de/gemma/pkg/models"
	"gemma.intevation.de/gemma/pkg/xlsx"

	"github.com/xuri/excelize/v2"
)

type Report struct {
	models.QueueConfigurationType
	Name models.SafePath `json:"name"`
}

const ReportJobKind JobKind = "report"

type reportJobCreator struct{}

const (
	selectReportUsersSQL = `
SELECT username, email_address
FROM users.list_users
WHERE report_reciever
ORDER BY country, username`

	selectCurrentUserSQL = `
SELECT current_user, email_address
FROM users.list_users
WHERE username = current_user`
)

var reportMailTmpl = template.Must(template.New("report-mail").
	Parse(`Dear {{ .Receiver }}

this is an automatically generated report from the Gemma system.
You got this mail because you are listed as a report receiver.
If you received it without consent please
contact {{ .Admin }} under {{ .AdminEmail }}.

Find attached {{ .Attachment }} containing the {{ .Report }} report from {{ .When }}.

Kind Regards`))

func init() { RegisterJobCreator(ReportJobKind, reportJobCreator{}) }

func (reportJobCreator) Description() string { return "report" }

func (reportJobCreator) AutoAccept() bool { return true }

func (reportJobCreator) Create() Job { return new(Report) }

func (reportJobCreator) Depends() [2][]string { return [2][]string{{}, {}} }

func (reportJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error {
	return nil
}

// RequiresRoles enforces to be a sys_admin to run this .
func (*Report) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} }

func (r *Report) Description() (string, error) { return string(r.Name), nil }

func (*Report) CleanUp() error { return nil }

func (r *Report) MarshalAttributes(attrs common.Attributes) error {
	if err := r.QueueConfigurationType.MarshalAttributes(attrs); err != nil {
		return err
	}
	attrs.Set("name", string(r.Name))
	return nil
}

func (r *Report) UnmarshalAttributes(attrs common.Attributes) error {
	if err := r.QueueConfigurationType.UnmarshalAttributes(attrs); err != nil {
		return err
	}
	name, found := attrs.Get("name")
	if !found {
		return errors.New("missing 'name' attribute")
	}
	r.Name = models.SafePath(name)
	if !r.Name.Valid() {
		return fmt.Errorf("'%s' is not a safe path", name)
	}
	return nil
}

func (r *Report) loadTemplate() (*excelize.File, *xlsx.Action, error) {
	path := config.ReportPath()
	if path == "" {
		return nil, nil, errors.New("no report dir configured")
	}

	if stat, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("report dir '%s' does not exists", path)
		}
		return nil, nil, err
	} else if !stat.Mode().IsDir() {
		return nil, nil, fmt.Errorf("report dir '%s' is not a directory", path)
	}

	xlsxFilename := filepath.Join(path, string(r.Name)+".xlsx")
	yamlFilename := filepath.Join(path, string(r.Name)+".yaml")

	for _, check := range []string{xlsxFilename, yamlFilename} {
		if _, err := os.Stat(check); err != nil {
			if os.IsNotExist(err) {
				return nil, nil, fmt.Errorf("'%s' does not exists", check)
			}
			return nil, nil, err
		}
	}

	template, err := excelize.OpenFile(xlsxFilename)
	if err != nil {
		return nil, nil, err
	}

	action, err := xlsx.ActionFromFile(yamlFilename)
	if err != nil {
		return nil, nil, err
	}

	return template, action, nil
}

func (r *Report) Do(
	ctx context.Context,
	importID int64,
	conn *sql.Conn,
	feedback Feedback,
) (interface{}, error) {

	start := time.Now()

	feedback.Info("Generating report %s.", r.Name)

	template, action, err := r.loadTemplate()
	if err != nil {
		return nil, err
	}

	tx, err := conn.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()

	// Fetch receivers
	var users []misc.EmailReceiver

	if err := func() error {
		rows, err := tx.QueryContext(ctx, selectReportUsersSQL)
		if err != nil {
			return err
		}
		defer rows.Close()

		for rows.Next() {
			var u misc.EmailReceiver
			if err := rows.Scan(&u.Name, &u.Address); err != nil {
				return err
			}
			users = append(users, u)
		}
		return rows.Err()
	}(); err != nil {
		return nil, err
	}

	if len(users) == 0 {
		feedback.Warn("No users found to send reports to.")
		return nil, nil
	}

	// Fetch admin who is responsible for the report.
	var admin misc.EmailReceiver
	if err := tx.QueryRowContext(
		ctx, selectCurrentUserSQL).Scan(&admin.Name, &admin.Address); err != nil {
		log.Errorf("cannot find sender: %v\n")
		return nil, fmt.Errorf("cannot find sender: %v", err)
	}

	// Generate the actual report.
	if err := action.Execute(ctx, tx, template); err != nil {
		log.Errorf("%v\n", err)
		return nil, fmt.Errorf("Generating report failed: %v", err)
	}

	var buf bytes.Buffer
	if _, err := template.WriteTo(&buf); err != nil {
		log.Errorf("%v\n", err)
		return nil, fmt.Errorf("generating report failed: %v", err)
	}

	feedback.Info("Sending report to %d receiver(s).", len(users))

	now := start.UTC().Format("2006-01-02")

	attached := string(r.Name) + "-" + now + ".xlsx"

	body := func(u misc.EmailReceiver) (string, error) {
		fill := struct {
			Receiver   string
			Attachment string
			Report     string
			When       string
			Admin      string
			AdminEmail string
		}{
			Receiver:   u.Name,
			Attachment: attached,
			Report:     string(r.Name),
			When:       now,
			Admin:      admin.Name,
			AdminEmail: admin.Address,
		}
		var sb strings.Builder
		if err := reportMailTmpl.Execute(&sb, &fill); err != nil {
			return "", err
		}
		return sb.String(), nil
	}

	errorHandler := func(r misc.EmailReceiver, err error) error {
		// We do not terminate the sending of the emails if
		// sending failed. We only log it.
		feedback.Warn("Sending report to %s failed: %v", r.Name, err)
		return nil
	}

	if err := misc.SendMailToAll(
		users,
		"Report "+string(r.Name)+" from "+now,
		body,
		[]misc.EmailAttachment{{
			Name:    attached,
			Content: buf.Bytes(),
		}},
		errorHandler,
	); err != nil {
		return nil, err
	}

	feedback.Info("Generating and sending report took %v.",
		time.Since(start))

	return nil, nil
}