view pkg/imports/report.go @ 5591:0011f50cf216 surveysperbottleneckid

Removed no longer used alternative api for surveys/ endpoint. As bottlenecks in the summary for SR imports are now identified by their id and no longer by the (not guarantied to be unique!) name, there is no longer the need to request survey data by the name+date tuple (which isn't reliable anyway). So the workaround was now reversed.
author Sascha Wilde <wilde@sha-bang.de>
date Wed, 06 Apr 2022 13:30:29 +0200
parents f2204f91d286
children ceecc205011c
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2021 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package imports

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"gemma.intevation.de/gemma/pkg/auth"
	"gemma.intevation.de/gemma/pkg/common"
	"gemma.intevation.de/gemma/pkg/config"
	"gemma.intevation.de/gemma/pkg/log"
	"gemma.intevation.de/gemma/pkg/misc"
	"gemma.intevation.de/gemma/pkg/models"
	"gemma.intevation.de/gemma/pkg/xlsx"

	"github.com/xuri/excelize/v2"
)

type Report struct {
	models.QueueConfigurationType
	Name models.SafePath `json:"name"`
}

const ReportJobKind JobKind = "report"

type reportJobCreator struct{}

const (
	selectReportUsersSQL = `
SELECT username, email_address
FROM users.list_users
WHERE report_reciever
ORDER BY country, username`

	selectCurrentUserSQL = `
SELECT current_user, email_address
FROM users.list_users
WHERE username = current_user`
)

var reportMailTmpl = template.Must(template.New("report-mail").
	Parse(`Dear {{ .Receiver }}

this is an automatically generated report from the Gemma system.
You got this mail because you are listed as a report receiver.
If you received it without consent please
contact {{ .Admin }} under {{ .AdminEmail }}.

Find attached {{ .Attachment }} containing the {{ .Report }} report from {{ .When }}.

Kind Regards`))

func init() { RegisterJobCreator(ReportJobKind, reportJobCreator{}) }

func (reportJobCreator) Description() string { return "report" }

func (reportJobCreator) AutoAccept() bool { return true }

func (reportJobCreator) Create() Job { return new(Report) }

func (reportJobCreator) Depends() [2][]string { return [2][]string{{}, {}} }

func (reportJobCreator) StageDone(context.Context, *sql.Tx, int64, Feedback) error {
	return nil
}

// RequiresRoles enforces to be a sys_admin to run this .
func (*Report) RequiresRoles() auth.Roles { return auth.Roles{"sys_admin"} }

func (r *Report) Description([]string) (string, error) { return string(r.Name), nil }

func (*Report) CleanUp() error { return nil }

func (r *Report) MarshalAttributes(attrs common.Attributes) error {
	if err := r.QueueConfigurationType.MarshalAttributes(attrs); err != nil {
		return err
	}
	attrs.Set("name", string(r.Name))
	return nil
}

func (r *Report) UnmarshalAttributes(attrs common.Attributes) error {
	if err := r.QueueConfigurationType.UnmarshalAttributes(attrs); err != nil {
		return err
	}
	name, found := attrs.Get("name")
	if !found {
		return errors.New("missing 'name' attribute")
	}
	r.Name = models.SafePath(name)
	if !r.Name.Valid() {
		return fmt.Errorf("'%s' is not a safe path", name)
	}
	return nil
}

func (r *Report) loadTemplate() (*excelize.File, *xlsx.Action, error) {
	path := config.ReportPath()
	if path == "" {
		return nil, nil, errors.New("no report dir configured")
	}

	if stat, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("report dir '%s' does not exists", path)
		}
		return nil, nil, err
	} else if !stat.Mode().IsDir() {
		return nil, nil, fmt.Errorf("report dir '%s' is not a directory", path)
	}

	xlsxFilename := filepath.Join(path, string(r.Name)+".xlsx")
	yamlFilename := filepath.Join(path, string(r.Name)+".yaml")

	for _, check := range []string{xlsxFilename, yamlFilename} {
		if _, err := os.Stat(check); err != nil {
			if os.IsNotExist(err) {
				return nil, nil, fmt.Errorf("'%s' does not exists", check)
			}
			return nil, nil, err
		}
	}

	template, err := excelize.OpenFile(xlsxFilename)
	if err != nil {
		return nil, nil, err
	}

	action, err := xlsx.ActionFromFile(yamlFilename)
	if err != nil {
		return nil, nil, err
	}

	return template, action, nil
}

func (r *Report) Do(
	ctx context.Context,
	importID int64,
	conn *sql.Conn,
	feedback Feedback,
) (interface{}, error) {

	start := time.Now()

	feedback.Info("Generating report %s.", r.Name)

	template, action, err := r.loadTemplate()
	if err != nil {
		return nil, err
	}

	tx, err := conn.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()

	// Fetch receivers
	var users []misc.EmailReceiver

	if err := func() error {
		rows, err := tx.QueryContext(ctx, selectReportUsersSQL)
		if err != nil {
			return err
		}
		defer rows.Close()

		for rows.Next() {
			var u misc.EmailReceiver
			if err := rows.Scan(&u.Name, &u.Address); err != nil {
				return err
			}
			users = append(users, u)
		}
		return rows.Err()
	}(); err != nil {
		return nil, err
	}

	if len(users) == 0 {
		feedback.Warn("No users found to send reports to.")
		return nil, nil
	}

	// Fetch admin who is responsible for the report.
	var admin misc.EmailReceiver
	if err := tx.QueryRowContext(
		ctx, selectCurrentUserSQL).Scan(&admin.Name, &admin.Address); err != nil {
		log.Errorf("cannot find sender: %v\n")
		return nil, fmt.Errorf("cannot find sender: %v", err)
	}

	// Generate the actual report.
	if err := action.Execute(ctx, tx, template); err != nil {
		log.Errorf("%v\n", err)
		return nil, fmt.Errorf("Generating report failed: %v", err)
	}

	var buf bytes.Buffer
	if _, err := template.WriteTo(&buf); err != nil {
		log.Errorf("%v\n", err)
		return nil, fmt.Errorf("generating report failed: %v", err)
	}

	feedback.Info("Sending report to %d receiver(s).", len(users))

	now := start.UTC().Format("2006-01-02")

	attached := string(r.Name) + "-" + now + ".xlsx"

	body := func(u misc.EmailReceiver) (string, error) {
		fill := struct {
			Receiver   string
			Attachment string
			Report     string
			When       string
			Admin      string
			AdminEmail string
		}{
			Receiver:   u.Name,
			Attachment: attached,
			Report:     string(r.Name),
			When:       now,
			Admin:      admin.Name,
			AdminEmail: admin.Address,
		}
		var sb strings.Builder
		if err := reportMailTmpl.Execute(&sb, &fill); err != nil {
			return "", err
		}
		return sb.String(), nil
	}

	errorHandler := func(r misc.EmailReceiver, err error) error {
		// We do not terminate the sending of the emails if
		// sending failed. We only log it.
		feedback.Warn("Sending report to %s failed: %v", r.Name, err)
		return nil
	}

	if err := misc.SendMailToAll(
		users,
		"Report "+string(r.Name)+" from "+now,
		body,
		[]misc.EmailAttachment{{
			Name:    attached,
			Content: buf.Bytes(),
		}},
		errorHandler,
	); err != nil {
		return nil, err
	}

	feedback.Info("Generating and sending report took %v.",
		time.Since(start))

	return nil, nil
}