Mercurial > kallithea
changeset 8833:aafca212c8e2
celery: move send_email task to a better home in notification model
Avoid bundling everything from many different layers in one big task library.
This is more feasible now when we don't need kallithea.CELERY_APP set at import
time.
| author | Mads Kiilerich <mads@kiilerich.com> |
|---|---|
| date | Tue, 29 Dec 2020 22:23:01 +0100 |
| parents | 1d3b67443aac |
| children | 516a43cbd814 |
| files | kallithea/controllers/admin/settings.py kallithea/lib/celery_app.py kallithea/model/async_tasks.py kallithea/model/notification.py kallithea/model/user.py kallithea/tests/functional/test_login.py kallithea/tests/models/test_notifications.py kallithea/tests/other/test_mail.py scripts/deps.py |
| diffstat | 9 files changed, 152 insertions(+), 155 deletions(-) [+] |
line wrap: on
line diff
--- a/kallithea/controllers/admin/settings.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/controllers/admin/settings.py Tue Dec 29 22:23:01 2020 +0100 @@ -44,7 +44,7 @@ from kallithea.lib.utils2 import safe_str from kallithea.lib.vcs import VCSError from kallithea.lib.webutils import url -from kallithea.model import async_tasks, db, meta +from kallithea.model import db, meta, notification from kallithea.model.forms import ApplicationSettingsForm, ApplicationUiSettingsForm, ApplicationVisualisationForm from kallithea.model.notification import EmailNotificationModel from kallithea.model.scm import ScmModel @@ -301,7 +301,7 @@ recipients = [test_email] if test_email else None - async_tasks.send_email(recipients, test_email_subj, + notification.send_email(recipients, test_email_subj, test_email_txt_body, test_email_html_body) webutils.flash(_('Send email task created'), category='success')
--- a/kallithea/lib/celery_app.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/lib/celery_app.py Tue Dec 29 22:23:01 2020 +0100 @@ -21,6 +21,7 @@ imports = [ 'kallithea.lib.indexers.daemon', 'kallithea.model.async_tasks', + 'kallithea.model.notification', 'kallithea.model.repo', ] task_always_eager = False
--- a/kallithea/model/async_tasks.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/model/async_tasks.py Tue Dec 29 22:23:01 2020 +0100 @@ -26,11 +26,7 @@ :license: GPLv3, see LICENSE.md for more details. """ -import email.message -import email.utils import os -import smtplib -import time import traceback from collections import OrderedDict from operator import itemgetter @@ -46,7 +42,7 @@ from kallithea.model import db, meta -__all__ = ['get_commits_stats', 'send_email'] +__all__ = ['get_commits_stats'] log = celery.utils.log.get_task_logger(__name__) @@ -219,120 +215,6 @@ log.info('Task with key %s already running', lockkey) -@celerylib.task -def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None): - """ - Sends an email with defined parameters from the .ini files. - - :param recipients: list of recipients, if this is None, the defined email - address from field 'email_to' and all admins is used instead - :param subject: subject of the mail - :param body: plain text body of the mail - :param html_body: html version of body - :param headers: dictionary of prepopulated e-mail headers - :param from_name: full name to be used as sender of this mail - often a - .full_name_or_username value - """ - assert isinstance(recipients, list), recipients - if headers is None: - headers = {} - else: - # do not modify the original headers object passed by the caller - headers = headers.copy() - - email_config = config - email_prefix = email_config.get('email_prefix', '') - if email_prefix: - subject = "%s %s" % (email_prefix, subject) - - if not recipients: - # if recipients are not defined we send to email_config + all admins - recipients = [u.email for u in db.User.query() - .filter(db.User.admin == True).all()] - if email_config.get('email_to') is not None: - recipients += email_config.get('email_to').split(',') - - # If there are still no recipients, there are no admins and no address - # configured in email_to, so return. - if not recipients: - log.error("No recipients specified and no fallback available.") - return - - log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients)) - - # SMTP sender - app_email_from = email_config.get('app_email_from', 'Kallithea') - # 'From' header - if from_name is not None: - # set From header based on from_name but with a generic e-mail address - # In case app_email_from is in "Some Name <e-mail>" format, we first - # extract the e-mail address. - envelope_addr = author_email(app_email_from) - headers['From'] = '"%s" <%s>' % ( - email.utils.quote('%s (no-reply)' % from_name), - envelope_addr) - - smtp_server = email_config.get('smtp_server') - smtp_port = email_config.get('smtp_port') - smtp_use_tls = asbool(email_config.get('smtp_use_tls')) - smtp_use_ssl = asbool(email_config.get('smtp_use_ssl')) - smtp_auth = email_config.get('smtp_auth') # undocumented - overrule automatic choice of auth mechanism - smtp_username = email_config.get('smtp_username') - smtp_password = email_config.get('smtp_password') - - logmsg = ("Mail details:\n" - "recipients: %s\n" - "headers: %s\n" - "subject: %s\n" - "body:\n%s\n" - "html:\n%s\n" - % (' '.join(recipients), headers, subject, body, html_body)) - - if smtp_server: - log.debug("Sending e-mail. " + logmsg) - else: - log.error("SMTP mail server not configured - cannot send e-mail.") - log.warning(logmsg) - return - - msg = email.message.EmailMessage() - msg['Subject'] = subject - msg['From'] = app_email_from # fallback - might be overridden by a header - msg['To'] = ', '.join(recipients) - msg['Date'] = email.utils.formatdate(time.time()) - - for key, value in headers.items(): - del msg[key] # Delete key first to make sure add_header will replace header (if any), no matter the casing - msg.add_header(key, value) - - msg.set_content(body) - msg.add_alternative(html_body, subtype='html') - - try: - if smtp_use_ssl: - smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port) - else: - smtp_serv = smtplib.SMTP(smtp_server, smtp_port) - - if smtp_use_tls: - smtp_serv.starttls() - - if smtp_auth: - smtp_serv.ehlo() # populate esmtp_features - smtp_serv.esmtp_features["auth"] = smtp_auth - - if smtp_username and smtp_password is not None: - smtp_serv.login(smtp_username, smtp_password) - - smtp_serv.sendmail(app_email_from, recipients, msg.as_string()) - smtp_serv.quit() - - log.info('Mail was sent to: %s' % recipients) - except: - log.error('Mail sending failed') - log.error(traceback.format_exc()) - - def __get_codes_stats(repo_name): scm_repo = db.Repository.get_by_repo_name(repo_name).scm_instance
--- a/kallithea/model/notification.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/model/notification.py Tue Dec 29 22:23:01 2020 +0100 @@ -27,14 +27,21 @@ """ import datetime +import email.message +import email.utils import logging +import smtplib +import time +import traceback -from tg import app_globals +from tg import app_globals, config from tg import tmpl_context as c from tg.i18n import ugettext as _ -from kallithea.lib import webutils -from kallithea.model import async_tasks, db +from kallithea.lib import celerylib, webutils +from kallithea.lib.utils2 import asbool +from kallithea.lib.vcs.utils import author_email +from kallithea.model import db log = logging.getLogger(__name__) @@ -133,7 +140,7 @@ # send email with notification to participants for rec_mail in sorted(rec_mails): - async_tasks.send_email([rec_mail], email_subject, email_txt_body, + send_email([rec_mail], email_subject, email_txt_body, email_html_body, headers, from_name=created_by_obj.full_name_or_username) @@ -228,3 +235,117 @@ log.debug('rendering tmpl %s with kwargs %s', base, _kwargs) return email_template.render_unicode(**_kwargs) + + +@celerylib.task +def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None): + """ + Sends an email with defined parameters from the .ini files. + + :param recipients: list of recipients, if this is None, the defined email + address from field 'email_to' and all admins is used instead + :param subject: subject of the mail + :param body: plain text body of the mail + :param html_body: html version of body + :param headers: dictionary of prepopulated e-mail headers + :param from_name: full name to be used as sender of this mail - often a + .full_name_or_username value + """ + assert isinstance(recipients, list), recipients + if headers is None: + headers = {} + else: + # do not modify the original headers object passed by the caller + headers = headers.copy() + + email_config = config + email_prefix = email_config.get('email_prefix', '') + if email_prefix: + subject = "%s %s" % (email_prefix, subject) + + if not recipients: + # if recipients are not defined we send to email_config + all admins + recipients = [u.email for u in db.User.query() + .filter(db.User.admin == True).all()] + if email_config.get('email_to') is not None: + recipients += email_config.get('email_to').split(',') + + # If there are still no recipients, there are no admins and no address + # configured in email_to, so return. + if not recipients: + log.error("No recipients specified and no fallback available.") + return + + log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients)) + + # SMTP sender + app_email_from = email_config.get('app_email_from', 'Kallithea') + # 'From' header + if from_name is not None: + # set From header based on from_name but with a generic e-mail address + # In case app_email_from is in "Some Name <e-mail>" format, we first + # extract the e-mail address. + envelope_addr = author_email(app_email_from) + headers['From'] = '"%s" <%s>' % ( + email.utils.quote('%s (no-reply)' % from_name), + envelope_addr) + + smtp_server = email_config.get('smtp_server') + smtp_port = email_config.get('smtp_port') + smtp_use_tls = asbool(email_config.get('smtp_use_tls')) + smtp_use_ssl = asbool(email_config.get('smtp_use_ssl')) + smtp_auth = email_config.get('smtp_auth') # undocumented - overrule automatic choice of auth mechanism + smtp_username = email_config.get('smtp_username') + smtp_password = email_config.get('smtp_password') + + logmsg = ("Mail details:\n" + "recipients: %s\n" + "headers: %s\n" + "subject: %s\n" + "body:\n%s\n" + "html:\n%s\n" + % (' '.join(recipients), headers, subject, body, html_body)) + + if smtp_server: + log.debug("Sending e-mail. " + logmsg) + else: + log.error("SMTP mail server not configured - cannot send e-mail.") + log.warning(logmsg) + return + + msg = email.message.EmailMessage() + msg['Subject'] = subject + msg['From'] = app_email_from # fallback - might be overridden by a header + msg['To'] = ', '.join(recipients) + msg['Date'] = email.utils.formatdate(time.time()) + + for key, value in headers.items(): + del msg[key] # Delete key first to make sure add_header will replace header (if any), no matter the casing + msg.add_header(key, value) + + msg.set_content(body) + msg.add_alternative(html_body, subtype='html') + + try: + if smtp_use_ssl: + smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port) + else: + smtp_serv = smtplib.SMTP(smtp_server, smtp_port) + + if smtp_use_tls: + smtp_serv.starttls() + + if smtp_auth: + smtp_serv.ehlo() # populate esmtp_features + smtp_serv.esmtp_features["auth"] = smtp_auth + + if smtp_username and smtp_password is not None: + smtp_serv.login(smtp_username, smtp_password) + + smtp_serv.sendmail(app_email_from, recipients, msg.as_string()) + smtp_serv.quit() + + log.info('Mail was sent to: %s' % recipients) + except: + log.error('Mail sending failed') + log.error(traceback.format_exc())
--- a/kallithea/model/user.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/model/user.py Tue Dec 29 22:23:01 2020 +0100 @@ -39,7 +39,7 @@ from kallithea.lib import hooks, webutils from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException from kallithea.lib.utils2 import check_password, generate_api_key, get_crypt_password, get_current_authuser -from kallithea.model import db, forms, meta +from kallithea.model import db, forms, meta, notification log = logging.getLogger(__name__) @@ -162,8 +162,6 @@ raise def create_registration(self, form_data): - from kallithea.model import notification - form_data['admin'] = False form_data['extern_type'] = db.User.DEFAULT_AUTH_TYPE form_data['extern_name'] = '' @@ -298,8 +296,6 @@ allowing users to copy-paste or manually enter the token from the email. """ - from kallithea.model import async_tasks, notification - user_email = data['email'] user = db.User.get_by_email(user_email) timestamp = int(time.time()) @@ -331,7 +327,7 @@ reset_token=token, reset_url=link) log.debug('sending email') - async_tasks.send_email([user_email], _("Password reset link"), body, html_body) + notification.send_email([user_email], _("Password reset link"), body, html_body) log.info('send new password mail to %s', user_email) else: log.debug("password reset email %s not found", user_email) @@ -364,7 +360,6 @@ return expected_token == token def reset_password(self, user_email, new_passwd): - from kallithea.model import async_tasks user = db.User.get_by_email(user_email) if user is not None: if not self.can_change_password(user): @@ -375,7 +370,7 @@ if new_passwd is None: raise Exception('unable to set new password') - async_tasks.send_email([user_email], + notification.send_email([user_email], _('Password reset notification'), _('The password to your account %s has been changed using password reset form.') % (user.username,)) log.info('send password reset mail to %s', user_email)
--- a/kallithea/tests/functional/test_login.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/tests/functional/test_login.py Tue Dec 29 22:23:01 2020 +0100 @@ -6,7 +6,7 @@ import mock from tg.util.webtest import test_context -import kallithea.model.async_tasks +import kallithea.model.notification from kallithea.lib import webutils from kallithea.lib.utils2 import check_password, generate_api_key from kallithea.model import db, meta, validators @@ -410,7 +410,7 @@ def mock_send_email(recipients, subject, body='', html_body='', headers=None, from_name=None): collected.append((recipients, subject, body, html_body)) - with mock.patch.object(kallithea.model.async_tasks, 'send_email', mock_send_email), \ + with mock.patch.object(kallithea.model.notification, 'send_email', mock_send_email), \ mock.patch.object(time, 'time', lambda: timestamp): response = self.app.post(base.url(controller='login', action='password_reset'),
--- a/kallithea/tests/models/test_notifications.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/tests/models/test_notifications.py Tue Dec 29 22:23:01 2020 +0100 @@ -5,7 +5,6 @@ from tg.util.webtest import test_context import kallithea.lib.celerylib -import kallithea.model.async_tasks from kallithea.lib import webutils from kallithea.model import db, meta from kallithea.model.notification import EmailNotificationModel, NotificationModel @@ -48,7 +47,7 @@ assert body == "hi there" assert '>hi there<' in html_body assert from_name == 'u1 u1' - with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email): + with mock.patch.object(kallithea.model.notification, 'send_email', send_email): NotificationModel().create(created_by=self.u1, body='hi there', recipients=usrs) @@ -73,7 +72,7 @@ l.append('<hr/>\n') with test_context(self.app): - with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email): + with mock.patch.object(kallithea.model.notification, 'send_email', send_email): pr_kwargs = dict( pr_nice_id='#7', pr_title='The Title', @@ -155,7 +154,7 @@ # Email type TYPE_PASSWORD_RESET has no corresponding notification type - test it directly: desc = 'TYPE_PASSWORD_RESET' kwargs = dict(user='John Doe', reset_token='decbf64715098db5b0bd23eab44bd792670ab746', reset_url='http://reset.com/decbf64715098db5b0bd23eab44bd792670ab746') - kallithea.model.async_tasks.send_email(['john@doe.com'], + kallithea.model.notification.send_email(['john@doe.com'], "Password reset link", EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'txt', **kwargs), EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'html', **kwargs),
--- a/kallithea/tests/other/test_mail.py Wed Dec 30 00:21:29 2020 +0100 +++ b/kallithea/tests/other/test_mail.py Tue Dec 29 22:23:01 2020 +0100 @@ -25,7 +25,7 @@ smtplib_mock.lastmsg = msg -@mock.patch('kallithea.model.async_tasks.smtplib', smtplib_mock) +@mock.patch('kallithea.model.notification.smtplib', smtplib_mock) class TestMail(base.TestController): def test_send_mail_trivial(self): @@ -40,8 +40,8 @@ 'smtp_server': mailserver, 'app_email_from': envelope_from, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body) assert smtplib_mock.lastdest == set(recipients) assert smtplib_mock.lastsender == envelope_from @@ -64,8 +64,8 @@ 'app_email_from': envelope_from, 'email_to': email_to, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body) assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL, email_to]) assert smtplib_mock.lastsender == envelope_from @@ -88,8 +88,8 @@ 'app_email_from': envelope_from, 'email_to': email_to, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body) assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL] + email_to.split(',')) assert smtplib_mock.lastsender == envelope_from @@ -110,8 +110,8 @@ 'smtp_server': mailserver, 'app_email_from': envelope_from, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body) assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL]) assert smtplib_mock.lastsender == envelope_from @@ -133,8 +133,8 @@ 'smtp_server': mailserver, 'app_email_from': envelope_from, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username) assert smtplib_mock.lastdest == set(recipients) assert smtplib_mock.lastsender == envelope_from @@ -157,8 +157,8 @@ 'smtp_server': mailserver, 'app_email_from': envelope_from, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username) + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username) assert smtplib_mock.lastdest == set(recipients) assert smtplib_mock.lastsender == envelope_from @@ -181,8 +181,8 @@ 'smtp_server': mailserver, 'app_email_from': envelope_from, } - with mock.patch('kallithea.model.async_tasks.config', config_mock): - kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, + with mock.patch('kallithea.model.notification.config', config_mock): + kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username, headers=headers) assert smtplib_mock.lastdest == set(recipients)
--- a/scripts/deps.py Wed Dec 30 00:21:29 2020 +0100 +++ b/scripts/deps.py Tue Dec 29 22:23:01 2020 +0100 @@ -158,7 +158,6 @@ ('kallithea.lib.utils', 'kallithea.model'), # clean up utils ('kallithea.lib.utils', 'kallithea.model.db'), ('kallithea.lib.utils', 'kallithea.model.scm'), -('kallithea.model.async_tasks', 'kallithea.model'), ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX ('kallithea.model', 'kallithea.lib.auth_modules'), # validators ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks