# HG changeset patch # User Mads Kiilerich # Date 1604938413 -3600 # Node ID 71a37439dcee62149382de39b759160e60bac7d9 # Parent 0383ed91d4ed6d5d4fbfcb6f3ce56df0f82381d3 lib: move urlification to webutils Less use of helpers in model. diff -r 0383ed91d4ed -r 71a37439dcee kallithea/controllers/feed.py --- a/kallithea/controllers/feed.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/controllers/feed.py Mon Nov 09 17:13:33 2020 +0100 @@ -89,7 +89,7 @@ desc_msg.append('changeset: %s' % (_url, cs.raw_id[:8])) desc_msg.append('
')
-        desc_msg.append(h.urlify_text(cs.message))
+        desc_msg.append(webutils.urlify_text(cs.message))
         desc_msg.append('\n')
         desc_msg.extend(changes)
         if asbool(kallithea.CONFIG.get('rss_include_diff', False)):
diff -r 0383ed91d4ed -r 71a37439dcee kallithea/lib/helpers.py
--- a/kallithea/lib/helpers.py	Mon Nov 09 16:42:43 2020 +0100
+++ b/kallithea/lib/helpers.py	Mon Nov 09 17:13:33 2020 +0100
@@ -45,9 +45,9 @@
 # SCM FILTERS available via h.
 #==============================================================================
 from kallithea.lib.vcs.utils import author_email, author_name
-from kallithea.lib.webutils import (HTML, MENTIONS_REGEX, Option, canonical_url, checkbox, chop_at, end_form, escape, form, format_byte_size, hidden,
-                                    html_escape, js, jshtml, link_to, literal, password, pop_flash_messages, radio, reset, safeid, select,
-                                    session_csrf_secret_name, session_csrf_secret_token, submit, text, textarea, truncate, url, url_re, wrap_paragraphs)
+from kallithea.lib.webutils import (HTML, Option, canonical_url, checkbox, chop_at, end_form, escape, form, format_byte_size, hidden, js, jshtml, link_to,
+                                    literal, password, pop_flash_messages, radio, render_w_mentions, reset, safeid, select, session_csrf_secret_name,
+                                    session_csrf_secret_token, submit, text, textarea, url, urlify_text, wrap_paragraphs)
 from kallithea.model import db
 from kallithea.model.changeset_status import ChangesetStatusModel
 
@@ -68,6 +68,7 @@
 assert password
 assert pop_flash_messages
 assert radio
+assert render_w_mentions
 assert reset
 assert safeid
 assert select
@@ -76,6 +77,7 @@
 assert submit
 assert text
 assert textarea
+assert urlify_text
 assert wrap_paragraphs
 # from kallithea.lib.auth
 assert HasPermissionAny
@@ -849,212 +851,6 @@
     return literal('
%s%s
' % (width, d_a, d_d)) -_URLIFY_RE = re.compile(r''' -# URL markup -(?P%s) | -# @mention markup -(?P%s) | -# Changeset hash markup -(?[0-9a-f]{12,40}) -(?!\w|[-_]) | -# Markup of *bold text* -(?: - (?:^|(?<=\s)) - (?P [*] (?!\s) [^*\n]* (?[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] | -\[license\ \=>\ *(?P[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] | -\[(?Prequires|recommends|conflicts|base)\ \=>\ *(?P[a-zA-Z0-9\-\/]*)\] | -\[(?:lang|language)\ \=>\ *(?P[a-zA-Z\-\/\#\+]*)\] | -\[(?P[a-z]+)\] -''' % (url_re.pattern, MENTIONS_REGEX.pattern), - re.VERBOSE | re.MULTILINE | re.IGNORECASE) - - -def urlify_text(s, repo_name=None, link_=None, truncate=None, stylize=False, truncatef=truncate): - """ - Parses given text message and make literal html with markup. - The text will be truncated to the specified length. - Hashes are turned into changeset links to specified repository. - URLs links to what they say. - Issues are linked to given issue-server. - If link_ is provided, all text not already linking somewhere will link there. - >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* markup/b>") - literal('Urlify http://example.com/ and 'https://example.com&apos; *and* <b>markup/b>') - """ - - def _replace(match_obj): - match_url = match_obj.group('url') - if match_url is not None: - return '%(url)s' % {'url': match_url} - mention = match_obj.group('mention') - if mention is not None: - return '%s' % mention - hash_ = match_obj.group('hash') - if hash_ is not None and repo_name is not None: - return '%(hash)s' % { - 'url': url('changeset_home', repo_name=repo_name, revision=hash_), - 'hash': hash_, - } - bold = match_obj.group('bold') - if bold is not None: - return '*%s*' % _urlify(bold[1:-1]) - if stylize: - seen = match_obj.group('seen') - if seen: - return '
see => %s
' % seen - license = match_obj.group('license') - if license: - return '' % (license, license) - tagtype = match_obj.group('tagtype') - if tagtype: - tagvalue = match_obj.group('tagvalue') - return '
%s => %s
' % (tagtype, tagtype, tagvalue, tagvalue) - lang = match_obj.group('lang') - if lang: - return '
%s
' % lang - tag = match_obj.group('tag') - if tag: - return '
%s
' % (tag, tag) - return match_obj.group(0) - - def _urlify(s): - """ - Extract urls from text and make html links out of them - """ - return _URLIFY_RE.sub(_replace, s) - - if truncate is None: - s = s.rstrip() - else: - s = truncatef(s, truncate, whole_word=True) - s = html_escape(s) - s = _urlify(s) - if repo_name is not None: - s = urlify_issues(s, repo_name) - if link_ is not None: - # make href around everything that isn't a href already - s = linkify_others(s, link_) - s = s.replace('\r\n', '
').replace('\n', '
') - # Turn HTML5 into more valid HTML4 as required by some mail readers. - # (This is not done in one step in html_escape, because character codes like - # { risk to be seen as an issue reference due to the presence of '#'.) - s = s.replace("'", "'") - return literal(s) - - -def linkify_others(t, l): - """Add a default link to html with links. - HTML doesn't allow nesting of links, so the outer link must be broken up - in pieces and give space for other links. - """ - urls = re.compile(r'(\)',) - links = [] - for e in urls.split(t): - if e.strip() and not urls.match(e): - links.append('%s' % (l, e)) - else: - links.append(e) - - return ''.join(links) - - -# Global variable that will hold the actual urlify_issues function body. -# Will be set on first use when the global configuration has been read. -_urlify_issues_f = None - - -def urlify_issues(newtext, repo_name): - """Urlify issue references according to .ini configuration""" - global _urlify_issues_f - if _urlify_issues_f is None: - assert kallithea.CONFIG['sqlalchemy.url'] # make sure config has been loaded - - # Build chain of urlify functions, starting with not doing any transformation - def tmp_urlify_issues_f(s): - return s - - issue_pat_re = re.compile(r'issue_pat(.*)') - for k in kallithea.CONFIG: - # Find all issue_pat* settings that also have corresponding server_link and prefix configuration - m = issue_pat_re.match(k) - if m is None: - continue - suffix = m.group(1) - issue_pat = kallithea.CONFIG.get(k) - issue_server_link = kallithea.CONFIG.get('issue_server_link%s' % suffix) - issue_sub = kallithea.CONFIG.get('issue_sub%s' % suffix) - issue_prefix = kallithea.CONFIG.get('issue_prefix%s' % suffix) - if issue_prefix: - log.error('found unsupported issue_prefix%s = %r - use issue_sub%s instead', suffix, issue_prefix, suffix) - if not issue_pat: - log.error('skipping incomplete issue pattern %r: it needs a regexp', k) - continue - if not issue_server_link: - log.error('skipping incomplete issue pattern %r: it needs issue_server_link%s', k, suffix) - continue - if issue_sub is None: # issue_sub can be empty but should be present - log.error('skipping incomplete issue pattern %r: it needs (a potentially empty) issue_sub%s', k, suffix) - continue - - # Wrap tmp_urlify_issues_f with substitution of this pattern, while making sure all loop variables (and compiled regexpes) are bound - try: - issue_re = re.compile(issue_pat) - except re.error as e: - log.error('skipping invalid issue pattern %r: %r -> %r %r. Error: %s', k, issue_pat, issue_server_link, issue_sub, str(e)) - continue - - log.debug('issue pattern %r: %r -> %r %r', k, issue_pat, issue_server_link, issue_sub) - - def issues_replace(match_obj, - issue_server_link=issue_server_link, issue_sub=issue_sub): - try: - issue_url = match_obj.expand(issue_server_link) - except (IndexError, re.error) as e: - log.error('invalid issue_url setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e)) - issue_url = issue_server_link - issue_url = issue_url.replace('{repo}', repo_name) - issue_url = issue_url.replace('{repo_name}', repo_name.split(kallithea.URL_SEP)[-1]) - # if issue_sub is empty use the matched issue reference verbatim - if not issue_sub: - issue_text = match_obj.group() - else: - try: - issue_text = match_obj.expand(issue_sub) - except (IndexError, re.error) as e: - log.error('invalid issue_sub setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e)) - issue_text = match_obj.group() - - return ( - '' - '%(text)s' - '' - ) % { - 'url': issue_url, - 'text': issue_text, - } - - def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f): - return issue_re.sub(issues_replace, chain_f(s)) - - # Set tmp function globally - atomically - _urlify_issues_f = tmp_urlify_issues_f - - return _urlify_issues_f(newtext) - - -def render_w_mentions(source, repo_name=None): - """ - Render plain text with revision hashes and issue references urlified - and with @mention highlighting. - """ - s = urlify_text(source, repo_name=repo_name) - return literal('
%s
' % s) - - def changeset_status(repo, revision): return ChangesetStatusModel().get_status(repo, revision) diff -r 0383ed91d4ed -r 71a37439dcee kallithea/lib/webutils.py --- a/kallithea/lib/webutils.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/lib/webutils.py Mon Nov 09 17:13:33 2020 +0100 @@ -327,3 +327,208 @@ ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz'] """ return MENTIONS_REGEX.findall(text) + + +_URLIFY_RE = re.compile(r''' +# URL markup +(?P%s) | +# @mention markup +(?P%s) | +# Changeset hash markup +(?[0-9a-f]{12,40}) +(?!\w|[-_]) | +# Markup of *bold text* +(?: + (?:^|(?<=\s)) + (?P [*] (?!\s) [^*\n]* (?[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] | +\[license\ \=>\ *(?P[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] | +\[(?Prequires|recommends|conflicts|base)\ \=>\ *(?P[a-zA-Z0-9\-\/]*)\] | +\[(?:lang|language)\ \=>\ *(?P[a-zA-Z\-\/\#\+]*)\] | +\[(?P[a-z]+)\] +''' % (url_re.pattern, MENTIONS_REGEX.pattern), + re.VERBOSE | re.MULTILINE | re.IGNORECASE) + + +def urlify_text(s, repo_name=None, link_=None, truncate=None, stylize=False, truncatef=truncate): + """ + Parses given text message and make literal html with markup. + The text will be truncated to the specified length. + Hashes are turned into changeset links to specified repository. + URLs links to what they say. + Issues are linked to given issue-server. + If link_ is provided, all text not already linking somewhere will link there. + >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* markup/b>") + literal('Urlify http://example.com/ and 'https://example.com&apos; *and* <b>markup/b>') + """ + + def _replace(match_obj): + match_url = match_obj.group('url') + if match_url is not None: + return '%(url)s' % {'url': match_url} + mention = match_obj.group('mention') + if mention is not None: + return '%s' % mention + hash_ = match_obj.group('hash') + if hash_ is not None and repo_name is not None: + return '%(hash)s' % { + 'url': url('changeset_home', repo_name=repo_name, revision=hash_), + 'hash': hash_, + } + bold = match_obj.group('bold') + if bold is not None: + return '*%s*' % _urlify(bold[1:-1]) + if stylize: + seen = match_obj.group('seen') + if seen: + return '
see => %s
' % seen + license = match_obj.group('license') + if license: + return '' % (license, license) + tagtype = match_obj.group('tagtype') + if tagtype: + tagvalue = match_obj.group('tagvalue') + return '
%s => %s
' % (tagtype, tagtype, tagvalue, tagvalue) + lang = match_obj.group('lang') + if lang: + return '
%s
' % lang + tag = match_obj.group('tag') + if tag: + return '
%s
' % (tag, tag) + return match_obj.group(0) + + def _urlify(s): + """ + Extract urls from text and make html links out of them + """ + return _URLIFY_RE.sub(_replace, s) + + if truncate is None: + s = s.rstrip() + else: + s = truncatef(s, truncate, whole_word=True) + s = html_escape(s) + s = _urlify(s) + if repo_name is not None: + s = _urlify_issues(s, repo_name) + if link_ is not None: + # make href around everything that isn't a href already + s = _linkify_others(s, link_) + s = s.replace('\r\n', '
').replace('\n', '
') + # Turn HTML5 into more valid HTML4 as required by some mail readers. + # (This is not done in one step in html_escape, because character codes like + # { risk to be seen as an issue reference due to the presence of '#'.) + s = s.replace("'", "'") + return literal(s) + + +def _linkify_others(t, l): + """Add a default link to html with links. + HTML doesn't allow nesting of links, so the outer link must be broken up + in pieces and give space for other links. + """ + urls = re.compile(r'(\)',) + links = [] + for e in urls.split(t): + if e.strip() and not urls.match(e): + links.append('%s' % (l, e)) + else: + links.append(e) + return ''.join(links) + + +# Global variable that will hold the actual _urlify_issues function body. +# Will be set on first use when the global configuration has been read. +_urlify_issues_f = None + + +def _urlify_issues(newtext, repo_name): + """Urlify issue references according to .ini configuration""" + global _urlify_issues_f + if _urlify_issues_f is None: + assert kallithea.CONFIG['sqlalchemy.url'] # make sure config has been loaded + + # Build chain of urlify functions, starting with not doing any transformation + def tmp_urlify_issues_f(s): + return s + + issue_pat_re = re.compile(r'issue_pat(.*)') + for k in kallithea.CONFIG: + # Find all issue_pat* settings that also have corresponding server_link and prefix configuration + m = issue_pat_re.match(k) + if m is None: + continue + suffix = m.group(1) + issue_pat = kallithea.CONFIG.get(k) + issue_server_link = kallithea.CONFIG.get('issue_server_link%s' % suffix) + issue_sub = kallithea.CONFIG.get('issue_sub%s' % suffix) + issue_prefix = kallithea.CONFIG.get('issue_prefix%s' % suffix) + if issue_prefix: + log.error('found unsupported issue_prefix%s = %r - use issue_sub%s instead', suffix, issue_prefix, suffix) + if not issue_pat: + log.error('skipping incomplete issue pattern %r: it needs a regexp', k) + continue + if not issue_server_link: + log.error('skipping incomplete issue pattern %r: it needs issue_server_link%s', k, suffix) + continue + if issue_sub is None: # issue_sub can be empty but should be present + log.error('skipping incomplete issue pattern %r: it needs (a potentially empty) issue_sub%s', k, suffix) + continue + + # Wrap tmp_urlify_issues_f with substitution of this pattern, while making sure all loop variables (and compiled regexpes) are bound + try: + issue_re = re.compile(issue_pat) + except re.error as e: + log.error('skipping invalid issue pattern %r: %r -> %r %r. Error: %s', k, issue_pat, issue_server_link, issue_sub, str(e)) + continue + + log.debug('issue pattern %r: %r -> %r %r', k, issue_pat, issue_server_link, issue_sub) + + def issues_replace(match_obj, + issue_server_link=issue_server_link, issue_sub=issue_sub): + try: + issue_url = match_obj.expand(issue_server_link) + except (IndexError, re.error) as e: + log.error('invalid issue_url setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e)) + issue_url = issue_server_link + issue_url = issue_url.replace('{repo}', repo_name) + issue_url = issue_url.replace('{repo_name}', repo_name.split(kallithea.URL_SEP)[-1]) + # if issue_sub is empty use the matched issue reference verbatim + if not issue_sub: + issue_text = match_obj.group() + else: + try: + issue_text = match_obj.expand(issue_sub) + except (IndexError, re.error) as e: + log.error('invalid issue_sub setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e)) + issue_text = match_obj.group() + + return ( + '' + '%(text)s' + '' + ) % { + 'url': issue_url, + 'text': issue_text, + } + + def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f): + return issue_re.sub(issues_replace, chain_f(s)) + + # Set tmp function globally - atomically + _urlify_issues_f = tmp_urlify_issues_f + + return _urlify_issues_f(newtext) + + +def render_w_mentions(source, repo_name=None): + """ + Render plain text with revision hashes and issue references urlified + and with @mention highlighting. + """ + s = urlify_text(source, repo_name=repo_name) + return literal('
%s
' % s) diff -r 0383ed91d4ed -r 71a37439dcee kallithea/model/notification.py --- a/kallithea/model/notification.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/model/notification.py Mon Nov 09 17:13:33 2020 +0100 @@ -33,6 +33,7 @@ from tg import tmpl_context as c from tg.i18n import ugettext as _ +from kallithea.lib import webutils from kallithea.lib.utils2 import fmt_date from kallithea.model import async_tasks, db @@ -65,7 +66,6 @@ :param with_email: send email with this notification :param email_kwargs: additional dict to pass as args to email template """ - import kallithea.lib.helpers as h email_kwargs = email_kwargs or {} if recipients and not getattr(recipients, '__iter__', False): raise Exception('recipients must be a list or iterable') @@ -103,7 +103,7 @@ # this is passed into template created_on = fmt_date(datetime.datetime.now()) html_kwargs = { - 'body': None if body is None else h.render_w_mentions(body, repo_name), + 'body': None if body is None else webutils.render_w_mentions(body, repo_name), 'when': created_on, 'user': created_by_obj.username, } diff -r 0383ed91d4ed -r 71a37439dcee kallithea/model/repo.py --- a/kallithea/model/repo.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/model/repo.py Mon Nov 09 17:13:33 2020 +0100 @@ -140,8 +140,7 @@ cs_cache.get('message')) def desc(desc): - import kallithea.lib.helpers as h - return h.urlify_text(desc, truncate=80, stylize=c.visual.stylify_metalabels) + return webutils.urlify_text(desc, truncate=80, stylize=c.visual.stylify_metalabels) def state(repo_state): return _render("repo_state", repo_state) diff -r 0383ed91d4ed -r 71a37439dcee kallithea/tests/functional/test_files.py --- a/kallithea/tests/functional/test_files.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/tests/functional/test_files.py Mon Nov 09 17:13:33 2020 +0100 @@ -3,7 +3,7 @@ import mimetypes import posixpath -import kallithea.lib.helpers +from kallithea.lib import webutils from kallithea.model import db, meta from kallithea.tests import base from kallithea.tests.fixture import Fixture @@ -96,7 +96,7 @@ def test_file_source(self): # Force the global cache to be populated now when we know the right .ini has been loaded. # (Without this, the test would fail.) - kallithea.lib.helpers._urlify_issues_f = None + webutils._urlify_issues_f = None self.log_user() response = self.app.get(base.url(controller='files', action='index', repo_name=base.HG_REPO, diff -r 0383ed91d4ed -r 71a37439dcee kallithea/tests/other/test_libs.py --- a/kallithea/tests/other/test_libs.py Mon Nov 09 16:42:43 2020 +0100 +++ b/kallithea/tests/other/test_libs.py Mon Nov 09 17:13:33 2020 +0100 @@ -111,7 +111,6 @@ assert asbool(str_bool) == expected def test_mention_extractor(self): - from kallithea.lib.webutils import extract_mentioned_usernames sample = ( "@first hi there @world here's my email username@example.com " "@lukaszb check @one_more22 it pls @ ttwelve @D[] @one@two@three " @@ -123,7 +122,7 @@ expected = set([ '2one_more22', 'first', 'lukaszb', 'one', 'one_more22', 'UPPER', 'cAmEL', 'john', 'marian.user', 'marco-polo', 'marco_polo', 'world']) - assert expected == set(extract_mentioned_usernames(sample)) + assert expected == set(webutils.extract_mentioned_usernames(sample)) @base.parametrize('age_args,expected', [ (dict(), 'just now'), @@ -197,7 +196,7 @@ "[requires => url] [lang => python] [just a tag]" "[,d] [ => ULR ] [obsolete] [desc]]" ) - res = h.urlify_text(sample, stylize=True) + res = webutils.urlify_text(sample, stylize=True) assert '
tag
' in res assert '
obsolete
' in res assert '
stale
' in res @@ -315,7 +314,7 @@ with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__', lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs, ): - assert h.urlify_text(sample, 'repo_name') == expected + assert webutils.urlify_text(sample, 'repo_name') == expected @base.parametrize('sample,expected,url_', [ ("", @@ -370,7 +369,7 @@ with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__', lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs, ): - assert h.urlify_text(sample, 'repo_name', stylize=True) == expected + assert webutils.urlify_text(sample, 'repo_name', stylize=True) == expected @base.parametrize('sample,expected', [ ("deadbeefcafe @mention, and http://foo.bar/ yo", @@ -383,7 +382,7 @@ with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__', lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs, ): - assert h.urlify_text(sample, 'repo_name', link_='#the-link') == expected + assert webutils.urlify_text(sample, 'repo_name', link_='#the-link') == expected @base.parametrize('issue_pat,issue_server,issue_sub,sample,expected', [ (r'#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1', @@ -471,9 +470,9 @@ 'issue_sub': issue_sub, } # force recreation of lazy function - with mock.patch('kallithea.lib.helpers._urlify_issues_f', None): + with mock.patch('kallithea.lib.webutils._urlify_issues_f', None): with mock.patch('kallithea.CONFIG', config_stub): - assert h.urlify_text(sample, 'repo_name') == expected + assert webutils.urlify_text(sample, 'repo_name') == expected @base.parametrize('sample,expected', [ ('abc X5', 'abc #5'), @@ -503,9 +502,9 @@ 'issue_server_link_absent_prefix': r'http://failmore/{repo}/\1', } # force recreation of lazy function - with mock.patch('kallithea.lib.helpers._urlify_issues_f', None): + with mock.patch('kallithea.lib.webutils._urlify_issues_f', None): with mock.patch('kallithea.CONFIG', config_stub): - assert h.urlify_text(sample, 'repo_name') == expected + assert webutils.urlify_text(sample, 'repo_name') == expected @base.parametrize('test,expected', [ ("", None),