# HG changeset patch # User Thomas De Schampheleire # Date 1598122423 -7200 # Node ID 3e9d079fcf914ef42c5e109bf389e02abea100ba # Parent 1dbe842f32c43f9ca9ebc8b7d9331bed8c750382# Parent eca0cb56a822690c58e6237e9787cd8cfa77fddc merge stable diff -r eca0cb56a822 -r 3e9d079fcf91 dev_requirements.txt --- a/dev_requirements.txt Sun Jul 26 00:03:12 2020 +0200 +++ b/dev_requirements.txt Sat Aug 22 20:53:43 2020 +0200 @@ -1,9 +1,9 @@ -pytest >= 4.6.6, < 5.4 +pytest >= 4.6.6, < 5.5 pytest-sugar >= 0.9.2, < 0.10 pytest-benchmark >= 3.2.2, < 3.3 pytest-localserver >= 0.5.0, < 0.6 mock >= 3.0.0, < 4.1 -Sphinx >= 1.8.0, < 2.4 +Sphinx >= 1.8.0, < 3.1 WebTest >= 2.0.6, < 2.1 -isort == 4.3.21 -pyflakes == 2.1.1 +isort == 5.1.2 +pyflakes == 2.2.0 diff -r eca0cb56a822 -r 3e9d079fcf91 development.ini --- a/development.ini Sun Jul 26 00:03:12 2020 +0200 +++ b/development.ini Sat Aug 22 20:53:43 2020 +0200 @@ -67,11 +67,11 @@ host = 0.0.0.0 port = 5000 -## WAITRESS ## +## Gearbox serve uses the Waitress web server ## use = egg:waitress#main -## number of worker threads +## avoid multi threading threads = 1 -## MAX BODY SIZE 100GB +## allow push of repos bigger than the default of 1 GB max_request_body_size = 107374182400 ## use poll instead of select, fixes fd limits, may not work on old ## windows systems. @@ -359,10 +359,10 @@ ## DB CONFIG ## ######################### -## SQLITE [default] sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60 - -## see sqlalchemy docs for other backends +#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea +#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4 +## Note: the mysql:// prefix should also be used for MariaDB sqlalchemy.pool_recycle = 3600 diff -r eca0cb56a822 -r 3e9d079fcf91 docs/contributing.rst --- a/docs/contributing.rst Sun Jul 26 00:03:12 2020 +0200 +++ b/docs/contributing.rst Sat Aug 22 20:53:43 2020 +0200 @@ -92,6 +92,17 @@ and the test suite creates repositories in the temporary directory. Linux systems with /tmp mounted noexec will thus fail. +Tests can be run on PostgreSQL like:: + + sudo -u postgres createuser 'kallithea-test' --pwprompt # password password + sudo -u postgres createdb 'kallithea-test' --owner 'kallithea-test' + REUSE_TEST_DB='postgresql://kallithea-test:password@localhost/kallithea-test' py.test + +Tests can be run on MariaDB/MySQL like:: + + echo "GRANT ALL PRIVILEGES ON \`kallithea-test\`.* TO 'kallithea-test'@'localhost' IDENTIFIED BY 'password'" | sudo -u mysql mysql + TEST_DB='mysql://kallithea-test:password@localhost/kallithea-test?charset=utf8mb4' py.test + You can also use ``tox`` to run the tests with all supported Python versions. When running tests, Kallithea generates a `test.ini` based on template values diff -r eca0cb56a822 -r 3e9d079fcf91 docs/overview.rst --- a/docs/overview.rst Sun Jul 26 00:03:12 2020 +0200 +++ b/docs/overview.rst Sat Aug 22 20:53:43 2020 +0200 @@ -30,13 +30,17 @@ database schema and insert the most basic information: the location of the repository store and an initial local admin user. -5. **Configure the web server.** +5. **Prepare front-end files** + Some front-end files must be fetched or created using ``npm`` tooling so + they can be served to the client as static files. + +6. **Configure the web server.** The web server must invoke the WSGI entrypoint for the Kallithea software using the ``.ini`` file (and thus the database). This makes the web application available so the local admin user can log in and tweak the configuration further. -6. **Configure users.** +7. **Configure users.** The initial admin user can create additional local users, or configure how users can be created and authenticated from other user directories. @@ -177,7 +181,7 @@ to get a configuration starting point for your choice of web server. (Gearbox will do like ``paste`` and use the WSGI application entry point - ``kallithea.config.middleware:make_app`` as specified in ``setup.py``.) + ``kallithea.config.application:make_app`` as specified in ``setup.py``.) - `Apache httpd`_ can serve WSGI applications directly using mod_wsgi_ and a simple Python file with the necessary configuration. This is a good option if diff -r eca0cb56a822 -r 3e9d079fcf91 docs/setup.rst --- a/docs/setup.rst Sun Jul 26 00:03:12 2020 +0200 +++ b/docs/setup.rst Sat Aug 22 20:53:43 2020 +0200 @@ -8,32 +8,68 @@ Setting up Kallithea -------------------- -First, you will need to create a Kallithea configuration file. Run the -following command to do so:: +Some further details to the steps mentioned in the overview. + +Create low level configuration file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - kallithea-cli config-create my.ini +First, you will need to create a Kallithea configuration file. The +configuration file is a ``.ini`` file that contains various low level settings +for Kallithea, e.g. configuration of how to use database, web server, email, +and logging. -This will create the file ``my.ini`` in the current directory. This -configuration file contains the various settings for Kallithea, e.g. -proxy port, email settings, usage of static files, cache, Celery -settings, and logging. Extra settings can be specified like:: +Run the following command to create the file ``my.ini`` in the current +directory:: + + kallithea-cli config-create my.ini http_server=waitress + +To get a good starting point for your configuration, specify the http server +you intend to use. It can be ``waitress``, ``gearbox``, ``gevent``, +``gunicorn``, or ``uwsgi``. (Apache ``mod_wsgi`` will not use this +configuration file, and it is fine to keep the default http_server configuration +unused. ``mod_wsgi`` is configured using ``httpd.conf`` directives and a WSGI +wrapper script.) + +Extra custom settings can be specified like:: kallithea-cli config-create my.ini host=8.8.8.8 "[handler_console]" formatter=color_formatter -Next, you need to create the databases used by Kallithea. It is recommended to -use PostgreSQL or SQLite (default). If you choose a database other than the -default, ensure you properly adjust the database URL in your ``my.ini`` -configuration file to use this other database. Kallithea currently supports -PostgreSQL, SQLite and MariaDB/MySQL databases. Create the database by running -the following command:: +Populate the database +^^^^^^^^^^^^^^^^^^^^^ + +Next, you need to create the databases used by Kallithea. Kallithea currently +supports PostgreSQL, SQLite and MariaDB/MySQL databases. It is recommended to +start out using SQLite (the default) and move to PostgreSQL if it becomes a +bottleneck or to get a "proper" database. MariaDB/MySQL is also supported. + +For PostgreSQL, run ``pip install psycopg2`` to get the database driver. Make +sure the PostgreSQL server is initialized and running. Make sure you have a +database user with password authentication with permissions to create databases +- for example by running:: + + sudo -u postgres createuser 'kallithea' --pwprompt --createdb + +For MariaDB/MySQL, run ``pip install mysqlclient`` to get the ``MySQLdb`` +database driver. Make sure the database server is initialized and running. Make +sure you have a database user with password authentication with permissions to +create the database - for example by running:: + + echo 'CREATE USER "kallithea"@"localhost" IDENTIFIED BY "password"' | sudo -u mysql mysql + echo 'GRANT ALL PRIVILEGES ON `kallithea`.* TO "kallithea"@"localhost"' | sudo -u mysql mysql + +Check and adjust ``sqlalchemy.url`` in your ``my.ini`` configuration file to use +this database. + +Create the database, tables, and initial content by running the following +command:: kallithea-cli db-create -c my.ini -This will prompt you for a "root" path. This "root" path is the location where -Kallithea will store all of its repositories on the current machine. After -entering this "root" path ``db-create`` will also prompt you for a username -and password for the initial admin account which ``db-create`` sets -up for you. +This will first prompt you for a "root" path. This "root" path is the location +where Kallithea will store all of its repositories on the current machine. This +location must be writable for the running Kallithea application. Next, +``db-create`` will prompt you for a username and password for the initial admin +account it sets up for you. The ``db-create`` values can also be given on the command line. Example:: @@ -48,11 +84,17 @@ location to its database. (Note: make sure you specify the correct path to the root). -.. note:: the given path for Mercurial_ repositories **must** be write - accessible for the application. It's very important since - the Kallithea web interface will work without write access, - but when trying to do a push it will fail with permission - denied errors unless it has write access. +.. note:: It is also possible to use an existing database. For example, + when using PostgreSQL without granting general createdb privileges to + the PostgreSQL kallithea user, set ``sqlalchemy.url = + postgresql://kallithea:password@localhost/kallithea`` and create the + database like:: + + sudo -u postgres createdb 'kallithea' --owner 'kallithea' + kallithea-cli db-create -c my.ini --reuse + +Prepare front-end files +^^^^^^^^^^^^^^^^^^^^^^^ Finally, the front-end files must be prepared. This requires ``npm`` version 6 or later, which needs ``node.js`` (version 12 or later). Prepare the front-end @@ -60,7 +102,11 @@ kallithea-cli front-end-build -You are now ready to use Kallithea. To run it simply execute:: +Running +^^^^^^^ + +You are now ready to use Kallithea. To run it using a gearbox web server, +simply execute:: gearbox serve -c my.ini diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/__init__.py --- a/kallithea/__init__.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/__init__.py Sat Aug 22 20:53:43 2020 +0200 @@ -34,7 +34,7 @@ if sys.version_info < (3, 6): raise Exception('Kallithea requires python 3.6 or later') -VERSION = (0, 6, 1) +VERSION = (0, 6, 99) BACKENDS = { 'hg': 'Mercurial repository', 'git': 'Git repository', diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/bin/kallithea_cli_base.py --- a/kallithea/bin/kallithea_cli_base.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/bin/kallithea_cli_base.py Sat Aug 22 20:53:43 2020 +0200 @@ -23,7 +23,7 @@ import paste.deploy import kallithea -import kallithea.config.middleware +import kallithea.config.application # kallithea_cli is usually invoked through the 'kallithea-cli' wrapper script @@ -77,7 +77,7 @@ logging.config.fileConfig(cp, {'__file__': path_to_ini_file, 'here': os.path.dirname(path_to_ini_file)}) if config_file_initialize_app: - kallithea.config.middleware.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf) + kallithea.config.application.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf) return annotated(*args, **kwargs) return cli_command(runtime_wrapper) return annotator diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/bin/kallithea_cli_db.py --- a/kallithea/bin/kallithea_cli_db.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/bin/kallithea_cli_db.py Sat Aug 22 20:53:43 2020 +0200 @@ -20,6 +20,8 @@ @cli_base.register_command(config_file=True) +@click.option('--reuse/--no-reuse', default=False, + help='Reuse and clean existing database instead of dropping and creating (default: no reuse)') @click.option('--user', help='Username of administrator account.') @click.option('--password', help='Password for administrator account.') @click.option('--email', help='Email address of administrator account.') @@ -28,7 +30,7 @@ @click.option('--force-no', is_flag=True, help='Answer no to every question.') @click.option('--public-access/--no-public-access', default=True, help='Enable/disable public access on this installation (default: enable)') -def db_create(user, password, email, repos, force_yes, force_no, public_access): +def db_create(user, password, email, repos, force_yes, force_no, public_access, reuse): """Initialize the database. Create all required tables in the database specified in the configuration @@ -57,7 +59,7 @@ ) dbmanage = DbManage(dbconf=dbconf, root=kallithea.CONFIG['here'], tests=False, cli_args=cli_args) - dbmanage.create_tables(override=True) + dbmanage.create_tables(reuse_database=reuse) repo_root_path = dbmanage.prompt_repo_root_path(None) dbmanage.create_settings(repo_root_path) dbmanage.create_default_user() @@ -67,7 +69,7 @@ Session().commit() # initial repository scan - kallithea.config.middleware.make_app( + kallithea.config.application.make_app( kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf) added, _ = kallithea.lib.utils.repo2db_mapper(kallithea.model.scm.ScmModel().repo_scan()) if added: diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/bin/kallithea_cli_ssh.py --- a/kallithea/bin/kallithea_cli_ssh.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/bin/kallithea_cli_ssh.py Sat Aug 22 20:53:43 2020 +0200 @@ -21,7 +21,7 @@ import kallithea import kallithea.bin.kallithea_cli_base as cli_base -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.lib.vcs.backends.git.ssh import GitSshHandler from kallithea.lib.vcs.backends.hg.ssh import MercurialSshHandler from kallithea.model.ssh_key import SshKeyModel, SshKeyModelException @@ -40,8 +40,7 @@ protocol access. The access will be granted as the specified user ID, and logged as using the specified key ID. """ - ssh_enabled = kallithea.CONFIG.get('ssh_enabled', False) - if not str2bool(ssh_enabled): + if not asbool(kallithea.CONFIG.get('ssh_enabled', False)): sys.stderr.write("SSH access is disabled.\n") return sys.exit(1) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/config/app_cfg.py --- a/kallithea/config/app_cfg.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/config/app_cfg.py Sat Aug 22 20:53:43 2020 +0200 @@ -28,82 +28,57 @@ from alembic.migration import MigrationContext from alembic.script.base import ScriptDirectory from sqlalchemy import create_engine -from tg.configuration import AppConfig -from tg.support.converters import asbool +from tg import FullStackApplicationConfigurator import kallithea.lib.locale import kallithea.model.base import kallithea.model.meta from kallithea.lib import celerypylons -from kallithea.lib.middleware.https_fixup import HttpsFixup -from kallithea.lib.middleware.permanent_repo_url import PermanentRepoUrl -from kallithea.lib.middleware.simplegit import SimpleGit -from kallithea.lib.middleware.simplehg import SimpleHg -from kallithea.lib.middleware.wrapper import RequestWrapper from kallithea.lib.utils import check_git_version, load_rcextensions, set_app_settings, set_indexer_config, set_vcs_config -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.model import db log = logging.getLogger(__name__) -class KallitheaAppConfig(AppConfig): - # Note: AppConfig has a misleading name, as it's not the application - # configuration, but the application configurator. The AppConfig values are - # used as a template to create the actual configuration, which might - # overwrite or extend the one provided by the configurator template. +base_config = FullStackApplicationConfigurator() - # To make it clear, AppConfig creates the config and sets into it the same - # values that AppConfig itself has. Then the values from the config file and - # gearbox options are loaded and merged into the configuration. Then an - # after_init_config(conf) method of AppConfig is called for any change that - # might depend on options provided by configuration files. +base_config.update_blueprint({ + 'package': kallithea, - def __init__(self): - super(KallitheaAppConfig, self).__init__() - - self['package'] = kallithea + # Rendering Engines Configuration + 'renderers': [ + 'json', + 'mako', + ], + 'default_renderer': 'mako', + 'use_dotted_templatenames': False, - self['prefer_toscawidgets2'] = False - self['use_toscawidgets'] = False - - self['renderers'] = [] - - # Enable json in expose - self['renderers'].append('json') + # Configure Sessions, store data as JSON to avoid pickle security issues + 'session.enabled': True, + 'session.data_serializer': 'json', - # Configure template rendering - self['renderers'].append('mako') - self['default_renderer'] = 'mako' - self['use_dotted_templatenames'] = False + # Configure the base SQLALchemy Setup + 'use_sqlalchemy': True, + 'model': kallithea.model.base, + 'DBSession': kallithea.model.meta.Session, - # Configure Sessions, store data as JSON to avoid pickle security issues - self['session.enabled'] = True - self['session.data_serializer'] = 'json' - - # Configure the base SQLALchemy Setup - self['use_sqlalchemy'] = True - self['model'] = kallithea.model.base - self['DBSession'] = kallithea.model.meta.Session + # Configure App without an authentication backend. + 'auth_backend': None, - # Configure App without an authentication backend. - self['auth_backend'] = None - - # Use custom error page for these errors. By default, Turbogears2 does not add - # 400 in this list. - # Explicitly listing all is considered more robust than appending to defaults, - # in light of possible future framework changes. - self['errorpage.status_codes'] = [400, 401, 403, 404] + # Use custom error page for these errors. By default, Turbogears2 does not add + # 400 in this list. + # Explicitly listing all is considered more robust than appending to defaults, + # in light of possible future framework changes. + 'errorpage.status_codes': [400, 401, 403, 404], - # Disable transaction manager -- currently Kallithea takes care of transactions itself - self['tm.enabled'] = False + # Disable transaction manager -- currently Kallithea takes care of transactions itself + 'tm.enabled': False, - # Set the default i18n source language so TG doesn't search beyond 'en' in Accept-Language. - self['i18n.lang'] = 'en' - - -base_config = KallitheaAppConfig() + # Set the default i18n source language so TG doesn't search beyond 'en' in Accept-Language. + 'i18n.lang': 'en', +}) # DebugBar, a debug toolbar for TurboGears2. # (https://github.com/TurboGears/tgext.debugbar) @@ -111,8 +86,8 @@ # 'debug = true' (not in production!) # See the Kallithea documentation for more information. try: + import kajiki # only to check its existence from tgext.debugbar import enable_debugbar - import kajiki # only to check its existence assert kajiki except ImportError: pass @@ -134,7 +109,7 @@ mercurial.encoding.encoding = hgencoding if config.get('ignore_alembic_revision', False): - log.warn('database alembic revision checking is disabled') + log.warning('database alembic revision checking is disabled') else: dbconf = config['sqlalchemy.url'] alembic_cfg = alembic.config.Config() @@ -160,7 +135,7 @@ # store some globals into kallithea kallithea.DEFAULT_USER_ID = db.User.get_default_user().user_id - if str2bool(config.get('use_celery')): + if asbool(config.get('use_celery')): kallithea.CELERY_APP = celerypylons.make_app() kallithea.CONFIG = config @@ -188,27 +163,3 @@ tg.hooks.register('configure_new_app', setup_configuration) - - -def setup_application(app): - config = app.config - - # we want our low level middleware to get to the request ASAP. We don't - # need any stack middleware in them - especially no StatusCodeRedirect buffering - app = SimpleHg(app, config) - app = SimpleGit(app, config) - - # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy - if any(asbool(config.get(x)) for x in ['https_fixup', 'force_https', 'use_htsts']): - app = HttpsFixup(app, config) - - app = PermanentRepoUrl(app, config) - - # Optional and undocumented wrapper - gives more verbose request/response logging, but has a slight overhead - if str2bool(config.get('use_wsgi_wrapper')): - app = RequestWrapper(app, config) - - return app - - -tg.hooks.register('before_config', setup_application) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/config/application.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kallithea/config/application.py Sat Aug 22 20:53:43 2020 +0200 @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""WSGI middleware initialization for the Kallithea application.""" + +from kallithea.config.app_cfg import base_config +from kallithea.lib.middleware.https_fixup import HttpsFixup +from kallithea.lib.middleware.permanent_repo_url import PermanentRepoUrl +from kallithea.lib.middleware.simplegit import SimpleGit +from kallithea.lib.middleware.simplehg import SimpleHg +from kallithea.lib.middleware.wrapper import RequestWrapper +from kallithea.lib.utils2 import asbool + + +__all__ = ['make_app'] + + +def wrap_app(app): + """Wrap the TG WSGI application in Kallithea middleware""" + config = app.config + + # we want our low level middleware to get to the request ASAP. We don't + # need any stack middleware in them - especially no StatusCodeRedirect buffering + app = SimpleHg(app, config) + app = SimpleGit(app, config) + + # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy + if any(asbool(config.get(x)) for x in ['https_fixup', 'force_https', 'use_htsts']): + app = HttpsFixup(app, config) + + app = PermanentRepoUrl(app, config) + + # Optional and undocumented wrapper - gives more verbose request/response logging, but has a slight overhead + if asbool(config.get('use_wsgi_wrapper')): + app = RequestWrapper(app, config) + + return app + + +def make_app(global_conf, **app_conf): + """ + Set up Kallithea with the settings found in the PasteDeploy configuration + file used. + + :param global_conf: The global settings for Kallithea (those + defined under the ``[DEFAULT]`` section). + :return: The Kallithea application with all the relevant middleware + loaded. + + This is the PasteDeploy factory for the Kallithea application. + + ``app_conf`` contains all the application-specific settings (those defined + under ``[app:main]``. + """ + assert app_conf.get('sqlalchemy.url') # must be called with a Kallithea .ini file, which for example must have this config option + assert global_conf.get('here') and global_conf.get('__file__') # app config should be initialized the paste way ... + + return base_config.make_wsgi_app(global_conf, app_conf, wrap_app=wrap_app) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/config/environment.py --- a/kallithea/config/environment.py Sun Jul 26 00:03:12 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""WSGI environment setup for Kallithea.""" - -from kallithea.config.app_cfg import base_config - - -__all__ = ['load_environment'] - -# Use base_config to setup the environment loader function -load_environment = base_config.make_load_environment() diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/config/middleware.py --- a/kallithea/config/middleware.py Sun Jul 26 00:03:12 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""WSGI middleware initialization for the Kallithea application.""" - -from kallithea.config.app_cfg import base_config -from kallithea.config.environment import load_environment - - -__all__ = ['make_app'] - -# Use base_config to setup the necessary PasteDeploy application factory. -# make_base_app will wrap the TurboGears2 app with all the middleware it needs. -make_base_app = base_config.setup_tg_wsgi_app(load_environment) - - -def make_app(global_conf, full_stack=True, **app_conf): - """ - Set up Kallithea with the settings found in the PasteDeploy configuration - file used. - - :param global_conf: The global settings for Kallithea (those - defined under the ``[DEFAULT]`` section). - :type global_conf: dict - :param full_stack: Should the whole TurboGears2 stack be set up? - :type full_stack: str or bool - :return: The Kallithea application with all the relevant middleware - loaded. - - This is the PasteDeploy factory for the Kallithea application. - - ``app_conf`` contains all the application-specific settings (those defined - under ``[app:main]``. - """ - assert app_conf.get('sqlalchemy.url') # must be called with a Kallithea .ini file, which for example must have this config option - assert global_conf.get('here') and global_conf.get('__file__') # app config should be initialized the paste way ... - return make_base_app(global_conf, full_stack=full_stack, **app_conf) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/admin/permissions.py --- a/kallithea/controllers/admin/permissions.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/admin/permissions.py Sat Aug 22 20:53:43 2020 +0200 @@ -61,18 +61,22 @@ super(PermissionsController, self)._before(*args, **kwargs) def __load_data(self): + # Permissions for the Default user on new repositories c.repo_perms_choices = [('repository.none', _('None'),), ('repository.read', _('Read'),), ('repository.write', _('Write'),), ('repository.admin', _('Admin'),)] + # Permissions for the Default user on new repository groups c.group_perms_choices = [('group.none', _('None'),), ('group.read', _('Read'),), ('group.write', _('Write'),), ('group.admin', _('Admin'),)] + # Permissions for the Default user on new user groups c.user_group_perms_choices = [('usergroup.none', _('None'),), ('usergroup.read', _('Read'),), ('usergroup.write', _('Write'),), ('usergroup.admin', _('Admin'),)] + # Registration - allow new Users to create an account c.register_choices = [ ('hg.register.none', _('Disabled')), @@ -80,26 +84,18 @@ _('Allowed with manual account activation')), ('hg.register.auto_activate', _('Allowed with automatic account activation')), ] - + # External auth account activation c.extern_activate_choices = [ ('hg.extern_activate.manual', _('Manual activation of external account')), ('hg.extern_activate.auto', _('Automatic activation of external account')), ] - + # Top level repository creation c.repo_create_choices = [('hg.create.none', _('Disabled')), ('hg.create.repository', _('Enabled'))] - - c.repo_create_on_write_choices = [ - ('hg.create.write_on_repogroup.true', _('Enabled')), - ('hg.create.write_on_repogroup.false', _('Disabled')), - ] - + # User group creation c.user_group_create_choices = [('hg.usergroup.create.false', _('Disabled')), ('hg.usergroup.create.true', _('Enabled'))] - - c.repo_group_create_choices = [('hg.repogroup.create.false', _('Disabled')), - ('hg.repogroup.create.true', _('Enabled'))] - + # Repository forking: c.fork_choices = [('hg.fork.none', _('Disabled')), ('hg.fork.repository', _('Enabled'))] @@ -112,8 +108,6 @@ [x[0] for x in c.group_perms_choices], [x[0] for x in c.user_group_perms_choices], [x[0] for x in c.repo_create_choices], - [x[0] for x in c.repo_create_on_write_choices], - [x[0] for x in c.repo_group_create_choices], [x[0] for x in c.user_group_create_choices], [x[0] for x in c.fork_choices], [x[0] for x in c.register_choices], @@ -157,15 +151,9 @@ if p.permission.permission_name.startswith('usergroup.'): defaults['default_user_group_perm'] = p.permission.permission_name - if p.permission.permission_name.startswith('hg.create.write_on_repogroup.'): - defaults['create_on_write'] = p.permission.permission_name - elif p.permission.permission_name.startswith('hg.create.'): defaults['default_repo_create'] = p.permission.permission_name - if p.permission.permission_name.startswith('hg.repogroup.'): - defaults['default_repo_group_create'] = p.permission.permission_name - if p.permission.permission_name.startswith('hg.usergroup.'): defaults['default_user_group_create'] = p.permission.permission_name diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/admin/repo_groups.py --- a/kallithea/controllers/admin/repo_groups.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/admin/repo_groups.py Sat Aug 22 20:53:43 2020 +0200 @@ -63,7 +63,7 @@ exclude is used for not moving group to itself TODO: also exclude descendants Note: only admin can create top level groups """ - repo_groups = AvailableRepoGroupChoices([], 'admin', extras) + repo_groups = AvailableRepoGroupChoices('admin', extras) exclude_group_ids = set(rg.group_id for rg in exclude) c.repo_groups = [rg for rg in repo_groups if rg[0] not in exclude_group_ids] diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/admin/repos.py --- a/kallithea/controllers/admin/repos.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/admin/repos.py Sat Aug 22 20:53:43 2020 +0200 @@ -39,7 +39,7 @@ import kallithea from kallithea.config.routing import url from kallithea.lib import helpers as h -from kallithea.lib.auth import HasPermissionAny, HasRepoPermissionLevelDecorator, LoginRequired, NotAnonymous +from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired, NotAnonymous from kallithea.lib.base import BaseRepoController, jsonify, render from kallithea.lib.exceptions import AttachedForksError from kallithea.lib.utils import action_logger @@ -76,14 +76,9 @@ return repo_obj def __load_defaults(self, repo=None): - top_perms = ['hg.create.repository'] - if HasPermissionAny('hg.create.write_on_repogroup.true')(): - repo_group_perm_level = 'write' - else: - repo_group_perm_level = 'admin' extras = [] if repo is None else [repo.group] - c.repo_groups = AvailableRepoGroupChoices(top_perms, repo_group_perm_level, extras) + c.repo_groups = AvailableRepoGroupChoices('write', extras) c.landing_revs_choices, c.landing_revs = ScmModel().get_repo_landing_revs(repo) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/changeset.py --- a/kallithea/controllers/changeset.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/changeset.py Sat Aug 22 20:53:43 2020 +0200 @@ -28,7 +28,7 @@ import binascii import logging import traceback -from collections import OrderedDict, defaultdict +from collections import OrderedDict from tg import request, response from tg import tmpl_context as c @@ -54,118 +54,6 @@ log = logging.getLogger(__name__) -def _update_with_GET(params, GET): - for k in ['diff1', 'diff2', 'diff']: - params[k] += GET.getall(k) - - -def anchor_url(revision, path, GET): - fid = h.FID(revision, path) - return h.url.current(anchor=fid, **dict(GET)) - - -def get_ignore_ws(fid, GET): - ig_ws_global = GET.get('ignorews') - ig_ws = [k for k in GET.getall(fid) if k.startswith('WS')] - if ig_ws: - try: - return int(ig_ws[0].split(':')[-1]) - except ValueError: - raise HTTPBadRequest() - return ig_ws_global - - -def _ignorews_url(GET, fileid=None): - fileid = str(fileid) if fileid else None - params = defaultdict(list) - _update_with_GET(params, GET) - lbl = _('Show whitespace') - ig_ws = get_ignore_ws(fileid, GET) - ln_ctx = get_line_ctx(fileid, GET) - # global option - if fileid is None: - if ig_ws is None: - params['ignorews'] += [1] - lbl = _('Ignore whitespace') - ctx_key = 'context' - ctx_val = ln_ctx - # per file options - else: - if ig_ws is None: - params[fileid] += ['WS:1'] - lbl = _('Ignore whitespace') - - ctx_key = fileid - ctx_val = 'C:%s' % ln_ctx - # if we have passed in ln_ctx pass it along to our params - if ln_ctx: - params[ctx_key] += [ctx_val] - - params['anchor'] = fileid - icon = h.literal('') - return h.link_to(icon, h.url.current(**params), title=lbl, **{'data-toggle': 'tooltip'}) - - -def get_line_ctx(fid, GET): - ln_ctx_global = GET.get('context') - if fid: - ln_ctx = [k for k in GET.getall(fid) if k.startswith('C')] - else: - _ln_ctx = [k for k in GET if k.startswith('C')] - ln_ctx = GET.get(_ln_ctx[0]) if _ln_ctx else ln_ctx_global - if ln_ctx: - ln_ctx = [ln_ctx] - - if ln_ctx: - retval = ln_ctx[0].split(':')[-1] - else: - retval = ln_ctx_global - - try: - return int(retval) - except Exception: - return 3 - - -def _context_url(GET, fileid=None): - """ - Generates url for context lines - - :param fileid: - """ - - fileid = str(fileid) if fileid else None - ig_ws = get_ignore_ws(fileid, GET) - ln_ctx = (get_line_ctx(fileid, GET) or 3) * 2 - - params = defaultdict(list) - _update_with_GET(params, GET) - - # global option - if fileid is None: - if ln_ctx > 0: - params['context'] += [ln_ctx] - - if ig_ws: - ig_ws_key = 'ignorews' - ig_ws_val = 1 - - # per file option - else: - params[fileid] += ['C:%s' % ln_ctx] - ig_ws_key = fileid - ig_ws_val = 'WS:%s' % 1 - - if ig_ws: - params[ig_ws_key] += [ig_ws_val] - - lbl = _('Increase diff context to %(num)s lines') % {'num': ln_ctx} - - params['anchor'] = fileid - icon = h.literal('') - return h.link_to(icon, h.url.current(**params), title=lbl, **{'data-toggle': 'tooltip'}) - - def create_cs_pr_comment(repo_name, revision=None, pull_request=None, allowed_to_change_status=True): """ Add a comment to the specified changeset or pull request, using POST values @@ -292,9 +180,6 @@ def _index(self, revision, method): c.pull_request = None - c.anchor_url = anchor_url - c.ignorews_url = _ignorews_url - c.context_url = _context_url c.fulldiff = request.GET.get('fulldiff') # for reporting number of changed files # get ranges of revisions if preset rev_range = revision.split('...')[:2] @@ -357,11 +242,10 @@ cs2 = changeset.raw_id cs1 = changeset.parents[0].raw_id if changeset.parents else EmptyChangeset().raw_id - context_lcl = get_line_ctx('', request.GET) - ign_whitespace_lcl = get_ignore_ws('', request.GET) - + ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET) + diff_context_size = h.get_diff_context_size(request.GET) raw_diff = diffs.get_diff(c.db_repo_scm_instance, cs1, cs2, - ignore_whitespace=ign_whitespace_lcl, context=context_lcl) + ignore_whitespace=ignore_whitespace_diff, context=diff_context_size) diff_limit = None if c.fulldiff else self.cut_off_limit file_diff_data = [] if method == 'show': diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/compare.py --- a/kallithea/controllers/compare.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/compare.py Sat Aug 22 20:53:43 2020 +0200 @@ -37,13 +37,12 @@ from webob.exc import HTTPBadRequest, HTTPFound, HTTPNotFound from kallithea.config.routing import url -from kallithea.controllers.changeset import _context_url, _ignorews_url from kallithea.lib import diffs from kallithea.lib import helpers as h from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired from kallithea.lib.base import BaseRepoController, render from kallithea.lib.graphmod import graph_data -from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes, safe_int +from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes from kallithea.model.db import Repository @@ -131,8 +130,8 @@ elif alias == 'git': if org_repo != other_repo: + from dulwich.client import SubprocessGitClient from dulwich.repo import Repo - from dulwich.client import SubprocessGitClient gitrepo = Repo(org_repo.path) SubprocessGitClient(thin_packs=False).fetch(other_repo.path, gitrepo) @@ -208,12 +207,8 @@ other_repo=c.a_repo.repo_name, other_ref_type=org_ref_type, other_ref_name=org_ref_name, merge=merge or '') - - # set callbacks for generating markup for icons - c.ignorews_url = _ignorews_url - c.context_url = _context_url - ignore_whitespace = request.GET.get('ignorews') == '1' - line_context = safe_int(request.GET.get('context'), 3) + ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET) + diff_context_size = h.get_diff_context_size(request.GET) c.a_rev = self._get_ref_rev(c.a_repo, org_ref_type, org_ref_name, returnempty=True) @@ -275,8 +270,8 @@ log.debug('running diff between %s and %s in %s', rev1, c.cs_rev, org_repo.scm_instance.path) raw_diff = diffs.get_diff(org_repo.scm_instance, rev1=rev1, rev2=c.cs_rev, - ignore_whitespace=ignore_whitespace, - context=line_context) + ignore_whitespace=ignore_whitespace_diff, + context=diff_context_size) diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit) c.limited_diff = diff_processor.limited_diff diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/feed.py --- a/kallithea/controllers/feed.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/feed.py Sat Aug 22 20:53:43 2020 +0200 @@ -39,7 +39,7 @@ from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired from kallithea.lib.base import BaseRepoController from kallithea.lib.diffs import DiffProcessor -from kallithea.lib.utils2 import safe_int, safe_str, str2bool +from kallithea.lib.utils2 import asbool, safe_int, safe_str log = logging.getLogger(__name__) @@ -92,7 +92,7 @@ desc_msg.append(h.urlify_text(cs.message)) desc_msg.append('\n') desc_msg.extend(changes) - if str2bool(CONFIG.get('rss_include_diff', False)): + if asbool(CONFIG.get('rss_include_diff', False)): desc_msg.append('\n\n') desc_msg.append(safe_str(raw_diff)) desc_msg.append('') diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/files.py --- a/kallithea/controllers/files.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/files.py Sat Aug 22 20:53:43 2020 +0200 @@ -39,14 +39,13 @@ from webob.exc import HTTPFound, HTTPNotFound from kallithea.config.routing import url -from kallithea.controllers.changeset import _context_url, _ignorews_url, anchor_url, get_ignore_ws, get_line_ctx from kallithea.lib import diffs from kallithea.lib import helpers as h from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired from kallithea.lib.base import BaseRepoController, jsonify, render from kallithea.lib.exceptions import NonRelativePathError from kallithea.lib.utils import action_logger -from kallithea.lib.utils2 import convert_line_endings, detect_mode, safe_int, safe_str, str2bool +from kallithea.lib.utils2 import asbool, convert_line_endings, detect_mode, safe_str from kallithea.lib.vcs.backends.base import EmptyChangeset from kallithea.lib.vcs.conf import settings from kallithea.lib.vcs.exceptions import (ChangesetDoesNotExistError, ChangesetError, EmptyRepositoryError, ImproperArchiveTypeError, NodeAlreadyExistsError, @@ -558,8 +557,8 @@ @LoginRequired(allow_default_user=True) @HasRepoPermissionLevelDecorator('read') def diff(self, repo_name, f_path): - ignore_whitespace = request.GET.get('ignorews') == '1' - line_context = safe_int(request.GET.get('context'), 3) + ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET) + diff_context_size = h.get_diff_context_size(request.GET) diff2 = request.GET.get('diff2', '') diff1 = request.GET.get('diff1', '') or diff2 c.action = request.GET.get('diff') @@ -567,9 +566,6 @@ c.f_path = f_path c.big_diff = False fulldiff = request.GET.get('fulldiff') - c.anchor_url = anchor_url - c.ignorews_url = _ignorews_url - c.context_url = _context_url c.changes = OrderedDict() c.changes[diff2] = [] @@ -577,7 +573,7 @@ # to reduce JS and callbacks if request.GET.get('show_rev'): - if str2bool(request.GET.get('annotate', 'False')): + if asbool(request.GET.get('annotate', 'False')): _url = url('files_annotate_home', repo_name=c.repo_name, revision=diff1, f_path=c.f_path) else: @@ -624,8 +620,8 @@ if c.action == 'download': raw_diff = diffs.get_gitdiff(node1, node2, - ignore_whitespace=ignore_whitespace, - context=line_context) + ignore_whitespace=ignore_whitespace_diff, + context=diff_context_size) diff_name = '%s_vs_%s.diff' % (diff1, diff2) response.content_type = 'text/plain' response.content_disposition = ( @@ -635,25 +631,21 @@ elif c.action == 'raw': raw_diff = diffs.get_gitdiff(node1, node2, - ignore_whitespace=ignore_whitespace, - context=line_context) + ignore_whitespace=ignore_whitespace_diff, + context=diff_context_size) response.content_type = 'text/plain' return raw_diff else: fid = h.FID(diff2, node2.path) - line_context_lcl = get_line_ctx(fid, request.GET) - ign_whitespace_lcl = get_ignore_ws(fid, request.GET) - diff_limit = None if fulldiff else self.cut_off_limit c.a_rev, c.cs_rev, a_path, diff, st, op = diffs.wrapped_diff(filenode_old=node1, filenode_new=node2, diff_limit=diff_limit, - ignore_whitespace=ign_whitespace_lcl, - line_context=line_context_lcl, + ignore_whitespace=ignore_whitespace_diff, + line_context=diff_context_size, enable_comments=False) c.file_diff_data = [(fid, fid, op, a_path, node2.path, diff, st)] - return render('files/file_diff.html') @LoginRequired(allow_default_user=True) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/forks.py --- a/kallithea/controllers/forks.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/forks.py Sat Aug 22 20:53:43 2020 +0200 @@ -38,7 +38,7 @@ import kallithea import kallithea.lib.helpers as h from kallithea.config.routing import url -from kallithea.lib.auth import HasPermissionAny, HasPermissionAnyDecorator, HasRepoPermissionLevel, HasRepoPermissionLevelDecorator, LoginRequired +from kallithea.lib.auth import HasPermissionAnyDecorator, HasRepoPermissionLevel, HasRepoPermissionLevelDecorator, LoginRequired from kallithea.lib.base import BaseRepoController, render from kallithea.lib.page import Page from kallithea.lib.utils2 import safe_int @@ -54,11 +54,7 @@ class ForksController(BaseRepoController): def __load_defaults(self): - if HasPermissionAny('hg.create.write_on_repogroup.true')(): - repo_group_perm_level = 'write' - else: - repo_group_perm_level = 'admin' - c.repo_groups = AvailableRepoGroupChoices(['hg.create.repository'], repo_group_perm_level) + c.repo_groups = AvailableRepoGroupChoices('write') c.landing_revs_choices, c.landing_revs = ScmModel().get_repo_landing_revs() diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/controllers/pullrequests.py --- a/kallithea/controllers/pullrequests.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/controllers/pullrequests.py Sat Aug 22 20:53:43 2020 +0200 @@ -36,7 +36,7 @@ from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPFound, HTTPNotFound from kallithea.config.routing import url -from kallithea.controllers.changeset import _context_url, _ignorews_url, create_cs_pr_comment, delete_cs_pr_comment +from kallithea.controllers.changeset import create_cs_pr_comment, delete_cs_pr_comment from kallithea.lib import diffs from kallithea.lib import helpers as h from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired @@ -569,10 +569,8 @@ c.cs_comments = c.cs_repo.get_comments(raw_ids) c.cs_statuses = c.cs_repo.statuses(raw_ids) - ignore_whitespace = request.GET.get('ignorews') == '1' - line_context = safe_int(request.GET.get('context'), 3) - c.ignorews_url = _ignorews_url - c.context_url = _context_url + ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET) + diff_context_size = h.get_diff_context_size(request.GET) fulldiff = request.GET.get('fulldiff') diff_limit = None if fulldiff else self.cut_off_limit @@ -581,7 +579,7 @@ c.a_rev, c.cs_rev, org_scm_instance.path) try: raw_diff = diffs.get_diff(org_scm_instance, rev1=c.a_rev, rev2=c.cs_rev, - ignore_whitespace=ignore_whitespace, context=line_context) + ignore_whitespace=ignore_whitespace_diff, context=diff_context_size) except ChangesetDoesNotExistError: raw_diff = safe_bytes(_("The diff can't be shown - the PR revisions could not be found.")) diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/front-end/kallithea-diff.less --- a/kallithea/front-end/kallithea-diff.less Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/front-end/kallithea-diff.less Sat Aug 22 20:53:43 2020 +0200 @@ -62,6 +62,7 @@ border-collapse: collapse; border-radius: 0px !important; width: 100%; + table-layout: fixed; /* line coloring */ .context { @@ -105,31 +106,26 @@ border-color: rgba(0, 0, 0, 0.3); } - /* line numbers */ - .lineno { - padding-left: 2px; - padding-right: 2px !important; - width: 30px; + /* line number columns */ + td.lineno { + width: 4em; border-right: 1px solid @panel-default-border !important; vertical-align: middle !important; - text-align: center; - } - .lineno.new { - text-align: right; - } - .lineno.old { - text-align: right; - } - .lineno a { - color: #aaa !important; font-size: 11px; font-family: @font-family-monospace; line-height: normal; - padding-left: 6px; - padding-right: 6px; - display: block; + text-align: center; + } + td.lineno[colspan="2"] { + width: 8em; } - .line:hover .lineno a { + td.lineno a { + color: #aaa !important; + display: inline-block; + min-width: 2em; + text-align: right; + } + tr.line:hover td.lineno a { color: #333 !important; } /** CODE **/ @@ -172,10 +168,7 @@ left: -8px; box-sizing: border-box; } -/* comment bubble, only visible when in a commentable diff */ -.commentable-diff tr.line.add:hover td .add-bubble, -.commentable-diff tr.line.del:hover td .add-bubble, -.commentable-diff tr.line.unmod:hover td .add-bubble { +.commentable-diff tr.line:hover td .add-bubble { display: block; z-index: 1; } diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/auth.py --- a/kallithea/lib/auth.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/auth.py Sat Aug 22 20:53:43 2020 +0200 @@ -149,7 +149,6 @@ # based on default permissions, just set everything to admin #================================================================== permissions[GLOBAL].add('hg.admin') - permissions[GLOBAL].add('hg.create.write_on_repogroup.true') # repositories for perm in default_repo_perms: @@ -242,7 +241,7 @@ # for each kind of global permissions, only keep the one with heighest weight kind_max_perm = {} - for perm in sorted(permissions[GLOBAL], key=lambda n: PERM_WEIGHTS[n]): + for perm in sorted(permissions[GLOBAL], key=lambda n: PERM_WEIGHTS.get(n, -1)): kind = perm.rsplit('.', 1)[0] kind_max_perm[kind] = perm permissions[GLOBAL] = set(kind_max_perm.values()) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/auth_modules/__init__.py --- a/kallithea/lib/auth_modules/__init__.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/auth_modules/__init__.py Sat Aug 22 20:53:43 2020 +0200 @@ -21,7 +21,7 @@ from kallithea.lib.auth import AuthUser, PasswordGenerator from kallithea.lib.compat import hybrid_property -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.model.db import Setting, User from kallithea.model.meta import Session from kallithea.model.user import UserModel @@ -350,7 +350,7 @@ plugin_settings[v["name"]] = setting.app_settings_value if setting else None log.debug('Settings for auth plugin %s: %s', plugin_name, plugin_settings) - if not str2bool(plugin_settings["enabled"]): + if not asbool(plugin_settings["enabled"]): log.info("Authentication plugin %s is disabled, skipping for %s", module, username) continue diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/auth_modules/auth_container.py --- a/kallithea/lib/auth_modules/auth_container.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/auth_modules/auth_container.py Sat Aug 22 20:53:43 2020 +0200 @@ -29,7 +29,7 @@ from kallithea.lib import auth_modules from kallithea.lib.compat import hybrid_property -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.model.db import Setting @@ -131,7 +131,7 @@ username = environ.get(header) log.debug('extracted %s:%s', header, username) - if username and str2bool(settings.get('clean_username')): + if username and asbool(settings.get('clean_username')): log.debug('Received username %s from container', username) username = self._clean_username(username) log.debug('New cleanup user is: %s', username) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/base.py --- a/kallithea/lib/base.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/base.py Sat Aug 22 20:53:43 2020 +0200 @@ -49,7 +49,7 @@ from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware from kallithea.lib.exceptions import UserCreationError from kallithea.lib.utils import get_repo_slug, is_valid_repo -from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_str, set_hook_environment, str2bool +from kallithea.lib.utils2 import AttributeDict, asbool, ascii_bytes, safe_int, safe_str, set_hook_environment from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError from kallithea.model import meta from kallithea.model.db import PullRequest, Repository, Setting, User @@ -375,14 +375,14 @@ c.visual = AttributeDict({}) ## DB stored - c.visual.show_public_icon = str2bool(rc_config.get('show_public_icon')) - c.visual.show_private_icon = str2bool(rc_config.get('show_private_icon')) - c.visual.stylify_metalabels = str2bool(rc_config.get('stylify_metalabels')) + c.visual.show_public_icon = asbool(rc_config.get('show_public_icon')) + c.visual.show_private_icon = asbool(rc_config.get('show_private_icon')) + c.visual.stylify_metalabels = asbool(rc_config.get('stylify_metalabels')) c.visual.page_size = safe_int(rc_config.get('dashboard_items', 100)) c.visual.admin_grid_items = safe_int(rc_config.get('admin_grid_items', 100)) - c.visual.repository_fields = str2bool(rc_config.get('repository_fields')) - c.visual.show_version = str2bool(rc_config.get('show_version')) - c.visual.use_gravatar = str2bool(rc_config.get('use_gravatar')) + c.visual.repository_fields = asbool(rc_config.get('repository_fields')) + c.visual.show_version = asbool(rc_config.get('show_version')) + c.visual.use_gravatar = asbool(rc_config.get('use_gravatar')) c.visual.gravatar_url = rc_config.get('gravatar_url') c.ga_code = rc_config.get('ga_code') @@ -404,9 +404,9 @@ c.clone_ssh_tmpl = rc_config.get('clone_ssh_tmpl') or Repository.DEFAULT_CLONE_SSH ## INI stored - c.visual.allow_repo_location_change = str2bool(config.get('allow_repo_location_change', True)) - c.visual.allow_custom_hooks_settings = str2bool(config.get('allow_custom_hooks_settings', True)) - c.ssh_enabled = str2bool(config.get('ssh_enabled', False)) + c.visual.allow_repo_location_change = asbool(config.get('allow_repo_location_change', True)) + c.visual.allow_custom_hooks_settings = asbool(config.get('allow_custom_hooks_settings', True)) + c.ssh_enabled = asbool(config.get('ssh_enabled', False)) c.instance_id = config.get('instance_id') c.issues_url = config.get('bugtracker', url('issues_url')) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/celerylib/__init__.py --- a/kallithea/lib/celerylib/__init__.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/celerylib/__init__.py Sat Aug 22 20:53:43 2020 +0200 @@ -28,7 +28,7 @@ import logging import os -from hashlib import md5 +from hashlib import sha1 from decorator import decorator from tg import config @@ -94,7 +94,7 @@ func_name = str(func.__name__) if hasattr(func, '__name__') else str(func) lockkey = 'task_%s.lock' % \ - md5(safe_bytes(func_name + '-' + '-'.join(str(x) for x in params))).hexdigest() + sha1(safe_bytes(func_name + '-' + '-'.join(str(x) for x in params))).hexdigest() return lockkey diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/celerylib/tasks.py --- a/kallithea/lib/celerylib/tasks.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/celerylib/tasks.py Sat Aug 22 20:53:43 2020 +0200 @@ -42,7 +42,7 @@ from kallithea.lib.hooks import log_create_repository from kallithea.lib.rcmail.smtp_mailer import SmtpMailer from kallithea.lib.utils import action_logger -from kallithea.lib.utils2 import ascii_bytes, str2bool +from kallithea.lib.utils2 import asbool, ascii_bytes from kallithea.lib.vcs.utils import author_email from kallithea.model.db import RepoGroup, Repository, Statistics, User @@ -289,9 +289,9 @@ passwd = email_config.get('smtp_password') mail_server = email_config.get('smtp_server') mail_port = email_config.get('smtp_port') - tls = str2bool(email_config.get('smtp_use_tls')) - ssl = str2bool(email_config.get('smtp_use_ssl')) - debug = str2bool(email_config.get('debug')) + tls = asbool(email_config.get('smtp_use_tls')) + ssl = asbool(email_config.get('smtp_use_ssl')) + debug = asbool(email_config.get('debug')) smtp_auth = email_config.get('smtp_auth') logmsg = ("Mail details:\n" @@ -323,8 +323,8 @@ @celerylib.task @celerylib.dbsession def create_repo(form_data, cur_user): + from kallithea.model.db import Setting from kallithea.model.repo import RepoModel - from kallithea.model.db import Setting DBS = celerylib.get_session() diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/colored_formatter.py --- a/kallithea/lib/colored_formatter.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/colored_formatter.py Sat Aug 22 20:53:43 2020 +0200 @@ -13,6 +13,7 @@ # along with this program. If not, see . import logging +import sys BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) @@ -65,15 +66,18 @@ def __init__(self, *args, **kwargs): # can't do super(...) here because Formatter is an old school class logging.Formatter.__init__(self, *args, **kwargs) + self.plain = not getattr(sys.stderr, 'isatty', lambda: False)() def format(self, record): """ Changes record's levelname to use with COLORS enum """ + def_record = logging.Formatter.format(self, record) + if self.plain: + return def_record levelname = record.levelname start = COLOR_SEQ % (COLORS[levelname]) - def_record = logging.Formatter.format(self, record) end = RESET_SEQ colored_record = ''.join([start, def_record, end]) @@ -85,14 +89,17 @@ def __init__(self, *args, **kwargs): # can't do super(...) here because Formatter is an old school class logging.Formatter.__init__(self, *args, **kwargs) + self.plain = not getattr(sys.stderr, 'isatty', lambda: False)() def format(self, record): """ Changes record's levelname to use with COLORS enum """ + def_record = format_sql(logging.Formatter.format(self, record)) + if self.plain: + return def_record start = COLOR_SEQ % (COLORS['SQL']) - def_record = format_sql(logging.Formatter.format(self, record)) end = RESET_SEQ colored_record = ''.join([start, def_record, end]) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/db_manage.py --- a/kallithea/lib/db_manage.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/db_manage.py Sat Aug 22 20:53:43 2020 +0200 @@ -37,11 +37,9 @@ from sqlalchemy.engine import create_engine from kallithea.model.base import init_model -from kallithea.model.db import Permission, RepoGroup, Repository, Setting, Ui, User, UserRepoGroupToPerm, UserToPerm -#from kallithea.model import meta +from kallithea.model.db import Repository, Setting, Ui, User from kallithea.model.meta import Base, Session from kallithea.model.permission import PermissionModel -from kallithea.model.repo_group import RepoGroupModel from kallithea.model.user import UserModel @@ -74,46 +72,48 @@ init_model(engine) self.sa = Session() - def create_tables(self, override=False): - """ - Create a auth database + def create_tables(self, reuse_database=False): """ - - log.info("Any existing database is going to be destroyed") - if self.tests: - destroy = True + Create database (optional) and tables. + If reuse_database is false, the database will be dropped (if it exists) + and a new one created. If true, the existing database will be reused + and cleaned for content. + """ + url = sqlalchemy.engine.url.make_url(self.dburi) + database = url.database + if reuse_database: + log.info("The content of the database %r will be destroyed and new tables created." % database) else: - destroy = self._ask_ok('Are you sure to destroy old database ? [y/n]') - if not destroy: - print('Nothing done.') - sys.exit(0) - if destroy: - # drop and re-create old schemas + log.info("The existing database %r will be destroyed and a new one created." % database) - url = sqlalchemy.engine.url.make_url(self.dburi) - database = url.database + if not self.tests: + if not self._ask_ok('Are you sure to destroy old database? [y/n]'): + print('Nothing done.') + sys.exit(0) - # Some databases enforce foreign key constraints and Base.metadata.drop_all() doesn't work + if reuse_database: + Base.metadata.drop_all() + else: if url.drivername == 'mysql': url.database = None # don't connect to the database (it might not exist) engine = sqlalchemy.create_engine(url) with engine.connect() as conn: - conn.execute('DROP DATABASE IF EXISTS ' + database) - conn.execute('CREATE DATABASE ' + database) + conn.execute('DROP DATABASE IF EXISTS `%s`' % database) + conn.execute('CREATE DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' % database) elif url.drivername == 'postgresql': from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT url.database = 'postgres' # connect to the system database (as the real one might not exist) engine = sqlalchemy.create_engine(url) with engine.connect() as conn: conn.connection.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - conn.execute('DROP DATABASE IF EXISTS ' + database) - conn.execute('CREATE DATABASE ' + database) + conn.execute('DROP DATABASE IF EXISTS "%s"' % database) + conn.execute('CREATE DATABASE "%s"' % database) else: + # Some databases enforce foreign key constraints and Base.metadata.drop_all() doesn't work, but this is # known to work on SQLite - possibly not on other databases with strong referential integrity Base.metadata.drop_all() - checkfirst = not override - Base.metadata.create_all(checkfirst=checkfirst) + Base.metadata.create_all(checkfirst=False) # Create an Alembic configuration and generate the version table, # "stamping" it with the most recent Alembic migration revision, to @@ -128,42 +128,6 @@ log.info('Created tables for %s', self.dbname) - def fix_repo_paths(self): - """ - Fixes a old kallithea version path into new one without a '*' - """ - - paths = Ui.query() \ - .filter(Ui.ui_key == '/') \ - .scalar() - - paths.ui_value = paths.ui_value.replace('*', '') - - self.sa.commit() - - def fix_default_user(self): - """ - Fixes a old default user with some 'nicer' default values, - used mostly for anonymous access - """ - def_user = User.query().filter_by(is_default_user=True).one() - - def_user.name = 'Anonymous' - def_user.lastname = 'User' - def_user.email = 'anonymous@kallithea-scm.org' - - self.sa.commit() - - def fix_settings(self): - """ - Fixes kallithea settings adds ga_code key for google analytics - """ - - hgsettings3 = Setting('ga_code', '') - - self.sa.add(hgsettings3) - self.sa.commit() - def admin_prompt(self, second=False): if not self.tests: import getpass @@ -199,11 +163,9 @@ self.create_user(username, password, email, True) else: log.info('creating admin and regular test users') - from kallithea.tests.base import TEST_USER_ADMIN_LOGIN, \ - TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \ - TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \ - TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \ - TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL + from kallithea.tests.base import (TEST_USER_ADMIN_EMAIL, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS, TEST_USER_REGULAR2_EMAIL, + TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR_LOGIN, + TEST_USER_REGULAR_PASS) self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, True) @@ -244,45 +206,6 @@ setting = Setting(k, v, t) self.sa.add(setting) - def fixup_groups(self): - def_usr = User.get_default_user() - for g in RepoGroup.query().all(): - g.group_name = g.get_new_name(g.name) - # get default perm - default = UserRepoGroupToPerm.query() \ - .filter(UserRepoGroupToPerm.group == g) \ - .filter(UserRepoGroupToPerm.user == def_usr) \ - .scalar() - - if default is None: - log.debug('missing default permission for group %s adding', g) - RepoGroupModel()._create_default_perms(g) - - def reset_permissions(self, username): - """ - Resets permissions to default state, useful when old systems had - bad permissions, we must clean them up - - :param username: - """ - default_user = User.get_by_username(username) - if not default_user: - return - - u2p = UserToPerm.query() \ - .filter(UserToPerm.user == default_user).all() - fixed = False - if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS): - for p in u2p: - Session().delete(p) - fixed = True - self.populate_default_permissions() - return fixed - - def update_repo_info(self): - for repo in Repository.query(): - repo.update_changeset_cache() - def prompt_repo_root_path(self, test_repo_path='', retries=3): _path = self.cli_args.get('repos_location') if retries == 3: diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/diffs.py --- a/kallithea/lib/diffs.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/diffs.py Sat Aug 22 20:53:43 2020 +0200 @@ -70,80 +70,72 @@ """ Return given diff as html table with customized css classes """ - def _link_to_if(condition, label, url): - """ - Generates a link if condition is meet or just the label if not. - """ - - if condition: - return '''''' % { - 'url': url, - 'label': label - } - else: - return label - _html_empty = True _html = [] _html.append('''\n''' % { 'table_class': table_class }) - for diff in parsed_lines: - for line in diff['chunks']: + for file_info in parsed_lines: + count_no_lineno = 0 # counter to allow comments on lines without new/old line numbers + for chunk in file_info['chunks']: _html_empty = False - for change in line: + for change in chunk: _html.append('''\n''' % { 'lc': line_class, 'action': change['action'] }) - anchor_old_id = '' - anchor_new_id = '' - anchor_old = "%(filename)s_o%(oldline_no)s" % { - 'filename': _safe_id(diff['filename']), - 'oldline_no': change['old_lineno'] - } - anchor_new = "%(filename)s_n%(oldline_no)s" % { - 'filename': _safe_id(diff['filename']), - 'oldline_no': change['new_lineno'] - } - cond_old = (change['old_lineno'] != '...' and - change['old_lineno']) - cond_new = (change['new_lineno'] != '...' and - change['new_lineno']) - no_lineno = (change['old_lineno'] == '...' and - change['new_lineno'] == '...') - if cond_old: - anchor_old_id = 'id="%s"' % anchor_old - if cond_new: - anchor_new_id = 'id="%s"' % anchor_new - ########################################################### - # OLD LINE NUMBER - ########################################################### - _html.append('''\t\n''') - ########################################################### - # NEW LINE NUMBER - ########################################################### - - if not no_lineno: + if change['old_lineno'] or change['new_lineno']: + ########################################################### + # OLD LINE NUMBER + ########################################################### + anchor_old = "%(filename)s_o%(oldline_no)s" % { + 'filename': _safe_id(file_info['filename']), + 'oldline_no': change['old_lineno'] + } + anchor_old_id = '' + if change['old_lineno']: + anchor_old_id = 'id="%s"' % anchor_old + _html.append('''\t\n''') + ########################################################### + # NEW LINE NUMBER + ########################################################### + anchor_new = "%(filename)s_n%(newline_no)s" % { + 'filename': _safe_id(file_info['filename']), + 'newline_no': change['new_lineno'] + } + anchor_new_id = '' + if change['new_lineno']: + anchor_new_id = 'id="%s"' % anchor_new _html.append('''\t\n''') + else: + ########################################################### + # NO LINE NUMBER + ########################################################### + anchor = "%(filename)s_%(count_no_lineno)s" % { + 'filename': _safe_id(file_info['filename']), + 'count_no_lineno': count_no_lineno, + } + count_no_lineno += 1 + _html.append('''\t\n''') ########################################################### @@ -453,7 +445,7 @@ return self.adds, self.removes -_escape_re = re.compile(r'(&)|(<)|(>)|(\t)|(\r)|(?<=.)( \n| $)') +_escape_re = re.compile(r'(&)|(<)|(>)|(\t)|(\r)|(?<=.)( \n| $)|(\t\n|\t$)') def _escaper(string): @@ -470,11 +462,13 @@ if groups[2]: return '>' if groups[3]: - return '\t' + return '\t' # Note: trailing tabs will get a longer match later if groups[4]: return '' if groups[5]: return ' ' + if groups[6]: + return '\t' assert False return _escape_re.sub(substitute, safe_str(string)) @@ -585,8 +579,8 @@ # skip context only if it's first line if int(gr[0]) > 1: lines.append({ - 'old_lineno': '...', - 'new_lineno': '...', + 'old_lineno': '', + 'new_lineno': '', 'action': 'context', 'line': line, }) @@ -630,8 +624,8 @@ # we need to append to lines, since this is not # counted in the line specs of diff lines.append({ - 'old_lineno': '...', - 'new_lineno': '...', + 'old_lineno': '', + 'new_lineno': '', 'action': 'context', 'line': line, }) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/helpers.py --- a/kallithea/lib/helpers.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/helpers.py Sat Aug 22 20:53:43 2020 +0200 @@ -48,7 +48,7 @@ from kallithea.lib.pygmentsutils import get_custom_lexer from kallithea.lib.utils2 import MENTIONS_REGEX, AttributeDict from kallithea.lib.utils2 import age as _age -from kallithea.lib.utils2 import credentials_filter, safe_bytes, safe_int, safe_str, str2bool, time_to_datetime +from kallithea.lib.utils2 import asbool, credentials_filter, safe_bytes, safe_int, safe_str, time_to_datetime from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError #============================================================================== @@ -213,12 +213,48 @@ """ Creates a unique ID for filenode based on it's hash of path and revision it's safe to use in urls + """ + return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12]) - :param raw_id: - :param path: - """ + +def get_ignore_whitespace_diff(GET): + """Return true if URL requested whitespace to be ignored""" + return bool(GET.get('ignorews')) + - return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12]) +def ignore_whitespace_link(GET, anchor=None): + """Return snippet with link to current URL with whitespace ignoring toggled""" + params = dict(GET) # ignoring duplicates + if get_ignore_whitespace_diff(GET): + params.pop('ignorews') + title = _("Show whitespace changes") + else: + params['ignorews'] = '1' + title = _("Ignore whitespace changes") + params['anchor'] = anchor + return link_to( + literal(''), + url.current(**params), + title=title, + **{'data-toggle': 'tooltip'}) + + +def get_diff_context_size(GET): + """Return effective context size requested in URL""" + return safe_int(GET.get('context'), default=3) + + +def increase_context_link(GET, anchor=None): + """Return snippet with link to current URL with double context size""" + context = get_diff_context_size(GET) * 2 + params = dict(GET) # ignoring duplicates + params['context'] = str(context) + params['anchor'] = anchor + return link_to( + literal(''), + url.current(**params), + title=_('Increase diff context to %(num)s lines') % {'num': context}, + **{'data-toggle': 'tooltip'}) class _FilesBreadCrumbs(object): @@ -526,7 +562,7 @@ """ from kallithea import CONFIG def_len = safe_int(CONFIG.get('show_sha_length', 12)) - show_rev = str2bool(CONFIG.get('show_revision_number', False)) + show_rev = asbool(CONFIG.get('show_revision_number', False)) raw_id = cs.raw_id[:def_len] if show_rev: @@ -596,6 +632,7 @@ """Find the user identified by 'author', return one of the users attributes, default to the username attribute, None if there is no user""" from kallithea.model.db import User + # if author is already an instance use it for extraction if isinstance(author, User): return getattr(author, show_attr) @@ -610,6 +647,7 @@ def person_by_id(id_, show_attr="username"): from kallithea.model.db import User + # maybe it's an ID ? if str(id_).isdigit() or isinstance(id_, int): id_ = int(id_) @@ -932,16 +970,14 @@ else: # if src is empty then there was no gravatar, so we use a font icon html = ("""""" - .format(cls=cls, size=size, src=src)) + .format(cls=cls, size=size)) return literal(html) def gravatar_url(email_address, size=30, default=''): - # doh, we need to re-import those to mock it later - from kallithea.config.routing import url - from kallithea.model.db import User from tg import tmpl_context as c + if not c.visual.use_gravatar: return "" @@ -951,6 +987,10 @@ if email_address == _def: return default + # re-import url so tests can mock it + from kallithea.config.routing import url + from kallithea.model.db import User + parsed_url = urllib.parse.urlparse(url.current(qualified=True)) url = (c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL) \ .replace('{email}', email_address) \ @@ -986,8 +1026,7 @@ :param stats: two element list of added/deleted lines of code """ - from kallithea.lib.diffs import NEW_FILENODE, DEL_FILENODE, \ - MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE + from kallithea.lib.diffs import BIN_FILENODE, CHMOD_FILENODE, DEL_FILENODE, MOD_FILENODE, NEW_FILENODE, RENAMED_FILENODE a, d = stats['added'], stats['deleted'] width = 100 diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/hooks.py --- a/kallithea/lib/hooks.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/hooks.py Sat Aug 22 20:53:43 2020 +0200 @@ -307,14 +307,15 @@ connect to the database. """ import paste.deploy - import kallithea.config.middleware + + import kallithea.config.application extras = get_hook_environment() path_to_ini_file = extras['config'] kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file) #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging - kallithea.config.middleware.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf) + kallithea.config.application.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf) # fix if it's not a bare repo if repo_path.endswith(os.sep + '.git'): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/inifile.py --- a/kallithea/lib/inifile.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/inifile.py Sat Aug 22 20:53:43 2020 +0200 @@ -119,9 +119,6 @@ #variable7 = 7.1 #variable8 = 8.0 - variable8 = None - variable9 = None - [fourth-section] fourth = "four" fourth_extra = 4 @@ -180,7 +177,7 @@ new_value = section_settings[key] if new_value == line_value: line = line.lstrip('#') - else: + elif new_value is not None: line += '\n%s = %s' % (key, new_value) section_settings.pop(key) return line @@ -189,8 +186,12 @@ # 3rd pass: # settings that haven't been consumed yet at is appended to section - if section_settings: - lines += '\n' + ''.join('%s = %s\n' % (key, value) for key, value in sorted(section_settings.items())) + append_lines = ''.join( + '%s = %s\n' % (key, value) + for key, value in sorted(section_settings.items()) + if value is not None) + if append_lines: + lines += '\n' + append_lines return sectionname + '\n' + re.sub('[ \t]+\n', '\n', lines) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/markup_renderer.py --- a/kallithea/lib/markup_renderer.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/markup_renderer.py Sat Aug 22 20:53:43 2020 +0200 @@ -74,13 +74,13 @@ :param text: """ - from hashlib import md5 + from hashlib import sha1 # Extract pre blocks. extractions = {} def pre_extraction_callback(matchobj): - digest = md5(matchobj.group(0)).hexdigest() + digest = sha1(matchobj.group(0)).hexdigest() extractions[digest] = matchobj.group(0) return "{gfm-extraction-%s}" % digest pattern = re.compile(r'
.*?
', re.MULTILINE | re.DOTALL) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/middleware/https_fixup.py --- a/kallithea/lib/middleware/https_fixup.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/middleware/https_fixup.py Sat Aug 22 20:53:43 2020 +0200 @@ -26,7 +26,7 @@ """ -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool class HttpsFixup(object): @@ -37,11 +37,11 @@ def __call__(self, environ, start_response): self.__fixup(environ) - debug = str2bool(self.config.get('debug')) + debug = asbool(self.config.get('debug')) is_ssl = environ['wsgi.url_scheme'] == 'https' def custom_start_response(status, headers, exc_info=None): - if is_ssl and str2bool(self.config.get('use_htsts')) and not debug: + if is_ssl and asbool(self.config.get('use_htsts')) and not debug: headers.append(('Strict-Transport-Security', 'max-age=8640000; includeSubDomains')) return start_response(status, headers, exc_info) @@ -66,7 +66,7 @@ org_proto = proto # if we have force, just override - if str2bool(self.config.get('force_https')): + if asbool(self.config.get('force_https')): proto = 'https' environ['wsgi.url_scheme'] = proto diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/middleware/pygrack.py --- a/kallithea/lib/middleware/pygrack.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/middleware/pygrack.py Sat Aug 22 20:53:43 2020 +0200 @@ -168,8 +168,9 @@ if git_command in ['git-receive-pack']: # updating refs manually after each push. # Needed for pre-1.7.0.4 git clients using regular HTTP mode. + from dulwich.server import update_server_info + from kallithea.lib.vcs import get_repo - from dulwich.server import update_server_info repo = get_repo(self.content_path) if repo: update_server_info(repo._repo) @@ -223,6 +224,6 @@ def make_wsgi_app(repo_name, repo_root): - from dulwich.web import LimitedInputFilter, GunzipFilter + from dulwich.web import GunzipFilter, LimitedInputFilter app = GitDirectory(repo_root, repo_name) return GunzipFilter(LimitedInputFilter(app)) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/paster_commands/template.ini.mako --- a/kallithea/lib/paster_commands/template.ini.mako Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/paster_commands/template.ini.mako Sat Aug 22 20:53:43 2020 +0200 @@ -69,7 +69,7 @@ port = ${port} %if http_server == 'gearbox': -<%text>## Gearbox default web server ## +<%text>## Gearbox serve uses the built-in development web server ## use = egg:gearbox#wsgiref <%text>## nr of worker threads to spawn threadpool_workers = 1 @@ -79,22 +79,22 @@ use_threadpool = true %elif http_server == 'gevent': -<%text>## Gearbox gevent web server ## +<%text>## Gearbox serve uses the gevent web server ## use = egg:gearbox#gevent %elif http_server == 'waitress': -<%text>## WAITRESS ## +<%text>## Gearbox serve uses the Waitress web server ## use = egg:waitress#main -<%text>## number of worker threads +<%text>## avoid multi threading threads = 1 -<%text>## MAX BODY SIZE 100GB +<%text>## allow push of repos bigger than the default of 1 GB max_request_body_size = 107374182400 <%text>## use poll instead of select, fixes fd limits, may not work on old <%text>## windows systems. #asyncore_use_poll = True %elif http_server == 'gunicorn': -<%text>## GUNICORN ## +<%text>## Gearbox serve uses the Gunicorn web server ## use = egg:gunicorn#main <%text>## number of process workers. You must set `instance_id = *` when this option <%text>## is set to more than one worker @@ -453,21 +453,22 @@ <%text>######################### %if database_engine == 'sqlite': -<%text>## SQLITE [default] sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60 - -%elif database_engine == 'postgres': -<%text>## POSTGRESQL -sqlalchemy.url = postgresql://user:pass@localhost/kallithea - -%elif database_engine == 'mysql': -<%text>## MySQL -sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8 +%else: +#sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60 +%endif +%if database_engine == 'postgres': +sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea +%else: +#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea +%endif +%if database_engine == 'mysql': +sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4 +%else: +#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4 +%endif <%text>## Note: the mysql:// prefix should also be used for MariaDB -%endif -<%text>## see sqlalchemy docs for other backends - sqlalchemy.pool_recycle = 3600 <%text>################################ diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/utils.py --- a/kallithea/lib/utils.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/utils.py Sat Aug 22 20:53:43 2020 +0200 @@ -466,7 +466,7 @@ enable_downloads = defs.get('repo_enable_downloads') private = defs.get('repo_private') - for name, repo in initial_repo_dict.items(): + for name, repo in sorted(initial_repo_dict.items()): group = map_groups(name) db_repo = repo_model.get_by_repo_name(name) # found repo that is on filesystem not in Kallithea database @@ -500,7 +500,7 @@ new_repo.update_changeset_cache() elif install_git_hooks: if db_repo.repo_type == 'git': - ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks) + ScmModel().install_git_hooks(db_repo.scm_instance, force=overwrite_git_hooks) removed = [] # remove from database those repositories that are not in the filesystem diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/utils2.py --- a/kallithea/lib/utils2.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/utils2.py Sat Aug 22 20:53:43 2020 +0200 @@ -38,6 +38,7 @@ import urlobject from tg.i18n import ugettext as _ from tg.i18n import ungettext +from tg.support.converters import asbool, aslist from webhelpers2.text import collapse, remove_formatting, strip_tags from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str # re-export @@ -51,6 +52,8 @@ # mute pyflakes "imported but unused" +assert asbool +assert aslist assert ascii_bytes assert ascii_str assert safe_bytes @@ -58,44 +61,6 @@ assert LazyProperty -def str2bool(_str): - """ - returns True/False value from given string, it tries to translate the - string into boolean - - :param _str: string value to translate into boolean - :rtype: boolean - :returns: boolean from given string - """ - if _str is None: - return False - if _str in (True, False): - return _str - _str = str(_str).strip().lower() - return _str in ('t', 'true', 'y', 'yes', 'on', '1') - - -def aslist(obj, sep=None, strip=True): - """ - Returns given string separated by sep as list - - :param obj: - :param sep: - :param strip: - """ - if isinstance(obj, (str)): - lst = obj.split(sep) - if strip: - lst = [v.strip() for v in lst] - return lst - elif isinstance(obj, (list, tuple)): - return obj - elif obj is None: - return [] - else: - return [obj] - - def convert_line_endings(line, mode): """ Converts a given line "line end" according to given mode @@ -366,9 +331,8 @@ :param repo: :param rev: """ - from kallithea.lib.vcs.backends.base import BaseRepository + from kallithea.lib.vcs.backends.base import BaseRepository, EmptyChangeset from kallithea.lib.vcs.exceptions import RepositoryError - from kallithea.lib.vcs.backends.base import EmptyChangeset if not isinstance(repo, BaseRepository): raise Exception('You must pass an Repository ' 'object as first argument got %s' % type(repo)) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/vcs/backends/hg/repository.py --- a/kallithea/lib/vcs/backends/hg/repository.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/vcs/backends/hg/repository.py Sat Aug 22 20:53:43 2020 +0200 @@ -272,7 +272,7 @@ self.get_changeset(rev1) self.get_changeset(rev2) if path: - file_filter = mercurial.match.exact(path) + file_filter = mercurial.match.exact([safe_bytes(path)]) else: file_filter = None diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/lib/vcs/utils/helpers.py --- a/kallithea/lib/vcs/utils/helpers.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/lib/vcs/utils/helpers.py Sat Aug 22 20:53:43 2020 +0200 @@ -112,8 +112,8 @@ except ImportError: return code from pygments import highlight - from pygments.lexers import guess_lexer_for_filename, ClassNotFound from pygments.formatters import TerminalFormatter + from pygments.lexers import ClassNotFound, guess_lexer_for_filename try: lexer = guess_lexer_for_filename(name, code) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/comment.py --- a/kallithea/model/comment.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/comment.py Sat Aug 22 20:53:43 2020 +0200 @@ -105,6 +105,7 @@ 'message': cs.message, 'message_short': h.shorter(cs.message, 50, firstline=True), 'cs_author': cs_author, + 'cs_author_username': cs_author.username, 'repo_name': repo.repo_name, 'short_id': h.short_id(revision), 'branch': cs.branch, diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/db.py --- a/kallithea/model/db.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/db.py Sat Aug 22 20:53:43 2020 +0200 @@ -46,7 +46,7 @@ import kallithea from kallithea.lib import ext_json from kallithea.lib.exceptions import DefaultUserException -from kallithea.lib.utils2 import (Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, str2bool, +from kallithea.lib.utils2 import (Optional, asbool, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, urlreadable) from kallithea.lib.vcs import get_backend from kallithea.lib.vcs.backends.base import EmptyChangeset @@ -61,10 +61,6 @@ # BASE CLASSES #============================================================================== -def _hash_key(k): - return hashlib.md5(safe_bytes(k)).hexdigest() - - class BaseDbModel(object): """ Base Model for all classes @@ -171,7 +167,6 @@ _table_args_default_dict = {'extend_existing': True, 'mysql_engine': 'InnoDB', - 'mysql_charset': 'utf8', 'sqlite_autoincrement': True, } @@ -185,7 +180,7 @@ 'str': safe_bytes, 'int': safe_int, 'unicode': safe_str, - 'bool': str2bool, + 'bool': asbool, 'list': functools.partial(aslist, sep=',') } DEFAULT_UPDATE_URL = '' @@ -312,8 +307,10 @@ @classmethod def get_server_info(cls): + import platform + import pkg_resources - import platform + from kallithea.lib.utils import check_git_version mods = [(p.project_name, p.version) for p in pkg_resources.working_set] info = { @@ -600,7 +597,8 @@ :param author: """ - from kallithea.lib.helpers import email, author_name + from kallithea.lib.helpers import author_name, email + # Valid email in the attribute passed, see if they're in the system _email = email(author) if _email: @@ -1164,7 +1162,7 @@ if with_pullrequests: data['pull_requests'] = repo.pull_requests_other rc_config = Setting.get_app_settings() - repository_fields = str2bool(rc_config.get('repository_fields')) + repository_fields = asbool(rc_config.get('repository_fields')) if repository_fields: for f in self.extra_fields: data[f.field_key_prefixed] = f.field_value @@ -1556,18 +1554,12 @@ ('usergroup.write', _('Default user has write access to new user groups')), ('usergroup.admin', _('Default user has admin access to new user groups')), - ('hg.repogroup.create.false', _('Only admins can create repository groups')), - ('hg.repogroup.create.true', _('Non-admins can create repository groups')), - ('hg.usergroup.create.false', _('Only admins can create user groups')), ('hg.usergroup.create.true', _('Non-admins can create user groups')), ('hg.create.none', _('Only admins can create top level repositories')), ('hg.create.repository', _('Non-admins can create top level repositories')), - ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')), - ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')), - ('hg.fork.none', _('Only admins can fork repositories')), ('hg.fork.repository', _('Non-admins can fork repositories')), @@ -1585,7 +1577,6 @@ 'group.read', 'usergroup.read', 'hg.create.repository', - 'hg.create.write_on_repogroup.true', 'hg.fork.repository', 'hg.register.manual_activate', 'hg.extern_activate.auto', @@ -1610,9 +1601,6 @@ 'usergroup.write': 3, 'usergroup.admin': 4, - 'hg.repogroup.create.false': 0, - 'hg.repogroup.create.true': 1, - 'hg.usergroup.create.false': 0, 'hg.usergroup.create.true': 1, @@ -1622,9 +1610,6 @@ 'hg.create.none': 0, 'hg.create.repository': 1, - 'hg.create.write_on_repogroup.false': 0, - 'hg.create.write_on_repogroup.true': 1, - 'hg.register.none': 0, 'hg.register.manual_activate': 1, 'hg.register.auto_activate': 2, diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/forms.py --- a/kallithea/model/forms.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/forms.py Sat Aug 22 20:53:43 2020 +0200 @@ -396,7 +396,6 @@ def DefaultPermissionsForm(repo_perms_choices, group_perms_choices, user_group_perms_choices, create_choices, - create_on_write_choices, repo_group_create_choices, user_group_create_choices, fork_choices, register_choices, extern_activate_choices): class _DefaultPermissionsForm(formencode.Schema): @@ -411,9 +410,7 @@ default_user_group_perm = v.OneOf(user_group_perms_choices) default_repo_create = v.OneOf(create_choices) - create_on_write = v.OneOf(create_on_write_choices) default_user_group_create = v.OneOf(user_group_create_choices) - #default_repo_group_create = v.OneOf(repo_group_create_choices) #not impl. yet default_fork = v.OneOf(fork_choices) default_register = v.OneOf(register_choices) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/notification.py --- a/kallithea/model/notification.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/notification.py Sat Aug 22 20:53:43 2020 +0200 @@ -126,12 +126,18 @@ email_html_body = EmailNotificationModel() \ .get_email_tmpl(type_, 'html', **html_kwargs) - # don't send email to person who created this comment - rec_objs = set(recipients_objs).difference(set([created_by_obj])) + # don't send email to the person who caused the notification, except for + # notifications about new pull requests where the author is explicitly + # added. + rec_mails = set(obj.email for obj in recipients_objs) + if type_ == NotificationModel.TYPE_PULL_REQUEST: + rec_mails.add(created_by_obj.email) + else: + rec_mails.discard(created_by_obj.email) - # send email with notification to all other participants - for rec in rec_objs: - tasks.send_email([rec.email], email_subject, email_txt_body, + # send email with notification to participants + for rec_mail in sorted(rec_mails): + tasks.send_email([rec_mail], email_subject, email_txt_body, email_html_body, headers, from_name=created_by_obj.full_name_or_username) @@ -159,7 +165,7 @@ self.TYPE_PULL_REQUEST_COMMENT: 'pull_request_comment', } self._subj_map = { - self.TYPE_CHANGESET_COMMENT: _('[Comment] %(repo_name)s changeset %(short_id)s "%(message_short)s" on %(branch)s'), + self.TYPE_CHANGESET_COMMENT: _('[Comment] %(repo_name)s changeset %(short_id)s "%(message_short)s" on %(branch)s by %(cs_author_username)s'), self.TYPE_MESSAGE: 'Test Message', # self.TYPE_PASSWORD_RESET self.TYPE_REGISTRATION: _('New user %(new_username)s registered'), diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/permission.py --- a/kallithea/model/permission.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/permission.py Sat Aug 22 20:53:43 2020 +0200 @@ -31,7 +31,7 @@ from sqlalchemy.exc import DatabaseError -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.model.db import Permission, Session, User, UserRepoGroupToPerm, UserRepoToPerm, UserToPerm, UserUserGroupToPerm @@ -97,7 +97,7 @@ try: # stage 1 set anonymous access if perm_user.is_default_user: - perm_user.active = str2bool(form_result['anonymous']) + perm_user.active = asbool(form_result['anonymous']) # stage 2 reset defaults and set them from form data def _make_new(usr, perm_name): @@ -119,8 +119,6 @@ 'default_group_perm', 'default_user_group_perm', 'default_repo_create', - 'create_on_write', # special case for create repos on write access to group - #'default_repo_group_create', # not implemented yet 'default_user_group_create', 'default_fork', 'default_register', diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/repo.py --- a/kallithea/model/repo.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/repo.py Sat Aug 22 20:53:43 2020 +0200 @@ -109,7 +109,8 @@ @classmethod def _render_datatable(cls, tmpl, *args, **kwargs): - from tg import tmpl_context as c, request, app_globals + from tg import app_globals, request + from tg import tmpl_context as c from tg.i18n import ugettext as _ _tmpl_lookup = app_globals.mako_lookup @@ -128,7 +129,9 @@ admin: return data for action column. """ _render = self._render_datatable - from tg import tmpl_context as c, request + from tg import request + from tg import tmpl_context as c + from kallithea.model.scm import ScmModel def repo_lnk(name, rtype, rstate, private, fork_of): @@ -666,7 +669,7 @@ elif repo_type == 'git': repo = backend(repo_path, create=True, src_url=clone_uri, bare=True) # add kallithea hook into this repo - ScmModel().install_git_hooks(repo=repo) + ScmModel().install_git_hooks(repo) else: raise Exception('Not supported repo_type %s expected hg/git' % repo_type) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/repo_group.py --- a/kallithea/model/repo_group.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/repo_group.py Sat Aug 22 20:53:43 2020 +0200 @@ -189,8 +189,8 @@ def _update_permissions(self, repo_group, perms_new=None, perms_updates=None, recursive=None, check_perms=True): + from kallithea.lib.auth import HasUserGroupPermissionLevel from kallithea.model.repo import RepoModel - from kallithea.lib.auth import HasUserGroupPermissionLevel if not perms_new: perms_new = [] diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/scm.py --- a/kallithea/model/scm.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/scm.py Sat Aug 22 20:53:43 2020 +0200 @@ -691,19 +691,19 @@ or sys.executable or '/usr/bin/env python3') - def install_git_hooks(self, repo, force_create=False): + def install_git_hooks(self, repo, force=False): """ Creates a kallithea hook inside a git repository :param repo: Instance of VCS repo - :param force_create: Create even if same name hook exists + :param force: Overwrite existing non-Kallithea hooks """ - loc = os.path.join(repo.path, 'hooks') + hooks_path = os.path.join(repo.path, 'hooks') if not repo.bare: - loc = os.path.join(repo.path, '.git', 'hooks') - if not os.path.isdir(loc): - os.makedirs(loc) + hooks_path = os.path.join(repo.path, '.git', 'hooks') + if not os.path.isdir(hooks_path): + os.makedirs(hooks_path) tmpl_post = b"#!%s\n" % safe_bytes(self._get_git_hook_interpreter()) tmpl_post += pkg_resources.resource_string( @@ -715,40 +715,36 @@ ) for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]: - _hook_file = os.path.join(loc, '%s-receive' % h_type) - has_hook = False + hook_file = os.path.join(hooks_path, '%s-receive' % h_type) + other_hook = False log.debug('Installing git hook in repo %s', repo) - if os.path.exists(_hook_file): + if os.path.exists(hook_file): # let's take a look at this hook, maybe it's kallithea ? log.debug('hook exists, checking if it is from kallithea') - with open(_hook_file, 'rb') as f: + with open(hook_file, 'rb') as f: data = f.read() matches = re.search(br'^KALLITHEA_HOOK_VER\s*=\s*(.*)$', data, flags=re.MULTILINE) if matches: - try: - ver = matches.groups()[0] - log.debug('Found Kallithea hook - it has KALLITHEA_HOOK_VER %r', ver) - has_hook = True - except Exception: - log.error(traceback.format_exc()) + ver = matches.groups()[0] + log.debug('Found Kallithea hook - it has KALLITHEA_HOOK_VER %r', ver) + else: + log.debug('Found non-Kallithea hook at %s', hook_file) + other_hook = True + + if other_hook and not force: + log.warning('skipping overwriting hook file %s', hook_file) else: - # there is no hook in this dir, so we want to create one - has_hook = True - - if has_hook or force_create: log.debug('writing %s hook file !', h_type) try: - with open(_hook_file, 'wb') as f: + with open(hook_file, 'wb') as f: tmpl = tmpl.replace(b'_TMPL_', safe_bytes(kallithea.__version__)) f.write(tmpl) - os.chmod(_hook_file, 0o755) + os.chmod(hook_file, 0o755) except IOError as e: - log.error('error writing %s: %s', _hook_file, e) - else: - log.debug('skipping writing hook file') + log.error('error writing hook %s: %s', hook_file, e) -def AvailableRepoGroupChoices(top_perms, repo_group_perm_level, extras=()): +def AvailableRepoGroupChoices(repo_group_perm_level, extras=()): """Return group_id,string tuples with choices for all the repo groups where the user has the necessary permissions. @@ -759,7 +755,7 @@ groups.append(None) else: groups = list(RepoGroupList(groups, perm_level=repo_group_perm_level)) - if top_perms and HasPermissionAny(*top_perms)('available repo groups'): + if HasPermissionAny('hg.create.repository')('available repo groups'): groups.append(None) for extra in extras: if not any(rg == extra for rg in groups): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/ssh_key.py --- a/kallithea/model/ssh_key.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/ssh_key.py Sat Aug 22 20:53:43 2020 +0200 @@ -29,7 +29,7 @@ from tg.i18n import ugettext as _ from kallithea.lib import ssh -from kallithea.lib.utils2 import str2bool +from kallithea.lib.utils2 import asbool from kallithea.lib.vcs.exceptions import RepositoryError from kallithea.model.db import User, UserSshKeys from kallithea.model.meta import Session @@ -95,7 +95,7 @@ return user_ssh_keys def write_authorized_keys(self): - if not str2bool(config.get('ssh_enabled', False)): + if not asbool(config.get('ssh_enabled', False)): log.error("Will not write SSH authorized_keys file - ssh_enabled is not configured") return authorized_keys = config.get('ssh_authorized_keys') diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/user.py --- a/kallithea/model/user.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/user.py Sat Aug 22 20:53:43 2020 +0200 @@ -59,8 +59,7 @@ if not cur_user: cur_user = getattr(get_current_authuser(), 'username', None) - from kallithea.lib.hooks import log_create_user, \ - check_allowed_create_user + from kallithea.lib.hooks import check_allowed_create_user, log_create_user _fd = form_data user_data = { 'username': _fd['username'], @@ -111,9 +110,8 @@ if not cur_user: cur_user = getattr(get_current_authuser(), 'username', None) - from kallithea.lib.auth import get_crypt_password, check_password - from kallithea.lib.hooks import log_create_user, \ - check_allowed_create_user + from kallithea.lib.auth import check_password, get_crypt_password + from kallithea.lib.hooks import check_allowed_create_user, log_create_user user_data = { 'username': username, 'password': password, 'email': email, 'firstname': firstname, 'lastname': lastname, @@ -168,8 +166,8 @@ raise def create_registration(self, form_data): + import kallithea.lib.helpers as h from kallithea.model.notification import NotificationModel - import kallithea.lib.helpers as h form_data['admin'] = False form_data['extern_type'] = User.DEFAULT_AUTH_TYPE @@ -317,9 +315,9 @@ allowing users to copy-paste or manually enter the token from the email. """ + import kallithea.lib.helpers as h from kallithea.lib.celerylib import tasks from kallithea.model.notification import EmailNotificationModel - import kallithea.lib.helpers as h user_email = data['email'] user = User.get_by_email(user_email) @@ -386,8 +384,8 @@ return expected_token == token def reset_password(self, user_email, new_passwd): + from kallithea.lib import auth from kallithea.lib.celerylib import tasks - from kallithea.lib import auth user = User.get_by_email(user_email) if user is not None: if not self.can_change_password(user): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/model/validators.py --- a/kallithea/model/validators.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/model/validators.py Sat Aug 22 20:53:43 2020 +0200 @@ -32,7 +32,7 @@ from kallithea.lib.compat import OrderedSet from kallithea.lib.exceptions import InvalidCloneUriException, LdapImportError from kallithea.lib.utils import is_valid_repo_uri -from kallithea.lib.utils2 import aslist, repo_name_slug, str2bool +from kallithea.lib.utils2 import asbool, aslist, repo_name_slug from kallithea.model import db from kallithea.model.db import RepoGroup, Repository, User, UserGroup @@ -456,12 +456,11 @@ gr_name = gr.group_name if gr is not None else None # None means ROOT location # create repositories with write permission on group is set to true - create_on_write = HasPermissionAny('hg.create.write_on_repogroup.true')() group_admin = HasRepoGroupPermissionLevel('admin')(gr_name, 'can write into group validator') group_write = HasRepoGroupPermissionLevel('write')(gr_name, 'can write into group validator') - forbidden = not (group_admin or (group_write and create_on_write)) + forbidden = not (group_admin or group_write) can_create_repos = HasPermissionAny('hg.admin', 'hg.create.repository') gid = (old_data['repo_group'].get('group_id') if (old_data and 'repo_group' in old_data) else None) @@ -568,7 +567,7 @@ 'g': 'users_group' }[k[0]] if member_name == User.DEFAULT_USER_NAME: - if str2bool(value.get('repo_private')): + if asbool(value.get('repo_private')): # set none for default when updating to # private repo protects against form manipulation v = EMPTY_PERM diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/admin/permissions/permissions_globals.html --- a/kallithea/templates/admin/permissions/permissions_globals.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/admin/permissions/permissions_globals.html Sat Aug 22 20:53:43 2020 +0200 @@ -58,13 +58,6 @@
- -
- ${h.select('create_on_write','',c.repo_create_on_write_choices,class_='form-control')} - ${_('With this, write permission to a repository group allows creating repositories inside that group. Without this, group write permissions mean nothing.')} -
-
-
${h.select('default_user_group_create','',c.user_group_create_choices,class_='form-control')} diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/changeset/changeset.html --- a/kallithea/templates/changeset/changeset.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/changeset/changeset.html Sat Aug 22 20:53:43 2020 +0200 @@ -47,8 +47,8 @@ - ${c.ignorews_url(request.GET)} - ${c.context_url(request.GET)} + ${h.ignore_whitespace_link(request.GET)} + ${h.increase_context_link(request.GET)}
diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/changeset/diff_block.html --- a/kallithea/templates/changeset/diff_block.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/changeset/diff_block.html Sat Aug 22 20:53:43 2020 +0200 @@ -61,16 +61,16 @@ - + - + - ${c.ignorews_url(request.GET, url_fid)} - ${c.context_url(request.GET, url_fid)} + ${h.ignore_whitespace_link(request.GET, id_fid)} + ${h.increase_context_link(request.GET, id_fid)}
${_('Show inline comments')} - ${h.checkbox('checkbox-show-inline-' + id_fid, checked="checked",class_="show-inline-comments",**{'data-id_for':id_fid})} + ${h.checkbox('checkbox-show-inline-' + id_fid, checked="checked",class_="show-inline-comments",**{'data-for':id_fid})}
@@ -137,7 +137,7 @@ if(target == null){ target = this; } - var boxid = $(target).data('id_for'); + var boxid = $(target).data('for'); if(target.checked){ $('#{0} .inline-comments'.format(boxid)).show(); }else{ diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/compare/compare_diff.html --- a/kallithea/templates/compare/compare_diff.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/compare/compare_diff.html Sat Aug 22 20:53:43 2020 +0200 @@ -60,9 +60,8 @@ % else: ${ungettext('%s file changed with %s insertions and %s deletions','%s files changed with %s insertions and %s deletions', len(c.file_diff_data)) % (len(c.file_diff_data),c.lines_added,c.lines_deleted)}: %endif - - ${c.ignorews_url(request.GET)} - ${c.context_url(request.GET)} + ${h.ignore_whitespace_link(request.GET)} + ${h.increase_context_link(request.GET)}
%if not c.file_diff_data: diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/files/diff_2way.html --- a/kallithea/templates/files/diff_2way.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/files/diff_2way.html Sat Aug 22 20:53:43 2020 +0200 @@ -48,10 +48,10 @@ + title="${_('Raw diff for this file')}"> + title="${_('Download diff for this file')}"> ${h.checkbox('ignorews', label=_('Ignore whitespace'))} ${h.checkbox('edit_mode', label=_('Edit'))}
diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/templates/index_base.html --- a/kallithea/templates/index_base.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/templates/index_base.html Sat Aug 22 20:53:43 2020 +0200 @@ -16,11 +16,10 @@ <% gr_name = c.group.group_name if c.group else None # create repositories with write permission on group is set to true - create_on_write = h.HasPermissionAny('hg.create.write_on_repogroup.true')() group_admin = h.HasRepoGroupPermissionLevel('admin')(gr_name, 'can write into group index page') group_write = h.HasRepoGroupPermissionLevel('write')(gr_name, 'can write into group index page') %> - %if h.HasPermissionAny('hg.admin','hg.create.repository')() or (group_admin or (group_write and create_on_write)): + %if h.HasPermissionAny('hg.admin','hg.create.repository')() or group_admin or group_write: %if c.group: ${_('Add Repository')} %if h.HasPermissionAny('hg.admin')() or h.HasRepoGroupPermissionLevel('admin')(c.group.group_name): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/api/api_base.py --- a/kallithea/tests/api/api_base.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/api/api_base.py Sat Aug 22 20:53:43 2020 +0200 @@ -36,7 +36,7 @@ from kallithea.model.user import UserModel from kallithea.model.user_group import UserGroupModel from kallithea.tests import base -from kallithea.tests.fixture import Fixture +from kallithea.tests.fixture import Fixture, raise_exception API_URL = '/_admin/api' @@ -63,10 +63,6 @@ jsonify = lambda obj: ext_json.loads(ext_json.dumps(obj)) -def crash(*args, **kwargs): - raise Exception('Total Crash !') - - def api_call(test_obj, params): response = test_obj.app.post(API_URL, content_type='application/json', params=params) @@ -149,7 +145,7 @@ assert 'trololo' == Optional.extract('trololo') def test_Optional_OAttr(self): - from kallithea.controllers.api.api import Optional, OAttr + from kallithea.controllers.api.api import OAttr, Optional option1 = Optional(OAttr('apiuser')) assert 'apiuser' == Optional.extract(option1) @@ -349,7 +345,7 @@ expected = {'added': [], 'removed': []} self._compare_ok(id_, expected, given=response.body) - @mock.patch.object(ScmModel, 'repo_scan', crash) + @mock.patch.object(ScmModel, 'repo_scan', raise_exception) def test_api_rescann_error(self): id_, params = _build_data(self.apikey, 'rescan_repos', ) response = api_call(self, params) @@ -439,7 +435,7 @@ finally: fixture.destroy_user(usr.user_id) - @mock.patch.object(UserModel, 'create_or_update', crash) + @mock.patch.object(UserModel, 'create_or_update', raise_exception) def test_api_create_user_when_exception_happened(self): username = 'test_new_api_user' @@ -473,7 +469,7 @@ expected = ret self._compare_ok(id_, expected, given=response.body) - @mock.patch.object(UserModel, 'delete', crash) + @mock.patch.object(UserModel, 'delete', raise_exception) def test_api_delete_user_when_exception_happened(self): usr = UserModel().create_or_update(username='test_user', password='qweqwe', @@ -561,7 +557,7 @@ expected = 'editing default user is forbidden' self._compare_error(id_, expected, given=response.body) - @mock.patch.object(UserModel, 'update_user', crash) + @mock.patch.object(UserModel, 'update_user', raise_exception) def test_api_update_user_when_exception_happens(self): usr = User.get_by_username(base.TEST_USER_ADMIN_LOGIN) ret = jsonify(usr.get_api_data()) @@ -1020,7 +1016,7 @@ self._compare_error(id_, expected, given=response.body) fixture.destroy_repo(repo_name) - @mock.patch.object(RepoModel, 'create', crash) + @mock.patch.object(RepoModel, 'create', raise_exception) def test_api_create_repo_exception_occurred(self): repo_name = 'api-repo' id_, params = _build_data(self.apikey, 'create_repo', @@ -1140,7 +1136,7 @@ finally: fixture.destroy_repo(repo_name) - @mock.patch.object(RepoModel, 'update', crash) + @mock.patch.object(RepoModel, 'update', raise_exception) def test_api_update_repo_exception_occurred(self): repo_name = 'api_update_me' fixture.create_repo(repo_name, repo_type=self.REPO_TYPE) @@ -1264,7 +1260,7 @@ repo_name = 'api_delete_me' fixture.create_repo(repo_name, repo_type=self.REPO_TYPE) try: - with mock.patch.object(RepoModel, 'delete', crash): + with mock.patch.object(RepoModel, 'delete', raise_exception): id_, params = _build_data(self.apikey, 'delete_repo', repoid=repo_name, ) response = api_call(self, params) @@ -1412,7 +1408,7 @@ expected = "repo `%s` already exist" % fork_name self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoModel, 'create_fork', crash) + @mock.patch.object(RepoModel, 'create_fork', raise_exception) def test_api_fork_repo_exception_occurred(self): fork_name = 'api-repo-fork' id_, params = _build_data(self.apikey, 'fork_repo', @@ -1484,7 +1480,7 @@ expected = "user group `%s` already exist" % TEST_USER_GROUP self._compare_error(id_, expected, given=response.body) - @mock.patch.object(UserGroupModel, 'create', crash) + @mock.patch.object(UserGroupModel, 'create', raise_exception) def test_api_get_user_group_exception_occurred(self): group_name = 'exception_happens' id_, params = _build_data(self.apikey, 'create_user_group', @@ -1520,7 +1516,7 @@ gr_name = updates['group_name'] fixture.destroy_user_group(gr_name) - @mock.patch.object(UserGroupModel, 'update', crash) + @mock.patch.object(UserGroupModel, 'update', raise_exception) def test_api_update_user_group_exception_occurred(self): gr_name = 'test_group' fixture.create_user_group(gr_name) @@ -1559,7 +1555,7 @@ expected = 'user group `%s` does not exist' % 'false-group' self._compare_error(id_, expected, given=response.body) - @mock.patch.object(UserGroupModel, 'add_user_to_group', crash) + @mock.patch.object(UserGroupModel, 'add_user_to_group', raise_exception) def test_api_add_user_to_user_group_exception_occurred(self): gr_name = 'test_group' fixture.create_user_group(gr_name) @@ -1592,7 +1588,7 @@ finally: fixture.destroy_user_group(gr_name) - @mock.patch.object(UserGroupModel, 'remove_user_from_group', crash) + @mock.patch.object(UserGroupModel, 'remove_user_from_group', raise_exception) def test_api_remove_user_from_user_group_exception_occurred(self): gr_name = 'test_group_3' gr = fixture.create_user_group(gr_name) @@ -1651,7 +1647,7 @@ usergroupid=gr_name) try: - with mock.patch.object(UserGroupModel, 'delete', crash): + with mock.patch.object(UserGroupModel, 'delete', raise_exception): response = api_call(self, params) expected = 'failed to delete user group ID:%s %s' % (gr_id, gr_name) self._compare_error(id_, expected, given=response.body) @@ -1693,7 +1689,7 @@ expected = 'permission `%s` does not exist' % perm self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoModel, 'grant_user_permission', crash) + @mock.patch.object(RepoModel, 'grant_user_permission', raise_exception) def test_api_grant_user_permission_exception_when_adding(self): perm = 'repository.read' id_, params = _build_data(self.apikey, @@ -1723,7 +1719,7 @@ } self._compare_ok(id_, expected, given=response.body) - @mock.patch.object(RepoModel, 'revoke_user_permission', crash) + @mock.patch.object(RepoModel, 'revoke_user_permission', raise_exception) def test_api_revoke_user_permission_exception_when_adding(self): id_, params = _build_data(self.apikey, 'revoke_user_permission', @@ -1771,7 +1767,7 @@ expected = 'permission `%s` does not exist' % perm self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoModel, 'grant_user_group_permission', crash) + @mock.patch.object(RepoModel, 'grant_user_group_permission', raise_exception) def test_api_grant_user_group_permission_exception_when_adding(self): perm = 'repository.read' id_, params = _build_data(self.apikey, @@ -1805,7 +1801,7 @@ } self._compare_ok(id_, expected, given=response.body) - @mock.patch.object(RepoModel, 'revoke_user_group_permission', crash) + @mock.patch.object(RepoModel, 'revoke_user_group_permission', raise_exception) def test_api_revoke_user_group_permission_exception_when_adding(self): id_, params = _build_data(self.apikey, 'revoke_user_group_permission', @@ -1907,7 +1903,7 @@ expected = 'permission `%s` does not exist' % perm self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoGroupModel, 'grant_user_permission', crash) + @mock.patch.object(RepoGroupModel, 'grant_user_permission', raise_exception) def test_api_grant_user_permission_to_repo_group_exception_when_adding(self): perm = 'group.read' id_, params = _build_data(self.apikey, @@ -1992,7 +1988,7 @@ expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoGroupModel, 'revoke_user_permission', crash) + @mock.patch.object(RepoGroupModel, 'revoke_user_permission', raise_exception) def test_api_revoke_user_permission_from_repo_group_exception_when_adding(self): id_, params = _build_data(self.apikey, 'revoke_user_permission_from_repo_group', @@ -2096,7 +2092,7 @@ expected = 'permission `%s` does not exist' % perm self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoGroupModel, 'grant_user_group_permission', crash) + @mock.patch.object(RepoGroupModel, 'grant_user_group_permission', raise_exception) def test_api_grant_user_group_permission_exception_when_adding_to_repo_group(self): perm = 'group.read' id_, params = _build_data(self.apikey, @@ -2180,7 +2176,7 @@ expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP self._compare_error(id_, expected, given=response.body) - @mock.patch.object(RepoGroupModel, 'revoke_user_group_permission', crash) + @mock.patch.object(RepoGroupModel, 'revoke_user_group_permission', raise_exception) def test_api_revoke_user_group_permission_from_repo_group_exception_when_adding(self): id_, params = _build_data(self.apikey, 'revoke_user_group_permission_from_repo_group', repogroupid=TEST_REPO_GROUP, @@ -2301,7 +2297,7 @@ } self._compare_ok(id_, expected, given=response.body) - @mock.patch.object(GistModel, 'create', crash) + @mock.patch.object(GistModel, 'create', raise_exception) def test_api_create_gist_exception_occurred(self): id_, params = _build_data(self.apikey_regular, 'create_gist', files={}) @@ -2333,7 +2329,7 @@ expected = 'gist `%s` does not exist' % (gist_id,) self._compare_error(id_, expected, given=response.body) - @mock.patch.object(GistModel, 'delete', crash) + @mock.patch.object(GistModel, 'delete', raise_exception) def test_api_delete_gist_exception_occurred(self): gist_id = fixture.create_gist().gist_access_id id_, params = _build_data(self.apikey, 'delete_gist', diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/conftest.py --- a/kallithea/tests/conftest.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/conftest.py Sat Aug 22 20:53:43 2020 +0200 @@ -59,8 +59,12 @@ 'formatter': 'color_formatter_sql', }, } - if os.environ.get('TEST_DB'): - ini_settings['[app:main]']['sqlalchemy.url'] = os.environ.get('TEST_DB') + create_database = os.environ.get('TEST_DB') # TODO: rename to 'CREATE_TEST_DB' + if create_database: + ini_settings['[app:main]']['sqlalchemy.url'] = create_database + reuse_database = os.environ.get('REUSE_TEST_DB') + if reuse_database: + ini_settings['[app:main]']['sqlalchemy.url'] = reuse_database test_ini_file = os.path.join(TESTS_TMP_PATH, 'test.ini') inifile.create(test_ini_file, None, ini_settings) @@ -70,7 +74,7 @@ # set KALLITHEA_NO_TMP_PATH=1 to disable re-creating the database and test repos if not int(os.environ.get('KALLITHEA_NO_TMP_PATH', 0)): - create_test_env(TESTS_TMP_PATH, context.config()) + create_test_env(TESTS_TMP_PATH, context.config(), reuse_database=bool(reuse_database)) # set KALLITHEA_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests if not int(os.environ.get('KALLITHEA_WHOOSH_TEST_DISABLE', 0)): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/fixture.py --- a/kallithea/tests/fixture.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/fixture.py Sat Aug 22 20:53:43 2020 +0200 @@ -50,8 +50,8 @@ FIXTURES = os.path.join(dirname(dirname(os.path.abspath(__file__))), 'tests', 'fixtures') -def error_function(*args, **kwargs): - raise Exception('Total Crash !') +def raise_exception(*args, **kwargs): + raise Exception('raise_exception raised exception') class Fixture(object): @@ -349,7 +349,7 @@ # Global test environment setup #============================================================================== -def create_test_env(repos_test_path, config): +def create_test_env(repos_test_path, config, reuse_database): """ Makes a fresh database and install test repository into tmp dir @@ -366,7 +366,7 @@ dbmanage = DbManage(dbconf=dbconf, root=config['here'], tests=True) - dbmanage.create_tables(override=True) + dbmanage.create_tables(reuse_database=reuse_database) # for tests dynamically set new root paths based on generated content dbmanage.create_settings(dbmanage.prompt_repo_root_path(repos_test_path)) dbmanage.create_default_user() diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/functional/test_admin_repos.py --- a/kallithea/tests/functional/test_admin_repos.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/functional/test_admin_repos.py Sat Aug 22 20:53:43 2020 +0200 @@ -14,7 +14,7 @@ from kallithea.model.repo_group import RepoGroupModel from kallithea.model.user import UserModel from kallithea.tests import base -from kallithea.tests.fixture import Fixture, error_function +from kallithea.tests.fixture import Fixture, raise_exception fixture = Fixture() @@ -605,7 +605,7 @@ RepoModel().delete(repo_name) Session().commit() - @mock.patch.object(RepoModel, '_create_filesystem_repo', error_function) + @mock.patch.object(RepoModel, '_create_filesystem_repo', raise_exception) def test_create_repo_when_filesystem_op_fails(self): self.log_user() repo_name = self.NEW_REPO diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/models/test_diff_parsers.py --- a/kallithea/tests/models/test_diff_parsers.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/models/test_diff_parsers.py Sat Aug 22 20:53:43 2020 +0200 @@ -295,7 +295,7 @@ l.append('%(action)-7s %(new_lineno)3s %(old_lineno)3s %(line)r\n' % d) s = ''.join(l) assert s == r''' -context ... ... '@@ -51,6 +51,13 @@\n' +context '@@ -51,6 +51,13 @@\n' unmod 51 51 '\tbegin();\n' unmod 52 52 '\t\n' add 53 '\tint foo;\n' diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/models/test_dump_html_mails.ref.html --- a/kallithea/tests/models/test_dump_html_mails.ref.html Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/models/test_dump_html_mails.ref.html Sat Aug 22 20:53:43 2020 +0200 @@ -7,7 +7,7 @@ 
 From: u1 u1 
 To: u2@example.com
-Subject: [Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch
+Subject: [Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch by u2
 
http://comment.org
@@ -166,7 +166,7 @@
 
 From: u1 u1 
 To: u2@example.com
-Subject: [Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch
+Subject: [Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch by u2
 

 

 
http://comment.org
@@ -325,7 +325,7 @@
 
 From: u1 u1 
 To: u2@example.com
-Subject: [Approved: Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch
+Subject: [Approved: Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch by u2
 

 

 
http://comment.org
@@ -502,7 +502,7 @@
 
 From: u1 u1 
 To: u2@example.com
-Subject: [Approved: Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch
+Subject: [Approved: Comment] repo/name changeset cafe1234 "This changeset did something cl..." on brunch by u2
 

 

 
http://comment.org
@@ -882,6 +882,197 @@
 
pull_request, is_mention=False

 
 From: u1 u1 
+To: u1@example.com
+Subject: [Review] repo/name PR #7 "The Title" from devbranch by u2
+

+

+
http://pr.org/7
+
+Added as Reviewer of Pull Request #7 "The Title" by Requesting User (root)
+
+
+Pull request #7 "The Title" by u2 u3 (u2)
+from https://dev.org/repo branch devbranch
+to http://mainline.com/repo branch trunk
+
+
+Description:
+
+This PR is 'awesome' because it does 
+ - please approve indented!
+
+
+Changesets:
+
+Introduce one and two
+Make one plus two equal tree
+
+
+View Pull Request: http://pr.org/7
+

+

+
+
+
+    
+    
+    
+
+
+
''' % { - 'a_id': anchor_old_id, - 'olc': no_lineno_class if no_lineno else old_lineno_class, - 'colspan': 'colspan="2"' if no_lineno else '' - }) - - _html.append('''%(link)s''' % { - 'link': _link_to_if(not no_lineno, change['old_lineno'], - '#%s' % anchor_old) - }) - _html.append('''''' % { + 'a_id': anchor_old_id, + 'olc': old_lineno_class, + }) + _html.append('''''' % { + 'label': change['old_lineno'], + 'url': '#%s' % anchor_old, + }) + _html.append('''''' % { 'a_id': anchor_new_id, 'nlc': new_lineno_class }) - - _html.append('''%(link)s''' % { - 'link': _link_to_if(True, change['new_lineno'], - '#%s' % anchor_new) + _html.append('''''' % { + 'label': change['new_lineno'], + 'url': '#%s' % anchor_new, + }) + _html.append('''''' % { + 'anchor': anchor, + 'olc': no_lineno_class, }) _html.append('''
+ + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
+ Added as Reviewer of Pull Request #7 "The Title" by Requesting User (root) +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Pull request + #7 "The Title" + by + u2 u3 (u2). +
+
+ from + https://dev.org/repo + branch + devbranch +
+ to + http://mainline.com/repo + branch + trunk +
+
+
+ Description: +
+
+ + + + + + + + + + + + +
+
This PR is 'awesome' because it does <stuff>
- please approve indented!
+
+
+
Changesets:
+
+ +
+
+ + + + + + + +
+ +
+ + View Pull Request + +
+ +
+
+
+
+
+ + +
+
+

pull_request, is_mention=False

+
+From: u1 u1 
 To: u2@example.com
 Subject: [Review] repo/name PR #7 "The Title" from devbranch by u2
@@ -1073,6 +1264,197 @@

pull_request, is_mention=True

 From: u1 u1 
+To: u1@example.com
+Subject: [Review] repo/name PR #7 "The Title" from devbranch by u2
+
+
+
http://pr.org/7
+
+Mention on Pull Request #7 "The Title" by Requesting User (root)
+
+
+Pull request #7 "The Title" by u2 u3 (u2)
+from https://dev.org/repo branch devbranch
+to http://mainline.com/repo branch trunk
+
+
+Description:
+
+This PR is 'awesome' because it does 
+ - please approve indented!
+
+
+Changesets:
+
+Introduce one and two
+Make one plus two equal tree
+
+
+View Pull Request: http://pr.org/7
+
+
+ + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
+ Mention on Pull Request #7 "The Title" by Requesting User (root) +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Pull request + #7 "The Title" + by + u2 u3 (u2). +
+
+ from + https://dev.org/repo + branch + devbranch +
+ to + http://mainline.com/repo + branch + trunk +
+
+
+ Description: +
+
+ + + + + + + + + + + + +
+
This PR is 'awesome' because it does <stuff>
- please approve indented!
+
+
+
Changesets:
+
+ +
+
+ + + + + + + +
+ +
+ + View Pull Request + +
+ +
+
+
+
+
+ + +
+
+

pull_request, is_mention=True

+
+From: u1 u1 
 To: u2@example.com
 Subject: [Review] repo/name PR #7 "The Title" from devbranch by u2
diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/models/test_notifications.py --- a/kallithea/tests/models/test_notifications.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/models/test_notifications.py Sat Aug 22 20:53:43 2020 +0200 @@ -103,6 +103,7 @@ status_change=[None, 'Approved'], cs_target_repo='http://example.com/repo_target', cs_url='http://changeset.com', + cs_author_username=User.get(self.u2).username, cs_author=User.get(self.u2))), (NotificationModel.TYPE_MESSAGE, 'This is the \'body\' of the "test" message\n - nothing interesting here except indentation.', diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/models/test_permissions.py --- a/kallithea/tests/models/test_permissions.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/models/test_permissions.py Sat Aug 22 20:53:43 2020 +0200 @@ -290,7 +290,7 @@ 'hg.register.manual_activate', 'hg.extern_activate.auto', 'repository.read', 'group.read', - 'usergroup.read', 'hg.create.write_on_repogroup.true']) + 'usergroup.read']) def test_inherit_sad_permissions_from_default_user(self): user_model = UserModel() @@ -307,7 +307,7 @@ 'hg.register.manual_activate', 'hg.extern_activate.auto', 'repository.read', 'group.read', - 'usergroup.read', 'hg.create.write_on_repogroup.true']) + 'usergroup.read']) def test_inherit_more_permissions_from_default_user(self): user_model = UserModel() @@ -333,7 +333,7 @@ 'hg.register.manual_activate', 'hg.extern_activate.auto', 'repository.read', 'group.read', - 'usergroup.read', 'hg.create.write_on_repogroup.true']) + 'usergroup.read']) def test_inherit_less_permissions_from_default_user(self): user_model = UserModel() @@ -359,7 +359,7 @@ 'hg.register.manual_activate', 'hg.extern_activate.auto', 'repository.read', 'group.read', - 'usergroup.read', 'hg.create.write_on_repogroup.true']) + 'usergroup.read']) def test_inactive_user_group_does_not_affect_global_permissions(self): # Add user to inactive user group, set specific permissions on user @@ -391,7 +391,7 @@ 'hg.extern_activate.auto', 'repository.read', 'group.read', 'usergroup.read', - 'hg.create.write_on_repogroup.true']) + ]) def test_inactive_user_group_does_not_affect_global_permissions_inverse(self): # Add user to inactive user group, set specific permissions on user @@ -423,7 +423,7 @@ 'hg.extern_activate.auto', 'repository.read', 'group.read', 'usergroup.read', - 'hg.create.write_on_repogroup.true']) + ]) def test_inactive_user_group_does_not_affect_repo_permissions(self): self.ug1 = fixture.create_user_group('G1') diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/other/test_libs.py --- a/kallithea/tests/other/test_libs.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/other/test_libs.py Sat Aug 22 20:53:43 2020 +0200 @@ -119,12 +119,10 @@ ('F', False), ('FALSE', False), ('0', False), - ('-1', False), - ('', False) ]) - def test_str2bool(self, str_bool, expected): - from kallithea.lib.utils2 import str2bool - assert str2bool(str_bool) == expected + def test_asbool(self, str_bool, expected): + from kallithea.lib.utils2 import asbool + assert asbool(str_bool) == expected def test_mention_extractor(self): from kallithea.lib.utils2 import extract_mentioned_usernames @@ -158,8 +156,9 @@ (dict(years= -3, months= -2), '3 years and 2 months ago'), ]) def test_age(self, age_args, expected): + from dateutil import relativedelta + from kallithea.lib.utils2 import age - from dateutil import relativedelta with test_context(self.app): n = datetime.datetime(year=2012, month=5, day=17) delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs) @@ -183,8 +182,9 @@ (dict(years= -4, months= -8), '5 years ago'), ]) def test_age_short(self, age_args, expected): + from dateutil import relativedelta + from kallithea.lib.utils2 import age - from dateutil import relativedelta with test_context(self.app): n = datetime.datetime(year=2012, month=5, day=17) delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs) @@ -202,8 +202,9 @@ (dict(years=1, months=1), 'in 1 year and 1 month') ]) def test_age_in_future(self, age_args, expected): + from dateutil import relativedelta + from kallithea.lib.utils2 import age - from dateutil import relativedelta with test_context(self.app): n = datetime.datetime(year=2012, month=5, day=17) delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs) @@ -299,6 +300,7 @@ :param text: """ import re + # quickly change expected url[] into a link url_pattern = re.compile(r'(?:url\[)(.+?)(?:\])') @@ -572,11 +574,11 @@ ('http://www.example.org/kallithea/repos/', 'abc/xyz/', 'http://www.example.org/kallithea/repos/abc/xyz/'), ]) def test_canonical_url(self, canonical, test, expected): - from kallithea.lib.helpers import canonical_url + # setup url(), used by canonical_url + import routes from tg import request - # setup url(), used by canonical_url - import routes + from kallithea.lib.helpers import canonical_url m = routes.Mapper() m.connect('about', '/about-page') url = routes.URLGenerator(m, {'HTTP_HOST': 'http_host.example.org'}) @@ -596,11 +598,12 @@ ('http://www.example.org/kallithea/repos/', 'www.example.org'), ]) def test_canonical_hostname(self, canonical, expected): - from kallithea.lib.helpers import canonical_hostname + import routes from tg import request + from kallithea.lib.helpers import canonical_hostname + # setup url(), used by canonical_hostname - import routes m = routes.Mapper() url = routes.URLGenerator(m, {'HTTP_HOST': 'http_host.example.org'}) diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/other/test_vcs_operations.py --- a/kallithea/tests/other/test_vcs_operations.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/other/test_vcs_operations.py Sat Aug 22 20:53:43 2020 +0200 @@ -308,7 +308,7 @@ if vt.repo_type == 'git': assert 'not found' in stderr or 'abort: Access to %r denied' % 'trololo' in stderr elif vt.repo_type == 'hg': - assert 'HTTP Error 404: Not Found' in stderr or 'abort: no suitable response from remote hg' in stderr and 'remote: abort: Access to %r denied' % 'trololo' in stdout + assert 'HTTP Error 404: Not Found' in stderr or 'abort: no suitable response from remote hg' in stderr and 'remote: abort: Access to %r denied' % 'trololo' in stdout + stderr @parametrize_vcs_test def test_push_new_repo(self, webserver, vt): @@ -358,6 +358,7 @@ # # # + Session.close() # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...) action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)] assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == ([ ('started_following_repo', 0), @@ -389,6 +390,7 @@ assert 'Repository size' in stdout assert 'Last revision is now' in stdout + Session.close() # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...) action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)] assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == \ [('pull', 0), ('push', 3)] @@ -403,6 +405,7 @@ clone_url = vt.repo_url_param(webserver, vt.repo_name) stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url) + Session.close() # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...) if vt.repo_type == 'git': assert 'FETCH_HEAD' in stderr @@ -426,7 +429,7 @@ if vt.repo_type == 'git': assert "abort: Access to './%s' denied" % vt.repo_name in stderr else: - assert "abort: Access to './%s' denied" % vt.repo_name in stdout + assert "abort: Access to './%s' denied" % vt.repo_name in stdout + stderr stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url.replace('/' + vt.repo_name, '/%s/' % vt.repo_name), ignoreReturnCode=True) if vt.repo_type == 'git': @@ -448,11 +451,10 @@ stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, files_no=1, clone_url=clone_url) - Session().commit() # expire test session to make sure SA fetch new Repository instances after last_changeset has been updated server side hook in other process - if vt.repo_type == 'git': _check_proper_git_push(stdout, stderr) + Session.close() # expire session to make sure SA fetches new Repository instances after last_changeset has been updated by server side hook in another process post_cached_tip = [repo.get_api_data()['last_changeset']['short_id'] for repo in Repository.query().filter(Repository.repo_name == testfork[vt.repo_type])] assert pre_cached_tip != post_cached_tip @@ -487,6 +489,7 @@ elif vt.repo_type == 'hg': assert 'abort: HTTP Error 403: Forbidden' in stderr or 'abort: push failed on remote' in stderr and 'remote: Push access to %r denied' % str(vt.repo_name) in stdout + Session.close() # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...) action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)] assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == \ [('pull', 0)] @@ -522,7 +525,7 @@ # The message apparently changed in Git 1.8.3, so match it loosely. assert re.search(r'\b403\b', stderr) or 'abort: User test_admin from 127.0.0.127 cannot be authorized' in stderr elif vt.repo_type == 'hg': - assert 'abort: HTTP Error 403: Forbidden' in stderr or 'remote: abort: User test_admin from 127.0.0.127 cannot be authorized' in stdout + assert 'abort: HTTP Error 403: Forbidden' in stderr or 'remote: abort: User test_admin from 127.0.0.127 cannot be authorized' in stdout + stderr finally: # release IP restrictions for ip in UserIpMap.query(): diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/scripts/manual_test_concurrency.py --- a/kallithea/tests/scripts/manual_test_concurrency.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/scripts/manual_test_concurrency.py Sat Aug 22 20:53:43 2020 +0200 @@ -37,7 +37,7 @@ from paste.deploy import appconfig from sqlalchemy import engine_from_config -from kallithea.config.environment import load_environment +import kallithea.config.application from kallithea.lib.auth import get_crypt_password from kallithea.model import meta from kallithea.model.base import init_model @@ -47,7 +47,7 @@ rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__))))) conf = appconfig('config:development.ini', relative_to=rel_path) -load_environment(conf.global_conf, conf.local_conf) +kallithea.config.application.make_app(conf.global_conf, **conf.local_conf) USER = TEST_USER_ADMIN_LOGIN PASS = TEST_USER_ADMIN_PASS diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/vcs/test_git.py --- a/kallithea/tests/vcs/test_git.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/vcs/test_git.py Sat Aug 22 20:53:43 2020 +0200 @@ -794,7 +794,7 @@ if os.path.exists(hook_path): os.remove(hook_path) - ScmModel().install_git_hooks(repo=self.repo) + ScmModel().install_git_hooks(self.repo) for hook, hook_path in self.kallithea_hooks.items(): assert os.path.exists(hook_path) @@ -808,7 +808,7 @@ with open(hook_path, "w") as f: f.write("KALLITHEA_HOOK_VER=0.0.0\nJUST_BOGUS") - ScmModel().install_git_hooks(repo=self.repo) + ScmModel().install_git_hooks(self.repo) for hook, hook_path in self.kallithea_hooks.items(): with open(hook_path) as f: @@ -823,7 +823,7 @@ with open(hook_path, "w") as f: f.write("#!/bin/bash\n#CUSTOM_HOOK") - ScmModel().install_git_hooks(repo=self.repo) + ScmModel().install_git_hooks(self.repo) for hook, hook_path in self.kallithea_hooks.items(): with open(hook_path) as f: @@ -838,7 +838,7 @@ with open(hook_path, "w") as f: f.write("#!/bin/bash\n#CUSTOM_HOOK") - ScmModel().install_git_hooks(repo=self.repo, force_create=True) + ScmModel().install_git_hooks(self.repo, force=True) for hook, hook_path in self.kallithea_hooks.items(): with open(hook_path) as f: diff -r eca0cb56a822 -r 3e9d079fcf91 kallithea/tests/vcs/test_workdirs.py --- a/kallithea/tests/vcs/test_workdirs.py Sun Jul 26 00:03:12 2020 +0200 +++ b/kallithea/tests/vcs/test_workdirs.py Sat Aug 22 20:53:43 2020 +0200 @@ -68,6 +68,7 @@ def test_checkout_branch(self): from kallithea.lib.vcs.exceptions import BranchDoesNotExistError + # first, 'foobranch' does not exist. with pytest.raises(BranchDoesNotExistError): self.repo.workdir.checkout_branch(branch='foobranch') diff -r eca0cb56a822 -r 3e9d079fcf91 scripts/i18n --- a/scripts/i18n Sun Jul 26 00:03:12 2020 +0200 +++ b/scripts/i18n Sat Aug 22 20:53:43 2020 +0200 @@ -19,7 +19,6 @@ import sys import click - import i18n_utils @@ -90,11 +89,8 @@ and then invoke merge/rebase/graft with the additional argument '--tool i18n'. """ - from mercurial import ( - context, - simplemerge, - ui as uimod, - ) + from mercurial import context, simplemerge + from mercurial import ui as uimod print('i18n normalized-merge: normalizing and merging %s' % output) diff -r eca0cb56a822 -r 3e9d079fcf91 setup.py --- a/setup.py Sun Jul 26 00:03:12 2020 +0200 +++ b/setup.py Sat Aug 22 20:53:43 2020 +0200 @@ -53,7 +53,7 @@ "FormEncode >= 1.3.1, < 1.4", "SQLAlchemy >= 1.2.9, < 1.4", "Mako >= 0.9.1, < 1.2", - "Pygments >= 2.2.0, < 2.6", + "Pygments >= 2.2.0, < 2.7", "Whoosh >= 2.7.1, < 2.8", "celery >= 4.3, < 4.5, != 4.4.4", # 4.4.4 is broken due to unexpressed dependency on 'future', see https://github.com/celery/celery/pull/6146 "Babel >= 1.3, < 2.9", @@ -63,9 +63,9 @@ "URLObject >= 2.3.4, < 2.5", "Routes >= 2.0, < 2.5", "dulwich >= 0.19.0, < 0.20", - "mercurial >= 5.2, < 5.5", + "mercurial >= 5.2, < 5.6", "decorator >= 4.2.1, < 4.5", - "Paste >= 2.0.3, < 3.4", + "Paste >= 2.0.3, < 3.5", "bleach >= 3.0, < 3.1.4", "Click >= 7.0, < 8", "ipaddr >= 2.2.0, < 2.3", @@ -156,6 +156,6 @@ kallithea-cli = kallithea.bin.kallithea_cli:cli [paste.app_factory] - main = kallithea.config.middleware:make_app + main = kallithea.config.application:make_app """, )