# HG changeset patch # User Thomas De Schampheleire # Date 1542567737 -3600 # Node ID 977e7aca38b16f9c099e6c846c63dc1d0bd08120 # Parent f10d56cb055cef27c4f8e9bd33507b87e45ebca9 cli: convert 'gearbox cleanup-repos' into 'kallithea-cli repo-purge-deleted' Some changes to user-facing text are done. diff -r f10d56cb055c -r 977e7aca38b1 docs/usage/general.rst --- a/docs/usage/general.rst Wed Oct 03 21:47:43 2018 +0200 +++ b/docs/usage/general.rst Sun Nov 18 20:02:17 2018 +0100 @@ -8,18 +8,18 @@ Repository deletion ------------------- -Currently when an admin or owner deletes a repository, Kallithea does +When an admin or owner deletes a repository, Kallithea does not physically delete said repository from the filesystem, but instead renames it in a special way so that it is not possible to push, clone or access the repository. There is a special command for cleaning up such archived repositories:: - gearbox cleanup-repos --older-than=30d -c my.ini + kallithea-cli repo-purge-deleted -c my.ini --older-than=30d This command scans for archived repositories that are older than 30 days, displays them, and asks if you want to delete them (unless given -the ``--dont-ask`` flag). If you host a large amount of repositories with +the ``--no-ask`` flag). If you host a large amount of repositories with forks that are constantly being deleted, it is recommended that you run this command via crontab. diff -r f10d56cb055c -r 977e7aca38b1 kallithea/bin/kallithea_cli_repo.py --- a/kallithea/bin/kallithea_cli_repo.py Wed Oct 03 21:47:43 2018 +0200 +++ b/kallithea/bin/kallithea_cli_repo.py Sun Nov 18 20:02:17 2018 +0100 @@ -22,9 +22,15 @@ import click import kallithea.bin.kallithea_cli_base as cli_base -from kallithea.lib.utils import repo2db_mapper -from kallithea.lib.utils2 import safe_unicode -from kallithea.model.db import Repository +import datetime +import os +import re +import shutil + +from kallithea.lib.paster_commands.common import ask_ok +from kallithea.lib.utils import repo2db_mapper, REMOVED_REPO_PAT +from kallithea.lib.utils2 import safe_unicode, safe_str +from kallithea.model.db import Repository, Ui from kallithea.model.meta import Session from kallithea.model.scm import ScmModel @@ -83,3 +89,89 @@ click.echo('Updated database with information about latest change in the following %s repositories:' % (len(repo_list))) click.echo('\n'.join(repo.repo_name for repo in repo_list)) + +@cli_base.register_command(config_file_initialize_app=True) +@click.option('--ask/--no-ask', default=True, help='Ask for confirmation or not. Default is --ask.') +@click.option('--older-than', + help="""Only purge repositories that have been removed at least the given time ago. + For example, '--older-than=30d' purges repositories deleted 30 days ago or longer. + Possible suffixes: d (days), h (hours), m (minutes), s (seconds).""") +def repo_purge_deleted(ask, older_than): + """Purge backups of deleted repositories. + + When a repository is deleted via the Kallithea web interface, the actual + data is still present on the filesystem but set aside using a special name. + This command allows to delete these files permanently. + """ + def _parse_older_than(val): + regex = re.compile(r'((?P\d+?)d)?((?P\d+?)h)?((?P\d+?)m)?((?P\d+?)s)?') + parts = regex.match(val) + if not parts: + return + parts = parts.groupdict() + time_params = {} + for (name, param) in parts.iteritems(): + if param: + time_params[name] = int(param) + return datetime.timedelta(**time_params) + + def _extract_date(name): + """ + Extract the date part from rm__ pattern of removed repos, + and convert it to datetime object + + :param name: + """ + date_part = name[4:19] # 4:19 since we don't parse milliseconds + return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S') + + repos_location = Ui.get_repos_location() + to_remove = [] + for dn_, dirs, f in os.walk(safe_str(repos_location)): + alldirs = list(dirs) + del dirs[:] + if ('.hg' in alldirs or + '.git' in alldirs or + '.svn' in alldirs or + 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)): + continue + for loc in alldirs: + if REMOVED_REPO_PAT.match(loc): + to_remove.append([os.path.join(dn_, loc), + _extract_date(loc)]) + else: + dirs.append(loc) + if dirs: + click.echo('Scanning: %s' % dn_) + + # filter older than (if present)! + now = datetime.datetime.now() + if older_than: + to_remove_filtered = [] + older_than_date = _parse_older_than(older_than) + for name, date_ in to_remove: + repo_age = now - date_ + if repo_age > older_than_date: + to_remove_filtered.append([name, date_]) + + to_remove = to_remove_filtered + click.echo('Purging %s deleted repos older than %s (%s)' + % (len(to_remove), older_than, older_than_date)) + else: + click.echo('Purging all %s deleted repos' % len(to_remove)) + + if not ask or not to_remove: + # don't ask just remove ! + remove = True + else: + remove = ask_ok('The following repositories will be removed completely:\n%s\n' + 'Do you want to proceed? [y/n] ' + % '\n'.join(['%s deleted on %s' % (safe_str(x[0]), safe_str(x[1])) + for x in to_remove])) + + if remove: + for path, date_ in to_remove: + click.echo('Purging repository %s' % path) + shutil.rmtree(path) + else: + click.echo('Nothing done, exiting...') diff -r f10d56cb055c -r 977e7aca38b1 kallithea/lib/paster_commands/cleanup.py --- a/kallithea/lib/paster_commands/cleanup.py Wed Oct 03 21:47:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -kallithea.lib.paster_commands.cleanup -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -cleanup-repos gearbox command for Kallithea - - -This file was forked by the Kallithea project in July 2014. -Original author and date, and relevant copyright and licensing information is below: -:created_on: Jul 14, 2012 -:author: marcink -:copyright: (c) 2013 RhodeCode GmbH, and others. -:license: GPLv3, see LICENSE.md for more details. -""" - - -import os -import re -import shutil -import datetime - -from kallithea.lib.paster_commands.common import ask_ok, BasePasterCommand -from kallithea.lib.utils import REMOVED_REPO_PAT -from kallithea.lib.utils2 import safe_str -from kallithea.model.db import Ui - - -class Command(BasePasterCommand): - """Kallithea: Cleanup of backup files of deleted repositories""" - - def _parse_older_than(self, val): - regex = re.compile(r'((?P\d+?)d)?((?P\d+?)h)?((?P\d+?)m)?((?P\d+?)s)?') - parts = regex.match(val) - if not parts: - return - parts = parts.groupdict() - time_params = {} - for (name, param) in parts.iteritems(): - if param: - time_params[name] = int(param) - return datetime.timedelta(**time_params) - - def _extract_date(self, name): - """ - Extract the date part from rm__ pattern of removed repos, - and convert it to datetime object - - :param name: - """ - date_part = name[4:19] # 4:19 since we don't parse milliseconds - return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S') - - def take_action(self, args): - repos_location = Ui.get_repos_location() - to_remove = [] - for dn_, dirs, f in os.walk(safe_str(repos_location)): - alldirs = list(dirs) - del dirs[:] - if ('.hg' in alldirs or - '.git' in alldirs or - '.svn' in alldirs or - 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)): - continue - for loc in alldirs: - if REMOVED_REPO_PAT.match(loc): - to_remove.append([os.path.join(dn_, loc), - self._extract_date(loc)]) - else: - dirs.append(loc) - if dirs: - print 'Scanning: %s' % dn_ - - # filter older than (if present)! - now = datetime.datetime.now() - older_than = args.older_than - if older_than: - to_remove_filtered = [] - older_than_date = self._parse_older_than(older_than) - for name, date_ in to_remove: - repo_age = now - date_ - if repo_age > older_than_date: - to_remove_filtered.append([name, date_]) - - to_remove = to_remove_filtered - print 'Removing %s deleted repos older than %s (%s)' \ - % (len(to_remove), older_than, older_than_date) - else: - print 'Removing all %s deleted repos' % len(to_remove) - if args.dont_ask or not to_remove: - # don't ask just remove ! - remove = True - else: - remove = ask_ok('the following repositories will be deleted completely:\n%s\n' - 'are you sure you want to remove them [y/n]?' - % '\n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1])) - for x in to_remove])) - - if remove: - for path, date_ in to_remove: - print 'Removing repository %s' % path - shutil.rmtree(path) - else: - print 'Nothing done, exiting...' - - def get_parser(self, prog_name): - parser = super(Command, self).get_parser(prog_name) - - parser.add_argument( - '--older-than', - action='store', - dest='older_than', - help=("only remove repos that have been removed " - "at least given time ago. " - "The default is to remove all removed repositories. " - "Possible suffixes: " - "d (days), h (hours), m (minutes), s (seconds). " - "For example --older-than=30d deletes repositories " - "removed more than 30 days ago.") - ) - - parser.add_argument( - '--dont-ask', - action="store_true", - dest="dont_ask", - help="remove repositories without asking for confirmation." - ) - - return parser diff -r f10d56cb055c -r 977e7aca38b1 setup.py --- a/setup.py Wed Oct 03 21:47:43 2018 +0200 +++ b/setup.py Sun Nov 18 20:02:17 2018 +0100 @@ -160,7 +160,6 @@ [gearbox.commands] celeryd=kallithea.lib.paster_commands.celeryd:Command - cleanup-repos=kallithea.lib.paster_commands.cleanup:Command install-iis=kallithea.lib.paster_commands.install_iis:Command make-index=kallithea.lib.paster_commands.make_index:Command make-rcext=kallithea.lib.paster_commands.make_rcextensions:Command