Mercurial > kallithea
changeset 2379:7ac09514a178 beta
created rhodecode-api binary script for working with api via cli
- created docs
- moved the backup script to bin folder
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Sun, 03 Jun 2012 20:35:13 +0200 |
| parents | 04ef27ce939e |
| children | 0c7dc3402efa |
| files | .hgignore docs/api/api.rst docs/changelog.rst rhodecode/bin/__init__.py rhodecode/bin/rhodecode_api.py rhodecode/bin/rhodecode_backup.py rhodecode/controllers/api/__init__.py rhodecode/controllers/api/api.py rhodecode/lib/backup_manager.py rhodecode/templates/admin/admin_log.html setup.py |
| diffstat | 10 files changed, 394 insertions(+), 119 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Sun Jun 03 20:24:02 2012 +0200 +++ b/.hgignore Sun Jun 03 20:35:13 2012 +0200 @@ -19,3 +19,4 @@ ^RhodeCode\.egg-info$ ^rc\.ini$ ^fabfile.py +^\.rhodecode$
--- a/docs/api/api.rst Sun Jun 03 20:24:02 2012 +0200 +++ b/docs/api/api.rst Sun Jun 03 20:35:13 2012 +0200 @@ -59,6 +59,47 @@ calling api *error* key from response will contain failure description and result will be null. + +API CLIENT +++++++++++ + +From version 1.4 RhodeCode adds a binary script that allows to easily +communicate with API. After installing RhodeCode a `rhodecode-api` script +will be available. + +To get started quickly simply run:: + + rhodecode-api _create_config --apikey=<youapikey> --apihost=<rhodecode host> + +This will create a file named .config in the directory you executed it storing +json config file with credentials. You can skip this step and always provide +both of the arguments to be able to communicate with server + + +after that simply run any api command for example get_repo:: + + rhodecode-api get_repo + + calling {"api_key": "<apikey>", "id": 75, "args": {}, "method": "get_repo"} to http://127.0.0.1:5000 + rhodecode said: + {'error': 'Missing non optional `repoid` arg in JSON DATA', + 'id': 75, + 'result': None} + +Ups looks like we forgot to add an argument + +Let's try again now giving the repoid as parameters:: + + rhodecode-api get_repo repoid:rhodecode + + calling {"api_key": "<apikey>", "id": 39, "args": {"repoid": "rhodecode"}, "method": "get_repo"} to http://127.0.0.1:5000 + rhodecode said: + {'error': None, + 'id': 39, + 'result': <json data...>} + + + API METHODS +++++++++++
--- a/docs/changelog.rst Sun Jun 03 20:24:02 2012 +0200 +++ b/docs/changelog.rst Sun Jun 03 20:35:13 2012 +0200 @@ -22,6 +22,7 @@ - #465 mentions autocomplete inside comments boxes - #469 added --update-only option to whoosh to re-index only given list of repos in index +- rhodecode-api CLI client fixes +++++
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rhodecode/bin/rhodecode_api.py Sun Jun 03 20:35:13 2012 +0200 @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +""" + rhodecode.bin.backup_manager + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Api CLI client for RhodeCode + + :created_on: Jun 3, 2012 + :author: marcink + :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> + :license: GPLv3, see COPYING for more details. +""" +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import with_statement +import os +import sys +import random +import urllib2 +import pprint +import argparse + +try: + from rhodecode.lib.ext_json import json +except ImportError: + try: + import simplejson as json + except ImportError: + import json + + +CONFIG_NAME = '.rhodecode' + + +class RcConf(object): + """ + RhodeCode config for API + + conf = RcConf() + conf['key'] + + """ + + def __init__(self, autoload=True, autocreate=False, config=None): + self._conf_name = CONFIG_NAME + self._conf = {} + if autocreate: + self.make_config(config) + if autoload: + self._conf = self.load_config() + + def __getitem__(self, key): + return self._conf[key] + + def __nonzero__(self): + if self._conf: + return True + return False + + def __eq__(self): + return self._conf.__eq__() + + def __repr__(self): + return 'RcConf<%s>' % self._conf.__repr__() + + def make_config(self, config): + """ + Saves given config as a JSON dump in the _conf_name location + + :param config: + :type config: + """ + with open(self._conf_name, 'wb') as f: + json.dump(config, f, indent=4) + sys.stdout.write('Updated conf\n') + + def update_config(self, new_config): + """ + Reads the JSON config updates it's values with new_config and + saves it back as JSON dump + + :param new_config: + """ + config = {} + try: + with open(self._conf_name, 'rb') as conf: + config = json.load(conf) + except IOError, e: + sys.stderr.write(str(e) + '\n') + + config.update(new_config) + self.make_config(config) + + def load_config(self): + """ + Loads config from file and returns loaded JSON object + """ + try: + with open(self._conf_name, 'rb') as conf: + return json.load(conf) + except IOError, e: + #sys.stderr.write(str(e) + '\n') + pass + + +def api_call(apikey, apihost, method=None, **kw): + """ + Api_call wrapper for RhodeCode + + :param apikey: + :param apihost: + :param method: + """ + def _build_data(random_id): + """ + Builds API data with given random ID + + :param random_id: + :type random_id: + """ + return { + "id": random_id, + "api_key": apikey, + "method": method, + "args": kw + } + + if not method: + raise Exception('please specify method name !') + id_ = random.randrange(1, 200) + req = urllib2.Request('%s/_admin/api' % apihost, + data=json.dumps(_build_data(id_)), + headers={'content-type': 'text/plain'}) + print 'calling %s to %s' % (req.get_data(), apihost) + ret = urllib2.urlopen(req) + json_data = json.loads(ret.read()) + id_ret = json_data['id'] + _formatted_json = pprint.pformat(json_data) + if id_ret == id_: + print 'rhodecode said:\n%s' % (_formatted_json) + else: + raise Exception('something went wrong. ' + 'ID mismatch got %s, expected %s | %s' % ( + id_ret, id_, _formatted_json)) + + +def argparser(argv): + usage = ("rhodecode_api [-h] [--apikey APIKEY] [--apihost APIHOST] " + "_create_config or METHOD <key:val> <key2:val> ...") + + parser = argparse.ArgumentParser(description='RhodeCode API cli', + usage=usage) + + ## config + group = parser.add_argument_group('config') + group.add_argument('--apikey', help='api access key') + group.add_argument('--apihost', help='api host') + + group = parser.add_argument_group('API') + group.add_argument('method', metavar='METHOD', type=str, + help='API method name to call followed by key:value attributes', + ) + + args, other = parser.parse_known_args() + return parser, args, other + + +def main(argv=None): + """ + Main execution function for cli + + :param argv: + :type argv: + """ + if argv is None: + argv = sys.argv + + conf = None + parser, args, other = argparser(argv) + + api_credentials_given = (args.apikey and args.apihost) + if args.method == '_create_config': + if not api_credentials_given: + raise parser.error('_create_config requires --apikey and --apihost') + conf = RcConf(autocreate=True, config={'apikey': args.apikey, + 'apihost': args.apihost}) + sys.stdout.write('Create new config in %s\n' % CONFIG_NAME) + + if not conf: + conf = RcConf(autoload=True) + if not conf: + if not api_credentials_given: + parser.error('Could not find config file and missing ' + '--apikey or --apihost in params') + + apikey = args.apikey or conf['apikey'] + host = args.apihost or conf['apihost'] + method = args.method + margs = dict(map(lambda s: s.split(':', 1), other)) + + api_call(apikey, host, method, **margs) + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rhodecode/bin/rhodecode_backup.py Sun Jun 03 20:35:13 2012 +0200 @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" + rhodecode.bin.backup_manager + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Repositories backup manager, it allows to backups all + repositories and send it to backup server using RSA key via ssh. + + :created_on: Feb 28, 2010 + :author: marcink + :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> + :license: GPLv3, see COPYING for more details. +""" +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import sys + +import logging +import tarfile +import datetime +import subprocess + +logging.basicConfig(level=logging.DEBUG, + format="%(asctime)s %(levelname)-5.5s %(message)s") + + +class BackupManager(object): + def __init__(self, repos_location, rsa_key, backup_server): + today = datetime.datetime.now().weekday() + 1 + self.backup_file_name = "rhodecode_repos.%s.tar.gz" % today + + self.id_rsa_path = self.get_id_rsa(rsa_key) + self.repos_path = self.get_repos_path(repos_location) + self.backup_server = backup_server + + self.backup_file_path = '/tmp' + + logging.info('starting backup for %s', self.repos_path) + logging.info('backup target %s', self.backup_file_path) + + def get_id_rsa(self, rsa_key): + if not os.path.isfile(rsa_key): + logging.error('Could not load id_rsa key file in %s', rsa_key) + sys.exit() + return rsa_key + + def get_repos_path(self, path): + if not os.path.isdir(path): + logging.error('Wrong location for repositories in %s', path) + sys.exit() + return path + + def backup_repos(self): + bckp_file = os.path.join(self.backup_file_path, self.backup_file_name) + tar = tarfile.open(bckp_file, "w:gz") + + for dir_name in os.listdir(self.repos_path): + logging.info('backing up %s', dir_name) + tar.add(os.path.join(self.repos_path, dir_name), dir_name) + tar.close() + logging.info('finished backup of mercurial repositories') + + def transfer_files(self): + params = { + 'id_rsa_key': self.id_rsa_path, + 'backup_file': os.path.join(self.backup_file_path, + self.backup_file_name), + 'backup_server': self.backup_server + } + cmd = ['scp', '-l', '40000', '-i', '%(id_rsa_key)s' % params, + '%(backup_file)s' % params, + '%(backup_server)s' % params] + + subprocess.call(cmd) + logging.info('Transfered file %s to %s', self.backup_file_name, cmd[4]) + + def rm_file(self): + logging.info('Removing file %s', self.backup_file_name) + os.remove(os.path.join(self.backup_file_path, self.backup_file_name)) + +if __name__ == "__main__": + + repo_location = '/home/repo_path' + backup_server = 'root@192.168.1.100:/backups/mercurial' + rsa_key = '/home/id_rsa' + + B_MANAGER = BackupManager(repo_location, rsa_key, backup_server) + B_MANAGER.backup_repos() + B_MANAGER.transfer_files() + B_MANAGER.rm_file()
--- a/rhodecode/controllers/api/__init__.py Sun Jun 03 20:24:02 2012 +0200 +++ b/rhodecode/controllers/api/__init__.py Sun Jun 03 20:35:13 2012 +0200 @@ -57,15 +57,16 @@ return str(self.message) -def jsonrpc_error(message, code=None): +def jsonrpc_error(message, retid=None, code=None): """ Generate a Response object with a JSON-RPC error body """ from pylons.controllers.util import Response - resp = Response(body=json.dumps(dict(id=None, result=None, error=message)), - status=code, - content_type='application/json') - return resp + return Response( + body=json.dumps(dict(id=retid, result=None, error=message)), + status=code, + content_type='application/json' + ) class JSONRPCController(WSGIController): @@ -94,9 +95,11 @@ Parse the request body as JSON, look up the method on the controller and if it exists, dispatch to it. """ + self._req_id = None if 'CONTENT_LENGTH' not in environ: log.debug("No Content-Length") - return jsonrpc_error(message="No Content-Length in request") + return jsonrpc_error(retid=self._req_id, + message="No Content-Length in request") else: length = environ['CONTENT_LENGTH'] or 0 length = int(environ['CONTENT_LENGTH']) @@ -104,7 +107,8 @@ if length == 0: log.debug("Content-Length is 0") - return jsonrpc_error(message="Content-Length is 0") + return jsonrpc_error(retid=self._req_id, + message="Content-Length is 0") raw_body = environ['wsgi.input'].read(length) @@ -112,7 +116,8 @@ json_body = json.loads(urllib.unquote_plus(raw_body)) except ValueError, e: # catch JSON errors Here - return jsonrpc_error(message="JSON parse error ERR:%s RAW:%r" \ + return jsonrpc_error(retid=self._req_id, + message="JSON parse error ERR:%s RAW:%r" \ % (e, urllib.unquote_plus(raw_body))) # check AUTH based on API KEY @@ -126,22 +131,26 @@ self._request_params) ) except KeyError, e: - return jsonrpc_error(message='Incorrect JSON query missing %s' % e) + return jsonrpc_error(retid=self._req_id, + message='Incorrect JSON query missing %s' % e) # check if we can find this session using api_key try: u = User.get_by_api_key(self._req_api_key) if u is None: - return jsonrpc_error(message='Invalid API KEY') + return jsonrpc_error(retid=self._req_id, + message='Invalid API KEY') auth_u = AuthUser(u.user_id, self._req_api_key) except Exception, e: - return jsonrpc_error(message='Invalid API KEY') + return jsonrpc_error(retid=self._req_id, + message='Invalid API KEY') self._error = None try: self._func = self._find_method() except AttributeError, e: - return jsonrpc_error(message=str(e)) + return jsonrpc_error(retid=self._req_id, + message=str(e)) # now that we have a method, add self._req_params to # self.kargs and dispatch control to WGIController @@ -164,9 +173,12 @@ USER_SESSION_ATTR = 'apiuser' if USER_SESSION_ATTR not in arglist: - return jsonrpc_error(message='This method [%s] does not support ' - 'authentication (missing %s param)' % - (self._func.__name__, USER_SESSION_ATTR)) + return jsonrpc_error( + retid=self._req_id, + message='This method [%s] does not support ' + 'authentication (missing %s param)' % ( + self._func.__name__, USER_SESSION_ATTR) + ) # get our arglist and check if we provided them as args for arg, default in func_kwargs.iteritems(): @@ -179,6 +191,7 @@ # NotImplementedType (default_empty) if (default == default_empty and arg not in self._request_params): return jsonrpc_error( + retid=self._req_id, message=( 'Missing non optional `%s` arg in JSON DATA' % arg )
--- a/rhodecode/controllers/api/api.py Sun Jun 03 20:24:02 2012 +0200 +++ b/rhodecode/controllers/api/api.py Sun Jun 03 20:35:13 2012 +0200 @@ -389,7 +389,7 @@ repo = RepoModel().get_repo(repoid) if repo is None: - raise JSONRPCError('unknown repository %s' % repo) + raise JSONRPCError('unknown repository "%s"' % (repo or repoid)) members = [] for user in repo.repo_to_perm:
--- a/rhodecode/lib/backup_manager.py Sun Jun 03 20:24:02 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -""" - rhodecode.lib.backup_manager - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Mercurial repositories backup manager, it allows to backups all - repositories and send it to backup server using RSA key via ssh. - - :created_on: Feb 28, 2010 - :author: marcink - :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> - :license: GPLv3, see COPYING for more details. -""" -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys - -import logging -import tarfile -import datetime -import subprocess - -logging.basicConfig(level=logging.DEBUG, - format="%(asctime)s %(levelname)-5.5s %(message)s") - - -class BackupManager(object): - def __init__(self, repos_location, rsa_key, backup_server): - today = datetime.datetime.now().weekday() + 1 - self.backup_file_name = "mercurial_repos.%s.tar.gz" % today - - self.id_rsa_path = self.get_id_rsa(rsa_key) - self.repos_path = self.get_repos_path(repos_location) - self.backup_server = backup_server - - self.backup_file_path = '/tmp' - - logging.info('starting backup for %s', self.repos_path) - logging.info('backup target %s', self.backup_file_path) - - def get_id_rsa(self, rsa_key): - if not os.path.isfile(rsa_key): - logging.error('Could not load id_rsa key file in %s', rsa_key) - sys.exit() - return rsa_key - - def get_repos_path(self, path): - if not os.path.isdir(path): - logging.error('Wrong location for repositories in %s', path) - sys.exit() - return path - - def backup_repos(self): - bckp_file = os.path.join(self.backup_file_path, self.backup_file_name) - tar = tarfile.open(bckp_file, "w:gz") - - for dir_name in os.listdir(self.repos_path): - logging.info('backing up %s', dir_name) - tar.add(os.path.join(self.repos_path, dir_name), dir_name) - tar.close() - logging.info('finished backup of mercurial repositories') - - def transfer_files(self): - params = { - 'id_rsa_key': self.id_rsa_path, - 'backup_file': os.path.join(self.backup_file_path, - self.backup_file_name), - 'backup_server': self.backup_server - } - cmd = ['scp', '-l', '40000', '-i', '%(id_rsa_key)s' % params, - '%(backup_file)s' % params, - '%(backup_server)s' % params] - - subprocess.call(cmd) - logging.info('Transfered file %s to %s', self.backup_file_name, cmd[4]) - - def rm_file(self): - logging.info('Removing file %s', self.backup_file_name) - os.remove(os.path.join(self.backup_file_path, self.backup_file_name)) - -if __name__ == "__main__": - - repo_location = '/home/repo_path' - backup_server = 'root@192.168.1.100:/backups/mercurial' - rsa_key = '/home/id_rsa' - - B_MANAGER = BackupManager(repo_location, rsa_key, backup_server) - B_MANAGER.backup_repos() - B_MANAGER.transfer_files() - B_MANAGER.rm_file()
--- a/rhodecode/templates/admin/admin_log.html Sun Jun 03 20:24:02 2012 +0200 +++ b/rhodecode/templates/admin/admin_log.html Sun Jun 03 20:35:13 2012 +0200 @@ -14,7 +14,7 @@ <td>${h.link_to(l.user.username,h.url('edit_user', id=l.user.user_id))}</td> <td>${h.action_parser(l)[0]()} <div class="journal_action_params"> - ${h.literal(h.action_parser(l)[1]())} + ${h.literal(h.action_parser(l)[1]())} </div> </td> <td>
--- a/setup.py Sun Jun 03 20:24:02 2012 +0200 +++ b/setup.py Sun Jun 03 20:35:13 2012 +0200 @@ -87,6 +87,9 @@ zip_safe=False, paster_plugins=['PasteScript', 'Pylons'], entry_points=""" + [console_scripts] + rhodecode-api = rhodecode.bin.rhodecode_api:main + [paste.app_factory] main = rhodecode.config.middleware:make_app