Mercurial > kallithea

annotate rhodecode/lib/celerylib/__init__.py @ 558:14559eb34003

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
more error catching on celery run_task
author Marcin Kuzminski <marcin@python-works.com>
date Thu, 07 Oct 2010 19:24:03 +0200
parents 1e757ac98988
children d85b0948e539 f6c613fba757
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
547
1e757ac98988 renamed project to rhodecode
Marcin Kuzminski <marcin@python-works.com>
parents: 510
diff changeset
1 from rhodecode.lib.pidlock import DaemonLock, LockHeld
467
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
2 from vcs.utils.lazy import LazyProperty
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
3 from decorator import decorator
467
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
4 import logging
474
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 467
diff changeset
5 import os
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 467
diff changeset
6 import sys
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 467
diff changeset
7 import traceback
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
8 from hashlib import md5
558
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
9 import socket
467
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
10 log = logging.getLogger(__name__)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
11
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
12 class ResultWrapper(object):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
13 def __init__(self, task):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
14 self.task = task
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
15
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
16 @LazyProperty
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
17 def result(self):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
18 return self.task
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
19
487
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 474
diff changeset
20 def run_task(task, *args, **kwargs):
467
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
21 try:
487
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 474
diff changeset
22 t = task.delay(*args, **kwargs)
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 474
diff changeset
23 log.info('running task %s', t.task_id)
467
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
24 return t
558
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
25 except socket.error, e:
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
26 if e.errno == 111:
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
27 log.debug('Unable to connect to celeryd. Sync execution')
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
28 else:
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
29 log.error(traceback.format_exc())
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
30 except KeyError, e:
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
31 log.debug('Unable to connect to celeryd. Sync execution')
487
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 474
diff changeset
32 except Exception, e:
558
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
33 log.error(traceback.format_exc())
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
34
14559eb34003 more error catching on celery run_task
Marcin Kuzminski <marcin@python-works.com>
parents: 547
diff changeset
35 return ResultWrapper(task(*args, **kwargs))
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
36
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
37
502
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
38 def locked_task(func):
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
39 def __wrapper(func, *fargs, **fkwargs):
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
40 params = list(fargs)
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
41 params.extend(['%s-%s' % ar for ar in fkwargs.items()])
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
42
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
43 lockkey = 'task_%s' % \
502
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
44 md5(str(func.__name__) + '-' + \
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
45 '-'.join(map(str, params))).hexdigest()
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
46 log.info('running task with lockkey %s', lockkey)
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
47 try:
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
48 l = DaemonLock(lockkey)
510
9bedaa073c23 fixed lockdecrator to return executed function data
Marcin Kuzminski <marcin@python-works.com>
parents: 506
diff changeset
49 ret = func(*fargs, **fkwargs)
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
50 l.release()
510
9bedaa073c23 fixed lockdecrator to return executed function data
Marcin Kuzminski <marcin@python-works.com>
parents: 506
diff changeset
51 return ret
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
52 except LockHeld:
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
53 log.info('LockHeld')
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
54 return 'Task with key %s already running' % lockkey
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
55
502
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 497
diff changeset
56 return decorator(__wrapper, func)
497
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
57
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
58
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
59
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
60
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
61
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
62
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
63
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 487
diff changeset
64