Mercurial > kallithea

comparison rhodecode/lib/indexers/daemon.py @ 2031:82a88013a3fd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge 1.3 into stable
author Marcin Kuzminski <marcin@python-works.com>
date Sun, 26 Feb 2012 17:25:09 +0200
parents bf263968da47 324ac367a4da
children dc2584ba5fbc
comparison
equal deleted inserted replaced
2005:ab0e122b38a7 2031:82a88013a3fd
5 5
6 A daemon will read from task table and run tasks 6 A daemon will read from task table and run tasks
7 7
8 :created_on: Jan 26, 2010 8 :created_on: Jan 26, 2010
9 :author: marcink 9 :author: marcink
10 :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> 10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details. 11 :license: GPLv3, see COPYING for more details.
12 """ 12 """
13 # This program is free software: you can redistribute it and/or modify 13 # This program is free software: you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by 14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation, either version 3 of the License, or 15 # the Free Software Foundation, either version 3 of the License, or
41 41
42 from rhodecode.model.scm import ScmModel 42 from rhodecode.model.scm import ScmModel
43 from rhodecode.lib import safe_unicode 43 from rhodecode.lib import safe_unicode
44 from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME 44 from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
45 45
46 from vcs.exceptions import ChangesetError, RepositoryError 46 from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
47 NodeDoesNotExistError
47 48
48 from whoosh.index import create_in, open_dir 49 from whoosh.index import create_in, open_dir
49
50 50
51 51
52 log = logging.getLogger('whooshIndexer') 52 log = logging.getLogger('whooshIndexer')
53 # create logger 53 # create logger
54 log.setLevel(logging.DEBUG) 54 log.setLevel(logging.DEBUG)
65 ch.setFormatter(formatter) 65 ch.setFormatter(formatter)
66 66
67 # add ch to logger 67 # add ch to logger
68 log.addHandler(ch) 68 log.addHandler(ch)
69 69
70
70 class WhooshIndexingDaemon(object): 71 class WhooshIndexingDaemon(object):
71 """ 72 """
72 Daemon for atomic jobs 73 Daemon for atomic jobs
73 """ 74 """
74 75
75 def __init__(self, indexname='HG_INDEX', index_location=None, 76 def __init__(self, indexname=IDX_NAME, index_location=None,
76 repo_location=None, sa=None, repo_list=None): 77 repo_location=None, sa=None, repo_list=None):
77 self.indexname = indexname 78 self.indexname = indexname
78 79
79 self.index_location = index_location 80 self.index_location = index_location
80 if not index_location: 81 if not index_location:
91 for repo_name, repo in self.repo_paths.items(): 92 for repo_name, repo in self.repo_paths.items():
92 if repo_name in repo_list: 93 if repo_name in repo_list:
93 filtered_repo_paths[repo_name] = repo 94 filtered_repo_paths[repo_name] = repo
94 95
95 self.repo_paths = filtered_repo_paths 96 self.repo_paths = filtered_repo_paths
96
97 97
98 self.initial = False 98 self.initial = False
99 if not os.path.isdir(self.index_location): 99 if not os.path.isdir(self.index_location):
100 os.makedirs(self.index_location) 100 os.makedirs(self.index_location)
101 log.info('Cannot run incremental index since it does not' 101 log.info('Cannot run incremental index since it does not'
152 path=safe_unicode(path), 152 path=safe_unicode(path),
153 content=u_content, 153 content=u_content,
154 modtime=self.get_node_mtime(node), 154 modtime=self.get_node_mtime(node),
155 extension=node.extension) 155 extension=node.extension)
156 156
157
158 def build_index(self): 157 def build_index(self):
159 if os.path.exists(self.index_location): 158 if os.path.exists(self.index_location):
160 log.debug('removing previous index') 159 log.debug('removing previous index')
161 rmtree(self.index_location) 160 rmtree(self.index_location)
162 161
173 self.add_doc(writer, idx_path, repo, repo_name) 172 self.add_doc(writer, idx_path, repo, repo_name)
174 173
175 log.debug('>> COMMITING CHANGES <<') 174 log.debug('>> COMMITING CHANGES <<')
176 writer.commit(merge=True) 175 writer.commit(merge=True)
177 log.debug('>>> FINISHED BUILDING INDEX <<<') 176 log.debug('>>> FINISHED BUILDING INDEX <<<')
178
179 177
180 def update_index(self): 178 def update_index(self):
181 log.debug('STARTING INCREMENTAL INDEXING UPDATE') 179 log.debug('STARTING INCREMENTAL INDEXING UPDATE')
182 180
183 idx = open_dir(self.index_location, indexname=self.indexname) 181 idx = open_dir(self.index_location, indexname=self.indexname)
196 194
197 repo = self.repo_paths[fields['repository']] 195 repo = self.repo_paths[fields['repository']]
198 196
199 try: 197 try:
200 node = self.get_node(repo, indexed_path) 198 node = self.get_node(repo, indexed_path)
201 except ChangesetError: 199 except (ChangesetError, NodeDoesNotExistError):
202 # This file was deleted since it was indexed 200 # This file was deleted since it was indexed
203 log.debug('removing from index %s' % indexed_path) 201 log.debug('removing from index %s' % indexed_path)
204 writer.delete_by_term('path', indexed_path) 202 writer.delete_by_term('path', indexed_path)
205 203
206 else: 204 else: