Mercurial > kallithea

comparison rhodecode/lib/indexers/daemon.py @ 885:94f7585af8a1 beta

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fixes to #92, updated changelog
author Marcin Kuzminski <marcin@python-works.com>
date Tue, 28 Dec 2010 18:44:32 +0100
parents aac24db58ce8
children 0736230c7d91
comparison
equal deleted inserted replaced
884:322b53be49cc 885:94f7585af8a1
1 #!/usr/bin/env python 1 # -*- coding: utf-8 -*-
2 # encoding: utf-8 2 """
3 # whoosh indexer daemon for rhodecode 3 rhodecode.lib.indexers.daemon
4 # Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5 # 5
6 A deamon will read from task table and run tasks
7
8 :created_on: Jan 26, 2010
9 :author: marcink
10 :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details.
12 """
6 # This program is free software; you can redistribute it and/or 13 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License 14 # modify it under the terms of the GNU General Public License
8 # as published by the Free Software Foundation; version 2 15 # as published by the Free Software Foundation; version 2
9 # of the License or (at your opinion) any later version of the license. 16 # of the License or (at your opinion) any later version of the license.
10 # 17 #
15 # 22 #
16 # You should have received a copy of the GNU General Public License 23 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software 24 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 25 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19 # MA 02110-1301, USA. 26 # MA 02110-1301, USA.
20 """ 27
21 Created on Jan 26, 2010
22
23 @author: marcink
24 A deamon will read from task table and run tasks
25 """
26 import sys 28 import sys
27 import os 29 import os
30 import traceback
28 from os.path import dirname as dn 31 from os.path import dirname as dn
29 from os.path import join as jn 32 from os.path import join as jn
30 33
31 #to get the rhodecode import 34 #to get the rhodecode import
32 project_path = dn(dn(dn(dn(os.path.realpath(__file__))))) 35 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
97 index_paths_.add(jn(repo.path, f.path)) 100 index_paths_.add(jn(repo.path, f.path))
98 for dir in dirs: 101 for dir in dirs:
99 for f in files: 102 for f in files:
100 index_paths_.add(jn(repo.path, f.path)) 103 index_paths_.add(jn(repo.path, f.path))
101 104
102 except RepositoryError: 105 except RepositoryError, e:
106 log.debug(traceback.format_exc())
103 pass 107 pass
104 return index_paths_ 108 return index_paths_
105 109
106 def get_node(self, repo, path): 110 def get_node(self, repo, path):
107 n_path = path[len(repo.path) + 1:] 111 n_path = path[len(repo.path) + 1:]
116 the instance of vcs backend""" 120 the instance of vcs backend"""
117 node = self.get_node(repo, path) 121 node = self.get_node(repo, path)
118 122
119 #we just index the content of chosen files 123 #we just index the content of chosen files
120 if node.extension in INDEX_EXTENSIONS: 124 if node.extension in INDEX_EXTENSIONS:
121 log.debug(' >> %s [WITH CONTENT]' % path) 125
122 u_content = node.content 126 u_content = node.content
127 if not isinstance(u_content, unicode):
128 log.warning(' >> %s Could not get this content as unicode '
129 'replacing with empty content', path)
130 u_content = u''
131 else:
132 log.debug(' >> %s [WITH CONTENT]' % path)
133
123 else: 134 else:
124 log.debug(' >> %s' % path) 135 log.debug(' >> %s' % path)
125 #just index file name without it's content 136 #just index file name without it's content
126 u_content = u'' 137 u_content = u''
127 138
141 if not os.path.exists(self.index_location): 152 if not os.path.exists(self.index_location):
142 os.mkdir(self.index_location) 153 os.mkdir(self.index_location)
143 154
144 idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME) 155 idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
145 writer = idx.writer() 156 writer = idx.writer()
146 157 print self.repo_paths.values()
147 for cnt, repo in enumerate(self.repo_paths.values()): 158 for cnt, repo in enumerate(self.repo_paths.values()):
148 log.debug('building index @ %s' % repo.path) 159 log.debug('building index @ %s' % repo.path)
149 160
150 for idx_path in self.get_paths(repo): 161 for idx_path in self.get_paths(repo):
151 self.add_doc(writer, idx_path, repo) 162 self.add_doc(writer, idx_path, repo)