Mercurial > kallithea

comparison rhodecode/lib/indexers/daemon.py @ 2101:df96adcbb1f7 beta

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
code garden
author Marcin Kuzminski <marcin@python-works.com>
date Sun, 04 Mar 2012 19:57:42 +0000
parents 324ac367a4da
children 8ecfed1d8f8b
comparison
equal deleted inserted replaced
2100:f0649c7cf94a 2101:df96adcbb1f7
46 from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \ 46 from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
47 NodeDoesNotExistError 47 NodeDoesNotExistError
48 48
49 from whoosh.index import create_in, open_dir 49 from whoosh.index import create_in, open_dir
50 50
51 51 log = logging.getLogger('whoosh_indexer')
52 log = logging.getLogger('whooshIndexer')
53 # create logger
54 log.setLevel(logging.DEBUG)
55 log.propagate = False
56 # create console handler and set level to debug
57 ch = logging.StreamHandler()
58 ch.setLevel(logging.DEBUG)
59
60 # create formatter
61 formatter = logging.Formatter("%(asctime)s - %(name)s -"
62 " %(levelname)s - %(message)s")
63
64 # add formatter to ch
65 ch.setFormatter(formatter)
66
67 # add ch to logger
68 log.addHandler(ch)
69 52
70 53
71 class WhooshIndexingDaemon(object): 54 class WhooshIndexingDaemon(object):
72 """ 55 """
73 Daemon for atomic jobs 56 Daemon for atomic jobs
101 log.info('Cannot run incremental index since it does not' 84 log.info('Cannot run incremental index since it does not'
102 ' yet exist running full build') 85 ' yet exist running full build')
103 self.initial = True 86 self.initial = True
104 87
105 def get_paths(self, repo): 88 def get_paths(self, repo):
106 """recursive walk in root dir and return a set of all path in that dir 89 """
90 recursive walk in root dir and return a set of all path in that dir
107 based on repository walk function 91 based on repository walk function
108 """ 92 """
109 index_paths_ = set() 93 index_paths_ = set()
110 try: 94 try:
111 tip = repo.get_changeset('tip') 95 tip = repo.get_changeset('tip')
125 109
126 def get_node_mtime(self, node): 110 def get_node_mtime(self, node):
127 return mktime(node.last_changeset.date.timetuple()) 111 return mktime(node.last_changeset.date.timetuple())
128 112
129 def add_doc(self, writer, path, repo, repo_name): 113 def add_doc(self, writer, path, repo, repo_name):
130 """Adding doc to writer this function itself fetches data from 114 """
131 the instance of vcs backend""" 115 Adding doc to writer this function itself fetches data from
116 the instance of vcs backend
117 """
118
132 node = self.get_node(repo, path) 119 node = self.get_node(repo, path)
133 120
134 #we just index the content of chosen files, and skip binary files 121 # we just index the content of chosen files, and skip binary files
135 if node.extension in INDEX_EXTENSIONS and not node.is_binary: 122 if node.extension in INDEX_EXTENSIONS and not node.is_binary:
136 123
137 u_content = node.content 124 u_content = node.content
138 if not isinstance(u_content, unicode): 125 if not isinstance(u_content, unicode):
139 log.warning(' >> %s Could not get this content as unicode ' 126 log.warning(' >> %s Could not get this content as unicode '
140 'replacing with empty content', path) 127 'replacing with empty content' % path)
141 u_content = u'' 128 u_content = u''
142 else: 129 else:
143 log.debug(' >> %s [WITH CONTENT]' % path) 130 log.debug(' >> %s [WITH CONTENT]' % path)
144 131
145 else: 132 else:
146 log.debug(' >> %s' % path) 133 log.debug(' >> %s' % path)
147 #just index file name without it's content 134 # just index file name without it's content
148 u_content = u'' 135 u_content = u''
149 136
150 writer.add_document(owner=unicode(repo.contact), 137 writer.add_document(
151 repository=safe_unicode(repo_name), 138 owner=unicode(repo.contact),
152 path=safe_unicode(path), 139 repository=safe_unicode(repo_name),
153 content=u_content, 140 path=safe_unicode(path),
154 modtime=self.get_node_mtime(node), 141 content=u_content,
155 extension=node.extension) 142 modtime=self.get_node_mtime(node),
143 extension=node.extension
144 )
156 145
157 def build_index(self): 146 def build_index(self):
158 if os.path.exists(self.index_location): 147 if os.path.exists(self.index_location):
159 log.debug('removing previous index') 148 log.debug('removing previous index')
160 rmtree(self.index_location) 149 rmtree(self.index_location)