Mercurial > kallithea
changeset 683:341beaa9edba beta
Implemented whoosh index building as paster command.
docs update
fixed manifest.in for missing yui file.
Fixed setup to beta
added base for paster upgrade-db command
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Sat, 13 Nov 2010 02:29:46 +0100 |
| parents | 23c2a0e6df0b |
| children | 2abb398cd9a7 |
| files | MANIFEST.in development.ini docs/changelog.rst docs/setup.rst production.ini rhodecode/config/deployment.ini_tmpl rhodecode/lib/indexers/__init__.py rhodecode/lib/indexers/daemon.py setup.cfg setup.py |
| diffstat | 10 files changed, 57 insertions(+), 35 deletions(-) [+] |
line wrap: on
line diff
--- a/MANIFEST.in Sat Nov 13 00:23:36 2010 +0100 +++ b/MANIFEST.in Sat Nov 13 02:29:46 2010 +0100 @@ -7,7 +7,7 @@ recursive-include rhodecode/public/css * recursive-include rhodecode/public/images * #js -include rhodecode/public/js/yui2.js +include rhodecode/public/js/yui2a.js include rhodecode/public/js/excanvas.min.js include rhodecode/public/js/yui.flot.js include rhodecode/public/js/graph.js
--- a/development.ini Sat Nov 13 00:23:36 2010 +0100 +++ b/development.ini Sat Nov 13 02:29:46 2010 +0100 @@ -43,6 +43,7 @@ static_files = true lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index #################################### ### BEAKER CACHE ####
--- a/docs/changelog.rst Sat Nov 13 00:23:36 2010 +0100 +++ b/docs/changelog.rst Sat Nov 13 02:29:46 2010 +0100 @@ -17,6 +17,7 @@ - introduced new enhanced changelog for merges that shows more accurate results - gui optimizations, fixed application width to 1024px - numerous small bugfixes +- whoosh index moved to paster command 1.0.2 (**2010-11-12**) ----------------------
--- a/docs/setup.rst Sat Nov 13 00:23:36 2010 +0100 +++ b/docs/setup.rst Sat Nov 13 02:29:46 2010 +0100 @@ -41,21 +41,40 @@ remember to update these if needed. -Setting up Whoosh ------------------ +Setting up Whoosh full text search +---------------------------------- + +Index for whoosh can be build starting from version 1.1 using paster command +passing repo locations to index, as well as Your config file that stores +whoosh index files locations. There is possible to pass `-f` to the options +to enable full index rebuild. Without that indexing will run always in in +incremental mode. + +:: + paster make-index --repo-location=<location for repos> production.ini + +for full index rebuild You can use + +:: + paster make-index -f --repo-location=<location for repos> production.ini - For full text search You can either put crontab entry for +This command can be run even from crontab in order to do periodical +index builds and keep Your index always up to date. An example entry might +look like this + :: - python /var/www/rhodecode/<rhodecode_installation_path>/lib/indexers/daemon.py incremental <put_here_path_to_repos> + /path/to/python/bin/paster --repo-location=<location for repos> /path/to/rhodecode/production.ini -When using incremental mode whoosh will check last modification date of each file -and add it to reindex if newer file is available. Also indexing daemon checks -for removed files and removes them from index. Sometime You might want to rebuild -index from scrach, in admin pannel You can check `build from scratch` flag -and in standalone daemon You can pass `full` instead on incremental to build -remove previos index and build new one. +When using incremental(default) mode whoosh will check last modification date +of each file and add it to reindex if newer file is available. Also indexing +daemon checks for removed files and removes them from index. + +Sometime You might want to rebuild index from scratch. You can do that using +the `-f` flag passed to paster command or, in admin panel You can check +`build from scratch` flag. Nginx virtual host example --------------------------
--- a/production.ini Sat Nov 13 00:23:36 2010 +0100 +++ b/production.ini Sat Nov 13 02:29:46 2010 +0100 @@ -43,6 +43,7 @@ static_files = false lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index #################################### ### BEAKER CACHE ####
--- a/rhodecode/config/deployment.ini_tmpl Sat Nov 13 00:23:36 2010 +0100 +++ b/rhodecode/config/deployment.ini_tmpl Sat Nov 13 02:29:46 2010 +0100 @@ -43,6 +43,7 @@ static_files = true lang=en cache_dir = %(here)s/data +index_dir = %(here)s/data/index app_instance_uuid = ${app_instance_uuid} ####################################
--- a/rhodecode/lib/indexers/__init__.py Sat Nov 13 00:23:36 2010 +0100 +++ b/rhodecode/lib/indexers/__init__.py Sat Nov 13 02:29:46 2010 +0100 @@ -59,14 +59,14 @@ usage = "CONFIG_FILE" summary = "Creates index for full text search given configuration file" - group_name = "Whoosh indexing" - + group_name = "RhodeCode" + takes_config_file = -1 parser = command.Command.standard_parser(verbose=True) -# parser.add_option('--repo-location', -# action='store', -# dest='repo_location', -# help="Specifies repositories location to index", -# ) + parser.add_option('--repo-location', + action='store', + dest='repo_location', + help="Specifies repositories location to index REQUIRED", + ) parser.add_option('-f', action='store_true', dest='full_index', @@ -75,27 +75,23 @@ default=False) def command(self): config_name = self.args[0] - p = config_name.split('/') - if len(p) == 1: - root = '.' - else: - root = '/'.join(p[:-1]) - print root + root = '.' if len(p) == 1 else '/'.join(p[:-1]) config = ConfigParser.ConfigParser({'here':root}) config.read(config_name) - print dict(config.items('app:main'))['index_dir'] + index_location = dict(config.items('app:main'))['index_dir'] - #return + repo_location = self.options.repo_location - #======================================================================= + #====================================================================== # WHOOSH DAEMON - #======================================================================= + #====================================================================== from rhodecode.lib.pidlock import LockHeld, DaemonLock from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon try: l = DaemonLock() - WhooshIndexingDaemon(index_location=index_location)\ + WhooshIndexingDaemon(index_location=index_location, + repo_location=repo_location)\ .run(full_index=self.options.full_index) l.release() except LockHeld:
--- a/rhodecode/lib/indexers/daemon.py Sat Nov 13 00:23:36 2010 +0100 +++ b/rhodecode/lib/indexers/daemon.py Sat Nov 13 02:29:46 2010 +0100 @@ -78,9 +78,7 @@ if not repo_location: raise Exception('You have to provide repositories location') - - - self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True) + self.repo_paths = HgModel().repo_scan(self.repo_location, None, True) self.initial = False if not os.path.isdir(self.index_location): os.mkdir(self.index_location) @@ -89,8 +87,7 @@ self.initial = True def get_paths(self, repo): - """ - recursive walk in root dir and return a set of all path in that dir + """recursive walk in root dir and return a set of all path in that dir based on repository walk function """ index_paths_ = set() @@ -115,7 +112,8 @@ return mktime(node.last_changeset.date.timetuple()) def add_doc(self, writer, path, repo): - """Adding doc to writer""" + """Adding doc to writer this function itself fetches data from + the instance of vcs backend""" node = self.get_node(repo, path) #we just index the content of chosen files
--- a/setup.cfg Sat Nov 13 00:23:36 2010 +0100 +++ b/setup.cfg Sat Nov 13 02:29:46 2010 +0100 @@ -1,5 +1,5 @@ [egg_info] -tag_build = rc4 +tag_build = beta tag_svn_revision = true [easy_install]
--- a/setup.py Sat Nov 13 00:23:36 2010 +0100 +++ b/setup.py Sat Nov 13 02:29:46 2010 +0100 @@ -89,5 +89,10 @@ [paste.app_install] main = pylons.util:PylonsInstaller + + [paste.global_paster_command] + make-index = rhodecode.lib.indexers:MakeIndex + upgrade-db = rhodecode.lib.utils:UpgradeDb + """, )