Mercurial > kallithea
changeset 1995:b6c902d88472 beta
bumbed whoosh to 2.3.X series
- fixed some html issues on search results
author | Marcin Kuzminski <marcin@python-works.com> |
---|---|
date | Thu, 16 Feb 2012 05:41:58 +0200 |
parents | 4d3179d2adfe |
children | 0592987973eb |
files | requires.txt rhodecode/__init__.py rhodecode/controllers/search.py rhodecode/lib/indexers/__init__.py rhodecode/lib/indexers/daemon.py rhodecode/public/css/pygments.css rhodecode/templates/search/search_content.html |
diffstat | 7 files changed, 62 insertions(+), 46 deletions(-) [+] |
line wrap: on
line diff
--- a/requires.txt Tue Feb 14 23:33:08 2012 +0200 +++ b/requires.txt Thu Feb 16 05:41:58 2012 +0200 @@ -5,7 +5,7 @@ SQLAlchemy==0.7.4 Mako==0.5.0 pygments>=1.4 -whoosh<1.8 +whoosh<2.4 celery>=2.2.5,<2.3 babel python-dateutil>=1.5.0,<2.0.0
--- a/rhodecode/__init__.py Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/__init__.py Thu Feb 16 05:41:58 2012 +0200 @@ -44,7 +44,7 @@ "SQLAlchemy==0.7.4", "Mako==0.5.0", "pygments>=1.4", - "whoosh<1.8", + "whoosh<2.4", "celery>=2.2.5,<2.3", "babel", "python-dateutil>=1.5.0,<2.0.0",
--- a/rhodecode/controllers/search.py Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/controllers/search.py Thu Feb 16 05:41:58 2012 +0200 @@ -26,7 +26,7 @@ import traceback from pylons.i18n.translation import _ -from pylons import request, config, session, tmpl_context as c +from pylons import request, config, tmpl_context as c from rhodecode.lib.auth import LoginRequired from rhodecode.lib.base import BaseController, render @@ -76,7 +76,7 @@ cur_query = u'repository:%s %s' % (c.repo_name, cur_query) try: query = qp.parse(unicode(cur_query)) - + # extract words for highlight if isinstance(query, Phrase): highlight_items.update(query.words) elif isinstance(query, Prefix): @@ -92,18 +92,22 @@ log.debug(highlight_items) results = searcher.search(query) res_ln = len(results) - c.runtime = '%s results (%.3f seconds)' \ - % (res_ln, results.runtime) + c.runtime = '%s results (%.3f seconds)' % ( + res_ln, results.runtime + ) def url_generator(**kw): return update_params("?q=%s&type=%s" \ % (c.cur_query, c.cur_search), **kw) c.formated_results = Page( - ResultWrapper(search_type, searcher, matcher, - highlight_items), - page=p, item_count=res_ln, - items_per_page=10, url=url_generator) + ResultWrapper(search_type, searcher, matcher, + highlight_items), + page=p, + item_count=res_ln, + items_per_page=10, + url=url_generator + ) except QueryParserError: c.runtime = _('Invalid search query. Try quoting it.') @@ -117,5 +121,6 @@ log.error(traceback.format_exc()) c.runtime = _('An error occurred during this search operation') + # Return a rendered template return render('/search/search.html')
--- a/rhodecode/lib/indexers/__init__.py Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/lib/indexers/__init__.py Thu Feb 16 05:41:58 2012 +0200 @@ -37,38 +37,39 @@ from whoosh.fields import TEXT, ID, STORED, Schema, FieldType from whoosh.index import create_in, open_dir from whoosh.formats import Characters -from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter +from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter from webhelpers.html.builder import escape from sqlalchemy import engine_from_config -from vcs.utils.lazy import LazyProperty from rhodecode.model import init_model from rhodecode.model.scm import ScmModel from rhodecode.model.repo import RepoModel from rhodecode.config.environment import load_environment -from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP +from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, LazyProperty from rhodecode.lib.utils import BasePasterCommand, Command, add_cache -#EXTENSIONS WE WANT TO INDEX CONTENT OFF +# EXTENSIONS WE WANT TO INDEX CONTENT OFF INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() -#CUSTOM ANALYZER wordsplit + lowercase filter +# CUSTOM ANALYZER wordsplit + lowercase filter ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() #INDEX SCHEMA DEFINITION -SCHEMA = Schema(owner=TEXT(), - repository=TEXT(stored=True), - path=TEXT(stored=True), - content=FieldType(format=Characters(ANALYZER), - scorable=True, stored=True), - modtime=STORED(), extension=TEXT(stored=True)) - +SCHEMA = Schema( + owner=TEXT(), + repository=TEXT(stored=True), + path=TEXT(stored=True), + content=FieldType(format=Characters(), analyzer=ANALYZER, + scorable=True, stored=True), + modtime=STORED(), + extension=TEXT(stored=True) +) IDX_NAME = 'HG_INDEX' FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') -FRAGMENTER = SimpleFragmenter(200) +FRAGMENTER = ContextFragmenter(200) class MakeIndex(BasePasterCommand): @@ -136,7 +137,7 @@ self.searcher = searcher self.matcher = matcher self.highlight_items = highlight_items - self.fragment_size = 200 / 2 + self.fragment_size = 200 @LazyProperty def doc_ids(self): @@ -172,10 +173,10 @@ """ i, j = key.start, key.stop - slice = [] + slices = [] for docid in self.doc_ids[i:j]: - slice.append(self.get_full_content(docid)) - return slice + slices.append(self.get_full_content(docid)) + return slices def get_full_content(self, docid): res = self.searcher.stored_fields(docid[0]) @@ -183,9 +184,9 @@ + len(res['repository']):].lstrip('/') content_short = self.get_short_content(res, docid[1]) - res.update({'content_short':content_short, - 'content_short_hl':self.highlight(content_short), - 'f_path':f_path}) + res.update({'content_short': content_short, + 'content_short_hl': self.highlight(content_short), + 'f_path': f_path}) return res @@ -217,10 +218,12 @@ def highlight(self, content, top=5): if self.search_type != 'content': return '' - hl = highlight(escape(content), - self.highlight_items, - analyzer=ANALYZER, - fragmenter=FRAGMENTER, - formatter=FORMATTER, - top=top) + hl = highlight( + text=escape(content), + terms=self.highlight_items, + analyzer=ANALYZER, + fragmenter=FRAGMENTER, + formatter=FORMATTER, + top=top + ) return hl
--- a/rhodecode/lib/indexers/daemon.py Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/lib/indexers/daemon.py Thu Feb 16 05:41:58 2012 +0200 @@ -49,7 +49,6 @@ from whoosh.index import create_in, open_dir - log = logging.getLogger('whooshIndexer') # create logger log.setLevel(logging.DEBUG) @@ -68,12 +67,13 @@ # add ch to logger log.addHandler(ch) + class WhooshIndexingDaemon(object): """ Daemon for atomic jobs """ - def __init__(self, indexname='HG_INDEX', index_location=None, + def __init__(self, indexname=IDX_NAME, index_location=None, repo_location=None, sa=None, repo_list=None): self.indexname = indexname @@ -95,7 +95,6 @@ self.repo_paths = filtered_repo_paths - self.initial = False if not os.path.isdir(self.index_location): os.makedirs(self.index_location) @@ -155,7 +154,6 @@ modtime=self.get_node_mtime(node), extension=node.extension) - def build_index(self): if os.path.exists(self.index_location): log.debug('removing previous index') @@ -177,7 +175,6 @@ writer.commit(merge=True) log.debug('>>> FINISHED BUILDING INDEX <<<') - def update_index(self): log.debug('STARTING INCREMENTAL INDEXING UPDATE')
--- a/rhodecode/public/css/pygments.css Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/public/css/pygments.css Thu Feb 16 05:41:58 2012 +0200 @@ -65,10 +65,20 @@ div.code-body { background-color: #FFFFFF; } -div.code-body pre .match{ + +div.codeblock .code-header .search-path { + padding: 0px 0px 0px 10px; +} + +div.search-code-body { + background-color: #FFFFFF; + padding: 5px 0px 5px 10px; +} + +div.search-code-body pre .match{ background-color: #FAFFA6; } -div.code-body pre .break{ +div.search-code-body pre .break{ background-color: #DDE7EF; width: 100%; color: #747474;
--- a/rhodecode/templates/search/search_content.html Tue Feb 14 23:33:08 2012 +0200 +++ b/rhodecode/templates/search/search_content.html Thu Feb 16 05:41:58 2012 +0200 @@ -5,10 +5,11 @@ <div class="table"> <div id="body${cnt}" class="codeblock"> <div class="code-header"> - <div class="revision">${h.link_to(h.literal('%s » %s' % (sr['repository'],sr['f_path'])), - h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}</div> + <div class="search-path">${h.link_to(h.literal('%s » %s' % (sr['repository'],sr['f_path'])), + h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))} + </div> </div> - <div class="code-body"> + <div class="search-code-body"> <pre>${h.literal(sr['content_short_hl'])}</pre> </div> </div>