Mercurial > kallithea

changeset 1995:b6c902d88472 beta

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
bumbed whoosh to 2.3.X series - fixed some html issues on search results
author Marcin Kuzminski <marcin@python-works.com>
date Thu, 16 Feb 2012 05:41:58 +0200
parents 4d3179d2adfe
children 0592987973eb
files requires.txt rhodecode/__init__.py rhodecode/controllers/search.py rhodecode/lib/indexers/__init__.py rhodecode/lib/indexers/daemon.py rhodecode/public/css/pygments.css rhodecode/templates/search/search_content.html
diffstat 7 files changed, 62 insertions(+), 46 deletions(-) [+]
line wrap: on
line diff
--- a/requires.txt	Tue Feb 14 23:33:08 2012 +0200
+++ b/requires.txt	Thu Feb 16 05:41:58 2012 +0200
@@ -5,7 +5,7 @@
 SQLAlchemy==0.7.4
 Mako==0.5.0
 pygments>=1.4
-whoosh<1.8
+whoosh<2.4
 celery>=2.2.5,<2.3
 babel
 python-dateutil>=1.5.0,<2.0.0
--- a/rhodecode/__init__.py	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/__init__.py	Thu Feb 16 05:41:58 2012 +0200
@@ -44,7 +44,7 @@
     "SQLAlchemy==0.7.4",
     "Mako==0.5.0",
     "pygments>=1.4",
-    "whoosh<1.8",
+    "whoosh<2.4",
     "celery>=2.2.5,<2.3",
     "babel",
     "python-dateutil>=1.5.0,<2.0.0",
--- a/rhodecode/controllers/search.py	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/controllers/search.py	Thu Feb 16 05:41:58 2012 +0200
@@ -26,7 +26,7 @@
 import traceback
 
 from pylons.i18n.translation import _
-from pylons import request, config, session, tmpl_context as c
+from pylons import request, config, tmpl_context as c
 
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
@@ -76,7 +76,7 @@
                     cur_query = u'repository:%s %s' % (c.repo_name, cur_query)
                 try:
                     query = qp.parse(unicode(cur_query))
-
+                    # extract words for highlight
                     if isinstance(query, Phrase):
                         highlight_items.update(query.words)
                     elif isinstance(query, Prefix):
@@ -92,18 +92,22 @@
                     log.debug(highlight_items)
                     results = searcher.search(query)
                     res_ln = len(results)
-                    c.runtime = '%s results (%.3f seconds)' \
-                        % (res_ln, results.runtime)
+                    c.runtime = '%s results (%.3f seconds)' % (
+                        res_ln, results.runtime
+                    )
 
                     def url_generator(**kw):
                         return update_params("?q=%s&type=%s" \
                                            % (c.cur_query, c.cur_search), **kw)
 
                     c.formated_results = Page(
-                                ResultWrapper(search_type, searcher, matcher,
-                                              highlight_items),
-                                page=p, item_count=res_ln,
-                                items_per_page=10, url=url_generator)
+                        ResultWrapper(search_type, searcher, matcher,
+                                      highlight_items),
+                        page=p,
+                        item_count=res_ln,
+                        items_per_page=10,
+                        url=url_generator
+                    )
 
                 except QueryParserError:
                     c.runtime = _('Invalid search query. Try quoting it.')
@@ -117,5 +121,6 @@
                 log.error(traceback.format_exc())
                 c.runtime = _('An error occurred during this search operation')
 
+
         # Return a rendered template
         return render('/search/search.html')
--- a/rhodecode/lib/indexers/__init__.py	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/lib/indexers/__init__.py	Thu Feb 16 05:41:58 2012 +0200
@@ -37,38 +37,39 @@
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
-from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
+from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter
 
 from webhelpers.html.builder import escape
 from sqlalchemy import engine_from_config
-from vcs.utils.lazy import LazyProperty
 
 from rhodecode.model import init_model
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.config.environment import load_environment
-from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP
+from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, LazyProperty
 from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 
-#EXTENSIONS WE WANT TO INDEX CONTENT OFF
+# EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys()
 
-#CUSTOM ANALYZER wordsplit + lowercase filter
+# CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 
 #INDEX SCHEMA DEFINITION
-SCHEMA = Schema(owner=TEXT(),
-                repository=TEXT(stored=True),
-                path=TEXT(stored=True),
-                content=FieldType(format=Characters(ANALYZER),
-                             scorable=True, stored=True),
-                modtime=STORED(), extension=TEXT(stored=True))
-
+SCHEMA = Schema(
+    owner=TEXT(),
+    repository=TEXT(stored=True),
+    path=TEXT(stored=True),
+    content=FieldType(format=Characters(), analyzer=ANALYZER,
+                      scorable=True, stored=True),
+    modtime=STORED(),
+    extension=TEXT(stored=True)
+)
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
-FRAGMENTER = SimpleFragmenter(200)
+FRAGMENTER = ContextFragmenter(200)
 
 
 class MakeIndex(BasePasterCommand):
@@ -136,7 +137,7 @@
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
-        self.fragment_size = 200 / 2
+        self.fragment_size = 200
 
     @LazyProperty
     def doc_ids(self):
@@ -172,10 +173,10 @@
         """
         i, j = key.start, key.stop
 
-        slice = []
+        slices = []
         for docid in self.doc_ids[i:j]:
-            slice.append(self.get_full_content(docid))
-        return slice
+            slices.append(self.get_full_content(docid))
+        return slices
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
@@ -183,9 +184,9 @@
                              + len(res['repository']):].lstrip('/')
 
         content_short = self.get_short_content(res, docid[1])
-        res.update({'content_short':content_short,
-                    'content_short_hl':self.highlight(content_short),
-                    'f_path':f_path})
+        res.update({'content_short': content_short,
+                    'content_short_hl': self.highlight(content_short),
+                    'f_path': f_path})
 
         return res
 
@@ -217,10 +218,12 @@
     def highlight(self, content, top=5):
         if self.search_type != 'content':
             return ''
-        hl = highlight(escape(content),
-                 self.highlight_items,
-                 analyzer=ANALYZER,
-                 fragmenter=FRAGMENTER,
-                 formatter=FORMATTER,
-                 top=top)
+        hl = highlight(
+            text=escape(content),
+            terms=self.highlight_items,
+            analyzer=ANALYZER,
+            fragmenter=FRAGMENTER,
+            formatter=FORMATTER,
+            top=top
+        )
         return hl
--- a/rhodecode/lib/indexers/daemon.py	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/lib/indexers/daemon.py	Thu Feb 16 05:41:58 2012 +0200
@@ -49,7 +49,6 @@
 from whoosh.index import create_in, open_dir
 
 
-
 log = logging.getLogger('whooshIndexer')
 # create logger
 log.setLevel(logging.DEBUG)
@@ -68,12 +67,13 @@
 # add ch to logger
 log.addHandler(ch)
 
+
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic jobs
     """
 
-    def __init__(self, indexname='HG_INDEX', index_location=None,
+    def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, sa=None, repo_list=None):
         self.indexname = indexname
 
@@ -95,7 +95,6 @@
 
             self.repo_paths = filtered_repo_paths
 
-
         self.initial = False
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
@@ -155,7 +154,6 @@
                         modtime=self.get_node_mtime(node),
                         extension=node.extension)
 
-
     def build_index(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
@@ -177,7 +175,6 @@
         writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
-
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
 
--- a/rhodecode/public/css/pygments.css	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/public/css/pygments.css	Thu Feb 16 05:41:58 2012 +0200
@@ -65,10 +65,20 @@
 div.code-body {
 	background-color: #FFFFFF;
 }
-div.code-body pre .match{
+
+div.codeblock .code-header .search-path {
+	padding: 0px 0px 0px 10px;
+}
+
+div.search-code-body {
+    background-color: #FFFFFF;
+    padding: 5px 0px 5px 10px;
+}
+
+div.search-code-body pre .match{
 	background-color: #FAFFA6;
 }
-div.code-body pre .break{
+div.search-code-body pre .break{
 	background-color: #DDE7EF;
 	width: 100%;
 	color: #747474;
--- a/rhodecode/templates/search/search_content.html	Tue Feb 14 23:33:08 2012 +0200
+++ b/rhodecode/templates/search/search_content.html	Thu Feb 16 05:41:58 2012 +0200
@@ -5,10 +5,11 @@
     <div class="table">
         <div id="body${cnt}" class="codeblock">
             <div class="code-header">
-                <div class="revision">${h.link_to(h.literal('%s &raquo; %s' % (sr['repository'],sr['f_path'])),
-                h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}</div>
+                <div class="search-path">${h.link_to(h.literal('%s &raquo; %s' % (sr['repository'],sr['f_path'])),
+                h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}
+                </div>
             </div>
-            <div class="code-body">
+            <div class="search-code-body">
                 <pre>${h.literal(sr['content_short_hl'])}</pre>
             </div>
         </div>