diff pylons_app/lib/helpers.py @ 443:e5157e2a530e

added safe unicode funtion, and implemented it in whoosh indexer
author Marcin Kuzminski <marcin@python-works.com>
date Wed, 01 Sep 2010 23:38:03 +0200
parents 0d4fceb91c9c
children 0668919c307c
line wrap: on
line diff
--- a/pylons_app/lib/helpers.py	Wed Sep 01 23:32:47 2010 +0200
+++ b/pylons_app/lib/helpers.py	Wed Sep 01 23:38:03 2010 +0200
@@ -336,3 +336,19 @@
     gravatar_url += urllib.urlencode({'d':default, 's':str(size)})
 
     return gravatar_url
+
+def safe_unicode(str):
+    """safe unicode function. In case of UnicodeDecode error we try to return
+    unicode with errors replace, if this failes we return unicode with 
+    string_escape decoding """
+    
+    try:
+        u_str = unicode(str)
+    except UnicodeDecodeError:
+        try:
+            u_str = unicode(str, 'utf-8', 'replace')
+        except UnicodeDecodeError:
+            #incase we have a decode error just represent as byte string
+            u_str = unicode(str(str).encode('string_escape'))
+        
+    return u_str
\ No newline at end of file