2109
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
1 # -*- coding: utf-8 -*-
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
2 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
3 rhodecode.lib.utils
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
4 ~~~~~~~~~~~~~~~~~~~
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
5
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
6 Some simple helper functions
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
7
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
8 :created_on: Jan 5, 2011
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
9 :author: marcink
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
10 :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com>
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
11 :license: GPLv3, see COPYING for more details.
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
12 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
13 # This program is free software: you can redistribute it and/or modify
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
14 # it under the terms of the GNU General Public License as published by
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
15 # the Free Software Foundation, either version 3 of the License, or
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
16 # (at your option) any later version.
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
17 #
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
18 # This program is distributed in the hope that it will be useful,
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
21 # GNU General Public License for more details.
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
22 #
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
23 # You should have received a copy of the GNU General Public License
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
25
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
26 import re
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
27 from rhodecode.lib.vcs.utils.lazy import LazyProperty
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
28
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
29
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
30 def __get_lem():
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
31 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
32 Get language extension map based on what's inside pygments lexers
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
33 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
34 from pygments import lexers
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
35 from string import lower
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
36 from collections import defaultdict
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
37
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
38 d = defaultdict(lambda: [])
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
39
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
40 def __clean(s):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
41 s = s.lstrip('*')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
42 s = s.lstrip('.')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
43
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
44 if s.find('[') != -1:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
45 exts = []
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
46 start, stop = s.find('['), s.find(']')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
47
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
48 for suffix in s[start + 1:stop]:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
49 exts.append(s[:s.find('[')] + suffix)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
50 return map(lower, exts)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
51 else:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
52 return map(lower, [s])
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
53
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
54 for lx, t in sorted(lexers.LEXERS.items()):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
55 m = map(__clean, t[-2])
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
56 if m:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
57 m = reduce(lambda x, y: x + y, m)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
58 for ext in m:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
59 desc = lx.replace('Lexer', '')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
60 d[ext].append(desc)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
61
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
62 return dict(d)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
63
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
64 def str2bool(_str):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
65 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
66 returs True/False value from given string, it tries to translate the
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
67 string into boolean
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
68
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
69 :param _str: string value to translate into boolean
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
70 :rtype: boolean
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
71 :returns: boolean from given string
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
72 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
73 if _str is None:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
74 return False
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
75 if _str in (True, False):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
76 return _str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
77 _str = str(_str).strip().lower()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
78 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
79
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
80
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
81 def convert_line_endings(line, mode):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
82 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
83 Converts a given line "line end" accordingly to given mode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
84
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
85 Available modes are::
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
86 0 - Unix
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
87 1 - Mac
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
88 2 - DOS
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
89
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
90 :param line: given line to convert
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
91 :param mode: mode to convert to
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
92 :rtype: str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
93 :return: converted line according to mode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
94 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
95 from string import replace
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
96
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
97 if mode == 0:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
98 line = replace(line, '\r\n', '\n')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
99 line = replace(line, '\r', '\n')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
100 elif mode == 1:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
101 line = replace(line, '\r\n', '\r')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
102 line = replace(line, '\n', '\r')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
103 elif mode == 2:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
104 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
105 return line
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
106
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
107
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
108 def detect_mode(line, default):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
109 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
110 Detects line break for given line, if line break couldn't be found
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
111 given default value is returned
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
112
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
113 :param line: str line
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
114 :param default: default
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
115 :rtype: int
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
116 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
117 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
118 if line.endswith('\r\n'):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
119 return 2
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
120 elif line.endswith('\n'):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
121 return 0
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
122 elif line.endswith('\r'):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
123 return 1
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
124 else:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
125 return default
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
126
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
127
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
128 def generate_api_key(username, salt=None):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
129 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
130 Generates unique API key for given username, if salt is not given
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
131 it'll be generated from some random string
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
132
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
133 :param username: username as string
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
134 :param salt: salt to hash generate KEY
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
135 :rtype: str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
136 :returns: sha1 hash from username+salt
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
137 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
138 from tempfile import _RandomNameSequence
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
139 import hashlib
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
140
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
141 if salt is None:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
142 salt = _RandomNameSequence().next()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
143
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
144 return hashlib.sha1(username + salt).hexdigest()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
145
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
146
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
147 def safe_unicode(str_, from_encoding=None):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
148 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
149 safe unicode function. Does few trick to turn str_ into unicode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
150
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
151 In case of UnicodeDecode error we try to return it with encoding detected
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
152 by chardet library if it fails fallback to unicode with errors replaced
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
153
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
154 :param str_: string to decode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
155 :rtype: unicode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
156 :returns: unicode object
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
157 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
158 if isinstance(str_, unicode):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
159 return str_
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
160
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
161 if not from_encoding:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
162 import rhodecode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
163 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
164 from_encoding = DEFAULT_ENCODING
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
165
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
166 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
167 return unicode(str_)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
168 except UnicodeDecodeError:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
169 pass
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
170
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
171 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
172 return unicode(str_, from_encoding)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
173 except UnicodeDecodeError:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
174 pass
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
175
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
176 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
177 import chardet
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
178 encoding = chardet.detect(str_)['encoding']
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
179 if encoding is None:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
180 raise Exception()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
181 return str_.decode(encoding)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
182 except (ImportError, UnicodeDecodeError, Exception):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
183 return unicode(str_, from_encoding, 'replace')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
184
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
185
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
186 def safe_str(unicode_, to_encoding=None):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
187 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
188 safe str function. Does few trick to turn unicode_ into string
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
189
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
190 In case of UnicodeEncodeError we try to return it with encoding detected
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
191 by chardet library if it fails fallback to string with errors replaced
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
192
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
193 :param unicode_: unicode to encode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
194 :rtype: str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
195 :returns: str object
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
196 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
197
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
198 # if it's not basestr cast to str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
199 if not isinstance(unicode_, basestring):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
200 return str(unicode_)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
201
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
202 if isinstance(unicode_, str):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
203 return unicode_
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
204
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
205 if not to_encoding:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
206 import rhodecode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
207 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
208 to_encoding = DEFAULT_ENCODING
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
209
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
210 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
211 return unicode_.encode(to_encoding)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
212 except UnicodeEncodeError:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
213 pass
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
214
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
215 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
216 import chardet
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
217 encoding = chardet.detect(unicode_)['encoding']
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
218 print encoding
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
219 if encoding is None:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
220 raise UnicodeEncodeError()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
221
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
222 return unicode_.encode(encoding)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
223 except (ImportError, UnicodeEncodeError):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
224 return unicode_.encode(to_encoding, 'replace')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
225
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
226 return safe_str
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
227
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
228
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
229 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
230 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
231 Custom engine_from_config functions that makes sure we use NullPool for
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
232 file based sqlite databases. This prevents errors on sqlite. This only
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
233 applies to sqlalchemy versions < 0.7.0
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
234
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
235 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
236 import sqlalchemy
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
237 from sqlalchemy import engine_from_config as efc
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
238 import logging
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
239
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
240 if int(sqlalchemy.__version__.split('.')[1]) < 7:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
241
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
242 # This solution should work for sqlalchemy < 0.7.0, and should use
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
243 # proxy=TimerProxy() for execution time profiling
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
244
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
245 from sqlalchemy.pool import NullPool
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
246 url = configuration[prefix + 'url']
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
247
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
248 if url.startswith('sqlite'):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
249 kwargs.update({'poolclass': NullPool})
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
250 return efc(configuration, prefix, **kwargs)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
251 else:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
252 import time
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
253 from sqlalchemy import event
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
254 from sqlalchemy.engine import Engine
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
255
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
256 log = logging.getLogger('sqlalchemy.engine')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
257 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
258 engine = efc(configuration, prefix, **kwargs)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
259
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
260 def color_sql(sql):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
261 COLOR_SEQ = "\033[1;%dm"
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
262 COLOR_SQL = YELLOW
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
263 normal = '\x1b[0m'
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
264 return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
265
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
266 if configuration['debug']:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
267 #attach events only for debug configuration
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
268
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
269 def before_cursor_execute(conn, cursor, statement,
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
270 parameters, context, executemany):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
271 context._query_start_time = time.time()
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
272 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
273
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
274
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
275 def after_cursor_execute(conn, cursor, statement,
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
276 parameters, context, executemany):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
277 total = time.time() - context._query_start_time
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
278 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
279
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
280 event.listen(engine, "before_cursor_execute",
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
281 before_cursor_execute)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
282 event.listen(engine, "after_cursor_execute",
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
283 after_cursor_execute)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
284
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
285 return engine
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
286
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
287
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
288 def age(curdate):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
289 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
290 turns a datetime into an age string.
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
291
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
292 :param curdate: datetime object
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
293 :rtype: unicode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
294 :returns: unicode words describing age
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
295 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
296
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
297 from datetime import datetime
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
298 from webhelpers.date import time_ago_in_words
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
299
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
300 _ = lambda s: s
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
301
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
302 if not curdate:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
303 return ''
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
304
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
305 agescales = [(_(u"year"), 3600 * 24 * 365),
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
306 (_(u"month"), 3600 * 24 * 30),
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
307 (_(u"day"), 3600 * 24),
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
308 (_(u"hour"), 3600),
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
309 (_(u"minute"), 60),
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
310 (_(u"second"), 1), ]
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
311
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
312 age = datetime.now() - curdate
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
313 age_seconds = (age.days * agescales[2][1]) + age.seconds
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
314 pos = 1
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
315 for scale in agescales:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
316 if scale[1] <= age_seconds:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
317 if pos == 6:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
318 pos = 5
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
319 return '%s %s' % (time_ago_in_words(curdate,
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
320 agescales[pos][0]), _('ago'))
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
321 pos += 1
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
322
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
323 return _(u'just now')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
324
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
325
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
326 def uri_filter(uri):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
327 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
328 Removes user:password from given url string
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
329
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
330 :param uri:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
331 :rtype: unicode
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
332 :returns: filtered list of strings
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
333 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
334 if not uri:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
335 return ''
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
336
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
337 proto = ''
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
338
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
339 for pat in ('https://', 'http://'):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
340 if uri.startswith(pat):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
341 uri = uri[len(pat):]
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
342 proto = pat
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
343 break
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
344
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
345 # remove passwords and username
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
346 uri = uri[uri.find('@') + 1:]
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
347
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
348 # get the port
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
349 cred_pos = uri.find(':')
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
350 if cred_pos == -1:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
351 host, port = uri, None
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
352 else:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
353 host, port = uri[:cred_pos], uri[cred_pos + 1:]
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
354
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
355 return filter(None, [proto, host, port])
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
356
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
357
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
358 def credentials_filter(uri):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
359 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
360 Returns a url with removed credentials
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
361
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
362 :param uri:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
363 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
364
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
365 uri = uri_filter(uri)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
366 #check if we have port
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
367 if len(uri) > 2 and uri[2]:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
368 uri[2] = ':' + uri[2]
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
369
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
370 return ''.join(uri)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
371
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
372
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
373 def get_changeset_safe(repo, rev):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
374 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
375 Safe version of get_changeset if this changeset doesn't exists for a
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
376 repo it returns a Dummy one instead
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
377
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
378 :param repo:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
379 :param rev:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
380 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
381 from rhodecode.lib.vcs.backends.base import BaseRepository
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
382 from rhodecode.lib.vcs.exceptions import RepositoryError
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
383 if not isinstance(repo, BaseRepository):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
384 raise Exception('You must pass an Repository '
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
385 'object as first argument got %s', type(repo))
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
386
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
387 try:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
388 cs = repo.get_changeset(rev)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
389 except RepositoryError:
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
390 from rhodecode.lib.utils import EmptyChangeset
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
391 cs = EmptyChangeset(requested_revision=rev)
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
392 return cs
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
393
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
394
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
395 def extract_mentioned_users(s):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
396 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
397 Returns unique usernames from given string s that have @mention
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
398
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
399 :param s: string to get mentions
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
400 """
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
401 usrs = {}
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
402 for username in re.findall(r'(?:^@|\s@)(\w+)', s):
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
403 usrs[username] = username
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
404
|
Marcin Kuzminski <marcin@python-works.com>
parents:
diff
changeset
|
405 return sorted(usrs.keys())
|