Mercurial > kallithea

comparison scripts/deps.py @ 8772:52816813cbec

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
docs: describe, visualize, and verify internal code structure and layering Try to describe something that isn't entirely there yet. deps.py will help track and minimize violations through checks and visualization in deps.svg .
author Mads Kiilerich <mads@kiilerich.com>
date Fri, 13 Nov 2020 01:04:30 +0100
parents
children 341e4bb9e227
comparison
equal deleted inserted replaced
8771:f8971422795e 8772:52816813cbec
1 #!/usr/bin/env python3
2
3
4 import re
5 import sys
6
7
8 ignored_modules = set('''
9 argparse
10 base64
11 bcrypt
12 binascii
13 bleach
14 calendar
15 celery
16 celery
17 chardet
18 click
19 collections
20 configparser
21 copy
22 csv
23 ctypes
24 datetime
25 dateutil
26 decimal
27 decorator
28 difflib
29 distutils
30 docutils
31 email
32 errno
33 fileinput
34 functools
35 getpass
36 grp
37 hashlib
38 hmac
39 html
40 http
41 imp
42 importlib
43 inspect
44 io
45 ipaddr
46 IPython
47 isapi_wsgi
48 itertools
49 json
50 kajiki
51 ldap
52 logging
53 mako
54 markdown
55 mimetypes
56 mock
57 msvcrt
58 multiprocessing
59 operator
60 os
61 paginate
62 paginate_sqlalchemy
63 pam
64 paste
65 pkg_resources
66 platform
67 posixpath
68 pprint
69 pwd
70 pyflakes
71 pytest
72 pytest_localserver
73 random
74 re
75 routes
76 setuptools
77 shlex
78 shutil
79 smtplib
80 socket
81 ssl
82 stat
83 string
84 struct
85 subprocess
86 sys
87 tarfile
88 tempfile
89 textwrap
90 tgext
91 threading
92 time
93 traceback
94 traitlets
95 types
96 urllib
97 urlobject
98 uuid
99 warnings
100 webhelpers2
101 webob
102 webtest
103 whoosh
104 win32traceutil
105 zipfile
106 '''.split())
107
108 top_modules = set('''
109 kallithea.alembic
110 kallithea.bin
111 kallithea.config
112 kallithea.controllers
113 kallithea.templates.py
114 scripts
115 '''.split())
116
117 bottom_external_modules = set('''
118 tg
119 mercurial
120 sqlalchemy
121 alembic
122 formencode
123 pygments
124 dulwich
125 beaker
126 psycopg2
127 docs
128 setup
129 conftest
130 '''.split())
131
132 normal_modules = set('''
133 kallithea
134 kallithea.lib.celerylib.tasks
135 kallithea.lib
136 kallithea.lib.auth
137 kallithea.lib.auth_modules
138 kallithea.lib.base
139 kallithea.lib.celerylib
140 kallithea.lib.db_manage
141 kallithea.lib.helpers
142 kallithea.lib.hooks
143 kallithea.lib.indexers
144 kallithea.lib.utils
145 kallithea.lib.utils2
146 kallithea.lib.vcs
147 kallithea.lib.webutils
148 kallithea.model
149 kallithea.model.scm
150 kallithea.templates.py
151 '''.split())
152
153 shown_modules = normal_modules | top_modules
154
155 # break the chains somehow - this is a cleanup TODO list
156 known_violations = [
157 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade
158 ('kallithea.lib.utils', 'kallithea.model'), # clean up utils
159 ('kallithea.lib.utils', 'kallithea.model.db'),
160 ('kallithea.lib.utils', 'kallithea.model.scm'),
161 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.helpers'),
162 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.hooks'),
163 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.indexers'),
164 ('kallithea.lib.celerylib.tasks', 'kallithea.model'),
165 ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX
166 ('kallithea.model', 'kallithea.lib.auth_modules'), # validators
167 ('kallithea.model', 'kallithea.lib.helpers'),
168 ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks
169 ('kallithea.model', 'kallithea.model.scm'),
170 ('kallithea.model.scm', 'kallithea.lib.hooks'),
171 ]
172
173 extra_edges = [
174 ('kallithea.config', 'kallithea.controllers'), # through TG
175 ('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader
176 ]
177
178
179 def normalize(s):
180 """Given a string with dot path, return the string it should be shown as."""
181 parts = s.replace('.__init__', '').split('.')
182 short_2 = '.'.join(parts[:2])
183 short_3 = '.'.join(parts[:3])
184 short_4 = '.'.join(parts[:4])
185 if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
186 return 'scripts'
187 if short_3 == 'kallithea.model.meta':
188 return 'kallithea.model.db'
189 if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
190 return 'kallithea.lib.vcs.ssh'
191 if short_4 in shown_modules:
192 return short_4
193 if short_3 in shown_modules:
194 return short_3
195 if short_2 in shown_modules:
196 return short_2
197 if short_2 == 'kallithea.tests':
198 return None
199 if parts[0] in ignored_modules:
200 return None
201 assert parts[0] in bottom_external_modules, parts
202 return parts[0]
203
204
205 def main(filenames):
206 if not filenames or filenames[0].startswith('-'):
207 print('''\
208 Usage:
209 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
210 dot -Tsvg deps.dot > deps.svg
211 ''')
212 raise SystemExit(1)
213
214 files_imports = dict() # map filenames to its imports
215 import_deps = set() # set of tuples with module name and its imports
216 for fn in filenames:
217 with open(fn) as f:
218 s = f.read()
219
220 dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.')
221 file_imports = set()
222 for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE):
223 m_from, m_from_import, m_from_import2, m_import = m.groups()
224 if m_from:
225 pre = m_from + '.'
226 if pre.startswith('.'):
227 pre = dot_name.rsplit('.', 1)[0] + pre
228 importlist = m_from_import or m_from_import2
229 else:
230 pre = ''
231 importlist = m_import
232 for imp in importlist.split('#', 1)[0].split(','):
233 full_imp = pre + imp.strip().split(' as ', 1)[0]
234 file_imports.add(full_imp)
235 import_deps.add((dot_name, full_imp))
236 files_imports[fn] = file_imports
237
238 # dump out all deps for debugging and analysis
239 with open('deps.txt', 'w') as f:
240 for fn, file_imports in sorted(files_imports.items()):
241 for file_import in sorted(file_imports):
242 if file_import.split('.', 1)[0] in ignored_modules:
243 continue
244 f.write('%s: %s\n' % (fn, file_import))
245
246 # find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row
247 only_imported = set(
248 set(normalize(b) for a, b in import_deps) -
249 set(normalize(a) for a, b in import_deps) -
250 set([None, 'kallithea'])
251 )
252
253 normalized_dep_edges = set()
254 for dot_name, full_imp in import_deps:
255 a = normalize(dot_name)
256 b = normalize(full_imp)
257 if a is None or b is None or a == b:
258 continue
259 normalized_dep_edges.add((a, b))
260 #print((dot_name, full_imp, a, b))
261 normalized_dep_edges.update(extra_edges)
262
263 unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges)
264 assert not unseen_shown_modules, unseen_shown_modules
265
266 with open('deps.dot', 'w') as f:
267 f.write('digraph {\n')
268 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules)))
269 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported)))
270 for a, b in sorted(normalized_dep_edges):
271 f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else ''))
272 f.write('}\n')
273
274 # verify dependencies by untangling dependency chain bottom-up:
275 todo = set(normalized_dep_edges)
276 for x in known_violations:
277 todo.remove(x)
278
279 while todo:
280 depending = set(a for a, b in todo)
281 depended = set(b for a, b in todo)
282 drop = depended - depending
283 if not drop:
284 print('ERROR: cycles:', len(todo))
285 for x in sorted(todo):
286 print('%s,' % (x,))
287 raise SystemExit(1)
288 #for do_b in sorted(drop):
289 # print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b))
290 todo = set((a, b) for a, b in todo if b in depending)
291 #print()
292
293
294 if __name__ == '__main__':
295 main(sys.argv[1:])