comparison MoinMoin/i18n/check_i18n.py @ 0:77665d8e2254

tag of nonpublic@localhost--archive/moin--enterprise--1.5--base-0 (automatically generated log message) imported from: moin--main--1.5--base-0
author Thomas Waldmann <tw-public@gmx.de>
date Thu, 22 Sep 2005 15:09:50 +0000
parents
children 9fb4124ea499
comparison
equal deleted inserted replaced
-1:000000000000 0:77665d8e2254
1 #! /usr/bin/env python
2 # -*- coding: iso-8859-1 -*-
3 """check_i18n - compare texts in the source with the language files
4
5 Searches in the MoinMoin sources for calls of _() and tries to extract
6 the parameter. Then it checks the language modules if those parameters
7 are in the dictionary.
8
9 Usage: check_i18n.py [lang ...]
10
11 Without arguments, checks all languages in i18n or the specified
12 languages. Look into MoinMoin.i18n.__init__ for availeable language
13 names.
14
15 The script will run from the moin root directory, where the MoinMoin
16 package lives, or from MoinMoin/i18n where this script lives.
17
18 TextFinder class based on code by Seo Sanghyeon and the python compiler
19 package.
20
21 @copyright: 2003 Florian Festi, Nir Soffer, Thomas Waldmann
22 @license: GNU GPL, see COPYING for details.
23 """
24
25 output_encoding = 'utf-8'
26
27 # These lead to crashes (MemoryError - due to missing codecs?)
28 #blacklist_files = ["ja.py", "zh.py", "zh_tw.py"]
29 #blacklist_langs = ["ja", "zh", "zh-tw"]
30
31 # If you have cjkcodecs installed, use this:
32 blacklist_files = []
33 blacklist_langs = []
34
35 import sys, os, compiler
36 from compiler.ast import Name, Const, CallFunc, Getattr
37
38 class TextFinder:
39 """ Walk through AST tree and collect text from gettext calls
40
41 Find all calls to gettext function in the source tree and collect
42 the texts in a dict. Use compiler to create an abstract syntax tree
43 from each source file, then find the nodes for gettext function
44 call, and get the text from the call.
45
46 Localized texts are used usually translated during runtime by
47 gettext functions and apear in the source as
48 _('text...'). TextFinder class finds calls to the '_' function in
49 any namespace, or your prefered gettext function.
50
51 Note that TextFinder will only retrieve text from function calls
52 with a constant argument like _('text'). Calls like _('text' % locals()),
53 _('text 1' + 'text 2') are marked as bad call in the report, and the
54 text is not retrieved into the dictionary.
55
56 Note also that texts in source can appear several times in the same
57 file or different files, but they will only apear once in the
58 dictionary that this tool creates.
59
60 The dictionary value for each text is a dictionary of filenames each
61 containing a list of (best guess) lines numbers containning the text.
62 """
63
64 def __init__(self, name='_'):
65 """ Init with the gettext function name or '_'"""
66 self._name = name # getText function name
67 self._dictionary = {} # Unique texts in the found texts
68 self._found = 0 # All good calls including duplicates
69 self._bad = 0 # Bad calls: _('%s' % var) or _('a' + 'b')
70
71 def setFilename(self, filename):
72 """Remember the filename we are parsing"""
73 self._filename = filename
74
75 def visitModule(self, node):
76 """ Start the search from the top node of a module
77
78 This is the entry point into the search. When compiler.walk is
79 called it calls this method with the module node.
80
81 This is the place to initialize module specific data.
82 """
83 self._visited = {} # init node cache - we will visit each node once
84 self._lineno = 'NA' # init line number
85
86 # Start walking in the module node
87 self.walk(node)
88
89 def walk(self, node):
90 """ Walk through all nodes """
91 if self._visited.has_key(node):
92 # We visited this node already
93 return
94
95 self._visited[node] = 1
96 if not self.parseNode(node):
97 for child in node.getChildNodes():
98 self.walk(child)
99
100 def parseNode(self, node):
101 """ Parse function call nodes and collect text """
102
103 # Get the current line number. Since not all nodes have a line number
104 # we save the last line number - it should be close to the gettext call
105 if node.lineno != None:
106 self._lineno = node.lineno
107
108 if node.__class__ == CallFunc and node.args:
109 child = node.node
110 klass = child.__class__
111 if (# Standard call _('text')
112 (klass == Name and child.name == self._name) or
113 # A call to an object attribute: object._('text')
114 (klass == Getattr and child.attrname == self._name)):
115 if node.args[0].__class__ == Const:
116 # Good call with a constant _('text')
117 self.addText(node.args[0].value)
118 else:
119 self.addBadCall(node)
120 return 1
121 return 0
122
123 def addText(self, text):
124 """ Add text to dictionary and count found texts.
125
126 Note that number of texts in dictionary could be different from
127 the number of texts found, because some texts appear several
128 times in the code.
129
130 Each text value is a dictionary of filenames that contain the
131 text and each filename value is the list of line numbers with
132 the text. Missing line numbers are recorded as 'NA'.
133
134 self._lineno is the last line number we checked. It may be the line
135 number of the text, or near it.
136 """
137
138 self._found = self._found + 1
139
140 # Create key for this text if needed
141 if not self._dictionary.has_key(text):
142 self._dictionary[text] = {}
143
144 # Create key for this filename if needed
145 textInfo = self._dictionary[text]
146 if not textInfo.has_key(self._filename):
147 textInfo[self._filename] = [self._lineno]
148 else:
149 textInfo[self._filename].append(self._lineno)
150
151 def addBadCall(self, node):
152 """Called when a bad call like _('a' + 'b') is found"""
153 self._bad = self._bad + 1
154 print
155 print "<!> Warning: non-constant _ call:"
156 print " `%s`" % str(node)
157 print " `%s`:%s" % (self._filename, self._lineno)
158
159 # Accessors
160
161 def dictionary(self):
162 return self._dictionary
163
164 def bad(self):
165 return self._bad
166
167 def found(self):
168 return self._found
169
170
171 def visit(path, visitor):
172 visitor.setFilename(path)
173 tree = compiler.parseFile(path)
174 compiler.walk(tree, visitor)
175
176
177 # MoinMoin specific stuff follows
178
179
180 class Report:
181 """Language status report"""
182 def __init__(self, lang, sourceDict):
183 self.__lang = lang
184 self.__sourceDict = sourceDict
185 self.__langDict = None
186 self.__missing = {}
187 self.__unused = {}
188 self.__error = None
189 self.__ready = 0
190 self.create()
191
192 def loadLanguage(self):
193 filename = i18n.filename(self.__lang)
194 self.__langDict = pysupport.importName("MoinMoin.i18n." + filename, "text")
195
196 def create(self):
197 """Compare language text dict against source dict"""
198 self.loadLanguage()
199 if not self.__langDict:
200 self.__error = "Language %s not found!" % self.__lang
201 self.__ready = 1
202 return
203
204 # Collect missing texts
205 for text in self.__sourceDict:
206 if not self.__langDict.has_key(text):
207 self.__missing[text] = self.__sourceDict[text]
208
209 # Collect unused texts
210 for text in self.__langDict:
211 if not self.__sourceDict.has_key(text):
212 self.__unused[text] = self.__langDict[text]
213 self.__ready = 1
214
215 def summary(self):
216 """Return summary dict"""
217 summary = {
218 'name': i18n.languages[self.__lang][i18n.ENAME].encode(output_encoding),
219 'maintainer': i18n.languages[self.__lang][i18n.MAINTAINER],
220 'total' : len(self.__langDict),
221 'missing': len(self.__missing),
222 'unused': len(self.__unused),
223 'error': self.__error
224 }
225 return summary
226
227 def missing(self):
228 return self.__missing
229
230 def unused(self):
231 return self.__unused
232
233
234
235 if __name__ == '__main__':
236
237 import time
238
239 # Check that we run from the root directory where MoinMoin package lives
240 # or from the i18n directory when this script lives
241 if os.path.exists('MoinMoin/__init__.py'):
242 # Running from the root directory
243 MoinMoin_dir = os.curdir
244 elif os.path.exists(os.path.join(os.pardir, 'i18n')):
245 # Runing from i18n
246 MoinMoin_dir = os.path.join(os.pardir, os.pardir)
247 else:
248 print __doc__
249 sys.exit(1)
250
251 # Insert MoinMoin_dir into sys.path
252 sys.path.insert(0, MoinMoin_dir)
253 from MoinMoin import i18n
254 from MoinMoin.util import pysupport
255
256 textFinder = TextFinder()
257 found = 0
258 unique = 0
259 bad = 0
260
261 # Find gettext calls in the source
262 for root, dirs, files in os.walk(os.path.join(MoinMoin_dir, 'MoinMoin')):
263 for name in files:
264 if name.endswith('.py'):
265 if name in blacklist_files: continue
266 path = os.path.join(root, name)
267 #print '%(path)s:' % locals(),
268 visit(path, textFinder)
269
270 # Report each file's results
271 new_unique = len(textFinder.dictionary()) - unique
272 new_found = textFinder.found() - found
273 #print '%(new_unique)d (of %(new_found)d)' % locals()
274
275 # Warn about bad calls - these should be fixed!
276 new_bad = textFinder.bad() - bad
277 #if new_bad:
278 # print '### Warning: %(new_bad)d bad call(s)' % locals()
279
280 unique = unique + new_unique
281 bad = bad + new_bad
282 found = found + new_found
283
284 # Print report using wiki markup, so we can publish this on MoinDev
285 # !!! Todo:
286 # save executive summary for the wiki
287 # save separate report for each language to be sent to the
288 # language translator.
289 # Update the wiki using XML-RPC??
290
291 print "This page is generated by `MoinMoin/i18n/check_i18n.py`."
292 print "To recreate this report run `make check-i18n` and paste here"
293 print
294 print '----'
295 print
296 print '[[TableOfContents(2)]]'
297 print
298 print
299 print "= Translation Report ="
300 print
301 print "== Summary =="
302 print
303 print 'Created on %s' % time.asctime()
304 print
305
306 print ('\n%(unique)d unique texts in dictionary of %(found)d texts '
307 'in source.') % locals()
308 if bad:
309 print '\n%(bad)d bad calls.' % locals()
310 print
311
312 # Check languages from the command line or from moin.i18n against
313 # the source
314 if sys.argv[1:]:
315 languages = sys.argv[1:]
316 else:
317 languages = i18n.languages.keys()
318 for lang in blacklist_langs:
319 # problems, maybe due to encoding?
320 if lang in languages:
321 languages.remove(lang)
322 if 'en' in languages:
323 languages.remove('en') # there is no en lang file
324 languages.sort()
325
326 # Create report for all languages
327 report = {}
328 for lang in languages:
329 report[lang] = Report(lang, textFinder.dictionary())
330
331 # Print summary for all languages
332 print ("||<:>'''Language'''||<:>'''Texts'''||<:>'''Missing'''"
333 "||<:>'''Unused'''||")
334 for lang in languages:
335 print ("||%(name)s||<)>%(total)s||<)>%(missing)s||<)>%(unused)s||"
336 ) % report[lang].summary()
337
338 # Print details
339 for lang in languages:
340 dict = report[lang].summary()
341 print
342 print "== %(name)s ==" % dict
343 print
344 print "Maintainer: [[MailTo(%(maintainer)s)]]" % dict
345
346 # Print missing texts, if any
347 if report[lang].missing():
348 print """
349 === Missing texts ===
350
351 These items should ''definitely'' get fixed.
352
353 Maybe the corresponding english text in the source code was only changed
354 slightly, then you want to look for a similar text in the ''unused''
355 section below and modify i18n, so that it will match again.
356 """
357 for text in report[lang].missing():
358 print " 1. `%r`" % text
359
360 # Print unused texts, if any
361 if report[lang].unused():
362 print """
363 === Possibly unused texts ===
364
365 Be ''very careful'' and double-check before removing any of these
366 potentially unused items.
367
368 This program can't detect references done from wiki pages, from
369 UserPreferences options, from Icon titles etc.!
370 """
371 for text in report[lang].unused():
372 print " 1. `%r`" % text
373
374