update_strings.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. '''
  4. Update or create an Apple XCode project localization strings file.
  5. TODO: handle localization domains
  6. '''
  7. from __future__ import with_statement
  8. import sys
  9. import os
  10. import os.path
  11. import re
  12. import tempfile
  13. import subprocess
  14. import codecs
  15. import unittest
  16. import optparse
  17. import shutil
  18. import logging
  19. ENCODINGS = ['utf16', 'utf8']
  20. class LocalizedString(object):
  21. ''' A localized string from a strings file '''
  22. COMMENT_EXPR = re.compile(
  23. # Line start
  24. '^\w*'
  25. # Comment
  26. '/\* (?P<comment>.+) \*/'
  27. # End of line
  28. '\w*$'
  29. )
  30. LOCALIZED_STRING_EXPR = re.compile(
  31. # Line start
  32. '^'
  33. # Key
  34. '"(?P<key>.+)"'
  35. # Equals
  36. ' ?= ?'
  37. # Value
  38. '"(?P<value>.+)"'
  39. # Whitespace
  40. ';'
  41. # Comment
  42. '(?: /\* (?P<comment>.+) \*/)?'
  43. # End of line
  44. '$'
  45. )
  46. @classmethod
  47. def parse_comment(cls, comment):
  48. '''
  49. Extract the content of a comment line from a strings file.
  50. Returns the comment string or None if the line doesn't match.
  51. '''
  52. result = cls.COMMENT_EXPR.match(comment)
  53. if result != None:
  54. return result.group('comment')
  55. else:
  56. return None
  57. @classmethod
  58. def from_line(cls, line):
  59. '''
  60. Extract the content of a string line from a strings file.
  61. Returns a LocalizedString instance or None if the line doesn't match.
  62. TODO: handle whitespace restore
  63. '''
  64. result = cls.LOCALIZED_STRING_EXPR.match(line)
  65. if result != None:
  66. return cls(
  67. result.group('key'),
  68. result.group('value'),
  69. result.group('comment')
  70. )
  71. else:
  72. return None
  73. def __init__(self, key, value=None, comment=None):
  74. super(LocalizedString, self).__init__()
  75. self.key = key
  76. self.value = value
  77. self.comment = comment
  78. def is_raw(self):
  79. '''
  80. Return True if the localized string has not been translated.
  81. '''
  82. return self.value == self.key
  83. def __str__(self):
  84. if self.comment:
  85. return '"%s" = "%s"; /* %s */' % (
  86. self.key or '', self.value or '', self.comment
  87. )
  88. else:
  89. return '"%s" = "%s";' % (self.key or '', self.value or '')
  90. def strings_from_folder(folder_path, extensions=None, exclude=None):
  91. '''
  92. Recursively scan folder_path for files containing localizable strings.
  93. Run genstrings on these files and extract the strings.
  94. Returns a dictionnary of LocalizedString instances, indexed by key.
  95. '''
  96. localized_strings = {}
  97. code_file_paths = []
  98. if extensions == None:
  99. extensions = frozenset(['m', 'mm', 'swift'])
  100. if exclude == None:
  101. exclude = frozenset(['ImportedSources','Pods'])
  102. logging.debug('Scanning for source files in %s', folder_path)
  103. for dir_path, dir_names, file_names in os.walk(folder_path):
  104. dir_names[:] = [d for d in dir_names if d not in exclude]
  105. for file_name in file_names:
  106. extension = file_name.rpartition('.')[2]
  107. if extension in extensions:
  108. code_file_path = os.path.join(dir_path, file_name)
  109. code_file_paths.append(code_file_path)
  110. logging.debug('Found %d files', len(code_file_paths))
  111. logging.debug('Running genstrings')
  112. temp_folder_path = tempfile.mkdtemp()
  113. arguments = ['genstrings', '-u', '-o', temp_folder_path]
  114. arguments.extend(code_file_paths)
  115. logging.debug('Here are the argumengts %s', arguments)
  116. subprocess.call(arguments)
  117. temp_file_path = os.path.join(temp_folder_path, 'Localizable.strings')
  118. if os.path.exists(temp_file_path):
  119. logging.debug('Analysing genstrings content')
  120. localized_strings = strings_from_file(temp_file_path)
  121. os.remove(temp_file_path)
  122. else:
  123. logging.debug('No translations found')
  124. shutil.rmtree(temp_folder_path)
  125. return localized_strings
  126. def strings_from_file(file_path):
  127. '''
  128. Try to autodetect file encoding and call strings_from_encoded_file on the
  129. file at file_path.
  130. Returns a dictionnary of LocalizedString instances, indexed by key.
  131. Returns an empty dictionnary if the encoding is wrong.
  132. '''
  133. for current_encoding in ENCODINGS:
  134. try:
  135. return strings_from_encoded_file(file_path, current_encoding)
  136. except UnicodeError:
  137. pass
  138. logging.error(
  139. 'Cannot determine encoding for file %s among %s',
  140. file_path,
  141. ', '.join(ENCODINGS)
  142. )
  143. return {}
  144. def strings_from_encoded_file(file_path, encoding):
  145. '''
  146. Extract the strings from the file at file_path.
  147. Returns a dictionnary of LocalizedString instances, indexed by key.
  148. '''
  149. localized_strings = {}
  150. with codecs.open(file_path, 'r', encoding) as content:
  151. comment = None
  152. for line in content:
  153. line = line.strip()
  154. if not line:
  155. comment = None
  156. continue
  157. current_comment = LocalizedString.parse_comment(line)
  158. if current_comment:
  159. if current_comment != 'No comment provided by engineer.':
  160. comment = current_comment
  161. continue
  162. localized_string = LocalizedString.from_line(line)
  163. if localized_string:
  164. if not localized_string.comment:
  165. localized_string.comment = comment
  166. localized_strings[localized_string.key] = localized_string
  167. else:
  168. logging.error('Could not parse: %s', line.strip())
  169. return localized_strings
  170. def strings_to_file(localized_strings, file_path, encoding='utf16'):
  171. '''
  172. Write a strings file at file_path containing string in
  173. the localized_strings dictionnary.
  174. The strings are alphabetically sorted.
  175. '''
  176. with codecs.open(file_path, 'w', encoding) as output:
  177. for localized_string in sorted_strings_from_dict(localized_strings):
  178. output.write('%s\n' % localized_string)
  179. def update_file_with_strings(file_path, localized_strings):
  180. '''
  181. Try to autodetect file encoding and call update_encoded_file_with_strings
  182. on the file at file_path.
  183. The file at file_path must exist or this function will raise an exception.
  184. '''
  185. for current_encoding in ENCODINGS:
  186. try:
  187. return update_encoded_file_with_strings(
  188. file_path,
  189. localized_strings,
  190. current_encoding
  191. )
  192. except UnicodeError:
  193. pass
  194. logging.error(
  195. 'Cannot determine encoding for file %s among %s',
  196. file_path,
  197. ', '.join(ENCODINGS)
  198. )
  199. return {}
  200. def update_encoded_file_with_strings(
  201. file_path,
  202. localized_strings,
  203. encoding='utf16'
  204. ):
  205. '''
  206. Update file at file_path with translations from localized_strings, trying
  207. to preserve the initial formatting by only removing the old translations,
  208. updating the current ones and adding the new translations at the end of
  209. the file.
  210. The file at file_path must exist or this function will raise an exception.
  211. '''
  212. output_strings = []
  213. keys = set()
  214. with codecs.open(file_path, 'r', encoding) as content:
  215. for line in content:
  216. current_string = LocalizedString.from_line(line.strip())
  217. if current_string:
  218. key = current_string.key
  219. localized_string = localized_strings.get(key, None)
  220. if localized_string:
  221. keys.add(key)
  222. output_strings.append(unicode(localized_string))
  223. else:
  224. output_strings.append(line[:-1])
  225. new_strings = []
  226. for value in localized_strings.itervalues():
  227. if value.key not in keys:
  228. new_strings.append(unicode(value))
  229. if len(new_strings) != 0:
  230. output_strings.append('')
  231. output_strings.append('/* New strings */')
  232. new_strings.sort()
  233. output_strings.extend(new_strings)
  234. with codecs.open(file_path, 'w', encoding) as output:
  235. output.write('\n'.join(output_strings))
  236. # Always add a new line at the end of the file
  237. output.write('\n')
  238. def match_strings(scanned_strings, reference_strings):
  239. '''
  240. Complete scanned_strings with translations from reference_strings.
  241. Return the completed scanned_strings dictionnary.
  242. scanned_strings is not affected.
  243. Strings in reference_strings and not in scanned_strings are not copied.
  244. '''
  245. final_strings = {}
  246. for key, value in scanned_strings.iteritems():
  247. reference_value = reference_strings.get(key, None)
  248. if reference_value:
  249. if reference_value.is_raw():
  250. # Mark non-translated strings
  251. logging.debug('[raw] %s', key)
  252. final_strings[key] = value
  253. else:
  254. # Reference comment comes from the code
  255. reference_value.comment = value.comment
  256. final_strings[key] = reference_value
  257. else:
  258. logging.debug('[new] %s', key)
  259. final_strings[key] = value
  260. final_keys = set(final_strings.keys())
  261. for key in reference_strings.iterkeys():
  262. if key not in final_keys:
  263. logging.debug('[deleted] %s', key)
  264. return final_strings
  265. def merge_dictionaries(reference_dict, import_dict):
  266. '''
  267. Return a dictionnary containing key/values from reference_dict
  268. and import_dict.
  269. In case of conflict, the value from reference_dict is chosen.
  270. '''
  271. final_dict = reference_dict.copy()
  272. reference_dict_keys = set(reference_dict.keys())
  273. for key, value in import_dict.iteritems():
  274. if key not in reference_dict_keys:
  275. final_dict[key] = value
  276. return final_dict
  277. def sorted_strings_from_dict(strings):
  278. '''
  279. Return an array containing the string objects sorted alphabetically.
  280. '''
  281. keys = strings.keys()
  282. keys.sort()
  283. values = []
  284. for key in keys:
  285. values.append(strings[key])
  286. return values
  287. class Tests(unittest.TestCase):
  288. ''' Unit Tests '''
  289. def test_comment(self):
  290. ''' Test comment pattern '''
  291. result = LocalizedString.COMMENT_EXPR.match('/* Testing Comments */')
  292. self.assertNotEqual(result, None, 'Pattern not recognized')
  293. self.assertEqual(result.group('comment'), 'Testing Comments',
  294. 'Incorrect pattern content: [%s]' % result.group('comment')
  295. )
  296. def test_localized_string(self):
  297. ''' Test localized string pattern '''
  298. result = LocalizedString.LOCALIZED_STRING_EXPR.match(
  299. '"KEY" = "VALUE";'
  300. )
  301. self.assertNotEqual(result, None, 'Pattern not recognized')
  302. self.assertEqual(result.group('key'), 'KEY',
  303. 'Incorrect comment content: [%s]' % result.group('key')
  304. )
  305. self.assertEqual(result.group('value'), 'VALUE',
  306. 'Incorrect comment content: [%s]' % result.group('value')
  307. )
  308. self.assertEqual(result.group('comment'), None,
  309. 'Incorrect comment content: [%s]' % result.group('comment')
  310. )
  311. def test_localized_comment_string(self):
  312. ''' Test localized string with comment pattern '''
  313. result = LocalizedString.LOCALIZED_STRING_EXPR.match(
  314. '"KEY" = "VALUE"; /* COMMENT */'
  315. )
  316. self.assertNotEqual(result, None, 'Pattern not recognized')
  317. self.assertEqual(result.group('key'), 'KEY',
  318. 'Incorrect comment content: [%s]' % result.group('key')
  319. )
  320. self.assertEqual(result.group('value'), 'VALUE',
  321. 'Incorrect comment content: [%s]' % result.group('value')
  322. )
  323. self.assertEqual(result.group('comment'), 'COMMENT',
  324. 'Incorrect comment content: [%s]' % result.group('comment')
  325. )
  326. def main():
  327. ''' Parse the command line and do what it is telled to do '''
  328. parser = optparse.OptionParser(
  329. 'usage: %prog [options] Localizable.strings [source folders]'
  330. )
  331. parser.add_option(
  332. '-v',
  333. '--verbose',
  334. action='store_true',
  335. dest='verbose',
  336. default=False,
  337. help='Show debug messages'
  338. )
  339. parser.add_option(
  340. '',
  341. '--dry-run',
  342. action='store_true',
  343. dest='dry_run',
  344. default=False,
  345. help='Do not write to the strings file'
  346. )
  347. parser.add_option(
  348. '',
  349. '--import',
  350. dest='import_file',
  351. help='Import strings from FILENAME'
  352. )
  353. parser.add_option(
  354. '',
  355. '--overwrite',
  356. action='store_true',
  357. dest='overwrite',
  358. default=False,
  359. help='Overwrite the strings file, ignores original formatting'
  360. )
  361. parser.add_option(
  362. '',
  363. '--unittests',
  364. action='store_true',
  365. dest='unittests',
  366. default=False,
  367. help='Run unit tests (debug)'
  368. )
  369. (options, args) = parser.parse_args()
  370. logging.basicConfig(
  371. format='%(message)s',
  372. level=options.verbose and logging.DEBUG or logging.INFO
  373. )
  374. if options.unittests:
  375. suite = unittest.TestLoader().loadTestsFromTestCase(Tests)
  376. return unittest.TextTestRunner(verbosity=2).run(suite)
  377. if len(args) == 0:
  378. parser.error('Please specify a strings file')
  379. strings_file = args[0]
  380. input_folders = ['.']
  381. if len(args) > 1:
  382. input_folders = args[1:]
  383. scanned_strings = {}
  384. for input_folder in input_folders:
  385. if not os.path.isdir(input_folder):
  386. logging.error('Input path is not a folder: %s', input_folder)
  387. return 1
  388. # TODO: allow to specify file extensions to scan
  389. scanned_strings = merge_dictionaries(
  390. scanned_strings,
  391. strings_from_folder(input_folder)
  392. )
  393. if options.import_file:
  394. logging.debug(
  395. 'Reading import file: %s',
  396. options.import_file
  397. )
  398. reference_strings = strings_from_file(options.import_file)
  399. scanned_strings = match_strings(
  400. scanned_strings,
  401. reference_strings
  402. )
  403. if os.path.isfile(strings_file):
  404. logging.debug(
  405. 'Reading strings file: %s',
  406. strings_file
  407. )
  408. reference_strings = strings_from_file(
  409. strings_file
  410. )
  411. scanned_strings = match_strings(
  412. scanned_strings,
  413. reference_strings
  414. )
  415. if options.dry_run:
  416. logging.info(
  417. 'Dry run: the strings file has not been updated'
  418. )
  419. else:
  420. try:
  421. if os.path.exists(strings_file) and not options.overwrite:
  422. update_file_with_strings(strings_file, scanned_strings)
  423. else:
  424. strings_to_file(scanned_strings, strings_file)
  425. except IOError, exc:
  426. logging.error('Error writing to file %s: %s', strings_file, exc)
  427. return 1
  428. logging.info(
  429. 'Strings were generated in %s',
  430. strings_file
  431. )
  432. return 0
  433. if __name__ == '__main__':
  434. sys.exit(main())