#!/usr/bin/env python
usage = """usage: %prog [options] path1 path2 ...
diffstrings compares your primary locale with all your other locales to help you determine which new strings need to be translated. It outputs XML files which can be translated, and then merged back into your strings files.
The path arguments supplied to this script should be the path containing your source files. The file system will be searched to find the .lproj directories containing the Localizable.strings files that the script will read and write.
"""
import os.path, codecs, optparse, re, datetime
from xml.sax.saxutils import escape
from xml.dom import minidom
###################################################################################################
global xmlFormat
xmlFormat = {}
reLprojFileName = re.compile(r'(.+?)\.lproj')
reStringsFileName = re.compile(r'(.+?)\.strings')
reComment = re.compile(r'/\*(.*?)\*/')
reString = re.compile(r'\s*"((\\.|.)+?)"\s*=\s*"(.+?)";')
reVariable = re.compile(r'%(@|d|f|lld|\d+\.?f|\d+\.\d+f|\d+d)')
reEnumeratedVariable = re.compile('(%?\d+\$)')
defaultComment = "No comment provided by engineer."
###################################################################################################
def generateProjects(projects):
for project in projects:
project.generateStrings()
def diffProjects(projects, sourceLocaleName, focusedLocaleName=None, verbose=False, dryRun=False):
compiledStrings = {}
sourceMap = {}
for project in projects:
project.compileStrings(compiledStrings, focusedLocaleName)
project.compileSourceMap(sourceMap)
if sourceLocaleName not in compiledStrings:
print "WARNING: No '%s' locale found!" % sourceLocaleName
else:
sourceStrings = compiledStrings[sourceLocaleName]
if verbose:
print "\n", "=" * 80
print "* %s has %d total strings." \
% (sourceLocaleName, len(sourceStrings.strings))
print "=" * 80
for localeName, localizedStrings in compiledStrings.iteritems():
if localizedStrings != sourceStrings:
sourceStrings.diffReport(localizedStrings)
for localeName, localizedStrings in compiledStrings.iteritems():
if localizedStrings != sourceStrings:
translation = sourceStrings.diff(localizedStrings)
if not dryRun:
translation.save(".", sourceMap=sourceMap)
def mergeProjects(projects, sourceLocaleName, focusedLocaleName=None, verbose=False, dryRun=False):
translations = {}
for project in projects:
sourceStrings = project.condenseStringSourceFiles()
sourceStrings.save()
for localeName, localizedStrings in project.locales.iteritems():
if not focusedLocaleName or focusedLocaleName == localeName:
if localizedStrings.name in translations:
translation = translations[localizedStrings.name]
else:
translation = Translation(localizedStrings.name)
translation.open(".")
translations[localizedStrings.name] = translation
if translation.strings:
if verbose:
localizedStrings.mergeReport(sourceStrings, translation)
localizedStrings.mergeTranslation(sourceStrings, translation)
if not dryRun:
localizedStrings.save()
else:
if verbose:
print "no translation.strings for %s, sad" % localeName
###################################################################################################
class XcodeProject(object):
def __init__(self, path, sourceLocaleName):
self.path = path
self.sourceLocaleName = sourceLocaleName
self.sourceLocalePath = self.__findStringsFile(sourceLocaleName, self.path)
self.stringSourceFiles = list(self.__iterateSourceStrings())
self.locales = {}
for localeName, localizedStrings in self.__iterateLocalizableStrings():
self.locales[localeName] = localizedStrings
def condenseStringSourceFiles(self):
""" Copies all strings from all sources files into a single file."""
sourceStrings = LocalizableStrings(self.sourceLocaleName)
sourceStrings.path = self.__findSourceStringsPath()
if not sourceStrings.path:
sourceStrings.path = os.path.join(self.sourceLocalePath, "Localizable.strings")
for sourceFile in self.stringSourceFiles:
sourceStrings.update(sourceFile)
return sourceStrings
def compileStrings(self, compiledStrings, focusedLocaleName=None):
""" Copies all strings in a dictionary for each locale."""
if not self.sourceLocaleName in compiledStrings:
compiledStrings[self.sourceLocaleName] = LocalizableStrings(self.sourceLocaleName)
compiledStringsFile = compiledStrings[self.sourceLocaleName]
for sourceStrings in self.stringSourceFiles:
compiledStringsFile.update(sourceStrings)
if focusedLocaleName:
locales = {focusedLocaleName: self.locales[focusedLocaleName]}
else:
locales = self.locales
for localeName, sourceStrings in locales.iteritems():
if not localeName in compiledStrings:
compiledStringsFile = LocalizableStrings(localeName)
compiledStrings[localeName] = compiledStringsFile
else:
compiledStringsFile = compiledStrings[localeName]
compiledStringsFile.update(sourceStrings)
def compileSourceMap(self, sourceMap):
for sourceStrings in self.stringSourceFiles:
for source in sourceStrings.strings:
if not source in sourceMap:
sourceMap[source] = []
name,ext = os.path.splitext(os.path.basename(sourceStrings.path))
sourceMap[source].append(name)
def generateStrings(self):
buildPath = None
cwd = os.getcwd()
os.chdir(self.path)
extras = ""
if os.path.isdir(os.path.join(self.path, "Three20.xcodeproj")):
extras = " -s TTLocalizedString"
for fileName in os.listdir(self.path):
name,ext = os.path.splitext(fileName)
if ext == ".m":
if not buildPath:
buildPath = self.__makeBuildPath()
if not os.path.isdir(buildPath):
os.makedirs(buildPath)
command = "genstrings %s -o %s%s" % (fileName, buildPath, extras)
print " %s" % command
os.system(command)
resultPath = os.path.join(buildPath, "Localizable.strings")
if os.path.isfile(resultPath):
renamedPath = os.path.join(buildPath, "%s.strings" % name)
os.rename(resultPath, renamedPath)
os.chdir(cwd)
def __findStringsFile(self, localeName, searchPath):
dirName = "%s.lproj" % localeName
localeDirPath = os.path.join(searchPath, dirName)
if os.path.isdir(localeDirPath):
return localeDirPath
for name in os.listdir(searchPath):
path = os.path.join(searchPath, name)
if os.path.isdir(path):
localeDirPath = self.__findStringsFile(localeName, path)
if localeDirPath:
return localeDirPath
return None
def __iterateSourceStrings(self):
buildPath = self.__makeBuildPath()
if not os.path.exists(buildPath):
for path in self.__findSourceStrings():
yield path
else:
for fileName in os.listdir(buildPath):
name,ext = os.path.splitext(fileName)
if ext == ".strings":
strings = LocalizableStrings(self.sourceLocaleName)
filePath = os.path.join(buildPath, fileName)
strings.open(filePath)
yield strings
def __findSourceStringsPath(self):
for name in os.listdir(self.sourceLocalePath):
m = reStringsFileName.match(name)
if m:
return os.path.join(self.sourceLocalePath, name)
def __findSourceStrings(self):
for name in os.listdir(self.sourceLocalePath):
m = reStringsFileName.match(name)
if m:
strings = LocalizableStrings(self.sourceLocaleName)
filePath = os.path.join(self.sourceLocalePath, name)
strings.open(filePath)
yield strings
def __iterateLocalizableStrings(self):
dirPath = os.path.dirname(self.sourceLocalePath)
for dirName in os.listdir(dirPath):
m = reLprojFileName.match(dirName)
if m:
localeName = m.groups()[0]
if localeName != self.sourceLocaleName:
strings = LocalizableStrings(localeName)
localeDirPath = os.path.join(dirPath, dirName)
for name in os.listdir(localeDirPath):
m = reStringsFileName.match(name)
if m:
filePath = os.path.abspath(os.path.join(localeDirPath, name))
strings.open(filePath)
break
yield localeName, strings
def __makeBuildPath(self):
return os.path.join(self.path, "build", "i18n")
###################################################################################################
class LocalizableStrings(object):
def __init__(self, name):
self.name = name
self.path = None
self.strings = {}
self.comments = {}
def open(self, path):
if os.path.isfile(path):
self.path = path
self.__parse()
def save(self, path=None, suffix=""):
text = self.generate()
if text:
if path:
filePath = self.__makePath(path, suffix)
else:
filePath = self.path
print "***** Saving %s" % filePath
f = codecs.open(filePath, 'w', 'utf-16')
f.write(text)
f.close()
def generate(self):
lines = []
# This may not sort non-English strings sensibly, but the order itself
# doesn't matter - this is just so that the strings come out in some
# consistent order every time. (Less efficient, but oh well.)
for source in sorted(self.strings.keys()):
if source in self.comments:
comment = self.comments[source]
lines.append("/* %s */" % comment)
lines.append('"%s" = "%s";\n' % (source, self.strings[source]))
return "\n".join(lines)
def mergeTranslation(self, sourceStrings, translation):
for source in sourceStrings.strings:
sourceEnum = enumerateStringVariables(source)
if sourceEnum in translation.strings:
targetEnum = translation.strings[sourceEnum]
target = denumerateStringVariables(targetEnum)
self.strings[source] = target
def update(self, other):
self.strings.update(other.strings)
self.comments.update(other.comments)
def diff(self, localizedStrings):
translation = Translation(localizedStrings.name)
for source, target in self.strings.iteritems():
sourceEnum = enumerateStringVariables(source)
if source in localizedStrings.strings:
target = localizedStrings.strings[source]
translation.translated[sourceEnum] = True
targetEnum = enumerateStringVariables(target)
translation.strings[sourceEnum] = targetEnum
if source in self.comments:
translation.comments[sourceEnum] = self.comments[source]
return translation
def diffReport(self, localizedStrings):
name = localizedStrings.name
newStrings = list(self.__compare(localizedStrings))
obsoleteStrings = list(localizedStrings.__compare(self))
troubleStrings = list(self.__compareSizes(localizedStrings))
print "\n", "=" * 80
if not len(newStrings):
if len(obsoleteStrings):
print "%s is fully translated, but has %s obsolete strings" \
% (name, len(obsoleteStrings))
else:
print "%s is fully translated" % name
else:
existingCount = len(self.strings) - len(newStrings)
if len(obsoleteStrings):
print "%s has %s new strings, %s translated, and %s obsolete."\
% (name, len(newStrings), existingCount, len(obsoleteStrings))
else:
print "%s has %s new strings, with %s already translated."\
% (name, len(newStrings), existingCount)
print "=" * 80
if len(newStrings):
print "\n---- %s NEW STRINGS ---\n" % name
print "\n".join(newStrings)
if len(obsoleteStrings):
print "\n---- %s OBSOLETE STRINGS ---\n" % name
print "\n".join(obsoleteStrings)
if len(troubleStrings):
print "\n---- %s TROUBLE STRINGS ---\n" % name
for source, diff in sorted(troubleStrings, lambda a,b: cmp(b[1], a[1])):
print "%3d. %s " % (diff, codecs.encode(source, 'utf-8'))
print " %s " % codecs.encode(localizedStrings.strings[source], 'utf-8')
print "\n"
def mergeReport(self, sourceStrings, translation):
name = self.name
updatedStrings = []
ignoredStrings = []
for source in sourceStrings.strings:
sourceEnum = enumerateStringVariables(source)
if sourceEnum in translation.strings:
targetEnum = translation.strings[sourceEnum]
target = denumerateStringVariables(targetEnum)
if source not in self.strings or target != self.strings[source]:
updatedStrings.append(source)
else:
ignoredStrings.append(source)
print "\n", "=" * 80
print self.path
print "%d newly translated strings and %d untranslated strings" \
% (len(updatedStrings), len(ignoredStrings))
print "=" * 80
if len(updatedStrings):
print "\n---- %s NEWLY TRANSLATED STRINGS ---\n" % name
print "\n".join(updatedStrings)
if len(ignoredStrings):
print "\n---- %s UNTRANSLATED STRINGS ---\n" % name
print "\n".join(ignoredStrings)
def __makePath(self, path=".", suffix=""):
fileName = "Localizable%s.strings" % (suffix)
return os.path.abspath(os.path.join(path, fileName))
def __parse(self):
lastIdentical = False
lastComment = None
for line in openWithProperEncoding(self.path):
m = reString.search(line)
if m:
source = m.groups()[0]
target = m.groups()[2]
self.strings[source] = target
if lastComment:
self.comments[source] = lastComment
lastComment = None
if lastIdentical:
lastIdentical = False
else:
m = reComment.search(line)
if m:
comment = m.groups()[0].strip()
if comment != defaultComment:
lastComment = comment
def __compare(self, other, compareStrings=False):
for source, target in self.strings.iteritems():
if source in other.strings:
target = other.strings[source]
if compareStrings and target == source:
yield source
else:
yield source
def __compareSizes(self, other):
for source, target in self.strings.iteritems():
if source in other.strings:
target = other.strings[source]
ratio = float(len(target)) / len(source)
diff = len(target) - len(source)
if ratio > 1.3 and diff > 5:
yield (source, diff)
###################################################################################################
class Translation(object):
def __init__(self, name):
self.name = name
self.path = None
self.strings = {}
self.translated = {}
self.comments = {}
def open(self, path=".", suffix=""):
filePath = self.__makePath(path, suffix)
if os.path.isfile(filePath):
self.__parse(filePath)
def save(self, path=None, suffix="", sourceMap=None):
text = self.generate(sourceMap)
if text:
if path:
filePath = self.__makePath(path, suffix)
else:
filePath = self.path
print "***** Saving %s" % filePath
#print codecs.encode(text, 'utf-8')
f = codecs.open(filePath, 'w', 'utf-16')
f.write(text)
f.close()
def generate(self, sourceMap=None):
lines = []
global xmlFormat
prefix = xmlFormat['prefix']
lines.append('')
lines.append('<%sexternal>' % prefix)
lines.append(' ')
if xmlFormat['appName']:
lines.append(' %s' % xmlFormat['appName'])
lines.append(' %s' % datetime.datetime.now().strftime('%Y%m%d'))
lines.append(' %s' % self.name)
lines.append(' ')
for sourceFileName, sourceFileStrings in self.__invertSourceMap(sourceMap):
lines.append(" " % sourceFileName)
for source in sourceFileStrings:
target = self.strings[source]
lines.append(" ")
lines.append(" <%ssource>%s" % (prefix, escape(source), prefix))
if source in self.translated:
lines.append(" <%sxtarget>%s" % (prefix, escape(target), prefix))
else:
lines.append(" <%starget>%s" % (prefix, escape(target), prefix))
if source in self.comments:
lines.append(" <%sdescription>%s"
% (prefix, escape(self.comments[source]), prefix))
lines.append(" ")
lines.append('' % prefix)
return "\n".join(lines)
def __makePath(self, path=".", suffix=""):
fileName = "%s%s.xml" % (self.name, suffix)
return os.path.abspath(os.path.join(path, fileName))
def __parse(self, filePath):
self.path = filePath
global xmlFormat
prefix = xmlFormat['prefix']
document = minidom.parse(filePath)
for entry in document.documentElement.childNodes:
if entry.nodeType == 1:
source = None
target = None
translated = False
sources = entry.getElementsByTagName("%ssource" % prefix)
if len(sources):
source = sources[0]
source = source.childNodes[0].data
targets = entry.getElementsByTagName("%sxtarget" % prefix)
if not len(targets):
targets = entry.getElementsByTagName("%starget" % prefix)
translated = True
if len(targets):
target = targets[0]
target = target.childNodes[0].data
if source and target:
self.strings[source] = target
if translated:
self.translated[source] = True
def __invertSourceMap(self, sourceMap):
sourceFileMap = {}
for sourceEnum in self.strings:
source = denumerateStringVariables(sourceEnum)
if source in sourceMap:
sourcePaths = sourceMap[source]
for sourcePath in sourcePaths:
if sourcePath not in sourceFileMap:
sourceFileMap[sourcePath] = []
sourceFileMap[sourcePath].append(sourceEnum)
break
for sourceName, sourceFileStrings in sourceFileMap.iteritems():
sourceFileStrings.sort()
keys = sourceFileMap.keys()
keys.sort()
for key in keys:
yield key, sourceFileMap[key]
###################################################################################################
## Helpers
def openProjects(projectDirPaths, sourceLocaleName):
for projectDirPath in projectDirPaths:
yield XcodeProject(projectDirPath, sourceLocaleName)
def openWithProperEncoding(path):
if not os.path.isfile(path):
return []
try:
f = codecs.open(path, 'r', 'utf-16')
lines = f.read().splitlines()
f.close()
except UnicodeError,exc:
f = codecs.open(path, 'r', 'utf-8')
lines = f.read().splitlines()
f.close()
return lines
def enumerateStringVariables(s):
i = 1
for var in reVariable.findall(s):
s = s.replace("%%%s" % var, "%%%d$%s" % (i, var), 1)
i += 1
return s
def denumerateStringVariables(s):
for var in reEnumeratedVariable.findall(s):
s = s.replace(var, "%")
return s
###################################################################################################
## Main
def parseOptions():
parser = optparse.OptionParser(usage)
parser.set_defaults(locale="en", focus=None, build=False, merge=False, diff=False,
verbose=False, dryrun=False, appName="", prefix="")
parser.add_option("-l", "--locale", dest="locale", type="str",
help = "The name of your source locale. The default is 'en'.")
parser.add_option("-f", "--focus", dest="focus", type="str",
help = "The name of the locale to operate on, excluding all others.")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help = "Verbose reporting of activity.")
parser.add_option("-r", "--dryrun", dest="dryrun", action="store_true",
help = "Print the output of files instead of saving them.")
parser.add_option("-b", "--build", dest="build", action="store_true",
help = "Runs genstrings on each source file in each project.")
parser.add_option("-d", "--diff", dest="diff", action="store_true",
help="Generates a diff of each locale against the source locale. Each locale's diff will be stored in a file in the working directory named .xml.")
parser.add_option("-m", "--merge", dest="merge", action="store_true",
help="Merges strings from the .xml file in the working directory back into the Localized.strings files in each locale.")
parser.add_option("-p", "--prefix", dest="prefix", type="str",
help="The prefix to use on the xml tags.")
parser.add_option("-a", "--appname", dest="appName", type="str",
help="The name of the application to include in the xml metadata.")
options, arguments = parser.parse_args()
paths = ["."] if not len(arguments) else arguments
if not options.merge:
options.diff = True
return options, paths
def main():
options, projectPaths = parseOptions()
projectPaths = [os.path.abspath(os.path.expanduser(path)) for path in projectPaths]
global xmlFormat
xmlFormat['prefix'] = options.prefix
xmlFormat['appName'] = options.appName
projects = list(openProjects(projectPaths, options.locale))
if options.build:
print "******* Generating strings *******"
generateProjects(projects)
print ""
if options.merge:
print "******* Merging *******"
mergeProjects(projects, options.locale, options.focus, options.verbose, options.dryrun)
print ""
if options.diff:
print "******* Diffing *******"
diffProjects(projects, options.locale, options.focus, options.verbose, options.dryrun)
print ""
if __name__ == "__main__":
main()