// do this only for reference language!
public List<I18nItem> getDouplicateValues() {
Locale refLocale = I18nModule.getDefaultLocale();
List<I18nItem> doupList = new ArrayList<I18nItem>();
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
Map<String, String> tempKeyMap = new HashMap<String, String>();
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(refLocale, bundleName);
for (Iterator<Entry<Object, Object>> keyIter = properties.entrySet().iterator();
keyIter.hasNext(); ) {
Entry<Object, Object> keyEntry = keyIter.next();
String keyName = (String) keyEntry.getKey();
String keyValue = (String) keyEntry.getValue();
if (tempKeyMap.containsKey(keyName)) {
List<I18nItem> tmpItem =
i18nMgr.findI18nItemsByValueSearch(keyValue, refLocale, refLocale, bundleName, false);
doupList.addAll(tmpItem);
} else {
tempKeyMap.put(keyValue, keyName);
}
}
}
log.info("found " + doupList.size() + " douplicated values in keys");
return doupList;
}
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
public void removeXKeysTask(boolean reallyRemoveIt) {
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
Set<String> allLangs = getAllLanguages();
int counter = 0;
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Set<Object> keys = properties.keySet();
for (Object keyObj : keys) {
String key = (String) keyObj;
if (key.endsWith("X")) {
String value = properties.getProperty(key);
if (StringHelper.containsNonWhitespace(value)) {
log.warn(
"NONEMPTY XKEY detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
if (reallyRemoveIt) {
addKey(locale, bundleName, key.substring(0, key.length() - 1), value);
}
}
log.info(
"XKEY detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key);
logText.append(
i18nMgr.getPropertiesFile(
locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName))
+ " XKEY detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value
+ "\n");
if (reallyRemoveIt) {
deleteKey(locale, bundleName, key);
}
counter++;
}
}
}
}
if (reallyRemoveIt) {
log.info(counter + " X-Keys got removed!");
}
}
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
public void removeTodoKeysTask(boolean reallyRemoveIt) {
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
Set<String> allLangs = getAllLanguages();
int counter = 0;
String[] comparisonStrings = {"TODO"};
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Set<Object> keys = properties.keySet();
for (Object keyObj : keys) {
String key = (String) keyObj;
String value = properties.getProperty(key);
for (int i = 0; i < comparisonStrings.length; i++) {
int pos = value.toLowerCase().indexOf(comparisonStrings[i].toLowerCase());
if (pos != -1 && pos < 2 && !value.toLowerCase().equals("todos")) {
log.info(
"TODO-Key detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
if (value.length() > comparisonStrings[i].length() + 1) {
log.warn(
"this is a TODO-Key WITH TEXT::"
+ value.substring(comparisonStrings[i].length())
+ "::");
} else {
logText.append(
i18nMgr.getPropertiesFile(
locale,
bundleName,
I18nModule.getPropertyFilesBaseDir(locale, bundleName))
+ " TODO-Key detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value
+ "\n");
if (reallyRemoveIt) {
deleteKey(locale, bundleName, key);
}
}
counter++;
}
}
} // each key
} // each bundle
}
log.info(counter + " TODO-Keys got removed!");
}
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
public void removeEmptyKeysTask(boolean reallyRemoveIt) {
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
int counter = 0;
Set<String> allLangs = getAllLanguages();
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Set<Object> keys = properties.keySet();
for (Object keyObj : keys) {
String key = (String) keyObj;
String value = properties.getProperty(key);
if (!StringHelper.containsNonWhitespace(value)) {
log.info(
"empty Key detected in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
logText.append(
i18nMgr.getPropertiesFile(
locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName))
+ " empty Key detected in lang"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value
+ "\n");
if (reallyRemoveIt) {
deleteKey(locale, bundleName, key);
}
}
counter++;
} // each key
} // each bundle
}
log.info(counter + " empty Keys got removed!");
}
public void sortKeysTask(boolean reallySortIt) {
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
Set<String> allLangs = getAllLanguages();
int counter = 0;
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
if (reallySortIt) {
// since opened as SortedProperties, save will sort it. Nothing changed, just resorted
if (properties.size() != 0) {
i18nMgr.saveOrUpdateProperties(properties, locale, bundleName);
}
} else {
log.info("Sorting " + langKey + ":" + bundleName);
}
counter++;
}
}
log.info("Sorted " + counter + " properties files");
}
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
public void removeReferenceLanguageCopiesTask(boolean reallyRemoveIt) {
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
// don't remove EN and DE here, this is a shared Map!!
int counter = 0;
int aliasCounter = 0;
// prepare exclusion list
String exKeys =
FileUtils.load(
new File(
I18nModule.getTransToolApplicationLanguagesSrcDir()
+ "/org/olat/lms/commons/i18n/devtools/exclusionKeys.txt"),
"UTF-8");
String[] exArray = exKeys.split("\n");
List<String> exList = new ArrayList<String>(Arrays.asList(exArray));
Set<String> allLangs = getAllLanguages();
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
if (locale.toString().equals("de") || locale.toString().equals("en")) {
// don't compare with DE and EN itself
continue;
}
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Properties refPropDe =
i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("de"), bundleName);
Properties refPropEn =
i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("en"), bundleName);
Set<Object> keys = properties.keySet();
for (Object keyObj : keys) {
String key = (String) keyObj;
// dont handle if in exclusion list
if (!exList.contains(key)) {
String value = properties.getProperty(key);
// get ref-lang. value and compare:
boolean foundInReferenceDe = false;
boolean foundInReferenceEn = false;
if (value.equals(refPropDe.getProperty(key))) {
log.info(
"Value of Key found in reference Language DE. lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
foundInReferenceDe = true;
}
if (value.equals(refPropEn.getProperty(key))) {
log.info(
"Value of Key found in reference Language EN. lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
foundInReferenceEn = true;
}
// probably an alias if found in both ref. lang.
boolean readyToDelete = (foundInReferenceDe || foundInReferenceEn);
if (foundInReferenceDe && foundInReferenceEn) {
log.info(
"Matching value in both reference languages. lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value);
readyToDelete = false;
aliasCounter++;
}
if (readyToDelete && reallyRemoveIt) {
deleteKey(locale, bundleName, key);
}
if (readyToDelete) {
counter++;
logText.append(
i18nMgr.getPropertiesFile(
locale,
bundleName,
I18nModule.getPropertyFilesBaseDir(locale, bundleName))
+ " value of key found in reference -> remove lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value
+ "\n");
}
}
}
}
}
log.info(counter + " Keys found/deleted with values copied from only one reference languages!");
log.info(aliasCounter + " Keys which seems to be alias found and NOT deleted!");
}
/**
* once again check for keys in branch (lost keys) and move them to Head reallyCopy: set to true
* to create Props/keys in Head, false: only log them
*/
public void getLostTranslationsFromBranch(
boolean reallyCopy,
String[] referenceLanguages,
String pathToOlatBranch,
String pathToCoreBranch) {
List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles());
Set<String> allLangs = getAllLanguages();
// loop over all langs
int totalCounter = 0;
for (String langKey : allLangs) {
int langCounter = 0;
// ignore ref langs
boolean isRefLang = false;
for (String refLangKey : referenceLanguages) {
if (refLangKey.equals(langKey)) {
isRefLang = true;
break;
}
}
if (isRefLang) continue;
// load current language
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
int bundleCounter = 0;
// get valid keys from ref langs and this bundle
Set<String> allValidKeys = new HashSet<String>();
for (String refLangKey : referenceLanguages) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(
i18nMgr.getLocaleOrNull(refLangKey), bundleName);
if (properties == null) {
throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null);
} else {
for (Object keyObj : properties.keySet()) {
String key = (String) keyObj;
allValidKeys.add(key);
}
}
} // for
// check if bundle + this locale exists in branch
String bundlePath = bundleName.replace(".", "/");
String langPropFileName =
I18nModule.LOCAL_STRINGS_FILE_PREFIX + langKey + I18nModule.LOCAL_STRINGS_FILE_POSTFIX;
File bundleInOlat =
new File(
pathToOlatBranch
+ bundlePath
+ "/"
+ I18nManager.I18N_DIRNAME
+ "/"
+ langPropFileName);
File bundleInCore =
new File(
pathToCoreBranch
+ bundlePath
+ "/"
+ I18nManager.I18N_DIRNAME
+ "/"
+ langPropFileName);
File bundleToUse;
if (bundleInOlat.exists()) {
bundleToUse = bundleInOlat;
} else if (bundleInCore.exists()) {
bundleToUse = bundleInCore;
} else { // no bundle found in branch, its not even worth to look after keys
log.debug(
"getLostTrans: no OLD prop file found in BRANCH for locale: "
+ locale
+ " and bundle: "
+ bundleName
+ " => continue with next bundle");
continue;
}
// look after all valid keys in given lang in Head
Properties targetProperties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Set<Object> targetLangBundleKeys = targetProperties.keySet();
Properties oldProps = new Properties();
FileInputStream is;
try {
is = new FileInputStream(bundleToUse);
oldProps.load(is);
is.close();
} catch (Exception e) {
e.printStackTrace();
}
for (Object keyObj : allValidKeys) {
String key = (String) keyObj;
if (targetLangBundleKeys.contains(key)) {
// everything ok
} else {
// only work on keys found in reference lang (de/en) but not in this bundle
// try to load key from branch
if (oldProps.containsKey(key)) {
String oldValue = oldProps.getProperty(key);
if (StringHelper.containsNonWhitespace(oldValue)
&& !oldValue.trim().startsWith("TODO")) {
langCounter++;
bundleCounter++;
totalCounter++;
if (reallyCopy) {
addKey(locale, bundleName, key, oldValue);
} else {
log.debug(
"getLostTrans: add a key from BRANCH to locale: "
+ locale
+ " and bundle: "
+ bundleName
+ " key: "
+ key);
}
} else {
log.debug(
"getLostTrans: ignoring invalid value::'"
+ oldValue
+ "' from BRANCH to locale: "
+ locale
+ " and bundle: "
+ bundleName
+ " key: "
+ key);
}
}
}
} // for: keys
if (bundleCounter > 0)
log.info(
"Changed "
+ bundleCounter
+ " keys for locale: "
+ locale
+ " and bundle: "
+ bundleName);
} // for: bundles
if (langCounter > 0) log.info("Changed " + langCounter + " keys for locale: " + locale);
} // for: langs
if (totalCounter > 0) log.info("Changed " + totalCounter + " keys in total");
}
/**
* Check for keys that exist in target languages but not in EN or DE. Delete such keys in the
* target languages
*
* @param reallyRemoveIt true: really delete; false: verbose dry run
* @param referenceLanguages array that contains the language keys that serves as a reference
* (e.g. en and de)
* @param languages the languages that should be cleaned up
*/
public void removeDeletedKeys(
boolean reallyRemoveIt, String[] referenceLanguages, Set<String> languages) {
// first get all available keys from de and en language
Set<String> validCombinedKeys = new HashSet<String>();
// copy list to prevent concurrent modification exception
List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles());
for (String bundleName : allBundles) {
for (String refLangKey : referenceLanguages) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(
i18nMgr.getLocaleOrNull(refLangKey), bundleName);
if (properties == null) {
throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null);
} else {
for (Object keyObj : properties.keySet()) {
String key = (String) keyObj;
String combinedKey = bundleName + ":" + key;
validCombinedKeys.add(combinedKey);
}
}
}
}
log.info(
"removeDeletedKeys: found "
+ validCombinedKeys.size()
+ " valid keys in "
+ ArrayHelper.toString(referenceLanguages));
//
// For each language except DE and EN, go through all i18n files and
// remove keys not in the valid set
for (String langKey : languages) {
boolean isRefLang = false;
for (String refLangKey : referenceLanguages) {
if (refLangKey.equals(langKey)) {
isRefLang = true;
break;
}
}
if (isRefLang) continue;
// Not a reference language - delete from here
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
int propCount = properties.size();
// copy keys to prevent concurrent modification
Set<String> availableKeys = new HashSet<String>();
for (Object key : properties.keySet()) {
availableKeys.add((String) key);
}
for (String key : availableKeys) {
String combinedKey = bundleName + ":" + key;
if (!validCombinedKeys.contains(combinedKey)) {
if (reallyRemoveIt) {
log.info(
"Deleting "
+ langKey
+ ":"
+ bundleName
+ ":"
+ key
+ " - does not exist in "
+ ArrayHelper.toString(referenceLanguages));
properties.remove(key);
} else {
log.info(
"Should be deleted: "
+ langKey
+ ":"
+ bundleName
+ ":"
+ key
+ " - does not exist in "
+ ArrayHelper.toString(referenceLanguages));
}
}
}
int delCount = (propCount - properties.size());
if (reallyRemoveIt && delCount > 0) {
// only save when changed
i18nMgr.saveOrUpdateProperties(properties, locale, bundleName);
log.info(
"For language::"
+ langKey
+ " the in bundle:: "
+ bundleName
+ " deleted "
+ delCount
+ " keys");
}
// delete empty bundles
if (reallyRemoveIt && properties.size() == 0) {
i18nMgr.deleteProperties(locale, bundleName);
log.info(
"Bundle:: "
+ bundleName
+ " deleted for language "
+ langKey
+ "entirely because it was empty");
}
}
}
}
protected void changeReferencesInValues(
String originBundleName, String targetBundleName, String origKey, String targetKey) {
// operation:
boolean movePackage = false;
boolean renameKey = false;
if (!originBundleName.equals(targetBundleName)) movePackage = true;
if (!origKey.equals(targetKey)) renameKey = true;
int counter = 0;
Pattern resolvingKeyPattern =
Pattern.compile("\\$\\{?(" + originBundleName + ")+:([\\w\\.\\-]*[\\w\\-])\\}?");
List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
Set<String> allLangs = getAllLanguages();
for (String langKey : allLangs) {
Locale locale = i18nMgr.getLocaleOrNull(langKey);
for (String bundleName : allBundles) {
Properties properties =
i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
Set<Object> keys = properties.keySet();
for (Object keyObj : keys) {
String key = (String) keyObj;
String value = properties.getProperty(key);
Matcher matcher = resolvingKeyPattern.matcher(value);
int lastPos = 0;
while (matcher.find()) {
String matchedKey = matcher.group(2);
String matchedBundle = matcher.group(1);
if (matchedKey.equals(origKey)
&& ((matchedBundle == null && bundleName.equals(originBundleName))
|| originBundleName.equals(matchedBundle))) {
StringBuffer newValue = new StringBuffer();
newValue.append(value.substring(0, matcher.start()));
newValue.append("$");
if (movePackage) {
if (!targetBundleName.equals(matchedBundle)) {
newValue.append(targetBundleName);
}
}
newValue.append(":");
if (renameKey) {
newValue.append(targetKey);
} else {
newValue.append(origKey);
}
lastPos = matcher.end();
newValue.append(value.substring(lastPos));
log.info("Key:: " + key + " should get changed to value:: " + newValue.toString());
logText.append(
i18nMgr.getPropertiesFile(
locale,
bundleName,
I18nModule.getPropertyFilesBaseDir(locale, bundleName))
+ " update reference in lang::"
+ locale.getLanguage()
+ " bundle::"
+ bundleName
+ " key::"
+ key
+ " value::"
+ value
+ " \n\t to new value:: "
+ newValue.toString()
+ "\n");
counter++;
// changeValueForSingleKey(locale, bundleName, key, newValue.toString());
}
}
} // each key
}
}
log.info(counter + " values have been updated.");
}