// do this only for reference language! public List<I18nItem> getDouplicateValues() { Locale refLocale = I18nModule.getDefaultLocale(); List<I18nItem> doupList = new ArrayList<I18nItem>(); List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); Map<String, String> tempKeyMap = new HashMap<String, String>(); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(refLocale, bundleName); for (Iterator<Entry<Object, Object>> keyIter = properties.entrySet().iterator(); keyIter.hasNext(); ) { Entry<Object, Object> keyEntry = keyIter.next(); String keyName = (String) keyEntry.getKey(); String keyValue = (String) keyEntry.getValue(); if (tempKeyMap.containsKey(keyName)) { List<I18nItem> tmpItem = i18nMgr.findI18nItemsByValueSearch(keyValue, refLocale, refLocale, bundleName, false); doupList.addAll(tmpItem); } else { tempKeyMap.put(keyValue, keyName); } } } log.info("found " + doupList.size() + " douplicated values in keys"); return doupList; }
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */ public void removeXKeysTask(boolean reallyRemoveIt) { List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); Set<String> allLangs = getAllLanguages(); int counter = 0; for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Set<Object> keys = properties.keySet(); for (Object keyObj : keys) { String key = (String) keyObj; if (key.endsWith("X")) { String value = properties.getProperty(key); if (StringHelper.containsNonWhitespace(value)) { log.warn( "NONEMPTY XKEY detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); if (reallyRemoveIt) { addKey(locale, bundleName, key.substring(0, key.length() - 1), value); } } log.info( "XKEY detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key); logText.append( i18nMgr.getPropertiesFile( locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName)) + " XKEY detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value + "\n"); if (reallyRemoveIt) { deleteKey(locale, bundleName, key); } counter++; } } } } if (reallyRemoveIt) { log.info(counter + " X-Keys got removed!"); } }
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */ public void removeTodoKeysTask(boolean reallyRemoveIt) { List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); Set<String> allLangs = getAllLanguages(); int counter = 0; String[] comparisonStrings = {"TODO"}; for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Set<Object> keys = properties.keySet(); for (Object keyObj : keys) { String key = (String) keyObj; String value = properties.getProperty(key); for (int i = 0; i < comparisonStrings.length; i++) { int pos = value.toLowerCase().indexOf(comparisonStrings[i].toLowerCase()); if (pos != -1 && pos < 2 && !value.toLowerCase().equals("todos")) { log.info( "TODO-Key detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); if (value.length() > comparisonStrings[i].length() + 1) { log.warn( "this is a TODO-Key WITH TEXT::" + value.substring(comparisonStrings[i].length()) + "::"); } else { logText.append( i18nMgr.getPropertiesFile( locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName)) + " TODO-Key detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value + "\n"); if (reallyRemoveIt) { deleteKey(locale, bundleName, key); } } counter++; } } } // each key } // each bundle } log.info(counter + " TODO-Keys got removed!"); }
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */ public void removeEmptyKeysTask(boolean reallyRemoveIt) { List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); int counter = 0; Set<String> allLangs = getAllLanguages(); for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Set<Object> keys = properties.keySet(); for (Object keyObj : keys) { String key = (String) keyObj; String value = properties.getProperty(key); if (!StringHelper.containsNonWhitespace(value)) { log.info( "empty Key detected in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); logText.append( i18nMgr.getPropertiesFile( locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName)) + " empty Key detected in lang" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value + "\n"); if (reallyRemoveIt) { deleteKey(locale, bundleName, key); } } counter++; } // each key } // each bundle } log.info(counter + " empty Keys got removed!"); }
public void sortKeysTask(boolean reallySortIt) { List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); Set<String> allLangs = getAllLanguages(); int counter = 0; for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); if (reallySortIt) { // since opened as SortedProperties, save will sort it. Nothing changed, just resorted if (properties.size() != 0) { i18nMgr.saveOrUpdateProperties(properties, locale, bundleName); } } else { log.info("Sorting " + langKey + ":" + bundleName); } counter++; } } log.info("Sorted " + counter + " properties files"); }
/** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */ public void removeReferenceLanguageCopiesTask(boolean reallyRemoveIt) { List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); // don't remove EN and DE here, this is a shared Map!! int counter = 0; int aliasCounter = 0; // prepare exclusion list String exKeys = FileUtils.load( new File( I18nModule.getTransToolApplicationLanguagesSrcDir() + "/org/olat/lms/commons/i18n/devtools/exclusionKeys.txt"), "UTF-8"); String[] exArray = exKeys.split("\n"); List<String> exList = new ArrayList<String>(Arrays.asList(exArray)); Set<String> allLangs = getAllLanguages(); for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); if (locale.toString().equals("de") || locale.toString().equals("en")) { // don't compare with DE and EN itself continue; } for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Properties refPropDe = i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("de"), bundleName); Properties refPropEn = i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("en"), bundleName); Set<Object> keys = properties.keySet(); for (Object keyObj : keys) { String key = (String) keyObj; // dont handle if in exclusion list if (!exList.contains(key)) { String value = properties.getProperty(key); // get ref-lang. value and compare: boolean foundInReferenceDe = false; boolean foundInReferenceEn = false; if (value.equals(refPropDe.getProperty(key))) { log.info( "Value of Key found in reference Language DE. lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); foundInReferenceDe = true; } if (value.equals(refPropEn.getProperty(key))) { log.info( "Value of Key found in reference Language EN. lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); foundInReferenceEn = true; } // probably an alias if found in both ref. lang. boolean readyToDelete = (foundInReferenceDe || foundInReferenceEn); if (foundInReferenceDe && foundInReferenceEn) { log.info( "Matching value in both reference languages. lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value); readyToDelete = false; aliasCounter++; } if (readyToDelete && reallyRemoveIt) { deleteKey(locale, bundleName, key); } if (readyToDelete) { counter++; logText.append( i18nMgr.getPropertiesFile( locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName)) + " value of key found in reference -> remove lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value + "\n"); } } } } } log.info(counter + " Keys found/deleted with values copied from only one reference languages!"); log.info(aliasCounter + " Keys which seems to be alias found and NOT deleted!"); }
/** * once again check for keys in branch (lost keys) and move them to Head reallyCopy: set to true * to create Props/keys in Head, false: only log them */ public void getLostTranslationsFromBranch( boolean reallyCopy, String[] referenceLanguages, String pathToOlatBranch, String pathToCoreBranch) { List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles()); Set<String> allLangs = getAllLanguages(); // loop over all langs int totalCounter = 0; for (String langKey : allLangs) { int langCounter = 0; // ignore ref langs boolean isRefLang = false; for (String refLangKey : referenceLanguages) { if (refLangKey.equals(langKey)) { isRefLang = true; break; } } if (isRefLang) continue; // load current language Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { int bundleCounter = 0; // get valid keys from ref langs and this bundle Set<String> allValidKeys = new HashSet<String>(); for (String refLangKey : referenceLanguages) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively( i18nMgr.getLocaleOrNull(refLangKey), bundleName); if (properties == null) { throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null); } else { for (Object keyObj : properties.keySet()) { String key = (String) keyObj; allValidKeys.add(key); } } } // for // check if bundle + this locale exists in branch String bundlePath = bundleName.replace(".", "/"); String langPropFileName = I18nModule.LOCAL_STRINGS_FILE_PREFIX + langKey + I18nModule.LOCAL_STRINGS_FILE_POSTFIX; File bundleInOlat = new File( pathToOlatBranch + bundlePath + "/" + I18nManager.I18N_DIRNAME + "/" + langPropFileName); File bundleInCore = new File( pathToCoreBranch + bundlePath + "/" + I18nManager.I18N_DIRNAME + "/" + langPropFileName); File bundleToUse; if (bundleInOlat.exists()) { bundleToUse = bundleInOlat; } else if (bundleInCore.exists()) { bundleToUse = bundleInCore; } else { // no bundle found in branch, its not even worth to look after keys log.debug( "getLostTrans: no OLD prop file found in BRANCH for locale: " + locale + " and bundle: " + bundleName + " => continue with next bundle"); continue; } // look after all valid keys in given lang in Head Properties targetProperties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Set<Object> targetLangBundleKeys = targetProperties.keySet(); Properties oldProps = new Properties(); FileInputStream is; try { is = new FileInputStream(bundleToUse); oldProps.load(is); is.close(); } catch (Exception e) { e.printStackTrace(); } for (Object keyObj : allValidKeys) { String key = (String) keyObj; if (targetLangBundleKeys.contains(key)) { // everything ok } else { // only work on keys found in reference lang (de/en) but not in this bundle // try to load key from branch if (oldProps.containsKey(key)) { String oldValue = oldProps.getProperty(key); if (StringHelper.containsNonWhitespace(oldValue) && !oldValue.trim().startsWith("TODO")) { langCounter++; bundleCounter++; totalCounter++; if (reallyCopy) { addKey(locale, bundleName, key, oldValue); } else { log.debug( "getLostTrans: add a key from BRANCH to locale: " + locale + " and bundle: " + bundleName + " key: " + key); } } else { log.debug( "getLostTrans: ignoring invalid value::'" + oldValue + "' from BRANCH to locale: " + locale + " and bundle: " + bundleName + " key: " + key); } } } } // for: keys if (bundleCounter > 0) log.info( "Changed " + bundleCounter + " keys for locale: " + locale + " and bundle: " + bundleName); } // for: bundles if (langCounter > 0) log.info("Changed " + langCounter + " keys for locale: " + locale); } // for: langs if (totalCounter > 0) log.info("Changed " + totalCounter + " keys in total"); }
/** * Check for keys that exist in target languages but not in EN or DE. Delete such keys in the * target languages * * @param reallyRemoveIt true: really delete; false: verbose dry run * @param referenceLanguages array that contains the language keys that serves as a reference * (e.g. en and de) * @param languages the languages that should be cleaned up */ public void removeDeletedKeys( boolean reallyRemoveIt, String[] referenceLanguages, Set<String> languages) { // first get all available keys from de and en language Set<String> validCombinedKeys = new HashSet<String>(); // copy list to prevent concurrent modification exception List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles()); for (String bundleName : allBundles) { for (String refLangKey : referenceLanguages) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively( i18nMgr.getLocaleOrNull(refLangKey), bundleName); if (properties == null) { throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null); } else { for (Object keyObj : properties.keySet()) { String key = (String) keyObj; String combinedKey = bundleName + ":" + key; validCombinedKeys.add(combinedKey); } } } } log.info( "removeDeletedKeys: found " + validCombinedKeys.size() + " valid keys in " + ArrayHelper.toString(referenceLanguages)); // // For each language except DE and EN, go through all i18n files and // remove keys not in the valid set for (String langKey : languages) { boolean isRefLang = false; for (String refLangKey : referenceLanguages) { if (refLangKey.equals(langKey)) { isRefLang = true; break; } } if (isRefLang) continue; // Not a reference language - delete from here Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); int propCount = properties.size(); // copy keys to prevent concurrent modification Set<String> availableKeys = new HashSet<String>(); for (Object key : properties.keySet()) { availableKeys.add((String) key); } for (String key : availableKeys) { String combinedKey = bundleName + ":" + key; if (!validCombinedKeys.contains(combinedKey)) { if (reallyRemoveIt) { log.info( "Deleting " + langKey + ":" + bundleName + ":" + key + " - does not exist in " + ArrayHelper.toString(referenceLanguages)); properties.remove(key); } else { log.info( "Should be deleted: " + langKey + ":" + bundleName + ":" + key + " - does not exist in " + ArrayHelper.toString(referenceLanguages)); } } } int delCount = (propCount - properties.size()); if (reallyRemoveIt && delCount > 0) { // only save when changed i18nMgr.saveOrUpdateProperties(properties, locale, bundleName); log.info( "For language::" + langKey + " the in bundle:: " + bundleName + " deleted " + delCount + " keys"); } // delete empty bundles if (reallyRemoveIt && properties.size() == 0) { i18nMgr.deleteProperties(locale, bundleName); log.info( "Bundle:: " + bundleName + " deleted for language " + langKey + "entirely because it was empty"); } } } }
protected void changeReferencesInValues( String originBundleName, String targetBundleName, String origKey, String targetKey) { // operation: boolean movePackage = false; boolean renameKey = false; if (!originBundleName.equals(targetBundleName)) movePackage = true; if (!origKey.equals(targetKey)) renameKey = true; int counter = 0; Pattern resolvingKeyPattern = Pattern.compile("\\$\\{?(" + originBundleName + ")+:([\\w\\.\\-]*[\\w\\-])\\}?"); List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles(); Set<String> allLangs = getAllLanguages(); for (String langKey : allLangs) { Locale locale = i18nMgr.getLocaleOrNull(langKey); for (String bundleName : allBundles) { Properties properties = i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName); Set<Object> keys = properties.keySet(); for (Object keyObj : keys) { String key = (String) keyObj; String value = properties.getProperty(key); Matcher matcher = resolvingKeyPattern.matcher(value); int lastPos = 0; while (matcher.find()) { String matchedKey = matcher.group(2); String matchedBundle = matcher.group(1); if (matchedKey.equals(origKey) && ((matchedBundle == null && bundleName.equals(originBundleName)) || originBundleName.equals(matchedBundle))) { StringBuffer newValue = new StringBuffer(); newValue.append(value.substring(0, matcher.start())); newValue.append("$"); if (movePackage) { if (!targetBundleName.equals(matchedBundle)) { newValue.append(targetBundleName); } } newValue.append(":"); if (renameKey) { newValue.append(targetKey); } else { newValue.append(origKey); } lastPos = matcher.end(); newValue.append(value.substring(lastPos)); log.info("Key:: " + key + " should get changed to value:: " + newValue.toString()); logText.append( i18nMgr.getPropertiesFile( locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName)) + " update reference in lang::" + locale.getLanguage() + " bundle::" + bundleName + " key::" + key + " value::" + value + " \n\t to new value:: " + newValue.toString() + "\n"); counter++; // changeValueForSingleKey(locale, bundleName, key, newValue.toString()); } } } // each key } } log.info(counter + " values have been updated."); }