Пример #1
0
  // do this only for reference language!
  public List<I18nItem> getDouplicateValues() {
    Locale refLocale = I18nModule.getDefaultLocale();
    List<I18nItem> doupList = new ArrayList<I18nItem>();
    List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
    Map<String, String> tempKeyMap = new HashMap<String, String>();

    for (String bundleName : allBundles) {
      Properties properties =
          i18nMgr.getPropertiesWithoutResolvingRecursively(refLocale, bundleName);
      for (Iterator<Entry<Object, Object>> keyIter = properties.entrySet().iterator();
          keyIter.hasNext(); ) {
        Entry<Object, Object> keyEntry = keyIter.next();
        String keyName = (String) keyEntry.getKey();
        String keyValue = (String) keyEntry.getValue();
        if (tempKeyMap.containsKey(keyName)) {
          List<I18nItem> tmpItem =
              i18nMgr.findI18nItemsByValueSearch(keyValue, refLocale, refLocale, bundleName, false);
          doupList.addAll(tmpItem);
        } else {
          tempKeyMap.put(keyValue, keyName);
        }
      }
    }

    log.info("found " + doupList.size() + " douplicated values in keys");
    return doupList;
  }
Пример #2
0
 /** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
 public void removeXKeysTask(boolean reallyRemoveIt) {
   List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
   Set<String> allLangs = getAllLanguages();
   int counter = 0;
   for (String langKey : allLangs) {
     Locale locale = i18nMgr.getLocaleOrNull(langKey);
     for (String bundleName : allBundles) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
       Set<Object> keys = properties.keySet();
       for (Object keyObj : keys) {
         String key = (String) keyObj;
         if (key.endsWith("X")) {
           String value = properties.getProperty(key);
           if (StringHelper.containsNonWhitespace(value)) {
             log.warn(
                 "NONEMPTY XKEY detected in lang::"
                     + locale.getLanguage()
                     + " bundle::"
                     + bundleName
                     + " key::"
                     + key
                     + " value::"
                     + value);
             if (reallyRemoveIt) {
               addKey(locale, bundleName, key.substring(0, key.length() - 1), value);
             }
           }
           log.info(
               "XKEY detected in lang::"
                   + locale.getLanguage()
                   + " bundle::"
                   + bundleName
                   + " key::"
                   + key);
           logText.append(
               i18nMgr.getPropertiesFile(
                       locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName))
                   + " XKEY detected in lang::"
                   + locale.getLanguage()
                   + " bundle::"
                   + bundleName
                   + " key::"
                   + key
                   + " value::"
                   + value
                   + "\n");
           if (reallyRemoveIt) {
             deleteKey(locale, bundleName, key);
           }
           counter++;
         }
       }
     }
   }
   if (reallyRemoveIt) {
     log.info(counter + " X-Keys got removed!");
   }
 }
Пример #3
0
 /** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
 public void removeTodoKeysTask(boolean reallyRemoveIt) {
   List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
   Set<String> allLangs = getAllLanguages();
   int counter = 0;
   String[] comparisonStrings = {"TODO"};
   for (String langKey : allLangs) {
     Locale locale = i18nMgr.getLocaleOrNull(langKey);
     for (String bundleName : allBundles) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
       Set<Object> keys = properties.keySet();
       for (Object keyObj : keys) {
         String key = (String) keyObj;
         String value = properties.getProperty(key);
         for (int i = 0; i < comparisonStrings.length; i++) {
           int pos = value.toLowerCase().indexOf(comparisonStrings[i].toLowerCase());
           if (pos != -1 && pos < 2 && !value.toLowerCase().equals("todos")) {
             log.info(
                 "TODO-Key detected in lang::"
                     + locale.getLanguage()
                     + " bundle::"
                     + bundleName
                     + " key::"
                     + key
                     + " value::"
                     + value);
             if (value.length() > comparisonStrings[i].length() + 1) {
               log.warn(
                   "this is a TODO-Key WITH TEXT::"
                       + value.substring(comparisonStrings[i].length())
                       + "::");
             } else {
               logText.append(
                   i18nMgr.getPropertiesFile(
                           locale,
                           bundleName,
                           I18nModule.getPropertyFilesBaseDir(locale, bundleName))
                       + " TODO-Key detected in lang::"
                       + locale.getLanguage()
                       + " bundle::"
                       + bundleName
                       + " key::"
                       + key
                       + " value::"
                       + value
                       + "\n");
               if (reallyRemoveIt) {
                 deleteKey(locale, bundleName, key);
               }
             }
             counter++;
           }
         }
       } // each key
     } // each bundle
   }
   log.info(counter + " TODO-Keys got removed!");
 }
Пример #4
0
 /** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
 public void removeEmptyKeysTask(boolean reallyRemoveIt) {
   List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
   int counter = 0;
   Set<String> allLangs = getAllLanguages();
   for (String langKey : allLangs) {
     Locale locale = i18nMgr.getLocaleOrNull(langKey);
     for (String bundleName : allBundles) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
       Set<Object> keys = properties.keySet();
       for (Object keyObj : keys) {
         String key = (String) keyObj;
         String value = properties.getProperty(key);
         if (!StringHelper.containsNonWhitespace(value)) {
           log.info(
               "empty Key detected in lang::"
                   + locale.getLanguage()
                   + " bundle::"
                   + bundleName
                   + " key::"
                   + key
                   + " value::"
                   + value);
           logText.append(
               i18nMgr.getPropertiesFile(
                       locale, bundleName, I18nModule.getPropertyFilesBaseDir(locale, bundleName))
                   + " empty Key detected in lang"
                   + locale.getLanguage()
                   + " bundle::"
                   + bundleName
                   + " key::"
                   + key
                   + " value::"
                   + value
                   + "\n");
           if (reallyRemoveIt) {
             deleteKey(locale, bundleName, key);
           }
         }
         counter++;
       } // each key
     } // each bundle
   }
   log.info(counter + " empty Keys got removed!");
 }
Пример #5
0
 public void sortKeysTask(boolean reallySortIt) {
   List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
   Set<String> allLangs = getAllLanguages();
   int counter = 0;
   for (String langKey : allLangs) {
     Locale locale = i18nMgr.getLocaleOrNull(langKey);
     for (String bundleName : allBundles) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
       if (reallySortIt) {
         // since opened as SortedProperties, save will sort it. Nothing changed, just resorted
         if (properties.size() != 0) {
           i18nMgr.saveOrUpdateProperties(properties, locale, bundleName);
         }
       } else {
         log.info("Sorting " + langKey + ":" + bundleName);
       }
       counter++;
     }
   }
   log.info("Sorted " + counter + " properties files");
 }
Пример #6
0
  /** @param reallyRemoveIt true: really remove it; false: dry run, only produce logging */
  public void removeReferenceLanguageCopiesTask(boolean reallyRemoveIt) {
    List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
    // don't remove EN and DE here, this is a shared Map!!
    int counter = 0;
    int aliasCounter = 0;
    // prepare exclusion list
    String exKeys =
        FileUtils.load(
            new File(
                I18nModule.getTransToolApplicationLanguagesSrcDir()
                    + "/org/olat/lms/commons/i18n/devtools/exclusionKeys.txt"),
            "UTF-8");
    String[] exArray = exKeys.split("\n");
    List<String> exList = new ArrayList<String>(Arrays.asList(exArray));

    Set<String> allLangs = getAllLanguages();
    for (String langKey : allLangs) {
      Locale locale = i18nMgr.getLocaleOrNull(langKey);
      if (locale.toString().equals("de") || locale.toString().equals("en")) {
        // don't compare with DE and EN itself
        continue;
      }
      for (String bundleName : allBundles) {
        Properties properties =
            i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
        Properties refPropDe =
            i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("de"), bundleName);
        Properties refPropEn =
            i18nMgr.getPropertiesWithoutResolvingRecursively(new Locale("en"), bundleName);
        Set<Object> keys = properties.keySet();
        for (Object keyObj : keys) {
          String key = (String) keyObj;
          // dont handle if in exclusion list
          if (!exList.contains(key)) {
            String value = properties.getProperty(key);
            // get ref-lang. value and compare:
            boolean foundInReferenceDe = false;
            boolean foundInReferenceEn = false;
            if (value.equals(refPropDe.getProperty(key))) {
              log.info(
                  "Value of Key found in reference Language DE. lang::"
                      + locale.getLanguage()
                      + " bundle::"
                      + bundleName
                      + " key::"
                      + key
                      + " value::"
                      + value);
              foundInReferenceDe = true;
            }
            if (value.equals(refPropEn.getProperty(key))) {
              log.info(
                  "Value of Key found in reference Language EN. lang::"
                      + locale.getLanguage()
                      + " bundle::"
                      + bundleName
                      + " key::"
                      + key
                      + " value::"
                      + value);
              foundInReferenceEn = true;
            }
            // probably an alias if found in both ref. lang.
            boolean readyToDelete = (foundInReferenceDe || foundInReferenceEn);
            if (foundInReferenceDe && foundInReferenceEn) {
              log.info(
                  "Matching value in both reference languages. lang::"
                      + locale.getLanguage()
                      + " bundle::"
                      + bundleName
                      + " key::"
                      + key
                      + " value::"
                      + value);
              readyToDelete = false;
              aliasCounter++;
            }
            if (readyToDelete && reallyRemoveIt) {
              deleteKey(locale, bundleName, key);
            }
            if (readyToDelete) {
              counter++;
              logText.append(
                  i18nMgr.getPropertiesFile(
                          locale,
                          bundleName,
                          I18nModule.getPropertyFilesBaseDir(locale, bundleName))
                      + " value of key found in reference -> remove lang::"
                      + locale.getLanguage()
                      + " bundle::"
                      + bundleName
                      + " key::"
                      + key
                      + " value::"
                      + value
                      + "\n");
            }
          }
        }
      }
    }
    log.info(counter + " Keys found/deleted with values copied from only one reference languages!");
    log.info(aliasCounter + " Keys which seems to be alias found and NOT deleted!");
  }
Пример #7
0
  /**
   * once again check for keys in branch (lost keys) and move them to Head reallyCopy: set to true
   * to create Props/keys in Head, false: only log them
   */
  public void getLostTranslationsFromBranch(
      boolean reallyCopy,
      String[] referenceLanguages,
      String pathToOlatBranch,
      String pathToCoreBranch) {
    List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles());

    Set<String> allLangs = getAllLanguages();
    // loop over all langs
    int totalCounter = 0;
    for (String langKey : allLangs) {
      int langCounter = 0;
      // ignore ref langs
      boolean isRefLang = false;
      for (String refLangKey : referenceLanguages) {
        if (refLangKey.equals(langKey)) {
          isRefLang = true;
          break;
        }
      }
      if (isRefLang) continue;

      // load current language
      Locale locale = i18nMgr.getLocaleOrNull(langKey);
      for (String bundleName : allBundles) {
        int bundleCounter = 0;
        // get valid keys from ref langs and this bundle
        Set<String> allValidKeys = new HashSet<String>();
        for (String refLangKey : referenceLanguages) {
          Properties properties =
              i18nMgr.getPropertiesWithoutResolvingRecursively(
                  i18nMgr.getLocaleOrNull(refLangKey), bundleName);
          if (properties == null) {
            throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null);
          } else {
            for (Object keyObj : properties.keySet()) {
              String key = (String) keyObj;
              allValidKeys.add(key);
            }
          }
        } // for

        // check if bundle + this locale exists in branch
        String bundlePath = bundleName.replace(".", "/");
        String langPropFileName =
            I18nModule.LOCAL_STRINGS_FILE_PREFIX + langKey + I18nModule.LOCAL_STRINGS_FILE_POSTFIX;
        File bundleInOlat =
            new File(
                pathToOlatBranch
                    + bundlePath
                    + "/"
                    + I18nManager.I18N_DIRNAME
                    + "/"
                    + langPropFileName);
        File bundleInCore =
            new File(
                pathToCoreBranch
                    + bundlePath
                    + "/"
                    + I18nManager.I18N_DIRNAME
                    + "/"
                    + langPropFileName);
        File bundleToUse;
        if (bundleInOlat.exists()) {
          bundleToUse = bundleInOlat;
        } else if (bundleInCore.exists()) {
          bundleToUse = bundleInCore;
        } else { // no bundle found in branch, its not even worth to look after keys
          log.debug(
              "getLostTrans: no OLD prop file found in BRANCH for locale: "
                  + locale
                  + " and bundle: "
                  + bundleName
                  + " => continue with next bundle");
          continue;
        }

        // look after all valid keys in given lang in Head

        Properties targetProperties =
            i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
        Set<Object> targetLangBundleKeys = targetProperties.keySet();
        Properties oldProps = new Properties();
        FileInputStream is;
        try {
          is = new FileInputStream(bundleToUse);
          oldProps.load(is);
          is.close();
        } catch (Exception e) {
          e.printStackTrace();
        }
        for (Object keyObj : allValidKeys) {
          String key = (String) keyObj;
          if (targetLangBundleKeys.contains(key)) {
            // everything ok
          } else {
            // only work on keys found in reference lang (de/en) but not in this bundle
            // try to load key from branch
            if (oldProps.containsKey(key)) {
              String oldValue = oldProps.getProperty(key);
              if (StringHelper.containsNonWhitespace(oldValue)
                  && !oldValue.trim().startsWith("TODO")) {
                langCounter++;
                bundleCounter++;
                totalCounter++;
                if (reallyCopy) {
                  addKey(locale, bundleName, key, oldValue);
                } else {
                  log.debug(
                      "getLostTrans: add a key from BRANCH to locale: "
                          + locale
                          + " and bundle: "
                          + bundleName
                          + " key: "
                          + key);
                }
              } else {
                log.debug(
                    "getLostTrans: ignoring invalid value::'"
                        + oldValue
                        + "' from BRANCH to locale: "
                        + locale
                        + " and bundle: "
                        + bundleName
                        + " key: "
                        + key);
              }
            }
          }
        } // for: keys
        if (bundleCounter > 0)
          log.info(
              "Changed "
                  + bundleCounter
                  + " keys for locale: "
                  + locale
                  + " and bundle: "
                  + bundleName);
      } // for: bundles
      if (langCounter > 0) log.info("Changed " + langCounter + " keys for locale: " + locale);
    } // for: langs
    if (totalCounter > 0) log.info("Changed " + totalCounter + " keys in total");
  }
Пример #8
0
 /**
  * Check for keys that exist in target languages but not in EN or DE. Delete such keys in the
  * target languages
  *
  * @param reallyRemoveIt true: really delete; false: verbose dry run
  * @param referenceLanguages array that contains the language keys that serves as a reference
  *     (e.g. en and de)
  * @param languages the languages that should be cleaned up
  */
 public void removeDeletedKeys(
     boolean reallyRemoveIt, String[] referenceLanguages, Set<String> languages) {
   // first get all available keys from de and en language
   Set<String> validCombinedKeys = new HashSet<String>();
   // copy list to prevent concurrent modification exception
   List<String> allBundles = new ArrayList<String>(I18nModule.getBundleNamesContainingI18nFiles());
   for (String bundleName : allBundles) {
     for (String refLangKey : referenceLanguages) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(
               i18nMgr.getLocaleOrNull(refLangKey), bundleName);
       if (properties == null) {
         throw new OLATRuntimeException("Invalid reference language::" + refLangKey, null);
       } else {
         for (Object keyObj : properties.keySet()) {
           String key = (String) keyObj;
           String combinedKey = bundleName + ":" + key;
           validCombinedKeys.add(combinedKey);
         }
       }
     }
   }
   log.info(
       "removeDeletedKeys: found "
           + validCombinedKeys.size()
           + " valid keys in "
           + ArrayHelper.toString(referenceLanguages));
   //
   // For each language except DE and EN, go through all i18n files and
   // remove keys not in the valid set
   for (String langKey : languages) {
     boolean isRefLang = false;
     for (String refLangKey : referenceLanguages) {
       if (refLangKey.equals(langKey)) {
         isRefLang = true;
         break;
       }
     }
     if (isRefLang) continue;
     // Not a reference language - delete from here
     Locale locale = i18nMgr.getLocaleOrNull(langKey);
     for (String bundleName : allBundles) {
       Properties properties =
           i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
       int propCount = properties.size();
       // copy keys to prevent concurrent modification
       Set<String> availableKeys = new HashSet<String>();
       for (Object key : properties.keySet()) {
         availableKeys.add((String) key);
       }
       for (String key : availableKeys) {
         String combinedKey = bundleName + ":" + key;
         if (!validCombinedKeys.contains(combinedKey)) {
           if (reallyRemoveIt) {
             log.info(
                 "Deleting "
                     + langKey
                     + ":"
                     + bundleName
                     + ":"
                     + key
                     + " - does not exist in "
                     + ArrayHelper.toString(referenceLanguages));
             properties.remove(key);
           } else {
             log.info(
                 "Should be deleted: "
                     + langKey
                     + ":"
                     + bundleName
                     + ":"
                     + key
                     + " - does not exist in "
                     + ArrayHelper.toString(referenceLanguages));
           }
         }
       }
       int delCount = (propCount - properties.size());
       if (reallyRemoveIt && delCount > 0) {
         // only save when changed
         i18nMgr.saveOrUpdateProperties(properties, locale, bundleName);
         log.info(
             "For language::"
                 + langKey
                 + " the in bundle:: "
                 + bundleName
                 + " deleted "
                 + delCount
                 + " keys");
       }
       // delete empty bundles
       if (reallyRemoveIt && properties.size() == 0) {
         i18nMgr.deleteProperties(locale, bundleName);
         log.info(
             "Bundle:: "
                 + bundleName
                 + " deleted for language "
                 + langKey
                 + "entirely because it was empty");
       }
     }
   }
 }
Пример #9
0
  protected void changeReferencesInValues(
      String originBundleName, String targetBundleName, String origKey, String targetKey) {
    // operation:
    boolean movePackage = false;
    boolean renameKey = false;
    if (!originBundleName.equals(targetBundleName)) movePackage = true;
    if (!origKey.equals(targetKey)) renameKey = true;
    int counter = 0;
    Pattern resolvingKeyPattern =
        Pattern.compile("\\$\\{?(" + originBundleName + ")+:([\\w\\.\\-]*[\\w\\-])\\}?");

    List<String> allBundles = I18nModule.getBundleNamesContainingI18nFiles();
    Set<String> allLangs = getAllLanguages();
    for (String langKey : allLangs) {
      Locale locale = i18nMgr.getLocaleOrNull(langKey);

      for (String bundleName : allBundles) {
        Properties properties =
            i18nMgr.getPropertiesWithoutResolvingRecursively(locale, bundleName);
        Set<Object> keys = properties.keySet();

        for (Object keyObj : keys) {
          String key = (String) keyObj;
          String value = properties.getProperty(key);

          Matcher matcher = resolvingKeyPattern.matcher(value);
          int lastPos = 0;
          while (matcher.find()) {
            String matchedKey = matcher.group(2);
            String matchedBundle = matcher.group(1);
            if (matchedKey.equals(origKey)
                && ((matchedBundle == null && bundleName.equals(originBundleName))
                    || originBundleName.equals(matchedBundle))) {
              StringBuffer newValue = new StringBuffer();
              newValue.append(value.substring(0, matcher.start()));
              newValue.append("$");
              if (movePackage) {
                if (!targetBundleName.equals(matchedBundle)) {
                  newValue.append(targetBundleName);
                }
              }
              newValue.append(":");
              if (renameKey) {
                newValue.append(targetKey);
              } else {
                newValue.append(origKey);
              }

              lastPos = matcher.end();
              newValue.append(value.substring(lastPos));
              log.info("Key:: " + key + " should get changed to value:: " + newValue.toString());
              logText.append(
                  i18nMgr.getPropertiesFile(
                          locale,
                          bundleName,
                          I18nModule.getPropertyFilesBaseDir(locale, bundleName))
                      + " update reference in lang::"
                      + locale.getLanguage()
                      + " bundle::"
                      + bundleName
                      + " key::"
                      + key
                      + " value::"
                      + value
                      + " \n\t to new value:: "
                      + newValue.toString()
                      + "\n");
              counter++;
              // changeValueForSingleKey(locale, bundleName, key, newValue.toString());
            }
          }
        } // each key
      }
    }
    log.info(counter + " values have been updated.");
  }