Beispiel #1
0
  public void processArgs(String[] args) {
    UOption.parseArgs(args, options);
    if (options[HELP1].doesOccur || options[HELP2].doesOccur) {
      System.out.println(HELP_TEXT1);
      return;
    }

    String sourceDir = options[SOURCEDIR].value; // Utility.COMMON_DIRECTORY + "transforms/";
    String targetDir = options[DESTDIR].value; // Utility.GEN_DIRECTORY + "main/";
    String match = options[MATCH].value;
    skipComments = options[SKIP_COMMENTS].doesOccur;
    writeIndex = options[WRITE_INDEX].doesOccur;
    verbose = options[VERBOSE].doesOccur;

    try {
      if (writeIndex) {
        throw new InternalError("writeIndex not implemented.");
      } else {
        ElapsedTimer et = new ElapsedTimer();
        writeTransforms(sourceDir, match, targetDir + File.separator);
        System.out.println("ConvertTransforms: wrote " + fileCount + " files in " + et);
      }
    } catch (IOException ex) {
      RuntimeException e = new RuntimeException();
      e.initCause(ex.getCause());
      throw e;
    } finally {
      System.out.println("DONE");
    }
  }
Beispiel #2
0
/** Utility to generate the Tansliteration resource bundle files. */
public class ConvertTransforms extends CLDRConverterTool {

  private static final int HELP1 = 0,
      HELP2 = 1,
      SOURCEDIR = 2,
      DESTDIR = 3,
      MATCH = 4,
      SKIP_COMMENTS = 5,
      WRITE_INDEX = 6,
      VERBOSE = 7;

  private static final UOption[] options = {
    UOption.HELP_H(),
    UOption.HELP_QUESTION_MARK(),
    UOption.SOURCEDIR().setDefault(CLDRPaths.COMMON_DIRECTORY + "transforms/"),
    UOption.DESTDIR().setDefault(CLDRPaths.GEN_DIRECTORY + "icu-transforms/"),
    UOption.create("match", 'm', UOption.REQUIRES_ARG).setDefault(".*"),
    UOption.create("commentSkip", 'c', UOption.NO_ARG),
    UOption.create("writeIndex", 'x', UOption.NO_ARG),
    UOption.VERBOSE(),
  };

  static final String HELP_TEXT1 =
      "Use the following options"
          + XPathParts.NEWLINE
          + "-h or -?\t for this message"
          + XPathParts.NEWLINE
          + "-"
          + options[SOURCEDIR].shortName
          + "\t source directory. Default = -s"
          + CldrUtility.getCanonicalName(CLDRPaths.MAIN_DIRECTORY)
          + XPathParts.NEWLINE
          + "\tExample:-sC:\\Unicode-CVS2\\cldr\\common\\gen\\source\\"
          + XPathParts.NEWLINE
          + "-"
          + options[DESTDIR].shortName
          + "\t destination directory. Default = -d"
          + CldrUtility.getCanonicalName(CLDRPaths.GEN_DIRECTORY + "main/")
          + XPathParts.NEWLINE
          + "-m<regex>\t to restrict the files to what matches <regex>"
          + XPathParts.NEWLINE
      // "--writeIndex / -x   to write the index (trnsfiles.mk)"+ XPathParts.NEWLINE
      ;

  // TODO add options to set input and output directories, matching pattern
  public static void main(String[] args) throws Exception {
    ConvertTransforms ct = new ConvertTransforms();
    ct.processArgs(args);
  }

  private boolean skipComments;
  private boolean writeIndex = false;
  private boolean verbose = false;

  int fileCount = 0;

  public void writeTransforms(String inputDirectory, String matchingPattern, String outputDirectory)
      throws IOException {
    System.out.println(new File(inputDirectory).getCanonicalPath());
    Factory cldrFactory = Factory.make(inputDirectory, matchingPattern);
    Set<String> ids = cldrFactory.getAvailable();
    PrintWriter index = BagFormatter.openUTF8Writer(outputDirectory, "root.txt");
    doHeader(index, "//", "root.txt");
    try {
      index.println("root {");
      index.println("    RuleBasedTransliteratorIDs {");
      // addAlias(index, "Latin", "el", "", "Latin", "Greek", "UNGEGN");
      // addAlias(index, "el", "Latin", "", "Greek", "Latin", "UNGEGN");
      // addAlias(index, "Latin", "Jamo", "", "Latin", "ConjoiningJamo", "");
      addAlias(index, "Tone", "Digit", "", "Pinyin", "NumericPinyin", "");
      addAlias(index, "Digit", "Tone", "", "NumericPinyin", "Pinyin", "");
      // addAlias(index, "Simplified", "Traditional", "", "Hans", "Hant", "");
      // addAlias(index, "Traditional", "Simplified", "", "Hant", "Hans", "");
      for (String id : ids) {
        if (id.equals("All")) continue;
        try {
          convertFile(cldrFactory, id, outputDirectory, index);
        } catch (IOException e) {
          System.err.println("Failure in: " + id);
          throw e;
        }
      }
      index.println("    }");
      index.println("    TransliteratorNamePattern {");
      index.println("        // Format for the display name of a Transliterator.");
      index.println("        // This is the language-neutral form of this resource.");
      index.println("        \"{0,choice,0#|1#{1}|2#{1}-{2}}\" // Display name");
      index.println("    }");
      index.println("    // Transliterator display names");
      index.println("    // This is the English form of this resource.");
      index.println("    \"%Translit%Hex\"         { \"%Translit%Hex\" }");
      index.println("    \"%Translit%UnicodeName\" { \"%Translit%UnicodeName\" }");
      index.println("    \"%Translit%UnicodeChar\" { \"%Translit%UnicodeChar\" }");
      index.println("    TransliterateLATIN{        ");
      index.println("    \"\",");
      index.println("    \"\"");
      index.println("    }");
      index.println("}");
    } finally {
      index.close();
    }
  }

  public static PrintWriter makePrintWriter(ByteArrayOutputStream bytes) {
    try {
      OutputStreamWriter outStream = new OutputStreamWriter(bytes, "UTF-8");
      BufferedWriter buff = new BufferedWriter(outStream, 4 * 1024);
      PrintWriter p = new PrintWriter(buff);

      return p;
    } catch (Exception e) {
      System.err.println("Error: Could not create OutputStreamWriter.");
    }
    return null;
  }

  private void showComments(PrintWriter toilet, String value) {
    String[] lines = value.trim().split("\\r\\n?|\\n");
    for (String line : lines) {
      if (!line.startsWith("#")) {
        line = "# " + line;
      }
      toilet.println(line);
    }
  }

  private void convertFile(
      Factory cldrFactory, String id, String outputDirectory, PrintWriter index)
      throws IOException {
    PrintWriter output = null;
    String filename = null;
    CLDRFile cldrFile = cldrFactory.make(id, false);
    boolean first = true;
    for (Iterator<String> it = cldrFile.iterator("", cldrFile.getComparator()); it.hasNext(); ) {
      String path = it.next();
      if (path.indexOf("/version") >= 0 || path.indexOf("/generation") >= 0) {
        continue;
      }
      String value = cldrFile.getStringValue(path);
      if (first) {
        String fullPath = cldrFile.getFullXPath(path);
        filename = addIndexInfo(index, fullPath, id);
        if (filename == null) return; // not a transform file!
        output = BagFormatter.openUTF8Writer(outputDirectory, filename);
        doHeader(output, "#", filename);
        first = false;
      }
      if (path.indexOf("/comment") >= 0) {
        if (!skipComments) {
          showComments(output, value);
        }
      } else if (path.indexOf("/tRule") >= 0) {
        value = fixup.transliterate(value);
        output.println(value);
      } else {
        throw new IllegalArgumentException("Unknown element: " + path + "\t " + value);
      }
    }
    output.close();
  }

  public static final Transliterator fixup = Transliterator.getInstance("[:Mn:]any-hex/java");

  public static String replaceUnquoted(String value, String toReplace, String replacement) {
    // quick exit in most cases
    if (value.indexOf(toReplace) < 0) return value;

    String updatedValue = "";
    int segmentStart = 0;
    boolean inQuotes = false;
    boolean ignoreCharValue = false;
    int length = value.length();

    for (int pos = 0; pos < length; ++pos) {
      char curChar = (char) 0;

      if (ignoreCharValue) {
        ignoreCharValue = false;
      } else {
        curChar = value.charAt(pos);
      }

      if (curChar == '\\') {
        // escape, ignore the value of the next char (actually the next UTF16 code unit, but that
        // works here)
        ignoreCharValue = true;
      }
      boolean isLastChar = (pos + 1 >= length);
      if (curChar == '\'' || isLastChar) {
        // quote, begin or end of a quoted literal (in which no replacement takes place)
        if (inQuotes) {
          // End of a quoted segment; guaranteed to include at least opening quote.
          // Just add the segment (including current char) to updatedValue.
          updatedValue = updatedValue + value.substring(segmentStart, pos + 1);
          segmentStart = pos + 1;
        } else {
          if (isLastChar) ++pos;
          if (pos > segmentStart) {
            // End of a nonempty unquoted segment; perform requested replacements and
            // then add segment to updatedValue.
            String currentSegment = value.substring(segmentStart, pos);
            updatedValue = updatedValue + currentSegment.replace(toReplace, replacement);
            segmentStart = pos;
          }
        }
        inQuotes = !inQuotes;
      }
      // else the char just becomes part of the current segment
    }
    return updatedValue;
  }

  static XPathParts parts = new XPathParts();

  private String addIndexInfo(PrintWriter index, String path, String transID) {
    parts.set(path);
    Map<String, String> attributes = parts.findAttributes("transform");
    if (attributes == null) return null; // error, not a transform file
    String source = attributes.get("source");
    String target = attributes.get("target");
    String variant = attributes.get("variant");
    String direction = attributes.get("direction");
    String alias = attributes.get("alias");
    String backwardAlias = attributes.get("backwardAlias");
    String visibility = attributes.get("visibility");

    String status = "internal".equals(visibility) ? "internal" : "file";

    fileCount++;

    String id = source + "-" + target;
    String rid = target + "-" + source;
    String filename = source + "_" + target;
    if (variant != null) {
      id += "/" + variant;
      rid += "/" + variant;
      filename += "_" + variant;
    }
    filename += ".txt";

    if (direction.equals("both") || direction.equals("forward")) {
      if (verbose) {
        System.out.println("    " + id + "    " + filename + "    " + "FORWARD");
      }
      if (alias != null) {
        for (String ali : alias.trim().split("\\s+")) {
          addAlias(index, ali, id);
        }
      }
      index.println("        " + id + " {");
      index.println("            " + status + " {");
      index.println("                resource:process(transliterator) {\"" + filename + "\"}");
      index.println("                direction {\"FORWARD\"}");
      index.println("            }");
      index.println("        }");
    }
    if (direction.equals("both") || direction.equals("backward")) {
      if (verbose) {
        System.out.println("    " + rid + "    " + filename + "    " + "REVERSE");
      }
      if (backwardAlias != null) {
        for (String bali : backwardAlias.trim().split("\\s+")) {
          addAlias(index, bali, rid);
        }
      }
      index.println("        " + rid + " {");
      index.println("            " + status + " {");
      index.println("                resource:process(transliterator) {\"" + filename + "\"}");
      index.println("                direction {\"REVERSE\"}");
      index.println("            }");
      index.println("        }");
    }
    index.println();
    return filename;
  }

  void addAlias(
      PrintWriter index,
      String aliasSource,
      String aliasTarget,
      String aliasVariant,
      String originalSource,
      String originalTarget,
      String originalVariant) {
    // Spacedhan-Han {
    // alias {"null"}
    // }
    addAlias(
        index,
        getName(aliasSource, aliasTarget, aliasVariant),
        getName(originalSource, originalTarget, originalVariant));
  }

  private void addAlias(PrintWriter index, String alias, String original) {
    index.println("        " + alias + " {");
    index.println("            alias" + " {\"" + original + "\"}");
    index.println("        }");
  }

  String getName(String source, String target, String variant) {
    String id = source + "-" + target;
    if (variant != null && variant.length() != 0) {
      id += "/" + variant;
    }
    return id;
  }

  private void doHeader(PrintWriter output, String quoteSymbol, String filename) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy");
    output.print('\uFEFF');
    output.println(
        quoteSymbol
            + " ***************************************************************************");
    output.println(quoteSymbol + " *");
    output.println(
        quoteSymbol
            + " *  Copyright (C) 2004-"
            + sdf.format(new Date())
            + ", International Business Machines");
    output.println(
        quoteSymbol + " *  Corporation; Unicode, Inc.; and others.  All Rights Reserved.");
    output.println(quoteSymbol + " *");
    output.println(
        quoteSymbol
            + " ***************************************************************************");
    output.println(quoteSymbol + " File: " + filename);
    output.println(quoteSymbol + " Generated from CLDR ");
    output.println(quoteSymbol + "");
  }

  public void processArgs(String[] args) {
    UOption.parseArgs(args, options);
    if (options[HELP1].doesOccur || options[HELP2].doesOccur) {
      System.out.println(HELP_TEXT1);
      return;
    }

    String sourceDir = options[SOURCEDIR].value; // Utility.COMMON_DIRECTORY + "transforms/";
    String targetDir = options[DESTDIR].value; // Utility.GEN_DIRECTORY + "main/";
    String match = options[MATCH].value;
    skipComments = options[SKIP_COMMENTS].doesOccur;
    writeIndex = options[WRITE_INDEX].doesOccur;
    verbose = options[VERBOSE].doesOccur;

    try {
      if (writeIndex) {
        throw new InternalError("writeIndex not implemented.");
      } else {
        ElapsedTimer et = new ElapsedTimer();
        writeTransforms(sourceDir, match, targetDir + File.separator);
        System.out.println("ConvertTransforms: wrote " + fileCount + " files in " + et);
      }
    } catch (IOException ex) {
      RuntimeException e = new RuntimeException();
      e.initCause(ex.getCause());
      throw e;
    } finally {
      System.out.println("DONE");
    }
  }

  // fixData ONLY NEEDED TO FIX FILE PROBLEM
  /*
   * private void fixData(String inputDirectory, String matchingPattern, String outputDirectory) throws IOException {
   * File dir = new File(inputDirectory);
   * File[] files = dir.listFiles();
   * for (int i = 0; i < files.length; ++i) {
   * if (files[i].isDirectory()) continue;
   * BufferedReader input = BagFormatter.openUTF8Reader("", files[i].getCanonicalPath());
   * PrintWriter output = BagFormatter.openUTF8Writer("", outputDirectory + files[i].getName());
   * while (true) {
   * String line = input.readLine();
   * if (line == null) break;
   * if (line.indexOf("DOCTYPE") >= 0) {
   * line = line.replaceAll(" ldml ", " supplementalData ");
   * }
   * output.println(line);
   * }
   * input.close();
   * output.close();
   * }
   * }
   */

}