Esempio n. 1
0
  private void addRawDocumentsIntoPipeline(LocaleId locale) {
    String.format("adding RawDocuments (locale: [%s]) to pipeline", locale);

    for (L10nFile f : project.getFiles()) {

      RawDocument rd = null;
      if (!project.createTM) {
        rd = new RawDocument(f.getURI(), f.getEncoding(), project.sourceLocale, locale);
      } else {
        rd = new RawDocument(f.getURI(locale), f.getEncoding(locale), locale, locale);
      }
      rd.setFilterConfigId(f.getFilterConfigurationId());

      URI targetUri;
      if (!project.createTM) targetUri = new File(f.getLocalXliffPath(locale)).toURI();
      else {
        targetUri =
            new File(f.getLocalXliffPath_SourceWithLocaleCodeBeforeXliffExtension(locale)).toURI();
      }

      driver.addBatchItem(rd, targetUri, f.getEncoding(locale));

      projetLogger.debug(
          String.format(
              "[%s] added to the pipeline driver (target uri: [%s])",
              f.getURI(), targetUri.toString()));
    }
  }
Esempio n. 2
0
  public void extractTranslatedFilesFromXliffs() {

    IPipelineDriver driverX = new PipelineDriver();
    driverX.setFilterConfigurationMapper(fcMapper);

    // L10nFile file = files.get(0);
    // LocaleId locale = LocaleId.FRENCH;

    for (L10nFile file : project.getFiles()) {
      for (LocaleId locale : project.getTargetLocales()) {

        // represents source document (e.g. file.resx)
        RawDocument originalDoc =
            new RawDocument(
                new File(file.getLocalPath()).toURI(),
                file.getEncoding(),
                project.sourceLocale,
                locale);
        originalDoc.setFilterConfigId(file.getFilterConfigurationId());

        // reprosents translated xliff file
        RawDocument xliffDoc =
            new RawDocument(
                new File(file.getLocalXliffPath_SourceWithLocaleCodeBeforeXliffExtension(locale))
                    .toURI(),
                "UTF-8",
                project.sourceLocale,
                locale);

        BatchItemContext bic =
            new BatchItemContext(
                xliffDoc,
                Util.toURI(file.getLocalPath(locale)),
                file.getEncoding(locale),
                originalDoc);
        driverX.addBatchItem(bic);
      }
    }

    driverX.addStep(new OriginalDocumentXliffMergerStep());
    driverX.addStep(new RawDocumentWriterStep());

    project.getEventCannon().fireEvent(CtsAppEventType.ExtractingFromXliffStarted);
    driverX.processBatch();
    project.getEventCannon().fireEvent(CtsAppEventType.ExtractingFromXliffFinished);

    driverX.destroy();
  }
Esempio n. 3
0
  @Override
  protected Event handleRawDocument(Event event) {
    RawDocument rawDoc = (RawDocument) event.getResource();
    BufferedReader reader = null;
    OutputStreamWriter writer = null;
    try {
      // Try to detect the type of file from extension
      isXML = false;
      isHTML = false;
      String ext = Util.getExtension(inputURI.getPath());
      if (!Util.isEmpty(ext)) {
        isHTML = (ext.toLowerCase().indexOf(".htm") == 0);
        isXML = ext.equalsIgnoreCase(".xml");
      }

      // === Try to detect the encoding

      InputStream is = rawDoc.getStream();
      // First: guess from a possible BOM
      BOMNewlineEncodingDetector detector =
          new BOMNewlineEncodingDetector(is, rawDoc.getEncoding());
      detector.detectAndRemoveBom();
      rawDoc.setEncoding(detector.getEncoding());

      String inputEncoding = rawDoc.getEncoding();
      // Then try internal detection for XML/HTML type files
      if (!detector.isAutodetected()) {
        reader = new BufferedReader(rawDoc.getReader());
        reader.read(buffer);
        String detectedEncoding = checkDeclaration(inputEncoding);
        if (!detectedEncoding.equalsIgnoreCase(inputEncoding)) {
          inputEncoding = detectedEncoding;
        }
        reader.close();
      }

      // Open the input document
      // TODO: Where did we reset the reader - can't call this twice unless we reset it
      reader = new BufferedReader(rawDoc.getReader());
      logger.info("Input encoding: " + inputEncoding);

      // Open the output document
      File outFile;
      if (isLastOutputStep()) {
        outFile = rawDoc.createOutputFile(outputURI);
      } else {
        try {
          outFile = File.createTempFile("okp-enc_", ".tmp");
        } catch (Throwable e) {
          throw new OkapiIOException("Cannot create temporary output.", e);
        }
        outFile.deleteOnExit();
      }
      writer =
          new OutputStreamWriter(
              new BufferedOutputStream(new FileOutputStream(outFile)), outputEncoding);
      outputEncoder = Charset.forName(outputEncoding).newEncoder();
      logger.info("Output encoding: " + outputEncoding);
      Util.writeBOMIfNeeded(writer, params.BOMonUTF8, outputEncoding);

      int n;
      CharBuffer tmpBuf = CharBuffer.allocate(1);
      ByteBuffer encBuf;
      boolean canEncode;
      boolean checkDeclaration = true;

      while (true) {
        buffer.clear();
        // Start with previous buffer remains if needed
        if (prevBuf != null) {
          buffer.append(prevBuf);
        }
        // Read the next block
        n = reader.read(buffer);
        // Check if we need to stop here
        boolean needSplitCheck = true;
        if (n == -1) {
          // Make sure we do not start an endless loop by
          // re-checking the last previous buffer
          if (prevBuf != null) {
            needSplitCheck = false;
            prevBuf = null;
            buffer.limit(buffer.position());
          } else break; // No previous, no read: Done
        }

        if (checkDeclaration) {
          checkDeclaration(inputEncoding);
          checkDeclaration = false;
        }

        // Un-escape if requested
        if (pattern != null) {
          if (needSplitCheck) checkSplitSequence();
          unescape();
        }

        // Output
        n = buffer.position();
        buffer.position(0);
        for (int i = 0; i < n; i++) {
          if (!(canEncode = outputEncoder.canEncode(buffer.get(i)))) {
            if (params.reportUnsupported) {
              logger.warning(
                  String.format(
                      "Un-supported character: U+%04X ('%c')", (int) buffer.get(i), buffer.get(i)));
            }
          }

          if ((params.escapeAll && (buffer.get(i) > 127)) || !canEncode) {
            boolean fallBack = false;
            // Write escape form
            if (useCER) {
              String tmp = entities.getName(buffer.get(i));
              if (tmp == null) fallBack = true;
              else writer.write("&" + tmp + ";");
            } else {
              if (params.useBytes) { // Escape bytes
                if (canEncode) {
                  tmpBuf.put(0, buffer.get(i));
                  tmpBuf.position(0);
                  encBuf = outputEncoder.encode(tmpBuf);
                  for (int j = 0; j < encBuf.limit(); j++) {
                    writer.write(
                        String.format(
                            outFormat,
                            (encBuf.get(j) < 0 ? (0xFF ^ ~encBuf.get(j)) : encBuf.get(j))));
                  }
                } else fallBack = true;
              } else { // Escape character
                writer.write(String.format(outFormat, (int) buffer.get(i)));
              }
            }
            if (fallBack) { // Default escaping when nothing else works
              writer.write(String.format("&#x%X;", (int) buffer.get(i)));
            }
          } else { // Normal raw forms
            writer.write(buffer.get(i));
          }
        }
      }

      // Done: close the files
      reader.close();
      reader = null;
      writer.close();
      writer = null;
      rawDoc.finalizeOutput();

      // Set the new raw-document URI and the encoding (in case one was auto-detected)
      // Other info stays the same
      RawDocument newDoc =
          new RawDocument(
              outFile.toURI(), outputEncoding, rawDoc.getSourceLocale(), rawDoc.getTargetLocale());
      event.setResource(newDoc);

    } catch (FileNotFoundException e) {
      throw new RuntimeException(e);
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      try {
        if (writer != null) {
          writer.close();
          writer = null;
        }
        if (reader != null) {
          reader.close();
          reader = null;
        }
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }

    return event;
  }