Пример #1
0
  public static void write2LzoFile(String destLzoFilePath, Configuration conf, byte[] datas) {
    LzopCodec lzo = null;
    OutputStream out = null;

    try {
      lzo = new LzopCodec();
      lzo.setConf(conf);
      out = lzo.createOutputStream(new FileOutputStream(destLzoFilePath));
      out.write(datas);
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if (out != null) {
          out.close();
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
Пример #2
0
  private static boolean copyLocalFilesToOneHDFS(
      FileSystem filesystemIn,
      List<Path> todoNames,
      FileSystem filesystemOut,
      Path pathOut,
      String filenamePrefix,
      Algorithm alg,
      Configuration conf)
      throws IOException {

    int outputBufferSize = 10 * 1024 * 1024;
    int inputBufferSize = 10 * 1024 * 1024;
    int copyBufferSize = 10 * 1024 * 1024;

    // batch load data
    FSDataOutputStream out = null;
    OutputStream os = null;

    try {
      String postfix;
      if (alg == Algorithm.LZO) {
        postfix = ".lzo";
      } else if (alg == Algorithm.GZ) {
        postfix = ".gz";
      } else {
        postfix = ".txt";
      }

      Path fileOut = new Path(pathOut, filenamePrefix + postfix);
      if (filesystemOut.exists(fileOut)) {
        System.err.println("[WARN] output file exists already! remove it first!" + fileOut);
        return false;
      }

      out = filesystemOut.create(fileOut, true, outputBufferSize);
      if (alg == Algorithm.LZO) {
        LzopCodec codec = new LzopCodec();
        codec.setConf(conf);
        os = codec.createOutputStream(out);
      } else if (alg == Algorithm.GZ) {
        os = new GZIPOutputStream(out);
      } else {
        os = out;
      }

      boolean isDone = false;
      byte[] buffer = new byte[copyBufferSize];
      int idx = 0;
      for (Path p : todoNames) {
        FSDataInputStream fsdi = null;
        int readDataLen = 0;
        boolean read = false;
        boolean write = false;
        try {
          System.out.println("[INFO]processing: (" + idx + ")" + p);
          fsdi = filesystemIn.open(p, inputBufferSize);
          for (; ; ) {
            int n = fsdi.read(buffer);
            if (n < 0) {
              break;
            }
            read = true;
            os.write(buffer, 0, n);
            readDataLen += n;
            write = true;
          }
          isDone = true;
        } catch (IOException e) {
          System.err.println(
              "[WARN] error reading :"
                  + p.toString()
                  + ", read data len: "
                  + readDataLen
                  + ", hasRead:"
                  + read
                  + ", hasWrite:"
                  + write
                  + ", msg:"
                  + e.getMessage());
        } finally {
          if (fsdi != null) {
            try {
              fsdi.close();
            } catch (IOException ignored) {
            }
            fsdi = null;
            idx++;
          }
        }
        if (read && !write) {
          break;
        }
      }
      if (!isDone) {
        System.err.println("[ERROR] not all files are written!!!");
        return true;
      }
      return true;
    } finally {
      if (os != null) {
        try {
          os.close();
        } catch (IOException ignored) {
        }
      } else if (out != null) {
        try {
          out.close();
        } catch (IOException ignored) {
        }
      }
    }
  }