public static void unCompressLzo(String input, String output, Configuration conf) {
   LzopCodec lzo = null;
   InputStream is = null;
   InputStreamReader isr = null;
   BufferedReader reader = null;
   String line = null;
   BufferedWriter bw = null;
   try {
     bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output)));
     lzo = new LzopCodec();
     lzo.setConf(conf);
     is = lzo.createInputStream(new FileInputStream(input));
     isr = new InputStreamReader(is);
     reader = new BufferedReader(isr);
     while ((line = reader.readLine()) != null) {
       bw.write(line);
       bw.newLine();
     }
   } catch (FileNotFoundException e) {
     e.printStackTrace();
   } catch (IOException e) {
     e.printStackTrace();
   } finally {
     try {
       if (null != reader) reader.close();
       if (null != isr) isr.close();
       if (null != is) is.close();
       if (null != bw) bw.close();
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
 }
  public static List<String> readLzoFile(String lzoFilePath, Configuration conf) {
    LzopCodec lzo = null;
    InputStream is = null;
    InputStreamReader isr = null;
    BufferedReader reader = null;
    List<String> result = null;
    String line = null;

    try {
      lzo = new LzopCodec();
      lzo.setConf(conf);
      is = lzo.createInputStream(new FileInputStream(lzoFilePath));
      isr = new InputStreamReader(is);
      reader = new BufferedReader(isr);
      result = new ArrayList<String>();
      while ((line = reader.readLine()) != null) {
        result.add(line);
      }

    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if (reader != null) {
          reader.close();
        }
        if (isr != null) {
          isr.close();
        }
        if (is != null) {
          is.close();
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    return result;
  }
  public static void write2LzoFile(String destLzoFilePath, Configuration conf, byte[] datas) {
    LzopCodec lzo = null;
    OutputStream out = null;

    try {
      lzo = new LzopCodec();
      lzo.setConf(conf);
      out = lzo.createOutputStream(new FileOutputStream(destLzoFilePath));
      out.write(datas);
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if (out != null) {
          out.close();
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
Exemple #4
0
  private static boolean copyLocalFilesToOneHDFS(
      FileSystem filesystemIn,
      List<Path> todoNames,
      FileSystem filesystemOut,
      Path pathOut,
      String filenamePrefix,
      Algorithm alg,
      Configuration conf)
      throws IOException {

    int outputBufferSize = 10 * 1024 * 1024;
    int inputBufferSize = 10 * 1024 * 1024;
    int copyBufferSize = 10 * 1024 * 1024;

    // batch load data
    FSDataOutputStream out = null;
    OutputStream os = null;

    try {
      String postfix;
      if (alg == Algorithm.LZO) {
        postfix = ".lzo";
      } else if (alg == Algorithm.GZ) {
        postfix = ".gz";
      } else {
        postfix = ".txt";
      }

      Path fileOut = new Path(pathOut, filenamePrefix + postfix);
      if (filesystemOut.exists(fileOut)) {
        System.err.println("[WARN] output file exists already! remove it first!" + fileOut);
        return false;
      }

      out = filesystemOut.create(fileOut, true, outputBufferSize);
      if (alg == Algorithm.LZO) {
        LzopCodec codec = new LzopCodec();
        codec.setConf(conf);
        os = codec.createOutputStream(out);
      } else if (alg == Algorithm.GZ) {
        os = new GZIPOutputStream(out);
      } else {
        os = out;
      }

      boolean isDone = false;
      byte[] buffer = new byte[copyBufferSize];
      int idx = 0;
      for (Path p : todoNames) {
        FSDataInputStream fsdi = null;
        int readDataLen = 0;
        boolean read = false;
        boolean write = false;
        try {
          System.out.println("[INFO]processing: (" + idx + ")" + p);
          fsdi = filesystemIn.open(p, inputBufferSize);
          for (; ; ) {
            int n = fsdi.read(buffer);
            if (n < 0) {
              break;
            }
            read = true;
            os.write(buffer, 0, n);
            readDataLen += n;
            write = true;
          }
          isDone = true;
        } catch (IOException e) {
          System.err.println(
              "[WARN] error reading :"
                  + p.toString()
                  + ", read data len: "
                  + readDataLen
                  + ", hasRead:"
                  + read
                  + ", hasWrite:"
                  + write
                  + ", msg:"
                  + e.getMessage());
        } finally {
          if (fsdi != null) {
            try {
              fsdi.close();
            } catch (IOException ignored) {
            }
            fsdi = null;
            idx++;
          }
        }
        if (read && !write) {
          break;
        }
      }
      if (!isDone) {
        System.err.println("[ERROR] not all files are written!!!");
        return true;
      }
      return true;
    } finally {
      if (os != null) {
        try {
          os.close();
        } catch (IOException ignored) {
        }
      } else if (out != null) {
        try {
          out.close();
        } catch (IOException ignored) {
        }
      }
    }
  }