Exemplo n.º 1
0
  public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
      throws IOException {

    final FileSystem rfs = FileSystem.getLocal(job).getRaw();
    final DataInputStream in =
        new DataInputStream(
            SecureIOUtils.openForRead(
                new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
    try {
      final long length = rfs.getFileStatus(indexFileName).getLen();
      final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
      final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;

      buf = ByteBuffer.allocate(size);
      if (crc != null) {
        crc.reset();
        CheckedInputStream chk = new CheckedInputStream(in, crc);
        IOUtils.readFully(chk, buf.array(), 0, size);
        if (chk.getChecksum().getValue() != in.readLong()) {
          throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
        }
      } else {
        IOUtils.readFully(in, buf.array(), 0, size);
      }
      entries = buf.asLongBuffer();
    } finally {
      in.close();
    }
  }
Exemplo n.º 2
0
  private static void zipFiles(String[] fileNames) throws IOException {
    // 获取zip文件输出流
    FileOutputStream f = new FileOutputStream("test.zip");
    // 从文件输出流中获取数据校验和输出流,并设置Adler32
    CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32());
    // 从数据校验和输出流中获取Zip输出流
    ZipOutputStream zos = new ZipOutputStream(csum);
    // 从Zip输出流中获取缓冲输出流
    BufferedOutputStream out = new BufferedOutputStream(zos);
    // 设置Zip文件注释
    zos.setComment("测试 java zip stream");
    for (String file : fileNames) {
      System.out.println("写入文件: " + file);
      // 获取文件输入字符流
      BufferedReader in = new BufferedReader(new FileReader(file));
      // 想Zip处理写入新的文件条目,并流定位到数据开始处
      zos.putNextEntry(new ZipEntry(file));
      int c;
      while ((c = in.read()) > 0) out.write(c);
      in.close();
      // 刷新Zip输出流,将缓冲的流写入该流
      out.flush();
    }
    // 文件全部写入Zip输出流后,关闭
    out.close();

    // 输出数据校验和
    System.out.println("数据校验和: " + csum.getChecksum().getValue());
    System.out.println("读取zip文件");
    // 读取test.zip文件输入流
    FileInputStream fi = new FileInputStream("test.zip");
    // 从文件输入流中获取数据校验和流
    CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32());
    // 从数据校验和流中获取Zip解压流
    ZipInputStream in2 = new ZipInputStream(csumi);
    // 从Zip解压流中获取缓冲输入流
    BufferedInputStream bis = new BufferedInputStream(in2);
    // 创建文件条目
    ZipEntry zipEntry;
    while ((zipEntry = in2.getNextEntry()) != null) {
      System.out.println("读取文件: " + zipEntry);
      int x;
      while ((x = bis.read()) > 0) System.out.write(x);
    }
    if (fileNames.length == 1) System.out.println("数据校验和: " + csumi.getChecksum().getValue());
    bis.close();

    // 获取Zip文件
    ZipFile zf = new ZipFile("test.zip");
    // 获取文件条目枚举
    Enumeration e = zf.entries();
    while (e.hasMoreElements()) {
      // 从Zip文件的枚举中获取文件条目
      ZipEntry ze2 = (ZipEntry) e.nextElement();
      System.out.println("文件: " + ze2);
    }
  }
Exemplo n.º 3
0
  private static Long checksum(String filename) throws IOException {

    // Compute Adler-32 checksum
    CheckedInputStream cis = new CheckedInputStream(new FileInputStream(filename), new Adler32());
    byte[] tempBuf = new byte[128];

    while (cis.read(tempBuf) >= 0) {}

    return cis.getChecksum().getValue();
  }
 public long checksum(File f) throws FITTESTException {
   try {
     CheckedInputStream cis = new CheckedInputStream(new FileInputStream(f), new Adler32());
     BufferedInputStream bis = new BufferedInputStream(cis, BUFFER);
     byte data[] = new byte[BUFFER];
     while (bis.read(data, 0, BUFFER) != -1) ;
     bis.close();
     return cis.getChecksum().getValue();
   } catch (FileNotFoundException e) {
     throw new FITTESTException(e.getMessage());
   } catch (IOException e) {
     throw new FITTESTException(e.getMessage());
   }
 }
Exemplo n.º 5
0
  /**
   * This method will attempt to checksum the current JE db environment by computing the Adler-32
   * checksum on the latest JE log file available.
   *
   * @return The checksum of JE db environment or zero if checksum failed.
   */
  private long checksumDbEnv() {

    File parentDirectory = getFileForPath(cfg.getDBDirectory());
    File backendDirectory = new File(parentDirectory, cfg.getBackendId());

    List<File> jdbFiles = new ArrayList<File>();
    if (backendDirectory.isDirectory()) {
      jdbFiles =
          Arrays.asList(
              backendDirectory.listFiles(
                  new FilenameFilter() {
                    public boolean accept(File dir, String name) {
                      return name.endsWith(".jdb");
                    }
                  }));
    }

    if (!jdbFiles.isEmpty()) {
      Collections.sort(jdbFiles, Collections.reverseOrder());
      FileInputStream fis = null;
      try {
        fis = new FileInputStream(jdbFiles.get(0).toString());
        CheckedInputStream cis = new CheckedInputStream(fis, new Adler32());
        byte[] tempBuf = new byte[8192];
        while (cis.read(tempBuf) >= 0) {}

        return cis.getChecksum().getValue();
      } catch (Exception e) {
        if (debugEnabled()) {
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
        }
      } finally {
        if (fis != null) {
          try {
            fis.close();
          } catch (Exception e) {
            if (debugEnabled()) {
              TRACER.debugCaught(DebugLogLevel.ERROR, e);
            }
          }
        }
      }
    }

    return 0;
  }
Exemplo n.º 6
0
 public static void main(String[] args) throws IOException {
   FileOutputStream f = new FileOutputStream("test.zip");
   CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32());
   ZipOutputStream zos = new ZipOutputStream(csum);
   BufferedOutputStream out = new BufferedOutputStream(zos);
   zos.setComment("A test of Java Zipping");
   // No corresponding getComment(), though.
   for (String arg : args) {
     print("Writing file " + arg);
     BufferedReader in = new BufferedReader(new FileReader(arg));
     zos.putNextEntry(new ZipEntry(arg));
     int c;
     while ((c = in.read()) != -1) out.write(c);
     in.close();
     out.flush();
   }
   out.close();
   // Checksum valid only after the file has been closed!
   print("Checksum: " + csum.getChecksum().getValue());
   // Now extract the files:
   print("Reading file");
   FileInputStream fi = new FileInputStream("test.zip");
   CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32());
   ZipInputStream in2 = new ZipInputStream(csumi);
   BufferedInputStream bis = new BufferedInputStream(in2);
   ZipEntry ze;
   while ((ze = in2.getNextEntry()) != null) {
     print("Reading file " + ze);
     int x;
     while ((x = bis.read()) != -1) System.out.write(x);
   }
   if (args.length == 1) print("Checksum: " + csumi.getChecksum().getValue());
   bis.close();
   // Alternative way to open and read Zip files:
   ZipFile zf = new ZipFile("test.zip");
   Enumeration e = zf.entries();
   while (e.hasMoreElements()) {
     ZipEntry ze2 = (ZipEntry) e.nextElement();
     print("File: " + ze2);
     // ... and extract the data as before
   }
   /* if(args.length == 1) */
 }
Exemplo n.º 7
0
  /**
   * Calculates the CRC32 value of a file.
   *
   * @param file File to compute the CRC value
   * @return CRC value formatted in 8 length hexadecimal in lowercase
   */
  public static String calculateCRC32(File file) throws ChecksumException {
    if (file == null) {
      return null;
    }
    String hex = "";
    try {
      CheckedInputStream cis = new CheckedInputStream(new FileInputStream(file), new CRC32());
      byte[] buf = new byte[10240]; // 10mb

      while (cis.read(buf) >= 0) ;

      hex = Long.toHexString(cis.getChecksum().getValue());
      cis.close();
    } catch (IOException | NullPointerException e) {
      throw new ChecksumException("Unable to determine CRC32 value for file: " + file.getName());
    }
    for (int i = hex.length(); i < CRC32_LENGTH; i++) {
      hex = "0" + hex;
    }
    return hex;
  }
  private int extractZip(String zip_path) {
    ZipInputStream zip = null;
    try {
      zip = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip_path)));
    } catch (java.io.FileNotFoundException e) {
      sendMessage(-2, 0, "Failed to read from zip file: " + e.toString());
      return -1;
    }
    ;

    int num_file = 0;
    while (true) {
      num_file++;
      ZipEntry entry = null;
      try {
        entry = zip.getNextEntry();
      } catch (java.io.IOException e) {
        sendMessage(-2, 0, "Failed to get entry from zip file: " + e.toString());
        return -1;
      }
      if (entry == null) break;

      String path = extract_dir + "/" + entry.getName();
      if (entry.isDirectory()) {
        try {
          (new File(path)).mkdirs();
        } catch (SecurityException e) {
          sendMessage(-2, 0, "Failed to create directory: " + e.toString());
          return -1;
        }
        continue;
      }

      try {
        (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
      } catch (SecurityException e) {
        sendMessage(-2, 0, "Failed to create directory: " + e.toString());
        return -1;
      }
      ;

      if (extractZipEntry(path, zip, (int) entry.getSize(), "Extracting archives: " + num_file)
          != 0) return -1;

      try {
        CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
        while (check.read(buf) > 0) {}
        ;
        check.close();
        if (check.getChecksum().getValue() != entry.getCrc()) {
          File ff = new File(path);
          ff.delete();
          throw new Exception();
        }
      } catch (Exception e) {
        sendMessage(-2, 0, "CRC check failed");
        return -1;
      }
    }

    return 0;
  }
Exemplo n.º 9
0
  public boolean DownloadDataFile(final String DataDownloadUrl, final String DownloadFlagFileName) {
    String[] downloadUrls = DataDownloadUrl.split("[|]");
    if (downloadUrls.length < 2) return false;

    Resources res = Parent.getResources();

    String path = getOutFilePath(DownloadFlagFileName);
    InputStream checkFile = null;
    try {
      checkFile = new FileInputStream(path);
    } catch (FileNotFoundException e) {
    } catch (SecurityException e) {
    }
    ;
    if (checkFile != null) {
      try {
        byte b[] = new byte[Globals.DataDownloadUrl.getBytes("UTF-8").length + 1];
        int readed = checkFile.read(b);
        String compare = new String(b, 0, readed, "UTF-8");
        boolean matched = false;
        // System.out.println("Read URL: '" + compare + "'");
        for (int i = 1; i < downloadUrls.length; i++) {
          // System.out.println("Comparing: '" + downloadUrls[i] + "'");
          if (compare.compareTo(downloadUrls[i]) == 0) matched = true;
        }
        // System.out.println("Matched: " + String.valueOf(matched));
        if (!matched) throw new IOException();
        Status.setText(res.getString(R.string.download_unneeded));
        return true;
      } catch (IOException e) {
      }
      ;
    }
    checkFile = null;

    // Create output directory (not necessary for phone storage)
    //		System.out.println("Downloading data to: '" + outFilesDir + "'");
    try {
      (new File(outFilesDir)).mkdirs();
      OutputStream out = new FileOutputStream(getOutFilePath(".nomedia"));
      out.flush();
      out.close();
    } catch (SecurityException e) {
    } catch (FileNotFoundException e) {
    } catch (IOException e) {
    }
    ;

    HttpResponse response = null;
    HttpGet request;
    long totalLen = 0;
    CountingInputStream stream;
    byte[] buf = new byte[16384];
    boolean DoNotUnzip = false;
    boolean FileInAssets = false;
    String url = "";

    int downloadUrlIndex = 1;
    while (downloadUrlIndex < downloadUrls.length) {
      System.out.println("Processing download " + downloadUrls[downloadUrlIndex]);
      url = new String(downloadUrls[downloadUrlIndex]);
      DoNotUnzip = false;
      if (url.indexOf(":") == 0) {
        url = url.substring(url.indexOf(":", 1) + 1);
        DoNotUnzip = true;
      }
      Status.setText(res.getString(R.string.connecting_to, url));
      if (url.indexOf("http://") == -1 && url.indexOf("https://") == -1) // File inside assets
      {
        System.out.println("Fetching file from assets: " + url);
        FileInAssets = true;
        break;
      } else {
        System.out.println("Connecting to: " + url);
        request = new HttpGet(url);
        request.addHeader("Accept", "*/*");
        try {
          DefaultHttpClient client = HttpWithDisabledSslCertCheck();
          client.getParams().setBooleanParameter("http.protocol.handle-redirects", true);
          response = client.execute(request);
        } catch (IOException e) {
          System.out.println("Failed to connect to " + url);
          downloadUrlIndex++;
        }
        ;
        if (response != null) {
          if (response.getStatusLine().getStatusCode() != 200) {
            response = null;
            System.out.println("Failed to connect to " + url);
            downloadUrlIndex++;
          } else break;
        }
      }
    }
    if (FileInAssets) {
      System.out.println("Unpacking from assets: '" + url + "'");
      try {
        //				System.out.println("Unpacking from assets: '" + url + "' 1");
        stream = new CountingInputStream(Parent.getAssets().open(url), 8192);
        //				System.out.println("Unpacking from assets: '" + url + "' 2");
        while (stream.skip(65536) > 0) {}
        ;
        //				System.out.println("Unpacking from assets: '" + url + "' 3");
        totalLen = stream.getBytesRead();
        //				System.out.println("Unpacking from assets: '" + url + "' 4 totalLen = " +
        // String.valueOf(totalLen));
        stream.close();
        //				System.out.println("Unpacking from assets: '" + url + "' 5");
        stream = new CountingInputStream(Parent.getAssets().open(url), 8192);
        //				System.out.println("Unpacking from assets: '" + url + "' 6");
      } catch (IOException e) {
        System.out.println("Unpacking from assets '" + url + "' - error: " + e.toString());
        Status.setText(res.getString(R.string.error_dl_from, url));
        return false;
      }
    } else {
      if (response == null) {
        //				System.out.println("Error connecting to " + url);
        Status.setText(res.getString(R.string.failed_connecting_to, url));
        return false;
      }

      Status.setText(res.getString(R.string.dl_from, url));
      totalLen = response.getEntity().getContentLength();
      try {
        stream = new CountingInputStream(response.getEntity().getContent(), 8192);
      } catch (java.io.IOException e) {
        Status.setText(res.getString(R.string.error_dl_from, url));
        return false;
      }
    }

    if (DoNotUnzip) {
      path =
          getOutFilePath(
              downloadUrls[downloadUrlIndex].substring(
                  1, downloadUrls[downloadUrlIndex].indexOf(":", 1)));
      //			System.out.println("Saving file '" + path + "'");
      OutputStream out = null;
      try {
        try {
          (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
        } catch (SecurityException e) {
        }
        ;

        out = new FileOutputStream(path);
      } catch (FileNotFoundException e) {
        System.out.println(
            "Saving file '" + path + "' - error creating output file: " + e.toString());
      } catch (SecurityException e) {
        System.out.println(
            "Saving file '" + path + "' - error creating output file: " + e.toString());
      }
      ;
      if (out == null) {
        Status.setText(res.getString(R.string.error_write, path));
        System.out.println("Saving file '" + path + "' - error creating output file");
        return false;
      }

      try {
        int len = stream.read(buf);
        while (len >= 0) {
          if (len > 0) out.write(buf, 0, len);
          len = stream.read(buf);

          float percent = 0.0f;
          if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
          Status.setText(res.getString(R.string.dl_progress, percent, path));
        }
        out.flush();
        out.close();
        out = null;
      } catch (java.io.IOException e) {
        Status.setText(res.getString(R.string.error_write, path));
        System.out.println("Saving file '" + path + "' - error writing: " + e.toString());
        return false;
      }
      //			System.out.println("Saving file '" + path + "' done");
    } else {
      //			System.out.println("Reading from zip file aa'" + url + "'");
      ZipInputStream zip = new ZipInputStream(stream);

      while (true) {
        ZipEntry entry = null;
        try {
          entry = zip.getNextEntry();
          if (entry != null)
            System.out.println(
                "Reading from zip file '" + url + "' entry '" + entry.getName() + "'");
        } catch (java.io.IOException e) {
          Status.setText(res.getString(R.string.error_dl_from, url));
          System.out.println("Error reading from zip file '" + url + "': " + e.toString());
          return false;
        }
        if (entry == null) {
          System.out.println("Reading from zip file '" + url + "' finished");
          break;
        }
        if (entry.isDirectory()) {
          System.out.println("Creating dir '" + getOutFilePath(entry.getName()) + "'");
          try {
            (new File(getOutFilePath(entry.getName()))).mkdirs();
          } catch (SecurityException e) {
          }
          ;
          continue;
        }

        OutputStream out = null;
        path = getOutFilePath(entry.getName());

        //				System.out.println("Saving file a '" + path + "'");

        try {
          (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
        } catch (SecurityException e) {
        }
        ;

        try {
          CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
          while (check.read(buf, 0, buf.length) > 0) {}
          ;
          check.close();
          if (check.getChecksum().getValue() != entry.getCrc()) {
            File ff = new File(path);
            ff.delete();
            throw new Exception();
          }
          //					System.out.println("File '" + path + "' exists and passed CRC check - not
          // overwriting it");
          continue;
        } catch (Exception e) {
        }

        try {
          out = new FileOutputStream(path);
        } catch (FileNotFoundException e) {
          System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString());
        } catch (SecurityException e) {
          System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString());
        }
        ;
        if (out == null) {
          Status.setText(res.getString(R.string.error_write, path));
          System.out.println("Saving file '" + path + "' - cannot create file");
          return false;
        }

        float percent = 0.0f;
        if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
        Status.setText(res.getString(R.string.dl_progress, percent, path));

        try {
          int len = zip.read(buf);
          while (len >= 0) {
            if (len > 0) out.write(buf, 0, len);
            len = zip.read(buf);

            percent = 0.0f;
            if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
            Status.setText(res.getString(R.string.dl_progress, percent, path));
          }
          out.flush();
          out.close();
          out = null;
        } catch (java.io.IOException e) {
          Status.setText(res.getString(R.string.error_write, path));
          System.out.println(
              "Saving file '" + path + "' - error writing or downloading: " + e.toString());
          return false;
        }

        try {
          CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
          while (check.read(buf, 0, buf.length) > 0) {}
          ;
          check.close();
          if (check.getChecksum().getValue() != entry.getCrc()) {
            File ff = new File(path);
            ff.delete();
            throw new Exception();
          }
        } catch (Exception e) {
          Status.setText(res.getString(R.string.error_write, path));
          System.out.println("Saving file '" + path + "' - CRC check failed");
          return false;
        }
        System.out.println("Saving file '" + path + "' done");
      }
    }
    ;

    OutputStream out = null;
    path = getOutFilePath(DownloadFlagFileName);
    try {
      out = new FileOutputStream(path);
      out.write(downloadUrls[downloadUrlIndex].getBytes("UTF-8"));
      out.flush();
      out.close();
    } catch (FileNotFoundException e) {
    } catch (SecurityException e) {
    } catch (java.io.IOException e) {
      Status.setText(res.getString(R.string.error_write, path));
      return false;
    }
    ;
    Status.setText(res.getString(R.string.dl_finished));

    try {
      stream.close();
    } catch (java.io.IOException e) {
    }
    ;

    return true;
  };