public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner) throws IOException { final FileSystem rfs = FileSystem.getLocal(job).getRaw(); final DataInputStream in = new DataInputStream( SecureIOUtils.openForRead( new File(indexFileName.toUri().getPath()), expectedIndexOwner, null)); try { final long length = rfs.getFileStatus(indexFileName).getLen(); final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH; final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH; buf = ByteBuffer.allocate(size); if (crc != null) { crc.reset(); CheckedInputStream chk = new CheckedInputStream(in, crc); IOUtils.readFully(chk, buf.array(), 0, size); if (chk.getChecksum().getValue() != in.readLong()) { throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1); } } else { IOUtils.readFully(in, buf.array(), 0, size); } entries = buf.asLongBuffer(); } finally { in.close(); } }
private static void zipFiles(String[] fileNames) throws IOException { // 获取zip文件输出流 FileOutputStream f = new FileOutputStream("test.zip"); // 从文件输出流中获取数据校验和输出流,并设置Adler32 CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32()); // 从数据校验和输出流中获取Zip输出流 ZipOutputStream zos = new ZipOutputStream(csum); // 从Zip输出流中获取缓冲输出流 BufferedOutputStream out = new BufferedOutputStream(zos); // 设置Zip文件注释 zos.setComment("测试 java zip stream"); for (String file : fileNames) { System.out.println("写入文件: " + file); // 获取文件输入字符流 BufferedReader in = new BufferedReader(new FileReader(file)); // 想Zip处理写入新的文件条目,并流定位到数据开始处 zos.putNextEntry(new ZipEntry(file)); int c; while ((c = in.read()) > 0) out.write(c); in.close(); // 刷新Zip输出流,将缓冲的流写入该流 out.flush(); } // 文件全部写入Zip输出流后,关闭 out.close(); // 输出数据校验和 System.out.println("数据校验和: " + csum.getChecksum().getValue()); System.out.println("读取zip文件"); // 读取test.zip文件输入流 FileInputStream fi = new FileInputStream("test.zip"); // 从文件输入流中获取数据校验和流 CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32()); // 从数据校验和流中获取Zip解压流 ZipInputStream in2 = new ZipInputStream(csumi); // 从Zip解压流中获取缓冲输入流 BufferedInputStream bis = new BufferedInputStream(in2); // 创建文件条目 ZipEntry zipEntry; while ((zipEntry = in2.getNextEntry()) != null) { System.out.println("读取文件: " + zipEntry); int x; while ((x = bis.read()) > 0) System.out.write(x); } if (fileNames.length == 1) System.out.println("数据校验和: " + csumi.getChecksum().getValue()); bis.close(); // 获取Zip文件 ZipFile zf = new ZipFile("test.zip"); // 获取文件条目枚举 Enumeration e = zf.entries(); while (e.hasMoreElements()) { // 从Zip文件的枚举中获取文件条目 ZipEntry ze2 = (ZipEntry) e.nextElement(); System.out.println("文件: " + ze2); } }
private static Long checksum(String filename) throws IOException { // Compute Adler-32 checksum CheckedInputStream cis = new CheckedInputStream(new FileInputStream(filename), new Adler32()); byte[] tempBuf = new byte[128]; while (cis.read(tempBuf) >= 0) {} return cis.getChecksum().getValue(); }
public long checksum(File f) throws FITTESTException { try { CheckedInputStream cis = new CheckedInputStream(new FileInputStream(f), new Adler32()); BufferedInputStream bis = new BufferedInputStream(cis, BUFFER); byte data[] = new byte[BUFFER]; while (bis.read(data, 0, BUFFER) != -1) ; bis.close(); return cis.getChecksum().getValue(); } catch (FileNotFoundException e) { throw new FITTESTException(e.getMessage()); } catch (IOException e) { throw new FITTESTException(e.getMessage()); } }
/** * This method will attempt to checksum the current JE db environment by computing the Adler-32 * checksum on the latest JE log file available. * * @return The checksum of JE db environment or zero if checksum failed. */ private long checksumDbEnv() { File parentDirectory = getFileForPath(cfg.getDBDirectory()); File backendDirectory = new File(parentDirectory, cfg.getBackendId()); List<File> jdbFiles = new ArrayList<File>(); if (backendDirectory.isDirectory()) { jdbFiles = Arrays.asList( backendDirectory.listFiles( new FilenameFilter() { public boolean accept(File dir, String name) { return name.endsWith(".jdb"); } })); } if (!jdbFiles.isEmpty()) { Collections.sort(jdbFiles, Collections.reverseOrder()); FileInputStream fis = null; try { fis = new FileInputStream(jdbFiles.get(0).toString()); CheckedInputStream cis = new CheckedInputStream(fis, new Adler32()); byte[] tempBuf = new byte[8192]; while (cis.read(tempBuf) >= 0) {} return cis.getChecksum().getValue(); } catch (Exception e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } finally { if (fis != null) { try { fis.close(); } catch (Exception e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } } } return 0; }
public static void main(String[] args) throws IOException { FileOutputStream f = new FileOutputStream("test.zip"); CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32()); ZipOutputStream zos = new ZipOutputStream(csum); BufferedOutputStream out = new BufferedOutputStream(zos); zos.setComment("A test of Java Zipping"); // No corresponding getComment(), though. for (String arg : args) { print("Writing file " + arg); BufferedReader in = new BufferedReader(new FileReader(arg)); zos.putNextEntry(new ZipEntry(arg)); int c; while ((c = in.read()) != -1) out.write(c); in.close(); out.flush(); } out.close(); // Checksum valid only after the file has been closed! print("Checksum: " + csum.getChecksum().getValue()); // Now extract the files: print("Reading file"); FileInputStream fi = new FileInputStream("test.zip"); CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32()); ZipInputStream in2 = new ZipInputStream(csumi); BufferedInputStream bis = new BufferedInputStream(in2); ZipEntry ze; while ((ze = in2.getNextEntry()) != null) { print("Reading file " + ze); int x; while ((x = bis.read()) != -1) System.out.write(x); } if (args.length == 1) print("Checksum: " + csumi.getChecksum().getValue()); bis.close(); // Alternative way to open and read Zip files: ZipFile zf = new ZipFile("test.zip"); Enumeration e = zf.entries(); while (e.hasMoreElements()) { ZipEntry ze2 = (ZipEntry) e.nextElement(); print("File: " + ze2); // ... and extract the data as before } /* if(args.length == 1) */ }
/** * Calculates the CRC32 value of a file. * * @param file File to compute the CRC value * @return CRC value formatted in 8 length hexadecimal in lowercase */ public static String calculateCRC32(File file) throws ChecksumException { if (file == null) { return null; } String hex = ""; try { CheckedInputStream cis = new CheckedInputStream(new FileInputStream(file), new CRC32()); byte[] buf = new byte[10240]; // 10mb while (cis.read(buf) >= 0) ; hex = Long.toHexString(cis.getChecksum().getValue()); cis.close(); } catch (IOException | NullPointerException e) { throw new ChecksumException("Unable to determine CRC32 value for file: " + file.getName()); } for (int i = hex.length(); i < CRC32_LENGTH; i++) { hex = "0" + hex; } return hex; }
private int extractZip(String zip_path) { ZipInputStream zip = null; try { zip = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip_path))); } catch (java.io.FileNotFoundException e) { sendMessage(-2, 0, "Failed to read from zip file: " + e.toString()); return -1; } ; int num_file = 0; while (true) { num_file++; ZipEntry entry = null; try { entry = zip.getNextEntry(); } catch (java.io.IOException e) { sendMessage(-2, 0, "Failed to get entry from zip file: " + e.toString()); return -1; } if (entry == null) break; String path = extract_dir + "/" + entry.getName(); if (entry.isDirectory()) { try { (new File(path)).mkdirs(); } catch (SecurityException e) { sendMessage(-2, 0, "Failed to create directory: " + e.toString()); return -1; } continue; } try { (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs(); } catch (SecurityException e) { sendMessage(-2, 0, "Failed to create directory: " + e.toString()); return -1; } ; if (extractZipEntry(path, zip, (int) entry.getSize(), "Extracting archives: " + num_file) != 0) return -1; try { CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32()); while (check.read(buf) > 0) {} ; check.close(); if (check.getChecksum().getValue() != entry.getCrc()) { File ff = new File(path); ff.delete(); throw new Exception(); } } catch (Exception e) { sendMessage(-2, 0, "CRC check failed"); return -1; } } return 0; }
public boolean DownloadDataFile(final String DataDownloadUrl, final String DownloadFlagFileName) { String[] downloadUrls = DataDownloadUrl.split("[|]"); if (downloadUrls.length < 2) return false; Resources res = Parent.getResources(); String path = getOutFilePath(DownloadFlagFileName); InputStream checkFile = null; try { checkFile = new FileInputStream(path); } catch (FileNotFoundException e) { } catch (SecurityException e) { } ; if (checkFile != null) { try { byte b[] = new byte[Globals.DataDownloadUrl.getBytes("UTF-8").length + 1]; int readed = checkFile.read(b); String compare = new String(b, 0, readed, "UTF-8"); boolean matched = false; // System.out.println("Read URL: '" + compare + "'"); for (int i = 1; i < downloadUrls.length; i++) { // System.out.println("Comparing: '" + downloadUrls[i] + "'"); if (compare.compareTo(downloadUrls[i]) == 0) matched = true; } // System.out.println("Matched: " + String.valueOf(matched)); if (!matched) throw new IOException(); Status.setText(res.getString(R.string.download_unneeded)); return true; } catch (IOException e) { } ; } checkFile = null; // Create output directory (not necessary for phone storage) // System.out.println("Downloading data to: '" + outFilesDir + "'"); try { (new File(outFilesDir)).mkdirs(); OutputStream out = new FileOutputStream(getOutFilePath(".nomedia")); out.flush(); out.close(); } catch (SecurityException e) { } catch (FileNotFoundException e) { } catch (IOException e) { } ; HttpResponse response = null; HttpGet request; long totalLen = 0; CountingInputStream stream; byte[] buf = new byte[16384]; boolean DoNotUnzip = false; boolean FileInAssets = false; String url = ""; int downloadUrlIndex = 1; while (downloadUrlIndex < downloadUrls.length) { System.out.println("Processing download " + downloadUrls[downloadUrlIndex]); url = new String(downloadUrls[downloadUrlIndex]); DoNotUnzip = false; if (url.indexOf(":") == 0) { url = url.substring(url.indexOf(":", 1) + 1); DoNotUnzip = true; } Status.setText(res.getString(R.string.connecting_to, url)); if (url.indexOf("http://") == -1 && url.indexOf("https://") == -1) // File inside assets { System.out.println("Fetching file from assets: " + url); FileInAssets = true; break; } else { System.out.println("Connecting to: " + url); request = new HttpGet(url); request.addHeader("Accept", "*/*"); try { DefaultHttpClient client = HttpWithDisabledSslCertCheck(); client.getParams().setBooleanParameter("http.protocol.handle-redirects", true); response = client.execute(request); } catch (IOException e) { System.out.println("Failed to connect to " + url); downloadUrlIndex++; } ; if (response != null) { if (response.getStatusLine().getStatusCode() != 200) { response = null; System.out.println("Failed to connect to " + url); downloadUrlIndex++; } else break; } } } if (FileInAssets) { System.out.println("Unpacking from assets: '" + url + "'"); try { // System.out.println("Unpacking from assets: '" + url + "' 1"); stream = new CountingInputStream(Parent.getAssets().open(url), 8192); // System.out.println("Unpacking from assets: '" + url + "' 2"); while (stream.skip(65536) > 0) {} ; // System.out.println("Unpacking from assets: '" + url + "' 3"); totalLen = stream.getBytesRead(); // System.out.println("Unpacking from assets: '" + url + "' 4 totalLen = " + // String.valueOf(totalLen)); stream.close(); // System.out.println("Unpacking from assets: '" + url + "' 5"); stream = new CountingInputStream(Parent.getAssets().open(url), 8192); // System.out.println("Unpacking from assets: '" + url + "' 6"); } catch (IOException e) { System.out.println("Unpacking from assets '" + url + "' - error: " + e.toString()); Status.setText(res.getString(R.string.error_dl_from, url)); return false; } } else { if (response == null) { // System.out.println("Error connecting to " + url); Status.setText(res.getString(R.string.failed_connecting_to, url)); return false; } Status.setText(res.getString(R.string.dl_from, url)); totalLen = response.getEntity().getContentLength(); try { stream = new CountingInputStream(response.getEntity().getContent(), 8192); } catch (java.io.IOException e) { Status.setText(res.getString(R.string.error_dl_from, url)); return false; } } if (DoNotUnzip) { path = getOutFilePath( downloadUrls[downloadUrlIndex].substring( 1, downloadUrls[downloadUrlIndex].indexOf(":", 1))); // System.out.println("Saving file '" + path + "'"); OutputStream out = null; try { try { (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs(); } catch (SecurityException e) { } ; out = new FileOutputStream(path); } catch (FileNotFoundException e) { System.out.println( "Saving file '" + path + "' - error creating output file: " + e.toString()); } catch (SecurityException e) { System.out.println( "Saving file '" + path + "' - error creating output file: " + e.toString()); } ; if (out == null) { Status.setText(res.getString(R.string.error_write, path)); System.out.println("Saving file '" + path + "' - error creating output file"); return false; } try { int len = stream.read(buf); while (len >= 0) { if (len > 0) out.write(buf, 0, len); len = stream.read(buf); float percent = 0.0f; if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen; Status.setText(res.getString(R.string.dl_progress, percent, path)); } out.flush(); out.close(); out = null; } catch (java.io.IOException e) { Status.setText(res.getString(R.string.error_write, path)); System.out.println("Saving file '" + path + "' - error writing: " + e.toString()); return false; } // System.out.println("Saving file '" + path + "' done"); } else { // System.out.println("Reading from zip file aa'" + url + "'"); ZipInputStream zip = new ZipInputStream(stream); while (true) { ZipEntry entry = null; try { entry = zip.getNextEntry(); if (entry != null) System.out.println( "Reading from zip file '" + url + "' entry '" + entry.getName() + "'"); } catch (java.io.IOException e) { Status.setText(res.getString(R.string.error_dl_from, url)); System.out.println("Error reading from zip file '" + url + "': " + e.toString()); return false; } if (entry == null) { System.out.println("Reading from zip file '" + url + "' finished"); break; } if (entry.isDirectory()) { System.out.println("Creating dir '" + getOutFilePath(entry.getName()) + "'"); try { (new File(getOutFilePath(entry.getName()))).mkdirs(); } catch (SecurityException e) { } ; continue; } OutputStream out = null; path = getOutFilePath(entry.getName()); // System.out.println("Saving file a '" + path + "'"); try { (new File(path.substring(0, path.lastIndexOf("/")))).mkdirs(); } catch (SecurityException e) { } ; try { CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32()); while (check.read(buf, 0, buf.length) > 0) {} ; check.close(); if (check.getChecksum().getValue() != entry.getCrc()) { File ff = new File(path); ff.delete(); throw new Exception(); } // System.out.println("File '" + path + "' exists and passed CRC check - not // overwriting it"); continue; } catch (Exception e) { } try { out = new FileOutputStream(path); } catch (FileNotFoundException e) { System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString()); } catch (SecurityException e) { System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString()); } ; if (out == null) { Status.setText(res.getString(R.string.error_write, path)); System.out.println("Saving file '" + path + "' - cannot create file"); return false; } float percent = 0.0f; if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen; Status.setText(res.getString(R.string.dl_progress, percent, path)); try { int len = zip.read(buf); while (len >= 0) { if (len > 0) out.write(buf, 0, len); len = zip.read(buf); percent = 0.0f; if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen; Status.setText(res.getString(R.string.dl_progress, percent, path)); } out.flush(); out.close(); out = null; } catch (java.io.IOException e) { Status.setText(res.getString(R.string.error_write, path)); System.out.println( "Saving file '" + path + "' - error writing or downloading: " + e.toString()); return false; } try { CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32()); while (check.read(buf, 0, buf.length) > 0) {} ; check.close(); if (check.getChecksum().getValue() != entry.getCrc()) { File ff = new File(path); ff.delete(); throw new Exception(); } } catch (Exception e) { Status.setText(res.getString(R.string.error_write, path)); System.out.println("Saving file '" + path + "' - CRC check failed"); return false; } System.out.println("Saving file '" + path + "' done"); } } ; OutputStream out = null; path = getOutFilePath(DownloadFlagFileName); try { out = new FileOutputStream(path); out.write(downloadUrls[downloadUrlIndex].getBytes("UTF-8")); out.flush(); out.close(); } catch (FileNotFoundException e) { } catch (SecurityException e) { } catch (java.io.IOException e) { Status.setText(res.getString(R.string.error_write, path)); return false; } ; Status.setText(res.getString(R.string.dl_finished)); try { stream.close(); } catch (java.io.IOException e) { } ; return true; };