public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
final DataInputStream in =
new DataInputStream(
SecureIOUtils.openForRead(
new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
try {
final long length = rfs.getFileStatus(indexFileName).getLen();
final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;
buf = ByteBuffer.allocate(size);
if (crc != null) {
crc.reset();
CheckedInputStream chk = new CheckedInputStream(in, crc);
IOUtils.readFully(chk, buf.array(), 0, size);
if (chk.getChecksum().getValue() != in.readLong()) {
throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
}
} else {
IOUtils.readFully(in, buf.array(), 0, size);
}
entries = buf.asLongBuffer();
} finally {
in.close();
}
}
private static void zipFiles(String[] fileNames) throws IOException {
// 获取zip文件输出流
FileOutputStream f = new FileOutputStream("test.zip");
// 从文件输出流中获取数据校验和输出流,并设置Adler32
CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32());
// 从数据校验和输出流中获取Zip输出流
ZipOutputStream zos = new ZipOutputStream(csum);
// 从Zip输出流中获取缓冲输出流
BufferedOutputStream out = new BufferedOutputStream(zos);
// 设置Zip文件注释
zos.setComment("测试 java zip stream");
for (String file : fileNames) {
System.out.println("写入文件: " + file);
// 获取文件输入字符流
BufferedReader in = new BufferedReader(new FileReader(file));
// 想Zip处理写入新的文件条目，并流定位到数据开始处
zos.putNextEntry(new ZipEntry(file));
int c;
while ((c = in.read()) > 0) out.write(c);
in.close();
// 刷新Zip输出流，将缓冲的流写入该流
out.flush();
}
// 文件全部写入Zip输出流后，关闭
out.close();
// 输出数据校验和
System.out.println("数据校验和: " + csum.getChecksum().getValue());
System.out.println("读取zip文件");
// 读取test.zip文件输入流
FileInputStream fi = new FileInputStream("test.zip");
// 从文件输入流中获取数据校验和流
CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32());
// 从数据校验和流中获取Zip解压流
ZipInputStream in2 = new ZipInputStream(csumi);
// 从Zip解压流中获取缓冲输入流
BufferedInputStream bis = new BufferedInputStream(in2);
// 创建文件条目
ZipEntry zipEntry;
while ((zipEntry = in2.getNextEntry()) != null) {
System.out.println("读取文件： " + zipEntry);
int x;
while ((x = bis.read()) > 0) System.out.write(x);
}
if (fileNames.length == 1) System.out.println("数据校验和： " + csumi.getChecksum().getValue());
bis.close();
// 获取Zip文件
ZipFile zf = new ZipFile("test.zip");
// 获取文件条目枚举
Enumeration e = zf.entries();
while (e.hasMoreElements()) {
// 从Zip文件的枚举中获取文件条目
ZipEntry ze2 = (ZipEntry) e.nextElement();
System.out.println("文件： " + ze2);
}
}
private static Long checksum(String filename) throws IOException {
// Compute Adler-32 checksum
CheckedInputStream cis = new CheckedInputStream(new FileInputStream(filename), new Adler32());
byte[] tempBuf = new byte[128];
while (cis.read(tempBuf) >= 0) {}
return cis.getChecksum().getValue();
}
public long checksum(File f) throws FITTESTException {
try {
CheckedInputStream cis = new CheckedInputStream(new FileInputStream(f), new Adler32());
BufferedInputStream bis = new BufferedInputStream(cis, BUFFER);
byte data[] = new byte[BUFFER];
while (bis.read(data, 0, BUFFER) != -1) ;
bis.close();
return cis.getChecksum().getValue();
} catch (FileNotFoundException e) {
throw new FITTESTException(e.getMessage());
} catch (IOException e) {
throw new FITTESTException(e.getMessage());
}
}
/**
* This method will attempt to checksum the current JE db environment by computing the Adler-32
* checksum on the latest JE log file available.
*
* @return The checksum of JE db environment or zero if checksum failed.
*/
private long checksumDbEnv() {
File parentDirectory = getFileForPath(cfg.getDBDirectory());
File backendDirectory = new File(parentDirectory, cfg.getBackendId());
List<File> jdbFiles = new ArrayList<File>();
if (backendDirectory.isDirectory()) {
jdbFiles =
Arrays.asList(
backendDirectory.listFiles(
new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".jdb");
}
}));
}
if (!jdbFiles.isEmpty()) {
Collections.sort(jdbFiles, Collections.reverseOrder());
FileInputStream fis = null;
try {
fis = new FileInputStream(jdbFiles.get(0).toString());
CheckedInputStream cis = new CheckedInputStream(fis, new Adler32());
byte[] tempBuf = new byte[8192];
while (cis.read(tempBuf) >= 0) {}
return cis.getChecksum().getValue();
} catch (Exception e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
} finally {
if (fis != null) {
try {
fis.close();
} catch (Exception e) {
if (debugEnabled()) {
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
}
}
}
}
return 0;
}
public static void main(String[] args) throws IOException {
FileOutputStream f = new FileOutputStream("test.zip");
CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32());
ZipOutputStream zos = new ZipOutputStream(csum);
BufferedOutputStream out = new BufferedOutputStream(zos);
zos.setComment("A test of Java Zipping");
// No corresponding getComment(), though.
for (String arg : args) {
print("Writing file " + arg);
BufferedReader in = new BufferedReader(new FileReader(arg));
zos.putNextEntry(new ZipEntry(arg));
int c;
while ((c = in.read()) != -1) out.write(c);
in.close();
out.flush();
}
out.close();
// Checksum valid only after the file has been closed!
print("Checksum: " + csum.getChecksum().getValue());
// Now extract the files:
print("Reading file");
FileInputStream fi = new FileInputStream("test.zip");
CheckedInputStream csumi = new CheckedInputStream(fi, new Adler32());
ZipInputStream in2 = new ZipInputStream(csumi);
BufferedInputStream bis = new BufferedInputStream(in2);
ZipEntry ze;
while ((ze = in2.getNextEntry()) != null) {
print("Reading file " + ze);
int x;
while ((x = bis.read()) != -1) System.out.write(x);
}
if (args.length == 1) print("Checksum: " + csumi.getChecksum().getValue());
bis.close();
// Alternative way to open and read Zip files:
ZipFile zf = new ZipFile("test.zip");
Enumeration e = zf.entries();
while (e.hasMoreElements()) {
ZipEntry ze2 = (ZipEntry) e.nextElement();
print("File: " + ze2);
// ... and extract the data as before
}
/* if(args.length == 1) */
}
/**
* Calculates the CRC32 value of a file.
*
* @param file File to compute the CRC value
* @return CRC value formatted in 8 length hexadecimal in lowercase
*/
public static String calculateCRC32(File file) throws ChecksumException {
if (file == null) {
return null;
}
String hex = "";
try {
CheckedInputStream cis = new CheckedInputStream(new FileInputStream(file), new CRC32());
byte[] buf = new byte[10240]; // 10mb
while (cis.read(buf) >= 0) ;
hex = Long.toHexString(cis.getChecksum().getValue());
cis.close();
} catch (IOException | NullPointerException e) {
throw new ChecksumException("Unable to determine CRC32 value for file: " + file.getName());
}
for (int i = hex.length(); i < CRC32_LENGTH; i++) {
hex = "0" + hex;
}
return hex;
}
private int extractZip(String zip_path) {
ZipInputStream zip = null;
try {
zip = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip_path)));
} catch (java.io.FileNotFoundException e) {
sendMessage(-2, 0, "Failed to read from zip file: " + e.toString());
return -1;
}
;
int num_file = 0;
while (true) {
num_file++;
ZipEntry entry = null;
try {
entry = zip.getNextEntry();
} catch (java.io.IOException e) {
sendMessage(-2, 0, "Failed to get entry from zip file: " + e.toString());
return -1;
}
if (entry == null) break;
String path = extract_dir + "/" + entry.getName();
if (entry.isDirectory()) {
try {
(new File(path)).mkdirs();
} catch (SecurityException e) {
sendMessage(-2, 0, "Failed to create directory: " + e.toString());
return -1;
}
continue;
}
try {
(new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
} catch (SecurityException e) {
sendMessage(-2, 0, "Failed to create directory: " + e.toString());
return -1;
}
;
if (extractZipEntry(path, zip, (int) entry.getSize(), "Extracting archives: " + num_file)
!= 0) return -1;
try {
CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
while (check.read(buf) > 0) {}
;
check.close();
if (check.getChecksum().getValue() != entry.getCrc()) {
File ff = new File(path);
ff.delete();
throw new Exception();
}
} catch (Exception e) {
sendMessage(-2, 0, "CRC check failed");
return -1;
}
}
return 0;
}
public boolean DownloadDataFile(final String DataDownloadUrl, final String DownloadFlagFileName) {
String[] downloadUrls = DataDownloadUrl.split("[|]");
if (downloadUrls.length < 2) return false;
Resources res = Parent.getResources();
String path = getOutFilePath(DownloadFlagFileName);
InputStream checkFile = null;
try {
checkFile = new FileInputStream(path);
} catch (FileNotFoundException e) {
} catch (SecurityException e) {
}
;
if (checkFile != null) {
try {
byte b[] = new byte[Globals.DataDownloadUrl.getBytes("UTF-8").length + 1];
int readed = checkFile.read(b);
String compare = new String(b, 0, readed, "UTF-8");
boolean matched = false;
// System.out.println("Read URL: '" + compare + "'");
for (int i = 1; i < downloadUrls.length; i++) {
// System.out.println("Comparing: '" + downloadUrls[i] + "'");
if (compare.compareTo(downloadUrls[i]) == 0) matched = true;
}
// System.out.println("Matched: " + String.valueOf(matched));
if (!matched) throw new IOException();
Status.setText(res.getString(R.string.download_unneeded));
return true;
} catch (IOException e) {
}
;
}
checkFile = null;
// Create output directory (not necessary for phone storage)
// System.out.println("Downloading data to: '" + outFilesDir + "'");
try {
(new File(outFilesDir)).mkdirs();
OutputStream out = new FileOutputStream(getOutFilePath(".nomedia"));
out.flush();
out.close();
} catch (SecurityException e) {
} catch (FileNotFoundException e) {
} catch (IOException e) {
}
;
HttpResponse response = null;
HttpGet request;
long totalLen = 0;
CountingInputStream stream;
byte[] buf = new byte[16384];
boolean DoNotUnzip = false;
boolean FileInAssets = false;
String url = "";
int downloadUrlIndex = 1;
while (downloadUrlIndex < downloadUrls.length) {
System.out.println("Processing download " + downloadUrls[downloadUrlIndex]);
url = new String(downloadUrls[downloadUrlIndex]);
DoNotUnzip = false;
if (url.indexOf(":") == 0) {
url = url.substring(url.indexOf(":", 1) + 1);
DoNotUnzip = true;
}
Status.setText(res.getString(R.string.connecting_to, url));
if (url.indexOf("http://") == -1 && url.indexOf("https://") == -1) // File inside assets
{
System.out.println("Fetching file from assets: " + url);
FileInAssets = true;
break;
} else {
System.out.println("Connecting to: " + url);
request = new HttpGet(url);
request.addHeader("Accept", "*/*");
try {
DefaultHttpClient client = HttpWithDisabledSslCertCheck();
client.getParams().setBooleanParameter("http.protocol.handle-redirects", true);
response = client.execute(request);
} catch (IOException e) {
System.out.println("Failed to connect to " + url);
downloadUrlIndex++;
}
;
if (response != null) {
if (response.getStatusLine().getStatusCode() != 200) {
response = null;
System.out.println("Failed to connect to " + url);
downloadUrlIndex++;
} else break;
}
}
}
if (FileInAssets) {
System.out.println("Unpacking from assets: '" + url + "'");
try {
// System.out.println("Unpacking from assets: '" + url + "' 1");
stream = new CountingInputStream(Parent.getAssets().open(url), 8192);
// System.out.println("Unpacking from assets: '" + url + "' 2");
while (stream.skip(65536) > 0) {}
;
// System.out.println("Unpacking from assets: '" + url + "' 3");
totalLen = stream.getBytesRead();
// System.out.println("Unpacking from assets: '" + url + "' 4 totalLen = " +
// String.valueOf(totalLen));
stream.close();
// System.out.println("Unpacking from assets: '" + url + "' 5");
stream = new CountingInputStream(Parent.getAssets().open(url), 8192);
// System.out.println("Unpacking from assets: '" + url + "' 6");
} catch (IOException e) {
System.out.println("Unpacking from assets '" + url + "' - error: " + e.toString());
Status.setText(res.getString(R.string.error_dl_from, url));
return false;
}
} else {
if (response == null) {
// System.out.println("Error connecting to " + url);
Status.setText(res.getString(R.string.failed_connecting_to, url));
return false;
}
Status.setText(res.getString(R.string.dl_from, url));
totalLen = response.getEntity().getContentLength();
try {
stream = new CountingInputStream(response.getEntity().getContent(), 8192);
} catch (java.io.IOException e) {
Status.setText(res.getString(R.string.error_dl_from, url));
return false;
}
}
if (DoNotUnzip) {
path =
getOutFilePath(
downloadUrls[downloadUrlIndex].substring(
1, downloadUrls[downloadUrlIndex].indexOf(":", 1)));
// System.out.println("Saving file '" + path + "'");
OutputStream out = null;
try {
try {
(new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
} catch (SecurityException e) {
}
;
out = new FileOutputStream(path);
} catch (FileNotFoundException e) {
System.out.println(
"Saving file '" + path + "' - error creating output file: " + e.toString());
} catch (SecurityException e) {
System.out.println(
"Saving file '" + path + "' - error creating output file: " + e.toString());
}
;
if (out == null) {
Status.setText(res.getString(R.string.error_write, path));
System.out.println("Saving file '" + path + "' - error creating output file");
return false;
}
try {
int len = stream.read(buf);
while (len >= 0) {
if (len > 0) out.write(buf, 0, len);
len = stream.read(buf);
float percent = 0.0f;
if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
Status.setText(res.getString(R.string.dl_progress, percent, path));
}
out.flush();
out.close();
out = null;
} catch (java.io.IOException e) {
Status.setText(res.getString(R.string.error_write, path));
System.out.println("Saving file '" + path + "' - error writing: " + e.toString());
return false;
}
// System.out.println("Saving file '" + path + "' done");
} else {
// System.out.println("Reading from zip file aa'" + url + "'");
ZipInputStream zip = new ZipInputStream(stream);
while (true) {
ZipEntry entry = null;
try {
entry = zip.getNextEntry();
if (entry != null)
System.out.println(
"Reading from zip file '" + url + "' entry '" + entry.getName() + "'");
} catch (java.io.IOException e) {
Status.setText(res.getString(R.string.error_dl_from, url));
System.out.println("Error reading from zip file '" + url + "': " + e.toString());
return false;
}
if (entry == null) {
System.out.println("Reading from zip file '" + url + "' finished");
break;
}
if (entry.isDirectory()) {
System.out.println("Creating dir '" + getOutFilePath(entry.getName()) + "'");
try {
(new File(getOutFilePath(entry.getName()))).mkdirs();
} catch (SecurityException e) {
}
;
continue;
}
OutputStream out = null;
path = getOutFilePath(entry.getName());
// System.out.println("Saving file a '" + path + "'");
try {
(new File(path.substring(0, path.lastIndexOf("/")))).mkdirs();
} catch (SecurityException e) {
}
;
try {
CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
while (check.read(buf, 0, buf.length) > 0) {}
;
check.close();
if (check.getChecksum().getValue() != entry.getCrc()) {
File ff = new File(path);
ff.delete();
throw new Exception();
}
// System.out.println("File '" + path + "' exists and passed CRC check - not
// overwriting it");
continue;
} catch (Exception e) {
}
try {
out = new FileOutputStream(path);
} catch (FileNotFoundException e) {
System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString());
} catch (SecurityException e) {
System.out.println("Saving file '" + path + "' - cannot create file: " + e.toString());
}
;
if (out == null) {
Status.setText(res.getString(R.string.error_write, path));
System.out.println("Saving file '" + path + "' - cannot create file");
return false;
}
float percent = 0.0f;
if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
Status.setText(res.getString(R.string.dl_progress, percent, path));
try {
int len = zip.read(buf);
while (len >= 0) {
if (len > 0) out.write(buf, 0, len);
len = zip.read(buf);
percent = 0.0f;
if (totalLen > 0) percent = stream.getBytesRead() * 100.0f / totalLen;
Status.setText(res.getString(R.string.dl_progress, percent, path));
}
out.flush();
out.close();
out = null;
} catch (java.io.IOException e) {
Status.setText(res.getString(R.string.error_write, path));
System.out.println(
"Saving file '" + path + "' - error writing or downloading: " + e.toString());
return false;
}
try {
CheckedInputStream check = new CheckedInputStream(new FileInputStream(path), new CRC32());
while (check.read(buf, 0, buf.length) > 0) {}
;
check.close();
if (check.getChecksum().getValue() != entry.getCrc()) {
File ff = new File(path);
ff.delete();
throw new Exception();
}
} catch (Exception e) {
Status.setText(res.getString(R.string.error_write, path));
System.out.println("Saving file '" + path + "' - CRC check failed");
return false;
}
System.out.println("Saving file '" + path + "' done");
}
}
;
OutputStream out = null;
path = getOutFilePath(DownloadFlagFileName);
try {
out = new FileOutputStream(path);
out.write(downloadUrls[downloadUrlIndex].getBytes("UTF-8"));
out.flush();
out.close();
} catch (FileNotFoundException e) {
} catch (SecurityException e) {
} catch (java.io.IOException e) {
Status.setText(res.getString(R.string.error_write, path));
return false;
}
;
Status.setText(res.getString(R.string.dl_finished));
try {
stream.close();
} catch (java.io.IOException e) {
}
;
return true;
};