public void doGzip(String filePath) {
FileOutputStream fos = null;
GZIPOutputStream gos = null;
FileInputStream fis = null;
try {
fos = new FileOutputStream("C:/myGzip.gzip");
gos = new GZIPOutputStream(fos);
fis = new FileInputStream(filePath);
byte[] tmp = new byte[4 * 1024];
int size = 0;
while ((size = fis.read(tmp)) != -1) {
gos.write(tmp, 0, size);
}
gos.finish();
System.out.println("Done with GZip...");
} catch (IOException e) {
} finally {
try {
if (fis != null) fis.close();
if (gos != null) gos.close();
} catch (Exception ex) {
}
}
}
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
if (out != null) {
fout.close();
out.close();
}
}
public static MyResponse ok(Object content, boolean isEncryp) {
MyResponse response = new MyResponse();
if (isEncryp) {
response.encryp = true;
JsonMapper jsonMapper = new JsonMapper();
String jsonContent = jsonMapper.toJson(content);
try {
// gzip压缩
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(out);
gzip.write(jsonContent.getBytes());
gzip.close();
byte[] vi = Cryptos.generateIV();
byte[] encryByte = Cryptos.aesEncrypt(out.toByteArray(), response.getCurrentToken(), vi);
String encrypContent = Base64.encodeBase64String(encryByte);
String viStr = Base64.encodeBase64String(vi);
response.setContent(viStr + ":" + encrypContent);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
response.setContent(content);
}
response.setStatus(STATUS_OK);
return response;
}
/**
* @param ungzipped the bytes to be gzipped
* @return gzipped bytes
*/
private byte[] gzip(byte[] ungzipped) throws IOException {
final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(bytes);
gzipOutputStream.write(ungzipped);
gzipOutputStream.close();
return bytes.toByteArray();
}
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String encoding = request.getHeader("Accept-Encoding");
boolean supportsGzip = (encoding != null && encoding.toLowerCase().indexOf("gzip") > -1);
SessionTerminal st = (SessionTerminal) request.getSession(true).getAttribute("terminal");
if (st == null || st.isClosed()) {
st = new SessionTerminal();
request.getSession().setAttribute("terminal", st);
}
String str = request.getParameter("k");
String f = request.getParameter("f");
String dump = st.handle(str, f != null && f.length() > 0);
if (dump != null) {
if (supportsGzip) {
response.setHeader("Content-Encoding", "gzip");
response.setHeader("Content-Type", "text/html");
try {
GZIPOutputStream gzos = new GZIPOutputStream(response.getOutputStream());
gzos.write(dump.getBytes());
gzos.close();
} catch (IOException ie) {
// handle the error here
ie.printStackTrace();
}
} else {
response.getOutputStream().write(dump.getBytes());
}
}
}
private static byte[] gzip(String message) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream stream = new GZIPOutputStream(out);
stream.write(message.getBytes(CharsetUtil.UTF_8));
stream.close();
return out.toByteArray();
}
/**
* Do all required cleanup now that we're finished with the currently-open .tar.gz
*
* @exception java.io.IOException
*/
@Override
public void finished() throws IOException {
outputStream.close();
if (gzipOutputStream != null) {
gzipOutputStream.close();
}
}
public static String saveSession(String sessionName) throws IOException {
StringBuilder builder = new StringBuilder();
String filename = mStoragePath + '/' + sessionName + ".dss", session;
builder.append(SESSION_MAGIC + "\n");
// skip the network target
builder.append(mTargets.size() - 1).append("\n");
for (Target target : mTargets) {
if (target.getType() != Target.Type.NETWORK) target.serialize(builder);
}
builder.append(mCurrentTarget).append("\n");
session = builder.toString();
FileOutputStream ostream = new FileOutputStream(filename);
GZIPOutputStream gzip = new GZIPOutputStream(ostream);
gzip.write(session.getBytes());
gzip.close();
mSessionName = sessionName;
return filename;
}
// 生成tar并压缩成tar.gz
public static void WriteToTarGzip(String folderPath, String targzipFilePath) {
byte[] buf = new byte[1024]; // 设定读入缓冲区尺寸
try {
// 建立压缩文件输出流
FileOutputStream fout = new FileOutputStream(targzipFilePath);
// 建立tar压缩输出流
TarOutputStream tout = new TarOutputStream(fout);
addFiles(tout, folderPath);
tout.close();
fout.close();
// 建立压缩文件输出流
FileOutputStream gzFile = new FileOutputStream(targzipFilePath + ".gz");
// 建立gzip压缩输出流
GZIPOutputStream gzout = new GZIPOutputStream(gzFile);
// 打开需压缩文件作为文件输入流
FileInputStream tarin = new FileInputStream(targzipFilePath); // targzipFilePath是文件全路径
int len;
while ((len = tarin.read(buf)) != -1) {
gzout.write(buf, 0, len);
}
gzout.close();
gzFile.close();
tarin.close();
} catch (FileNotFoundException e) {
System.out.println(e);
} catch (IOException e) {
System.out.println(e);
}
File tarfile = new File(targzipFilePath);
tarfile.delete();
}
/**
* Ctor. The level of zipping is setable using the method ZipOutputStream.setLevel(0 - 9).
*
* @param sz The filename.
*/
public gzip(String sz) {
GZIPOutputStream gzos = null;
FileInputStream fis = null;
try {
gzos = new GZIPOutputStream(new FileOutputStream(sz + ".gz"));
fis = new FileInputStream(sz);
byte[] buffer = new byte[BLOCKSIZE];
for (int length; (length = fis.read(buffer, 0, BLOCKSIZE)) != -1; )
gzos.write(buffer, 0, length);
} catch (IOException e) {
System.err.println("Error: Couldn't compress " + sz);
} finally {
if (fis != null)
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
if (gzos != null)
try {
gzos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static Result processFile(Path prev, Path cur, Path outDir, int num, int gzipFrom)
throws IOException {
Result deltaHashes = new Result();
Path deltaFile = outDir.resolve(cur.getFileName().toString() + ".bpatch");
deleteIfExists(deltaFile);
deltaHashes.path = deltaFile;
boolean isGzipping = num >= gzipFrom;
try (HashingOutputStream hashingStream =
new HashingOutputStream(
Hashing.sha256(),
new BufferedOutputStream(newOutputStream(deltaFile, StandardOpenOption.CREATE_NEW)))) {
GZIPOutputStream zipStream = null;
GDiffWriter writer;
if (isGzipping) {
// Just constructing this object writes to the stream.
zipStream = new GZIPOutputStream(hashingStream);
writer = new GDiffWriter(zipStream);
} else {
writer = new GDiffWriter(hashingStream);
}
Delta delta = new Delta();
deltaHashes.preHash = sha256(readAllBytes(prev));
delta.compute(prev.toFile(), cur.toFile(), writer);
if (isGzipping) zipStream.close();
deltaHashes.patchHash = hashingStream.hash().asBytes();
deltaHashes.postHash = sha256(readAllBytes(cur));
}
long size = Files.size(deltaFile);
deltaHashes.patchSize = size;
println("... done: %s (%.2fkb) %s", deltaFile, size / 1024.0, isGzipping ? "zipped" : "");
return deltaHashes;
}
InMemorySourceRepository(@WillClose ZipInputStream in) throws IOException {
try {
while (true) {
ZipEntry e = in.getNextEntry();
if (e == null) break;
if (!e.isDirectory()) {
String name = e.getName();
long size = e.getSize();
if (size > Integer.MAX_VALUE)
throw new IOException(name + " is too big at " + size + " bytes");
ByteArrayOutputStream out;
if (size <= 0) out = new ByteArrayOutputStream();
else out = new ByteArrayOutputStream((int) size);
GZIPOutputStream gOut = new GZIPOutputStream(out);
IO.copy(in, gOut);
gOut.close();
byte data[] = out.toByteArray();
contents.put(name, data);
lastModified.put(name, e.getTime());
}
in.closeEntry();
}
} finally {
Util.closeSilently(in);
}
}
/**
* Submit an HTTP post to the given url, sending the data specified
*
* @param url url to post to
* @param length number of bytes in the dataToSend to, er, send
* @param dataToSend stream of bytes to be sent
*/
public static boolean postData(URL url, long length, InputStream dataToSend) {
try {
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setDoInput(true);
con.setDoOutput(true);
con.setUseCaches(false);
con.setRequestMethod("POST");
con.setRequestProperty("Content-length", "" + length);
OutputStream out = con.getOutputStream();
byte buf[] = new byte[1 * 1024];
int read;
long sent = 0;
GZIPOutputStream zipOut = new GZIPOutputStream(out);
while ((read = dataToSend.read(buf)) != -1) {
zipOut.write(buf, 0, read);
sent += read;
if (sent >= length) break;
}
zipOut.flush();
zipOut.finish();
zipOut.close();
out.close();
int rv = con.getResponseCode();
_log.debug("Posted " + sent + " bytes: " + rv);
return length == sent;
} catch (IOException ioe) {
_log.error("Error posting the data", ioe);
return false;
}
}
protected void sendResponseAppropriately(
CallingContext context, HttpServletRequest req, HttpServletResponse resp, String response)
throws IOException {
String encoding = req.getHeader("Accept-Encoding");
resp.setContentType("application/json");
if (encoding != null && encoding.indexOf("gzip") >= 0) {
resp.setHeader("Content-Encoding", "gzip");
OutputStream o = null;
GZIPOutputStream gz = null;
try {
o = resp.getOutputStream();
gz = new GZIPOutputStream(o);
gz.write(response.getBytes("UTF-8"));
} finally {
if (gz != null) {
gz.close();
}
if (o != null) {
o.close();
}
}
} else {
resp.getWriter().append(response);
resp.flushBuffer();
}
}
/** Returns an gzipped copy of the input array. */
public static final byte[] zip(byte[] in) {
try {
// compress using GZIPOutputStream
ByteArrayOutputStream byteOut =
new ByteArrayOutputStream(in.length / EXPECTED_COMPRESSION_RATIO);
GZIPOutputStream outStream = new GZIPOutputStream(byteOut);
try {
outStream.write(in);
} catch (Exception e) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
}
try {
outStream.close();
} catch (IOException e) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
}
return byteOut.toByteArray();
} catch (IOException e) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
return null;
}
}
protected void makeGzip(File file) throws IOException {
BufferedInputStream bin = null;
GZIPOutputStream gzos = null;
try {
FileInputStream fin = new FileInputStream(file);
bin = new BufferedInputStream(fin);
FileOutputStream fos = new FileOutputStream(file + ".gz");
gzos = new GZIPOutputStream(fos);
byte[] buf = new byte[1024];
int len;
while ((len = bin.read(buf)) > -1) {
gzos.write(buf, 0, len);
}
} catch (IOException e) {
LOGGER.fatal("Error while gzipping!", e);
} finally {
if (bin != null) {
bin.close();
}
if (gzos != null) {
gzos.close();
}
}
}
public byte[] compress(byte[] buffer) throws IOException {
ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(arrayOutputStream);
ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer);
try {
byte[] buf = new byte[unit];
int len = 0;
while ((len = inputStream.read(buf)) != -1) {
gzip.write(buf, 0, len);
}
gzip.finish();
byte[] result = arrayOutputStream.toByteArray();
return result;
} catch (IOException e) {
throw e;
} finally {
gzip.close();
arrayOutputStream.close();
inputStream.close();
}
}
@Override
public void run() {
DataInputStream in = new DataInputStream(inStream);
try {
GZIPOutputStream out = new GZIPOutputStream(outStream);
byte[] buf = new byte[256];
int numbercount = 0;
while (!suspending && !done) {
numbercount = in.read(buf);
if (numbercount == -1) {
done = true;
break;
}
out.write(buf, 0, numbercount);
out.flush();
try {
Thread.sleep(9000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
out.close();
} catch (IOException e) {
e.printStackTrace();
}
inStream.closeStream();
inStream.setMigrated(true);
outStream.closeStream();
outStream.setMigrated(true);
suspending = false;
}
// adds file to S3 Data store or offline cache (if working offline)
public String addPointSet(File pointSetFile, String pointSetId) throws IOException {
if (pointSetId == null) throw new NullPointerException("null point set id");
File renamedPointSetFile = new File(POINT_DIR, pointSetId + ".json");
if (renamedPointSetFile.exists()) return pointSetId;
FileUtils.copyFile(pointSetFile, renamedPointSetFile);
if (!this.workOffline) {
// only upload if it doesn't exist
try {
s3.getObjectMetadata(pointsetBucket, pointSetId + ".json.gz");
} catch (AmazonServiceException e) {
// gzip compression in storage, not because we're worried about file size but to speed file
// transfer
FileInputStream fis = new FileInputStream(pointSetFile);
File tempFile = File.createTempFile(pointSetId, ".json.gz");
FileOutputStream fos = new FileOutputStream(tempFile);
GZIPOutputStream gos = new GZIPOutputStream(fos);
try {
ByteStreams.copy(fis, gos);
} finally {
gos.close();
fis.close();
}
s3.putObject(pointsetBucket, pointSetId + ".json.gz", tempFile);
tempFile.delete();
}
}
return pointSetId;
}
@Override
public byte[] packLog(long index, int itemsToPack) {
if (index < this.startIndex.get() || index >= this.nextIndex.get()) {
throw new IllegalArgumentException("index out of range");
}
try {
ByteArrayOutputStream memoryStream = new ByteArrayOutputStream();
GZIPOutputStream gzipStream = new GZIPOutputStream(memoryStream);
PreparedStatement ps = this.connection.prepareStatement(SELECT_RANGE_SQL);
ps.setLong(1, index);
ps.setLong(2, index + itemsToPack);
ResultSet rs = ps.executeQuery();
while (rs.next()) {
byte[] value = rs.getBytes(4);
int size = value.length + Long.BYTES + 1 + Integer.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(size);
buffer.putInt(size);
buffer.putLong(rs.getLong(2));
buffer.put(rs.getByte(3));
buffer.put(value);
gzipStream.write(buffer.array());
}
rs.close();
gzipStream.flush();
memoryStream.flush();
gzipStream.close();
return memoryStream.toByteArray();
} catch (Throwable error) {
this.logger.error("failed to pack log entries", error);
throw new RuntimeException("log store error", error);
}
}
// 测试压缩
// @Test
public void testCompress() throws Exception {
GZIPOutputStream gOut = new GZIPOutputStream(new FileOutputStream(destDirectory));
FileInputStream in = new FileInputStream(sourceTar);
IOUtils.copy(in, gOut);
gOut.close();
in.close();
}
public byte[] gzip(byte b[]) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
GZIPOutputStream gout = new GZIPOutputStream(bout);
gout.write(b);
gout.close();
return bout.toByteArray();
}
// 不能对每层都包含文件和目录的多层次目录结构打包
public static void compressedFiles_Gzip(String folderPath, String targzipFilePath) {
File srcPath = new File(folderPath);
int length = srcPath.listFiles().length;
byte[] buf = new byte[1024]; // 设定读入缓冲区尺寸
File[] files = srcPath.listFiles();
try {
File targetFile = new File(targzipFilePath);
File parent = targetFile.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
// 建立压缩文件输出流
FileOutputStream fout = new FileOutputStream(targetFile);
// 建立tar压缩输出流
TarOutputStream tout = new TarOutputStream(fout);
for (int i = 0; i < length; i++) {
String filename = srcPath.getPath() + File.separator + files[i].getName();
// 打开需压缩文件作为文件输入流
FileInputStream fin = new FileInputStream(filename); // filename是文件全路径
TarEntry tarEn = new TarEntry(files[i]); // 此处必须使用new
// TarEntry(File
// file);
// tarEn.setName(files[i].getName());
// //此处需重置名称，默认是带全路径的，否则打包后会带全路径
tout.putNextEntry(tarEn);
int num;
while ((num = fin.read(buf)) != -1) {
tout.write(buf, 0, num);
}
tout.closeEntry();
fin.close();
}
tout.close();
fout.close();
// 建立压缩文件输出流
FileOutputStream gzFile = new FileOutputStream(targzipFilePath + ".gz");
// 建立gzip压缩输出流
GZIPOutputStream gzout = new GZIPOutputStream(gzFile);
// 打开需压缩文件作为文件输入流
FileInputStream tarin = new FileInputStream(targzipFilePath); // targzipFilePath是文件全路径
int len;
while ((len = tarin.read(buf)) != -1) {
gzout.write(buf, 0, len);
}
gzout.close();
gzFile.close();
tarin.close();
// 因为只要tar.gz文件，所以删除.tar文件
del(targzipFilePath);
} catch (FileNotFoundException e) {
System.out.println(e);
} catch (IOException e) {
System.out.println(e);
}
}
/**
* End the current request. It is acceptable to write extra bytes using buffer.doWrite during the
* execution of this method.
*/
@Override
public long end() throws IOException {
if (compressionStream == null) {
compressionStream = new FlushableGZIPOutputStream(fakeOutputStream);
}
compressionStream.finish();
compressionStream.close();
return ((OutputFilter) buffer).end();
}
@JRubyMethod(name = "close")
public IRubyObject close() throws IOException {
if (!closed) {
io.close();
}
this.closed = true;
return getRuntime().getNil();
}
// http://stackoverflow.com/questions/6717165/how-can-i-compress-a-string-using-gzipoutputstream-and-vice-versa
public static byte[] compress(String string) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream(string.length());
GZIPOutputStream gos = new GZIPOutputStream(os);
gos.write(string.getBytes());
gos.close();
byte[] compressed = os.toByteArray();
os.close();
return compressed;
}
public static InputStream gzip(final InputStream inputStream) throws IOException {
Assert.notNull(inputStream, "inputStream");
InputOutputStream inputOutputStream = new InputOutputStream();
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(inputOutputStream);
IOUtils.copy(inputStream, gzipOutputStream);
gzipOutputStream.close();
return inputOutputStream.getInputStream();
}
/**
* Serializes an object and returns the Base64-encoded version of that serialized object. If the
* object cannot be serialized or there is another error, the method will return <tt>null</tt>.
*
* <p>Valid options:
*
* <pre>
* GZIP: gzip-compresses object before encoding it.
* DONT_BREAK_LINES: don't break lines at 76 characters
* <i>Note: Technically, this makes your encoding non-compliant.</i>
* </pre>
*
* <p>Example: <code>encodeObject( myObj, Base64.GZIP )</code> or
*
* <p>Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
*
* @param serializableObject The object to encode
* @param options Specified options
* @return The Base64-encoded object
* @see Base64#GZIP
* @see Base64#DONT_BREAK_LINES
* @since 2.0
*/
public static String encodeObject(java.io.Serializable serializableObject, int options) {
// Streams
java.io.ByteArrayOutputStream baos = null;
java.io.OutputStream b64os = null;
java.io.ObjectOutputStream oos = null;
java.util.zip.GZIPOutputStream gzos = null;
// Isolate options
int gzip = (options & GZIP);
int dontBreakLines = (options & DONT_BREAK_LINES);
try {
// ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream
baos = new java.io.ByteArrayOutputStream();
b64os = new Base64.OutputStream(baos, ENCODE | dontBreakLines);
// GZip?
if (gzip == GZIP) {
gzos = new java.util.zip.GZIPOutputStream(b64os);
oos = new java.io.ObjectOutputStream(gzos);
} // end if: gzip
else oos = new java.io.ObjectOutputStream(b64os);
oos.writeObject(serializableObject);
} // end try
catch (java.io.IOException e) {
e.printStackTrace();
return null;
} // end catch
finally {
try {
oos.close();
} catch (Exception e) {
}
try {
gzos.close();
} catch (Exception e) {
}
try {
b64os.close();
} catch (Exception e) {
}
try {
baos.close();
} catch (Exception e) {
}
} // end finally
// Return value according to relevant encoding.
try {
return new String(baos.toByteArray(), PREFERRED_ENCODING);
} // end try
catch (java.io.UnsupportedEncodingException uue) {
return new String(baos.toByteArray());
} // end catch
} // end encode
public byte[] compresBytesToGzip(byte[] bytes) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream gzos = new GZIPOutputStream(baos);
gzos.write(bytes, 0, bytes.length);
gzos.close();
return baos.toByteArray();
}
private ByteString gzip(final String s) throws IOException {
final ByteArrayOutputStream buf = new ByteArrayOutputStream();
final GZIPOutputStream out = new GZIPOutputStream(buf);
try {
out.write(s.getBytes(StandardCharsets.UTF_8));
} finally {
out.close();
}
return ByteString.fromArray(buf.toByteArray());
}