public HTTPResponseOutputStream(OutputStream raw) {
super(raw);
_context = I2PAppContext.getGlobalContext();
// all createRateStat in I2PTunnelHTTPClient.startRunning()
_log = _context.logManager().getLog(getClass());
_headerBuffer = _cache.acquire();
_buf1 = new byte[1];
}
/**
* grow (and free) the buffer as necessary
* @throws IOException if the headers are too big
*/
private void ensureCapacity() throws IOException {
if (_headerBuffer.getValid() >= MAX_HEADER_SIZE)
throw new IOException("Max header size exceeded: " + MAX_HEADER_SIZE);
if (_headerBuffer.getValid() + 1 >= _headerBuffer.getData().length) {
int newSize = (int)(_headerBuffer.getData().length * 1.5);
ByteArray newBuf = new ByteArray(new byte[newSize]);
System.arraycopy(_headerBuffer.getData(), 0, newBuf.getData(), 0, _headerBuffer.getValid());
newBuf.setValid(_headerBuffer.getValid());
newBuf.setOffset(0);
// if we changed the ByteArray size, don't put it back in the cache
if (_headerBuffer.getData().length == CACHE_SIZE)
_cache.release(_headerBuffer);
_headerBuffer = newBuf;
}
}
/**
* This does the transparent gzip decompression on the client side.
* Extended in I2PTunnelHTTPServer to do the compression on the server side.
*
* Simple stream for delivering an HTTP response to
* the client, trivially filtered to make sure "Connection: close"
* is always in the response. Perhaps add transparent handling of the
* Content-encoding: x-i2p-gzip, adjusting the headers to say Content-encoding: identity?
* Content-encoding: gzip is trivial as well, but Transfer-encoding: chunked makes it
* more work than is worthwhile at the moment.
*
*/
class HTTPResponseOutputStream extends FilterOutputStream {
private final I2PAppContext _context;
private final Log _log;
protected ByteArray _headerBuffer;
private boolean _headerWritten;
private final byte _buf1[];
protected boolean _gzip;
protected long _dataExpected;
protected String _contentType;
private static final int CACHE_SIZE = 8*1024;
private static final ByteCache _cache = ByteCache.getInstance(8, CACHE_SIZE);
// OOM DOS prevention
private static final int MAX_HEADER_SIZE = 64*1024;
public HTTPResponseOutputStream(OutputStream raw) {
super(raw);
_context = I2PAppContext.getGlobalContext();
// all createRateStat in I2PTunnelHTTPClient.startRunning()
_log = _context.logManager().getLog(getClass());
_headerBuffer = _cache.acquire();
_buf1 = new byte[1];
}
@Override
public void write(int c) throws IOException {
_buf1[0] = (byte)c;
write(_buf1, 0, 1);
}
@Override
public void write(byte buf[]) throws IOException {
write(buf, 0, buf.length);
}
@Override
public void write(byte buf[], int off, int len) throws IOException {
if (_headerWritten) {
out.write(buf, off, len);
//out.flush();
return;
}
for (int i = 0; i < len; i++) {
ensureCapacity();
_headerBuffer.getData()[_headerBuffer.getValid()] = buf[off+i];
_headerBuffer.setValid(_headerBuffer.getValid()+1);
if (headerReceived()) {
writeHeader();
_headerWritten = true;
if (i + 1 < len) {
// write out the remaining
out.write(buf, off+i+1, len-i-1);
//out.flush();
}
return;
}
}
}
/**
* grow (and free) the buffer as necessary
* @throws IOException if the headers are too big
*/
private void ensureCapacity() throws IOException {
if (_headerBuffer.getValid() >= MAX_HEADER_SIZE)
throw new IOException("Max header size exceeded: " + MAX_HEADER_SIZE);
if (_headerBuffer.getValid() + 1 >= _headerBuffer.getData().length) {
int newSize = (int)(_headerBuffer.getData().length * 1.5);
ByteArray newBuf = new ByteArray(new byte[newSize]);
System.arraycopy(_headerBuffer.getData(), 0, newBuf.getData(), 0, _headerBuffer.getValid());
newBuf.setValid(_headerBuffer.getValid());
newBuf.setOffset(0);
// if we changed the ByteArray size, don't put it back in the cache
if (_headerBuffer.getData().length == CACHE_SIZE)
_cache.release(_headerBuffer);
_headerBuffer = newBuf;
}
}
/** are the headers finished? */
private boolean headerReceived() {
if (_headerBuffer.getValid() < 3) return false;
byte first = _headerBuffer.getData()[_headerBuffer.getValid()-3];
byte second = _headerBuffer.getData()[_headerBuffer.getValid()-2];
byte third = _headerBuffer.getData()[_headerBuffer.getValid()-1];
return (isNL(second) && isNL(third)) || // \n\n
(isNL(first) && isNL(third)); // \n\r\n
}
/**
* Possibly tweak that first HTTP response line (HTTP/1.0 200 OK, etc).
* Overridden on server side.
*
*/
protected String filterResponseLine(String line) {
return line;
}
/** we ignore any potential \r, since we trim it on write anyway */
private static final byte NL = '\n';
private static boolean isNL(byte b) { return (b == NL); }
/** ok, received, now munge & write it */
private void writeHeader() throws IOException {
String responseLine = null;
boolean connectionSent = false;
boolean proxyConnectionSent = false;
int lastEnd = -1;
for (int i = 0; i < _headerBuffer.getValid(); i++) {
if (isNL(_headerBuffer.getData()[i])) {
if (lastEnd == -1) {
responseLine = new String(_headerBuffer.getData(), 0, i+1); // includes NL
responseLine = filterResponseLine(responseLine);
responseLine = (responseLine.trim() + "\r\n");
out.write(responseLine.getBytes());
} else {
for (int j = lastEnd+1; j < i; j++) {
if (_headerBuffer.getData()[j] == ':') {
int keyLen = j-(lastEnd+1);
int valLen = i-(j+1);
if ( (keyLen <= 0) || (valLen < 0) )
throw new IOException("Invalid header @ " + j);
String key = new String(_headerBuffer.getData(), lastEnd+1, keyLen);
String val = null;
if (valLen == 0)
val = "";
else
val = new String(_headerBuffer.getData(), j+2, valLen).trim();
if (_log.shouldLog(Log.INFO))
_log.info("Response header [" + key + "] = [" + val + "]");
String lcKey = key.toLowerCase(Locale.US);
if ("connection".equals(lcKey)) {
out.write("Connection: close\r\n".getBytes());
connectionSent = true;
} else if ("proxy-connection".equals(lcKey)) {
out.write("Proxy-Connection: close\r\n".getBytes());
proxyConnectionSent = true;
} else if ("content-encoding".equals(lcKey) && "x-i2p-gzip".equals(val.toLowerCase(Locale.US))) {
_gzip = true;
} else if ("proxy-authenticate".equals(lcKey)) {
// filter this hop-by-hop header; outproxy authentication must be configured in I2PTunnelHTTPClient
// see e.g. http://blog.c22.cc/2013/03/11/privoxy-proxy-authentication-credential-exposure-cve-2013-2503/
} else {
if ("content-length".equals(lcKey)) {
// save for compress decision on server side
try {
_dataExpected = Long.parseLong(val);
} catch (NumberFormatException nfe) {}
} else if ("content-type".equals(lcKey)) {
// save for compress decision on server side
_contentType = val;
} else if ("set-cookie".equals(lcKey)) {
String lcVal = val.toLowerCase(Locale.US);
if (lcVal.contains("domain=b32.i2p") ||
lcVal.contains("domain=.b32.i2p")) {
// Strip privacy-damaging "supercookie" for b32.i2p
// Let's presume the user agent ignores a cookie for "i2p"
// See RFC 6265 and http://publicsuffix.org/
if (_log.shouldLog(Log.INFO))
_log.info("Stripping \"" + key + ": " + val + "\" from response ");
break;
}
}
out.write((key.trim() + ": " + val.trim() + "\r\n").getBytes());
}
break;
}
}
}
lastEnd = i;
}
}
if (!connectionSent)
out.write("Connection: close\r\n".getBytes());
if (!proxyConnectionSent)
out.write("Proxy-Connection: close\r\n".getBytes());
finishHeaders();
boolean shouldCompress = shouldCompress();
if (_log.shouldLog(Log.INFO))
_log.info("After headers: gzip? " + _gzip + " compress? " + shouldCompress);
// done, shove off
if (_headerBuffer.getData().length == CACHE_SIZE)
_cache.release(_headerBuffer);
else
_headerBuffer = null;
if (shouldCompress) {
beginProcessing();
}
}
protected boolean shouldCompress() { return _gzip; }
protected void finishHeaders() throws IOException {
out.write("\r\n".getBytes()); // end of the headers
}
@Override
public void close() throws IOException {
out.close();
}
protected void beginProcessing() throws IOException {
//out.flush();
PipedInputStream pi = BigPipedInputStream.getInstance();
PipedOutputStream po = new PipedOutputStream(pi);
// Run in the client thread pool, as there should be an unused thread
// there after the accept().
// Overridden in I2PTunnelHTTPServer, where it does not use the client pool.
try {
I2PTunnelClientBase.getClientExecutor().execute(new Pusher(pi, out));
} catch (RejectedExecutionException ree) {
// shouldn't happen
throw ree;
}
out = po;
}
private class Pusher implements Runnable {
private final InputStream _inRaw;
private final OutputStream _out;
public Pusher(InputStream in, OutputStream out) {
_inRaw = in;
_out = out;
}
public void run() {
ReusableGZIPInputStream _in = null;
long written = 0;
ByteArray ba = null;
try {
_in = ReusableGZIPInputStream.acquire();
// blocking
_in.initialize(_inRaw);
ba = _cache.acquire();
byte buf[] = ba.getData();
int read = -1;
while ( (read = _in.read(buf)) != -1) {
if (_log.shouldLog(Log.DEBUG))
_log.debug("Read " + read + " and writing it to the browser/streams");
_out.write(buf, 0, read);
_out.flush();
written += read;
}
if (_log.shouldLog(Log.INFO))
_log.info("Decompressed: " + written + ", " + _in.getTotalRead() + "/" + _in.getTotalExpanded());
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error decompressing: " + written + ", " + (_in != null ? _in.getTotalRead() + "/" + _in.getTotalExpanded() : ""), ioe);
} catch (OutOfMemoryError oom) {
_log.error("OOM in HTTP Decompressor", oom);
} finally {
if (_log.shouldLog(Log.INFO) && (_in != null))
_log.info("After decompression, written=" + written +
" read=" + _in.getTotalRead()
+ ", expanded=" + _in.getTotalExpanded() + ", remaining=" + _in.getRemaining()
+ ", finished=" + _in.getFinished());
if (ba != null)
_cache.release(ba);
if (_out != null) try {
_out.close();
} catch (IOException ioe) {}
}
if (_in != null) {
double compressed = _in.getTotalRead();
double expanded = _in.getTotalExpanded();
ReusableGZIPInputStream.release(_in);
if (compressed > 0 && expanded > 0) {
// only update the stats if we did something
double ratio = compressed/expanded;
_context.statManager().addRateData("i2ptunnel.httpCompressionRatio", (int)(100d*ratio), 0);
_context.statManager().addRateData("i2ptunnel.httpCompressed", (long)compressed, 0);
_context.statManager().addRateData("i2ptunnel.httpExpanded", (long)expanded, 0);
}
}
}
}
/*******
public static void main(String args[]) {
String simple = "HTTP/1.1 200 OK\n" +
"foo: bar\n" +
"baz: bat\n" +
"\n" +
"hi ho, this is the body";
String filtered = "HTTP/1.1 200 OK\n" +
"Connection: keep-alive\n" +
"foo: bar\n" +
"baz: bat\n" +
"\n" +
"hi ho, this is the body";
String winfilter= "HTTP/1.1 200 OK\r\n" +
"Connection: keep-alive\r\n" +
"foo: bar\r\n" +
"baz: bat\r\n" +
"\r\n" +
"hi ho, this is the body";
String minimal = "HTTP/1.1 200 OK\n" +
"\n" +
"hi ho, this is the body";
String winmin = "HTTP/1.1 200 OK\r\n" +
"\r\n" +
"hi ho, this is the body";
String invalid1 = "HTTP/1.1 200 OK\n";
String invalid2 = "HTTP/1.1 200 OK";
String invalid3 = "HTTP 200 OK\r\n";
String invalid4 = "HTTP 200 OK\r";
String invalid5 = "HTTP/1.1 200 OK\r\n" +
"I am broken, and I smell\r\n" +
"\r\n";
String invalid6 = "HTTP/1.1 200 OK\r\n" +
":I am broken, and I smell\r\n" +
"\r\n";
String invalid7 = "HTTP/1.1 200 OK\n" +
"I am broken, and I smell:\n" +
":asdf\n" +
":\n" +
"\n";
String large = "HTTP/1.1 200 OK\n" +
"Last-modified: Tue, 25 Nov 2003 12:05:38 GMT\n" +
"Expires: Tue, 25 Nov 2003 12:05:38 GMT\n" +
"Content-length: 32\n" +
"\n" +
"hi ho, this is the body";
String blankval = "HTTP/1.0 200 OK\n" +
"A:\n" +
"\n";
test("Simple", simple, true);
test("Filtered", filtered, true);
test("Filtered windows", winfilter, true);
test("Minimal", minimal, true);
test("Windows", winmin, true);
test("Large", large, true);
test("Blank whitespace", blankval, true);
test("Invalid (short headers)", invalid1, true);
test("Invalid (no headers)", invalid2, true);
test("Invalid (windows with short headers)", invalid3, true);
test("Invalid (windows no headers)", invalid4, true);
test("Invalid (bad headers)", invalid5, true);
test("Invalid (bad headers2)", invalid6, false);
test("Invalid (bad headers3)", invalid7, false);
}
private static void test(String name, String orig, boolean shouldPass) {
System.out.println("====Testing: " + name + "\n" + orig + "\n------------");
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
HTTPResponseOutputStream resp = new HTTPResponseOutputStream(baos);
resp.write(orig.getBytes());
resp.flush();
String received = new String(baos.toByteArray());
System.out.println(received);
} catch (Exception e) {
if (shouldPass)
e.printStackTrace();
else
System.out.println("Properly fails with " + e.getMessage());
}
}
******/
}
/** ok, received, now munge & write it */
private void writeHeader() throws IOException {
String responseLine = null;
boolean connectionSent = false;
boolean proxyConnectionSent = false;
int lastEnd = -1;
for (int i = 0; i < _headerBuffer.getValid(); i++) {
if (isNL(_headerBuffer.getData()[i])) {
if (lastEnd == -1) {
responseLine = new String(_headerBuffer.getData(), 0, i+1); // includes NL
responseLine = filterResponseLine(responseLine);
responseLine = (responseLine.trim() + "\r\n");
out.write(responseLine.getBytes());
} else {
for (int j = lastEnd+1; j < i; j++) {
if (_headerBuffer.getData()[j] == ':') {
int keyLen = j-(lastEnd+1);
int valLen = i-(j+1);
if ( (keyLen <= 0) || (valLen < 0) )
throw new IOException("Invalid header @ " + j);
String key = new String(_headerBuffer.getData(), lastEnd+1, keyLen);
String val = null;
if (valLen == 0)
val = "";
else
val = new String(_headerBuffer.getData(), j+2, valLen).trim();
if (_log.shouldLog(Log.INFO))
_log.info("Response header [" + key + "] = [" + val + "]");
String lcKey = key.toLowerCase(Locale.US);
if ("connection".equals(lcKey)) {
out.write("Connection: close\r\n".getBytes());
connectionSent = true;
} else if ("proxy-connection".equals(lcKey)) {
out.write("Proxy-Connection: close\r\n".getBytes());
proxyConnectionSent = true;
} else if ("content-encoding".equals(lcKey) && "x-i2p-gzip".equals(val.toLowerCase(Locale.US))) {
_gzip = true;
} else if ("proxy-authenticate".equals(lcKey)) {
// filter this hop-by-hop header; outproxy authentication must be configured in I2PTunnelHTTPClient
// see e.g. http://blog.c22.cc/2013/03/11/privoxy-proxy-authentication-credential-exposure-cve-2013-2503/
} else {
if ("content-length".equals(lcKey)) {
// save for compress decision on server side
try {
_dataExpected = Long.parseLong(val);
} catch (NumberFormatException nfe) {}
} else if ("content-type".equals(lcKey)) {
// save for compress decision on server side
_contentType = val;
} else if ("set-cookie".equals(lcKey)) {
String lcVal = val.toLowerCase(Locale.US);
if (lcVal.contains("domain=b32.i2p") ||
lcVal.contains("domain=.b32.i2p")) {
// Strip privacy-damaging "supercookie" for b32.i2p
// Let's presume the user agent ignores a cookie for "i2p"
// See RFC 6265 and http://publicsuffix.org/
if (_log.shouldLog(Log.INFO))
_log.info("Stripping \"" + key + ": " + val + "\" from response ");
break;
}
}
out.write((key.trim() + ": " + val.trim() + "\r\n").getBytes());
}
break;
}
}
}
lastEnd = i;
}
}
if (!connectionSent)
out.write("Connection: close\r\n".getBytes());
if (!proxyConnectionSent)
out.write("Proxy-Connection: close\r\n".getBytes());
finishHeaders();
boolean shouldCompress = shouldCompress();
if (_log.shouldLog(Log.INFO))
_log.info("After headers: gzip? " + _gzip + " compress? " + shouldCompress);
// done, shove off
if (_headerBuffer.getData().length == CACHE_SIZE)
_cache.release(_headerBuffer);
else
_headerBuffer = null;
if (shouldCompress) {
beginProcessing();
}
}