/**
* Load UTF8withBOM or any ansi text file.
*
* @param filename
* @return
* @throws java.io.IOException
*/
public static String loadFileAsString(String filename) throws java.io.IOException {
final int BUFLEN = 1024;
BufferedInputStream is = new BufferedInputStream(new FileInputStream(filename), BUFLEN);
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream(BUFLEN);
byte[] bytes = new byte[BUFLEN];
boolean isUTF8 = false;
int read, count = 0;
while ((read = is.read(bytes)) != -1) {
if (count == 0
&& bytes[0] == (byte) 0xEF
&& bytes[1] == (byte) 0xBB
&& bytes[2] == (byte) 0xBF) {
isUTF8 = true;
baos.write(bytes, 3, read - 3); // drop UTF8 bom marker
} else {
baos.write(bytes, 0, read);
}
count += read;
}
return isUTF8 ? new String(baos.toByteArray(), "UTF-8") : new String(baos.toByteArray());
} finally {
try {
is.close();
} catch (Exception ex) {
}
}
}
public void run() {
URL url;
Base64Encoder base64 = new Base64Encoder();
try {
url = new URL(urlString);
} catch (MalformedURLException e) {
System.err.println("Invalid URL");
return;
}
try {
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Authorization", "Basic " + base64.encode(user + ":" + pass));
httpIn = new BufferedInputStream(conn.getInputStream(), 8192);
} catch (IOException e) {
System.err.println("Unable to connect: " + e.getMessage());
return;
}
int prev = 0;
int cur = 0;
try {
while (keepAlive && (cur = httpIn.read()) >= 0) {
if (prev == 0xFF && cur == 0xD8) {
jpgOut = new ByteArrayOutputStream(8192);
jpgOut.write((byte) prev);
}
if (jpgOut != null) {
jpgOut.write((byte) cur);
}
if (prev == 0xFF && cur == 0xD9) {
synchronized (curFrame) {
curFrame = jpgOut.toByteArray();
}
frameAvailable = true;
jpgOut.close();
}
prev = cur;
}
} catch (IOException e) {
System.err.println("I/O Error: " + e.getMessage());
}
try {
jpgOut.close();
httpIn.close();
} catch (IOException e) {
System.err.println("Error closing streams: " + e.getMessage());
}
conn.disconnect();
}
private Icon makeIcon(final String gifFile) throws IOException {
/* Copy resource into a byte array. This is
* necessary because several browsers consider
* Class.getResource a security risk because it
* can be used to load additional classes.
* Class.getResourceAsStream just returns raw
* bytes, which we can convert to an image.
*/
InputStream resource = MyImageView.class.getResourceAsStream(gifFile);
if (resource == null) {
System.err.println(MyImageView.class.getName() + "/" + gifFile + " not found!!.");
return null;
}
BufferedInputStream in = new BufferedInputStream(resource);
ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
byte[] buffer = new byte[1024];
int n;
while ((n = in.read(buffer)) > 0) {
out.write(buffer, 0, n);
}
in.close();
out.flush();
buffer = out.toByteArray();
if (buffer.length == 0) {
System.err.println("warning: " + gifFile + " is zero-length");
return null;
}
return new ImageIcon(buffer);
}
public void run() {
byte[] tmp = new byte[10000];
int read;
try {
FileInputStream fis = new FileInputStream(fileToSend.getFile());
read = fis.read(tmp);
while (read != -1) {
baos.write(tmp, 0, read);
read = fis.read(tmp);
System.out.println(read);
}
fis.close();
baos.writeTo(sWriter);
sWriter.flush();
baos.flush();
baos.close();
System.out.println("fileSent");
} catch (IOException e) {
e.printStackTrace();
} finally {
this.closeSocket();
}
}
/** @throws IOException If failed. */
private void initFavicon() throws IOException {
assert favicon == null;
InputStream in = getClass().getResourceAsStream("favicon.ico");
if (in != null) {
BufferedInputStream bis = new BufferedInputStream(in);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try {
byte[] buf = new byte[2048];
while (true) {
int n = bis.read(buf);
if (n == -1) break;
bos.write(buf, 0, n);
}
favicon = bos.toByteArray();
} finally {
U.closeQuiet(bis);
}
}
}
private Class<?> getClassFromStream(
final InputStream stream, final String classname, final File container)
throws IOException, SecurityException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
int bytesRead = -1;
final byte[] buffer = new byte[8192];
while ((bytesRead = stream.read(buffer, 0, 8192)) != -1) {
baos.write(buffer, 0, bytesRead);
}
final byte[] classData = baos.toByteArray();
return this.defineClassFromData(container, classData, classname);
}
private byte[] readFully(InputStream istream) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
byte[] buf = new byte[1024];
int num = 0;
while ((num = istream.read(buf)) != -1) {
bout.write(buf, 0, num);
}
byte[] ret = bout.toByteArray();
return ret;
}
byte[] messageToBuffer(Message msg) throws Exception {
ObjectOutputStream out;
// BufferedOutputStream bos;
out_stream.reset();
// bos=new BufferedOutputStream(out_stream);
out_stream.write(Version.version_id, 0, Version.version_id.length); // write the version
// bos.write(Version.version_id, 0, Version.version_id.length); // write the version
out = new ObjectOutputStream(out_stream);
// out=new ObjectOutputStream(bos);
msg.writeExternal(out);
out.flush(); // needed if out buffers its output to out_stream
return out_stream.toByteArray();
}
/** TIFF Adobe ZIP support contributed by Jason Newton. */
public byte[] zipUncompress(byte[] input) {
ByteArrayOutputStream imageBuffer = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
Inflater decompressor = new Inflater();
decompressor.setInput(input);
try {
while (!decompressor.finished()) {
int rlen = decompressor.inflate(buffer);
imageBuffer.write(buffer, 0, rlen);
}
} catch (DataFormatException e) {
IJ.log(e.toString());
}
decompressor.end();
return imageBuffer.toByteArray();
}
private Class findRawClass(String className) throws ClassNotFoundException {
try {
String resourcePath = className.replace('.', '/') + ".class";
InputStream resourceStream = getResourceAsStream(resourcePath);
ByteArrayOutputStream rawByteStream = new ByteArrayOutputStream();
byte[] buf = new byte[4096];
int bytesread = 0;
while ((bytesread = resourceStream.read(buf)) >= 0) {
rawByteStream.write(buf, 0, bytesread);
}
resourceStream.close();
byte[] rawBytes = rawByteStream.toByteArray();
return super.defineClass(className, rawBytes, 0, rawBytes.length);
} catch (Exception exc) {
throw new ClassNotFoundException(className, exc);
}
}
public static byte[] loadResourceIntoByteArray(String resourcePath) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
InputStream inputStream = null;
try {
inputStream = Skygge.class.getResourceAsStream(resourcePath);
byte[] byteChunk = new byte[8192];
int n;
while ((n = inputStream.read(byteChunk)) > 0) {
outputStream.write(byteChunk, 0, n);
}
} finally {
if (inputStream != null) {
inputStream.close();
}
}
return outputStream.toByteArray();
}
public static byte[] loadUrlIntoByteArray(String urlString) throws IOException {
URL url = new URL(urlString);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
InputStream inputStream = null;
try {
inputStream = url.openStream();
byte[] byteChunk = new byte[8192];
int n;
while ((n = inputStream.read(byteChunk)) > 0) {
outputStream.write(byteChunk, 0, n);
}
} finally {
if (inputStream != null) {
inputStream.close();
}
}
return outputStream.toByteArray();
}
public Image getImage(String sImage) {
Image imReturn = null;
try {
if (jar == null) {
imReturn = this.toolkit.createImage(this.getClass().getClassLoader().getResource(sImage));
} else {
//
BufferedInputStream bis = new BufferedInputStream(jar.getInputStream(jar.getEntry(sImage)));
ByteArrayOutputStream buffer = new ByteArrayOutputStream(4096);
int b;
while ((b = bis.read()) != -1) {
buffer.write(b);
}
byte[] imageBuffer = buffer.toByteArray();
imReturn = this.toolkit.createImage(imageBuffer);
bis.close();
buffer.close();
}
} catch (IOException ex) {
}
return imReturn;
}
/**
* ** Reads the bytes from the specifed socket until an eod-of-stream error occurs, or ** until
* the maximum number of bytes has bee read. ** @param socket The socket from which bytes are read
* ** @param baos The ByteArrayOutputStream to which the bytes are written ** @param maxLength The
* number of bytes to read from the socket ** @return The number of bytes read if no exception has
* occurred ** @throws IOException if an error occured or the server has stopped
*/
protected static int socketReadBytes(Socket socket, ByteArrayOutputStream baos, int maxLength)
throws IOException {
if (socket == null) {
return 0;
} else if (maxLength == 0) {
return 0;
} else {
int dataLen = 0;
InputStream input = socket.getInputStream();
while ((maxLength < 0) || (dataLen < maxLength)) {
int ch = input.read();
if (ch < 0) {
// we've reached the end of input
return dataLen;
} else {
if (baos != null) {
baos.write(ch);
}
dataLen++;
}
}
return dataLen;
}
}
public void execute() {
CoreEvent event = null;
URLConnection connection = null;
InputStream inputStream = null;
int httpStatus = 0;
HTTPHeader[] headers = null;
try {
connection = url.openConnection();
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).setInstanceFollowRedirects(false);
}
connection.setRequestProperty("User-agent", site.getUserAgent());
context.preHandle(connection, site);
long start = System.currentTimeMillis();
connection.connect();
if (connection instanceof HttpURLConnection) {
httpStatus = ((HttpURLConnection) connection).getResponseCode();
switch (httpStatus) {
case HttpURLConnection.HTTP_MOVED_PERM:
case HttpURLConnection.HTTP_MOVED_TEMP:
String redirectURL = connection.getHeaderField("location");
notifyEvent(
url, new URLFoundEvent(context, url, URLUtil.normalize(new URL(redirectURL))));
break;
default:
break;
}
}
inputStream = new BufferedInputStream(connection.getInputStream());
ByteArrayOutputStream os = new ByteArrayOutputStream();
InputStream is = new BufferedInputStream(inputStream);
// int size = connection.getContentLength();
int size = 0;
try {
int i = is.read();
while (i != -1) {
size++;
os.write(i);
i = is.read();
}
} catch (IOException e) {
LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception during fetch", e);
}
String contentType = connection.getContentType();
int timeMs = (int) (System.currentTimeMillis() - start);
headers = HTTPHeaderUtil.getHeaders(connection);
if (httpStatus >= 200 && httpStatus < 303) {
event =
new URLSpideredOkEvent(
context,
url,
httpStatus,
connection,
contentType,
timeMs,
size,
os.toByteArray(),
headers);
} else {
event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, null);
}
context.postHandle(connection, site);
} catch (FileNotFoundException e) {
headers = HTTPHeaderUtil.getHeaders(connection);
event = new URLSpideredErrorEvent(context, url, 404, connection, headers, e);
} catch (Exception e) {
LogFactory.getLog(this.getClass()).error("exception during spidering", e);
event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, e);
} finally {
notifyEvent(url, event);
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception closing inputstream", e);
}
}
}
}
private byte[] load(String url, int max_millis_to_wait) {
// limit the subset here as we're looping waiting for something to be alive and we can't afford
// to take ages getting back to the start
long start = System.currentTimeMillis();
while (true) {
outer:
for (int i = 45100; i <= 45108; i++) {
long now = System.currentTimeMillis();
if (now < start) {
start = now;
}
if (now - start > max_millis_to_wait) {
return (null);
}
Socket sock = null;
try {
sock = new Socket();
sock.connect(new InetSocketAddress("127.0.0.1", i), 500);
sock.setSoTimeout(5000);
PrintWriter pw = new PrintWriter(sock.getOutputStream());
pw.println("GET " + url + " HTTP/1.1" + NL + NL);
pw.flush();
InputStream is = sock.getInputStream();
String header = "";
byte[] buffer = new byte[1];
while (true) {
int len = is.read(buffer);
if (len <= 0) {
break outer;
}
header += new String(buffer, 0, len);
if (header.endsWith(NL + NL)) {
break;
}
}
int pos = header.indexOf(NL);
String first_line = header.substring(0, pos);
if (first_line.indexOf("200") == -1) {
continue;
}
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
buffer = new byte[2048];
while (true) {
int len = is.read(buffer);
if (len <= 0) {
break;
}
baos.write(buffer, 0, len);
if (baos.size() > 512 * 1024) {
break outer;
}
}
return (baos.toByteArray());
} catch (Throwable e) {
} finally {
if (sock != null) {
try {
sock.close();
} catch (Throwable e) {
}
}
}
}
}
}
public ByteArrayOutputStream receive() throws SocketException {
// The complete sequence of bytes received from the external process
ByteArrayOutputStream receivedBytes = new ByteArrayOutputStream();
// The header received from the external process
int header = -1;
// The number of payload bytes received from the external process for a
// packet
int numberRead = 0;
// The total number of payload bytes received from the external process
// for
// a packet, if not all are received immediately.
int totalNumberRead = 0;
// The payload received from the external process for each packet
byte[] payload;
while (true) {
// Read a header byte from the input stream
try {
header = (int) input.read();
} catch (IOException e) {
e.printStackTrace();
}
// If the header shows that the socket has closed ...
if (header == -1) {
logger.info("#" + index + "已经关闭");
throw new SocketException("Closed");
}
// If the header indicates another packet to follow ...
else if (header >= 127) {
// ... create 127 bytes of payload storage ...
payload = new byte[127];
}
// ... else create storage of the appropriate size
else payload = new byte[header];
// Read the payload bytes from the input stream
// Reset the total bytes received to 0 for this iteration
totalNumberRead = 0;
// Loop until all data has been read for this packet.
while (totalNumberRead < payload.length && numberRead != -1) {
try {
// Try to read all bytes in this packet
numberRead = input.read(payload, totalNumberRead, payload.length - totalNumberRead);
// If some bytes were read ...
if (numberRead != -1)
// ... record this many bytes as having been read
totalNumberRead = totalNumberRead + numberRead;
} catch (SocketException e) {
throw new SocketException("Socket closed while receiving data.");
} catch (IOException e) {
e.printStackTrace();
}
}
// If not all bytes could be read ...
if ((totalNumberRead < header || numberRead == -1) && header != 255)
// ... throw a SocketException
throw new SocketException("Error receiving data.");
// Write the payload data to the complete sequence of received bytes
receivedBytes.write(payload, 0, payload.length);
// If no more bytes to follow, break from the loop.
if (header <= 127) break;
}
// Return the received bytes
return receivedBytes;
}
