/**
* Validate streams generate the same output.
*
* @param expIn Expected input stream.
* @param actIn Actual input stream.
* @param expSize Expected size of the streams.
* @param seek Seek to use async position-based reading or {@code null} to use simple continuous
* reading.
* @throws IOException In case of any IO exception.
*/
private void assertEqualStreams(
InputStream expIn, GridGgfsInputStream actIn, @Nullable Long expSize, @Nullable Long seek)
throws IOException {
if (seek != null) expIn.skip(seek);
int bufSize = 2345;
byte buf1[] = new byte[bufSize];
byte buf2[] = new byte[bufSize];
long pos = 0;
long start = System.currentTimeMillis();
while (true) {
int read = (int) Math.min(bufSize, expSize - pos);
int i1;
if (seek == null) i1 = actIn.read(buf1, 0, read);
else if (seek % 2 == 0) i1 = actIn.read(pos + seek, buf1, 0, read);
else {
i1 = read;
actIn.readFully(pos + seek, buf1, 0, read);
}
// Read at least 0 byte, but don't read more then 'i1' or 'read'.
int i2 = expIn.read(buf2, 0, Math.max(0, Math.min(i1, read)));
if (i1 != i2) {
fail(
"Expects the same data [read="
+ read
+ ", pos="
+ pos
+ ", seek="
+ seek
+ ", i1="
+ i1
+ ", i2="
+ i2
+ ']');
}
if (i1 == -1) break; // EOF
// i1 == bufSize => compare buffers.
// i1 < bufSize => Compare part of buffers, rest of buffers are equal from previous
// iteration.
assertTrue(
"Expects the same data [read="
+ read
+ ", pos="
+ pos
+ ", seek="
+ seek
+ ", i1="
+ i1
+ ", i2="
+ i2
+ ']',
Arrays.equals(buf1, buf2));
if (read == 0) break; // Nothing more to read.
pos += i1;
}
if (expSize != null) assertEquals(expSize.longValue(), pos);
long time = System.currentTimeMillis() - start;
if (time != 0 && log.isInfoEnabled()) {
log.info(
String.format(
"Streams were compared in continuous reading " + "[size=%7d, rate=%3.1f MB/sec]",
expSize, expSize * 1000. / time / 1024 / 1024));
}
}
private boolean unpackFile(ZipFile zip, byte[] buf, ZipEntry fileEntry, String name)
throws IOException, FileNotFoundException {
if (fileEntry == null) fileEntry = zip.getEntry(name);
if (fileEntry == null)
throw new FileNotFoundException("Can't find " + name + " in " + zip.getName());
File outFile = new File(sGREDir, name);
if (outFile.lastModified() == fileEntry.getTime() && outFile.length() == fileEntry.getSize())
return false;
File dir = outFile.getParentFile();
if (!dir.exists()) dir.mkdirs();
InputStream fileStream;
fileStream = zip.getInputStream(fileEntry);
OutputStream outStream = new FileOutputStream(outFile);
while (fileStream.available() > 0) {
int read = fileStream.read(buf, 0, buf.length);
outStream.write(buf, 0, read);
}
fileStream.close();
outStream.close();
outFile.setLastModified(fileEntry.getTime());
return true;
}
public void run() {
try {
Thread.sleep(10);
byte[] buf = getBuf();
URL url = new URL("http://127.0.0.1:" + port + "/test");
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setDoOutput(true);
con.setDoInput(true);
con.setRequestMethod("POST");
con.setRequestProperty(
"Content-Type",
"Multipart/Related; type=\"application/xop+xml\"; boundary=\"----=_Part_0_6251267.1128549570165\"; start-info=\"text/xml\"");
OutputStream out = con.getOutputStream();
out.write(buf);
out.close();
InputStream in = con.getInputStream();
byte[] newBuf = readFully(in);
in.close();
if (buf.length != newBuf.length) {
System.out.println("Doesn't match");
error = true;
}
synchronized (lock) {
++received;
if ((received % 1000) == 0) {
System.out.println("Received=" + received);
}
}
} catch (Exception e) {
// e.printStackTrace();
System.out.print(".");
error = true;
}
}
private static KeyStore readKeyStore(String name) throws Exception {
File file = new File(PATH, name);
InputStream in = new FileInputStream(file);
KeyStore ks = KeyStore.getInstance("JKS");
ks.load(in, passwd);
in.close();
return ks;
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
String filePickerResult = "";
if (data != null && resultCode == RESULT_OK) {
try {
ContentResolver cr = getContentResolver();
Uri uri = data.getData();
Cursor cursor =
GeckoApp.mAppContext
.getContentResolver()
.query(uri, new String[] {OpenableColumns.DISPLAY_NAME}, null, null, null);
String name = null;
if (cursor != null) {
try {
if (cursor.moveToNext()) {
name = cursor.getString(0);
}
} finally {
cursor.close();
}
}
String fileName = "tmp_";
String fileExt = null;
int period;
if (name == null || (period = name.lastIndexOf('.')) == -1) {
String mimeType = cr.getType(uri);
fileExt = "." + GeckoAppShell.getExtensionFromMimeType(mimeType);
} else {
fileExt = name.substring(period);
fileName = name.substring(0, period);
}
File file = File.createTempFile(fileName, fileExt, sGREDir);
FileOutputStream fos = new FileOutputStream(file);
InputStream is = cr.openInputStream(uri);
byte[] buf = new byte[4096];
int len = is.read(buf);
while (len != -1) {
fos.write(buf, 0, len);
len = is.read(buf);
}
fos.close();
filePickerResult = file.getAbsolutePath();
} catch (Exception e) {
Log.e(LOG_FILE_NAME, "showing file picker", e);
}
}
try {
mFilePickerResult.put(filePickerResult);
} catch (InterruptedException e) {
Log.i(LOG_FILE_NAME, "error returning file picker result", e);
}
}
void handleRequest(InputStream in, OutputStream out) throws IOException {
boolean newline = false;
StringBuilder sb = new StringBuilder();
while (true) {
int ch = in.read();
if (ch < 0) {
throw new EOFException();
}
sb.append((char) ch);
if (ch == '\r') {
// empty
} else if (ch == '\n') {
if (newline) {
// 2nd newline in a row, end of request
break;
}
newline = true;
} else {
newline = false;
}
}
String request = sb.toString();
if (request.startsWith("GET / HTTP/1.") == false) {
throw new IOException("Invalid request: " + request);
}
out.write("HTTP/1.0 200 OK\r\n\r\n".getBytes());
}
public void run() {
try {
URL url = new URL(protocol + "://localhost:" + port + "/test1/" + f);
HttpURLConnection urlc = (HttpURLConnection) url.openConnection();
if (urlc instanceof HttpsURLConnection) {
HttpsURLConnection urlcs = (HttpsURLConnection) urlc;
urlcs.setHostnameVerifier(
new HostnameVerifier() {
public boolean verify(String s, SSLSession s1) {
return true;
}
});
urlcs.setSSLSocketFactory(ctx.getSocketFactory());
}
byte[] buf = new byte[4096];
if (fixedLen) {
urlc.setRequestProperty("XFixed", "yes");
}
InputStream is = urlc.getInputStream();
File temp = File.createTempFile("Test1", null);
temp.deleteOnExit();
OutputStream fout = new BufferedOutputStream(new FileOutputStream(temp));
int c, count = 0;
while ((c = is.read(buf)) != -1) {
count += c;
fout.write(buf, 0, c);
}
is.close();
fout.close();
if (count != size) {
throw new RuntimeException("wrong amount of data returned");
}
String orig = root + "/" + f;
compare(new File(orig), temp);
temp.delete();
} catch (Exception e) {
e.printStackTrace();
fail = true;
}
}
@Override
public HTTPResponse call() throws Exception {
final HttpURLConnection hc = (HttpURLConnection) uri.toURL().openConnection();
hc.setReadTimeout(SOCKET_TIMEOUT);
try {
while (!stop) {
try {
final int code = hc.getResponseCode();
final InputStream input = hc.getInputStream();
final ByteList bl = new ByteList();
for (int i; (i = input.read()) != -1; ) bl.add(i);
return new HTTPResponse(code, bl.toString());
} catch (final SocketTimeoutException e) {
}
}
return null;
} finally {
hc.disconnect();
}
}
private static byte[] readFully(InputStream istream) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
byte[] buf = new byte[1024];
int num = 0;
if (istream != null) {
while ((num = istream.read(buf)) != -1) {
bout.write(buf, 0, num);
}
}
byte[] ret = bout.toByteArray();
return ret;
}
/**
* Checks availability of a classpath resource.
*
* @param name Resource name.
* @return {@code true} if resource is available and ready for read, {@code false} otherwise.
*/
private boolean resourceAvailable(String name) {
InputStream cfgStream =
Thread.currentThread().getContextClassLoader().getResourceAsStream(name);
if (cfgStream == null) {
log.error("Classpath resource not found: " + name);
return false;
}
try {
// Read a single byte to force actual content access by JVM.
cfgStream.read();
return true;
} catch (IOException e) {
log.error("Failed to read classpath resource: " + name, e);
return false;
} finally {
U.close(cfgStream, log);
}
}
protected void runMayThrow() throws Exception {
byte[] compressedWithCRC;
while (chunks.hasNext()) {
CompressionMetadata.Chunk chunk = chunks.next();
int readLength = chunk.length + 4; // read with CRC
compressedWithCRC = new byte[readLength];
int bufferRead = 0;
while (bufferRead < readLength)
bufferRead += source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
dataBuffer.put(compressedWithCRC);
}
}
@Override
public void run() {
byte[] buf = new byte[100];
try {
int len;
while ((len = in.read(buf)) > 0) {
String output = new String(buf, 0, len);
Thread t = Thread.currentThread();
System.out.println(
"thread " + t.getName() + " " + t.getId() + ", read " + len + " bytes: " + output);
}
} catch (IOException e) {
logger.log(Level.SEVERE, "Failed to read", e);
} finally {
try {
in.close();
} catch (IOException e) {
logger.log(Level.SEVERE, "Failed to close", e);
}
}
}
void sendRequest(InputStream in, OutputStream out) throws IOException {
out.write("GET / HTTP/1.0\r\n\r\n".getBytes());
out.flush();
StringBuilder sb = new StringBuilder();
while (true) {
int ch = in.read();
if (ch < 0) {
break;
}
sb.append((char) ch);
}
String response = sb.toString();
if (response.startsWith("HTTP/1.0 200 ") == false) {
throw new IOException("Invalid response: " + response);
}
}
static void compare(File f1, File f2) throws IOException {
InputStream i1 = new BufferedInputStream(new FileInputStream(f1));
InputStream i2 = new BufferedInputStream(new FileInputStream(f2));
int c1, c2;
try {
while ((c1 = i1.read()) != -1) {
c2 = i2.read();
if (c1 != c2) {
throw new RuntimeException("file compare failed 1");
}
}
if (i2.read() != -1) {
throw new RuntimeException("file compare failed 2");
}
} finally {
i1.close();
i2.close();
}
}
/**
* Test file creation.
*
* @param path Path to file to store.
* @param size Size of file to store.
* @param salt Salt for file content generation.
* @throws Exception In case of any exception.
*/
private void testCreateFile(final GridGgfsPath path, final long size, final int salt)
throws Exception {
info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']');
final AtomicInteger cnt = new AtomicInteger(0);
final Collection<GridGgfsPath> cleanUp = new ConcurrentLinkedQueue<>();
long time =
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
int id = cnt.incrementAndGet();
GridGgfsPath f = new GridGgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : ""));
try (GridGgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) {
assertNotNull(out);
cleanUp.add(f); // Add all created into cleanup list.
U.copy(new GridGgfsTestInputStream(size, salt), out);
}
return null;
}
},
WRITING_THREADS_CNT,
"perform-multi-thread-writing");
if (time > 0) {
double rate = size * 1000. / time / 1024 / 1024;
info(
String.format(
"Write file [path=%s, size=%d kB, rate=%2.1f MB/s]",
path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate));
}
info("Read and validate saved file: " + path);
final InputStream expIn = new GridGgfsTestInputStream(size, salt);
final GridGgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10);
// Validate continuous reading of whole file.
assertEqualStreams(expIn, actIn, size, null);
// Validate random seek and reading.
final Random rnd = new Random();
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
long skip = Math.abs(rnd.nextLong() % (size + 1));
long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400));
assertEqualStreams(new GridGgfsTestInputStream(size, salt), actIn, range, skip);
return null;
}
},
READING_THREADS_CNT,
"validate-multi-thread-reading");
expIn.close();
actIn.close();
info("Get stored file info: " + path);
GridGgfsFile desc = fs.info(path);
info("Validate stored file info: " + desc);
assertNotNull(desc);
if (log.isDebugEnabled()) log.debug("File descriptor: " + desc);
Collection<GridGgfsBlockLocation> aff = fs.affinity(path, 0, desc.length());
assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty());
int blockSize = desc.blockSize();
assertEquals("File size", size, desc.length());
assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize);
// assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString());
// assertEquals("Permission sticky bit marks this is file", false,
// desc.getPermission().getStickyBit());
assertEquals("Type", true, desc.isFile());
assertEquals("Type", false, desc.isDirectory());
info("Cleanup files: " + cleanUp);
for (GridGgfsPath f : cleanUp) {
fs.delete(f, true);
assertNull(fs.info(f));
}
}
int readCharacter() throws Exception {
return in.read();
}