@Test
public void testStream_bytesRefHash() throws Exception {
final BytesRefHash brh = new BytesRefHash();
brh.add(new BytesRef("foo"));
brh.add(new BytesRef("bar"));
brh.add(new BytesRef("baz"));
Assert.assertEquals("Not all terms streamed.", 3L, StreamUtils.stream(brh).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(brh).filter(br -> br.bytesEquals(new BytesRef("foo"))).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(brh).filter(br -> br.bytesEquals(new BytesRef("bar"))).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(brh).filter(br -> br.bytesEquals(new BytesRef("baz"))).count());
Assert.assertEquals(
"Unknown term found.",
0L,
StreamUtils.stream(brh)
.filter(
t ->
!t.bytesEquals(new BytesRef("foo"))
&& !t.bytesEquals(new BytesRef("bar"))
&& !t.bytesEquals(new BytesRef("baz")))
.count());
}
@Test
public void testStream_bytesRefArray() throws Exception {
final BytesRefArray bArr = new BytesRefArray(Counter.newCounter(false));
bArr.append(new BytesRef("foo"));
bArr.append(new BytesRef("bar"));
bArr.append(new BytesRef("baz"));
Assert.assertEquals("Not all items streamed.", 3L, StreamUtils.stream(bArr).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("foo"))).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("bar"))).count());
Assert.assertEquals(
"Term not found.",
1L,
StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("baz"))).count());
Assert.assertEquals(
"Unknown term found.",
0L,
StreamUtils.stream(bArr)
.filter(
t ->
!t.bytesEquals(new BytesRef("foo"))
&& !t.bytesEquals(new BytesRef("bar"))
&& !t.bytesEquals(new BytesRef("baz")))
.count());
}
/**
* Reads the entire file into a byte array.
*
* @throws RuntimeException if the file handle represents a directory, doesn't exist, or could not
* be read.
*/
public byte[] readBytes() {
InputStream input = read();
try {
return StreamUtils.copyStreamToByteArray(input, estimateLength());
} catch (IOException ex) {
throw new RuntimeException("Error reading file: " + this, ex);
} finally {
StreamUtils.closeQuietly(input);
}
}
/**
* Reads the remaining bytes from the specified stream and writes them to this file. The stream is
* closed. Parent directories will be created if necessary.
*
* @param append If false, this file will be overwritten if it exists, otherwise it will be
* appended.
* @throws RuntimeException if this file handle represents a directory, if it is a {@link
* FileType#Classpath} or {@link FileType#Internal} file, or if it could not be written.
*/
public void write(InputStream input, boolean append) {
OutputStream output = null;
try {
output = write(append);
StreamUtils.copyStream(input, output, 4096);
} catch (Exception ex) {
throw new RuntimeException("Error stream writing to file: " + file + " (" + type + ")", ex);
} finally {
StreamUtils.closeQuietly(input);
StreamUtils.closeQuietly(output);
}
}
@Test
public void testStream_docIdSet() throws Exception {
final Builder disBuilder = new Builder(10);
disBuilder.add(1).add(3).add(6).add(7).add(8).add(10);
final DocIdSet dis = disBuilder.build();
Assert.assertEquals("Not all document ids streamed.", 6L, StreamUtils.stream(dis).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 1).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 3).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 6).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 7).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 8).count());
Assert.assertEquals(
"Document id count mismatch.", 1L, StreamUtils.stream(dis).filter(id -> id == 10).count());
Assert.assertEquals(
"Unknown document id found.",
0L,
StreamUtils.stream(dis)
.filter(id -> id != 1 && id != 3 && id != 6 && id != 7 && id != 8 && id != 10)
.count());
}
@Test
public void testStream_bits() throws Exception {
final FixedBitSet fbs = new FixedBitSet(11);
fbs.set(1);
fbs.set(3);
fbs.set(6);
fbs.set(7);
fbs.set(8);
fbs.set(10);
final Bits bits = fbs;
Assert.assertEquals("Not all bits streamed.", 6L, StreamUtils.stream(bits).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 1).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 3).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 6).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 7).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 8).count());
Assert.assertEquals(
"Bit not found.", 1L, StreamUtils.stream(bits).filter(id -> id == 10).count());
Assert.assertEquals(
"Unknown document id found.",
0L,
StreamUtils.stream(bits)
.filter(id -> id != 1 && id != 3 && id != 6 && id != 7 && id != 8 && id != 10)
.count());
}
@Test
public void testStream_docIdSet_empty() throws Exception {
final Builder disBuilder = new Builder(10);
final DocIdSet dis = disBuilder.build();
Assert.assertEquals("Too much document ids streamed.", 0L, StreamUtils.stream(dis).count());
}
@SuppressWarnings("ConstantConditions")
@Test
public void testStream_docIdSetIterator_null() throws Exception {
// expect an empty stream
Assert.assertEquals(
"Too much document ids streamed.", 0L, StreamUtils.stream((DocIdSetIterator) null).count());
}
/** 提交QQ账号和密码到服务器 */
private void requestNet() {
try {
String path =
"http://192.168.1.253:8080/web/LoginServlet"
+ "?qq="
+ URLEncoder.encode(qq, "utf-8")
+ "&pwd="
+ URLEncoder.encode(pwd, "utf-8");
// 1. 打开浏览器
HttpClient client = new DefaultHttpClient();
// 2. 输入Url网址
HttpGet url = new HttpGet(path);
// 3. 敲回车
HttpResponse response = client.execute(url);
int code = response.getStatusLine().getStatusCode();
if (code == 200) {
InputStream is = response.getEntity().getContent();
String text = StreamUtils.parserStream2String(is);
showToastInThread(text);
} else {
showToastInThread("code:" + code);
}
} catch (Exception e) {
e.printStackTrace();
showToastInThread("哥哥，异常了!!!");
}
}
protected StreamingPropertyValueRef saveStreamingPropertyValue(
final String rowKey, final Property property, StreamingPropertyValue propertyValue) {
try {
HdfsLargeDataStore largeDataStore =
new HdfsLargeDataStore(this.fileSystem, this.dataDir, rowKey, property);
LimitOutputStream out =
new LimitOutputStream(largeDataStore, maxStreamingPropertyValueTableDataSize);
try {
StreamUtils.copy(propertyValue.getInputStream(), out);
} finally {
out.close();
}
if (out.hasExceededSizeLimit()) {
LOGGER.debug(
"saved large file to \"%s\" (length: %d)",
largeDataStore.getFullHdfsPath(), out.getLength());
return new StreamingPropertyValueHdfsRef(
largeDataStore.getRelativeFileName(), propertyValue);
} else {
return saveStreamingPropertyValueSmall(rowKey, property, out.getSmall(), propertyValue);
}
} catch (IOException ex) {
throw new VertexiumException(ex);
}
}
/** @see com.rapidminer.operator.OperatorChain#doWork() */
@Override
public void doWork() throws OperatorException {
List<Operator> nested = this.getImmediateChildren();
log.info("This StreamProcess has {} nested operators", nested.size());
for (Operator op : nested) {
log.info(" op: {}", op);
if (op instanceof DataStreamOperator) {
log.info("Resetting stream-operator {}", op);
((DataStreamOperator) op).reset();
}
}
log.info("Starting some work in doWork()");
ExampleSet exampleSet = input.getData(ExampleSet.class);
log.info("input is an example set with {} examples", exampleSet.size());
int i = 0;
Iterator<Example> it = exampleSet.iterator();
while (it.hasNext()) {
Example example = it.next();
log.info("Processing example {}", i);
DataObject datum = StreamUtils.wrap(example);
log.info("Wrapped data-object is: {}", datum);
dataStream.deliver(datum);
getSubprocess(0).execute();
inApplyLoop();
i++;
}
// super.doWork();
log.info("doWork() is finished.");
}
@Override
public Object fromBody(TypedInput typedInput, Type type) throws ConversionException {
try {
return StreamUtils.readInputStream(typedInput.in(), "utf-8");
} catch (IOException exception) {
exception.printStackTrace();
}
return null;
}
public Element parse(InputStream input) throws IOException {
try {
return parse(new InputStreamReader(input, "UTF-8"));
} catch (IOException ex) {
throw new SerializationException(ex);
} finally {
StreamUtils.closeQuietly(input);
}
}
/**
* 지정된 url에서 이미지를 가져옴
*
* @param urls 가져올 이미지의 url
* @return 가져온 이미지
* @throws IOException
*/
public static Bitmap bitmapFromURL(String bitmapUrl, BitmapFactory.Options opt)
throws IOException {
DataInputStream bis = new DataInputStream(StreamUtils.inStreamFromURL(bitmapUrl, null));
Bitmap b;
if (opt != null) b = BitmapFactory.decodeStream(bis, null, opt);
else b = BitmapFactory.decodeStream(bis);
bis.close();
return b;
}
@Test
public void testNonClosing() {
final MockCloseCountingOutputStream aX =
new MockCloseCountingOutputStream(new NonBlockingByteArrayOutputStream());
StreamUtils.copyInputStreamToOutputStreamAndCloseOS(
new NonBlockingByteArrayInputStream(
CharsetManager.getAsBytes("abc", CCharset.CHARSET_ISO_8859_1_OBJ)),
new NonClosingOutputStream(aX));
assertEquals(0, aX.getCloseCount());
}
/**
* Writes the specified bytes to the file. Parent directories will be created if necessary.
*
* @param append If false, this file will be overwritten if it exists, otherwise it will be
* appended.
* @throws RuntimeException if this file handle represents a directory, if it is a {@link
* FileType#Classpath} or {@link FileType#Internal} file, or if it could not be written.
*/
public void writeBytes(byte[] bytes, int offset, int length, boolean append) {
OutputStream output = write(append);
try {
output.write(bytes, offset, length);
} catch (IOException ex) {
throw new RuntimeException("Error writing file: " + file + " (" + type + ")", ex);
} finally {
StreamUtils.closeQuietly(output);
}
}
/**
* Writes the specified string to the file as UTF-8. Parent directories will be created if
* necessary.
*
* @param append If false, this file will be overwritten if it exists, otherwise it will be
* appended.
* @param charset May be null to use the default charset.
* @throws RuntimeException if this file handle represents a directory, if it is a {@link
* FileType#Classpath} or {@link FileType#Internal} file, or if it could not be written.
*/
public void writeString(String string, boolean append, String charset) {
Writer writer = null;
try {
writer = writer(append, charset);
writer.write(string);
} catch (Exception ex) {
throw new RuntimeException("Error writing file: " + file + " (" + type + ")", ex);
} finally {
StreamUtils.closeQuietly(writer);
}
}
/**
* Returns the length in bytes of this file, or 0 if this file is a directory, does not exist, or
* the size cannot otherwise be determined.
*/
public long length() {
if (type == FileType.Classpath || (type == FileType.Internal && !file.exists())) {
InputStream input = read();
try {
return input.available();
} catch (Exception ignored) {
} finally {
StreamUtils.closeQuietly(input);
}
return 0;
}
return file().length();
}
/** Cluster data from an XML file (local). */
public void clusterFromFile() throws IOException {
final Map<String, String> attributes = new LinkedHashMap<String, String>();
System.out.println("## Clustering documents from a local file");
/*
* Note the optional query attribute, we can provide it to avoid creation of
* trivial clusters.
*/
attributes.put(
"dcs.c2stream",
new String(StreamUtils.readFullyAndClose(new FileInputStream(XML_FILE_PATH)), "UTF-8"));
attributes.put("query", "data mining");
displayResults(httpPoster.post(dcsURI, attributes));
}
@Override
public void run() {
try {
// TODO Auto-generated method stub
// 在线程内存访问网络 HttpURLConnection
// 拿到一个流
URL urlObj = new URL(url);
// 生成Bitmap
HttpURLConnection conn = (HttpURLConnection) urlObj.openConnection();
conn.setRequestMethod("GET");
conn.setConnectTimeout(5000);
if (200 == conn.getResponseCode()) {
InputStream input = conn.getInputStream();
// 使用工具快速生成bitmap对象
byte[] bytes = StreamUtils.readInputStream(input);
// Bitmap bitmap = BitmapFactory.decodeStream(input);
Bitmap bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.length);
if (bitmap != null) {
Message msg = new Message();
msg.what = 200;
HashMap<String, Object> data = new HashMap<String, Object>();
data.put("imagview", imageView); // 停止出现图片错乱问题
data.put("bitmap", bitmap);
msg.obj = data;
handler.sendMessage(msg); // -->handlemesssage();
// Log.i("wzx", "imageview");
// Log.i("wzx", "gridview");
// Log.i("wzx", "listview");
// 保存到集合
cache.put(url, bitmap);
// 保存到文件目录
writeToLocal(url, bitmap);
}
return;
}
} catch (Exception e) {
// TODO: handle exception
}
Message msg = new Message();
msg.what = 404;
handler.sendMessage(msg);
}
/**
* Reads the entire file into the byte array. The byte array must be big enough to hold the file's
* data.
*
* @param bytes the array to load the file into
* @param offset the offset to start writing bytes
* @param size the number of bytes to read, see {@link #length()}
* @return the number of read bytes
*/
public int readBytes(byte[] bytes, int offset, int size) {
InputStream input = read();
int position = 0;
try {
while (true) {
int count = input.read(bytes, offset + position, size - position);
if (count <= 0) {
break;
}
position += count;
}
} catch (IOException ex) {
throw new RuntimeException("Error reading file: " + this, ex);
} finally {
StreamUtils.closeQuietly(input);
}
return position - offset;
}
public static String getResponse(URL url) {
String response = "";
try {
URLConnection connect = url.openConnection();
connect.setRequestProperty("Accept", "application/json");
byte[] data = StreamUtils.streamToBytes(connect.getInputStream());
response = new String(data);
} catch (IOException e) {
response = "Something went wrong";
e.printStackTrace();
Toast.makeText(
MainActivity.context,
"Something went wrong while retrieving the data. Please check your "
+ "connection and try again.",
Toast.LENGTH_LONG)
.show();
}
return response;
}
public Element parse(Reader reader) throws IOException {
try {
char[] data = new char[1024];
int offset = 0;
while (true) {
int length = reader.read(data, offset, data.length - offset);
if (length == -1) break;
if (length == 0) {
char[] newData = new char[data.length * 2];
System.arraycopy(data, 0, newData, 0, data.length);
data = newData;
} else offset += length;
}
return parse(data, 0, offset);
} catch (IOException ex) {
throw new SerializationException(ex);
} finally {
StreamUtils.closeQuietly(reader);
}
}
/**
* Reads the entire file into a string using the specified charset.
*
* @param charset If null the default charset is used.
* @throws RuntimeException if the file handle represents a directory, doesn't exist, or could not
* be read.
*/
public String readString(String charset) {
StringBuilder output = new StringBuilder(estimateLength());
InputStreamReader reader = null;
try {
if (charset == null) {
reader = new InputStreamReader(read());
} else {
reader = new InputStreamReader(read(), charset);
}
char[] buffer = new char[256];
while (true) {
int length = reader.read(buffer);
if (length == -1) {
break;
}
output.append(buffer, 0, length);
}
} catch (IOException ex) {
throw new RuntimeException("Error reading layout file: " + this, ex);
} finally {
StreamUtils.closeQuietly(reader);
}
return output.toString();
}
@Test
public void testStream_termsEnum() throws Exception {
final BytesRef[] terms = {new BytesRef("foo"), new BytesRef("bar"), new BytesRef("baz")};
final class TEnum extends TermsEnum {
int idx = 0;
@Override
public SeekStatus seekCeil(final BytesRef text) {
throw new UnsupportedOperationException();
}
@Override
public void seekExact(final long ord) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef term() {
throw new UnsupportedOperationException();
}
@Override
public long ord() {
throw new UnsupportedOperationException();
}
@Override
public int docFreq() {
throw new UnsupportedOperationException();
}
@Override
public long totalTermFreq() {
throw new UnsupportedOperationException();
}
@Override
public PostingsEnum postings(final Bits liveDocs, final PostingsEnum reuse, final int flags) {
throw new UnsupportedOperationException();
}
@Nullable
@Override
public BytesRef next() {
if (this.idx < terms.length) {
return terms[this.idx++];
}
return null;
}
}
Assert.assertEquals(
"Not all terms streamed.", (long) terms.length, StreamUtils.stream(new TEnum()).count());
Assert.assertEquals(
"Term count mismatch.",
1L,
StreamUtils.stream(new TEnum()).filter(t -> t.bytesEquals(new BytesRef("foo"))).count());
Assert.assertEquals(
"Term count mismatch.",
1L,
StreamUtils.stream(new TEnum()).filter(t -> t.bytesEquals(new BytesRef("bar"))).count());
Assert.assertEquals(
"Term count mismatch.",
1L,
StreamUtils.stream(new TEnum()).filter(t -> t.bytesEquals(new BytesRef("baz"))).count());
Assert.assertEquals(
"Unknown term found.",
0L,
StreamUtils.stream(new TEnum())
.filter(
t ->
!t.bytesEquals(new BytesRef("foo"))
&& !t.bytesEquals(new BytesRef("bar"))
&& !t.bytesEquals(new BytesRef("baz")))
.count());
}
@Test
public void testStream_bytesRefHash_empty() throws Exception {
final BytesRefHash brh = new BytesRefHash();
Assert.assertEquals("Too much document ids streamed.", 0L, StreamUtils.stream(brh).count());
}
@Test
public void testStream_bits_empty() throws Exception {
Assert.assertEquals(
"Too much bits streamed.", 0L, StreamUtils.stream((Bits) EMPTY_BITSET).count());
}
private String parseBodyContents() {
body = StreamUtils.getStreamContents(getStream());
return body;
}
@Test
public void testStream_bytesRefArray_empty() throws Exception {
final BytesRefArray bArr = new BytesRefArray(Counter.newCounter(false));
Assert.assertEquals("Too much items streamed.", 0L, StreamUtils.stream(bArr).count());
}
