@Test
public void testGetTimestamp() throws Exception {
List<Query.Type> types = Arrays.asList(Query.Type.DATETIME, Query.Type.TIMESTAMP);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("2008-01-02 14:15:16.123456".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("2008-01-02 14:15:16.123456")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(Timestamp.valueOf("2008-01-02 14:15:16.123456"), row.getObject("col1"));
Assert.assertEquals(
Timestamp.valueOf("2008-01-02 14:15:16.123456"), row.getTimestamp("col1"));
Timestamp ts = new Timestamp(1199283316000L);
ts.setNanos(123456000);
Assert.assertEquals(ts, row.getTimestamp("col1", GMT));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getTimestamp("null"));
Assert.assertTrue(row.wasNull());
}
}
}
@Test
public void testGetInt() throws Exception {
List<Query.Type> types =
Arrays.asList(
Query.Type.INT8,
Query.Type.UINT8,
Query.Type.INT16,
Query.Type.UINT16,
Query.Type.INT24,
Query.Type.UINT24,
Query.Type.INT32);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("12345".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("12345")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(12345, row.getInt("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(0, row.getInt("null"));
Assert.assertTrue(row.wasNull());
Assert.assertEquals(null, row.getObject("null", Integer.class));
Assert.assertTrue(row.wasNull());
}
}
}
@Test
public void testGetBytes() throws Exception {
List<Query.Type> types =
Arrays.asList(
Query.Type.TEXT,
Query.Type.BLOB,
Query.Type.VARCHAR,
Query.Type.VARBINARY,
Query.Type.CHAR,
Query.Type.BINARY,
Query.Type.BIT);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("hello world".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("hello world")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertArrayEquals("hello world".getBytes("UTF-8"), row.getBytes("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getBytes("null"));
Assert.assertTrue(row.wasNull());
}
}
}
@Test
public void testGetBytes() throws Exception {
List<Field.Type> types =
Arrays.asList(
Field.Type.TYPE_VARCHAR,
Field.Type.TYPE_BIT,
Field.Type.TYPE_TINY_BLOB,
Field.Type.TYPE_MEDIUM_BLOB,
Field.Type.TYPE_LONG_BLOB,
Field.Type.TYPE_BLOB,
Field.Type.TYPE_VAR_STRING,
Field.Type.TYPE_STRING,
Field.Type.TYPE_GEOMETRY);
for (Field.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(type).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("hello world")))
.build())) {
cursor.next();
Assert.assertArrayEquals("hello world".getBytes("UTF-8"), cursor.getBytes("col0"));
}
}
}
public int dump(String filenameOrDirectory, PrintStream out, TimeZone timeZone)
throws IOException {
int logsFound = 0;
for (String fileName : filenamesOf(filenameOrDirectory, getLogPrefix())) {
logsFound++;
out.println("=== " + fileName + " ===");
StoreChannel fileChannel = fileSystem.open(new File(fileName), "r");
ByteBuffer buffer = ByteBuffer.allocateDirect(9 + Xid.MAXGTRIDSIZE + Xid.MAXBQUALSIZE * 10);
long logVersion, prevLastCommittedTx;
try {
long[] header = VersionAwareLogEntryReader.readLogHeader(buffer, fileChannel, false);
logVersion = header[0];
prevLastCommittedTx = header[1];
} catch (IOException ex) {
out.println("Unable to read timestamp information, no records in logical log.");
out.println(ex.getMessage());
fileChannel.close();
throw ex;
}
out.println(
"Logical log version: "
+ logVersion
+ " with prev committed tx["
+ prevLastCommittedTx
+ "]");
LogDeserializer deserializer = new LogDeserializer(buffer, instantiateCommandReaderFactory());
PrintingConsumer consumer = new PrintingConsumer(out, timeZone);
try (Cursor<LogEntry, IOException> cursor = deserializer.cursor(fileChannel)) {
while (cursor.next(consumer)) ;
}
}
return logsFound;
}
public List<DBObject> getSourceBrowsers() {
DBObject groupFields = new BasicDBObject("_id", "$device");
groupFields.put("sum", new BasicDBObject("$sum", "$sum"));
DBObject group = new BasicDBObject("$group", groupFields);
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));
List<DBObject> pipeline = Arrays.asList(group, sort);
// allowDiskUse
AggregationOptions options =
AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
Cursor cursor = device.aggregate(pipeline, options);
List<DBObject> list = new ArrayList<DBObject>();
while (cursor.hasNext()) {
DBObject object = cursor.next();
DBObject newObj = new BasicDBObject();
newObj.put("device", object.get("_id"));
newObj.put("sum", object.get("sum"));
list.add(newObj);
}
cursor.close();
return list;
}
@Test
public void testGetDate() throws Exception {
List<Query.Type> types = Arrays.asList(Query.Type.DATE);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("2008-01-02".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("2008-01-02")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(Date.valueOf("2008-01-02"), row.getObject("col1"));
Assert.assertEquals(Date.valueOf("2008-01-02"), row.getDate("col1"));
Assert.assertEquals(new Date(1199232000000L), row.getDate("col1", GMT));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getDate("null"));
Assert.assertTrue(row.wasNull());
}
}
}
@Test
public void testGetBigDecimal() throws Exception {
List<Query.Type> types = Arrays.asList(Query.Type.DECIMAL);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("1234.56789".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("1234.56789")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(
new BigDecimal(BigInteger.valueOf(123456789), 5), row.getBigDecimal("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getBigDecimal("null"));
Assert.assertTrue(row.wasNull());
}
}
}
public boolean next() throws SQLException {
// TODO: for timeout, see IteratorResultSet.next
if (cursor.next()) {
++row;
return true;
} else {
afterLast = true;
return false;
}
}
public Cursor findFirstOrLast(Session session, boolean first) throws SQLException {
Cursor cursor;
if (first) {
cursor = mainIndex.find(session, Long.MIN_VALUE, Long.MAX_VALUE, false);
} else {
long x = mainIndex.getLastKey();
cursor = mainIndex.find(session, x, x, false);
}
cursor.next();
return cursor;
}
@Test
public void testNull() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(Field.Type.TYPE_NULL).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("1234")))
.build())) {
cursor.next();
Assert.assertEquals(null, cursor.getObject("col0"));
}
}
@Test
public void testGetDateTime() throws Exception {
List<Field.Type> types = Arrays.asList(Field.Type.TYPE_DATETIME, Field.Type.TYPE_TIMESTAMP);
for (Field.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(type).build())
.addRows(
Row.newBuilder().addValues(ByteString.copyFromUtf8("2008-01-02 14:15:16")))
.build())) {
cursor.next();
Assert.assertEquals(new DateTime(2008, 1, 2, 14, 15, 16), cursor.getDateTime("col0"));
}
}
types = Arrays.asList(Field.Type.TYPE_DATE, Field.Type.TYPE_NEWDATE);
for (Field.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(type).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("2008-01-02")))
.build())) {
cursor.next();
Assert.assertEquals(new DateTime(2008, 1, 2, 0, 0, 0), cursor.getDateTime("col0"));
}
}
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(Field.Type.TYPE_TIME).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("12:34:56")))
.build())) {
cursor.next();
Assert.assertEquals(new DateTime(1970, 1, 1, 12, 34, 56), cursor.getDateTime("col0"));
}
}
@Test
public void testGetFloat() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(
Field.newBuilder().setName("col0").setType(Field.Type.TYPE_FLOAT).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("2.5")))
.build())) {
cursor.next();
Assert.assertEquals(2.5f, cursor.getFloat("col0"), 0.01f);
}
}
@Test
public void testGetLong() throws Exception {
List<Field.Type> types = Arrays.asList(Field.Type.TYPE_LONG, Field.Type.TYPE_LONGLONG);
for (Field.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(type).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("12345")))
.build())) {
cursor.next();
Assert.assertEquals(12345L, cursor.getLong("col0"));
}
}
}
private void verify() throws IOException, InterruptedException {
long sum = 0;
Cursor cursor = accounts.first();
Account account;
while ((account = (Account) cursor.next()) != null) {
LOG.log(Level.INFO, "{0}", account);
sum += account.balance();
}
db.commitTransaction();
if (sum == 0) {
LOG.log(Level.INFO, "OK!");
} else {
LOG.log(Level.WARNING, "Test failed, sum = {0}", sum);
}
}
@Override
public boolean next() {
while (true) {
if (cursor == null) {
nextCursor();
if (cursor == null) {
return false;
}
}
if (cursor.next()) {
return true;
}
cursor = null;
}
}
@Test
public void testGetBigDecimal() throws Exception {
List<Field.Type> types = Arrays.asList(Field.Type.TYPE_DECIMAL, Field.Type.TYPE_NEWDECIMAL);
for (Field.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col0").setType(type).build())
.addRows(Row.newBuilder().addValues(ByteString.copyFromUtf8("1234.56789")))
.build())) {
cursor.next();
Assert.assertEquals(
new BigDecimal(BigInteger.valueOf(123456789), 5), cursor.getBigDecimal("col0"));
}
}
}
@Test
public void testNull() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("null").setType(Query.Type.NULL_TYPE).build())
.addRows(
Query.Row.newBuilder().addLengths(-1) // SQL NULL
)
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(null, row.getObject("null"));
Assert.assertTrue(row.wasNull());
}
}
/**
* Create a cursor reading data from messageId onwards. To read the oldest message appearing in
* the file use {@link #getFirstMessageId()} as the message ID. To read the newest use {@link
* #getNextMessageId()}. If messageId is 'between' messages it is advanced to the next message.
* The cursor is not thread safe.
*/
@SuppressWarnings("StatementWithEmptyBody")
public MessageCursor cursor(long messageId) throws IOException {
long nextMessageId = getNextMessageId();
if (messageId < firstMessageId || messageId > nextMessageId) {
throw new IllegalArgumentException("messageId " + messageId + " not in " + this);
}
if (messageId == nextMessageId) return new Cursor(messageId); // at EOF
long pos = getBucket(findBucket(messageId)).getFirstMessageId();
Cursor c = new Cursor(pos);
if (pos < messageId) { // skip messages until we get to the one we want
while (c.next() && c.getNextId() < messageId) ;
}
return c;
}
// 获得一天的转换漏斗
public List<DBObject> getConversionFunnel(String date) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
DBObject match = null;
try {
match = new BasicDBObject("$match", new BasicDBObject("date", sdf.parse(date)));
} catch (ParseException e) {
}
DBObject groupFields = new BasicDBObject("_id", "");
groupFields.put("step1sum", new BasicDBObject("$sum", "$step1"));
groupFields.put("step2sum", new BasicDBObject("$sum", "$step2"));
groupFields.put("step3sum", new BasicDBObject("$sum", "$step3"));
DBObject group = new BasicDBObject("$group", groupFields);
// DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));
List<DBObject> pipeline = Arrays.asList(match, group);
// allowDiskUse
AggregationOptions options =
AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
Cursor cursor = CF.aggregate(pipeline, options);
List<DBObject> list = new ArrayList<DBObject>();
while (cursor.hasNext()) {
DBObject object = cursor.next();
DBObject newObj = new BasicDBObject();
// DBObject _id = (DBObject)object.get("_id");
// newObj.put("date",sdf.format(_id.get("$date")));
DBObject newObj1 = new BasicDBObject();
newObj1.put("step", "访问本站");
newObj1.put("num", object.get("step1sum"));
list.add(newObj1);
DBObject newObj2 = new BasicDBObject();
newObj2.put("step", "浏览商品");
newObj2.put("num", object.get("step2sum"));
list.add(newObj2);
DBObject newObj3 = new BasicDBObject();
newObj3.put("step", "完成交易");
newObj3.put("num", object.get("step3sum"));
list.add(newObj3);
}
cursor.close();
return list;
}
@Test
public void testGetULong() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(
Field.newBuilder()
.setName("col0")
.setType(Field.Type.TYPE_LONGLONG)
.setFlags(Field.Flag.VT_UNSIGNED_FLAG_VALUE)
.build())
.addRows(
Row.newBuilder().addValues(ByteString.copyFromUtf8("18446744073709551615")))
.build())) {
cursor.next();
Assert.assertEquals(UnsignedLong.fromLongBits(-1), cursor.getULong("col0"));
}
}
@Override
public Cursor findFirstOrLast(Session session, boolean first) {
if (closed) {
throw DbException.throwInternalError();
}
if (first) {
// TODO optimization: this loops through NULL
Cursor cursor = find(session, null, null);
while (cursor.next()) {
SearchRow row = cursor.getSearchRow();
Value v = row.getValue(columnIds[0]);
if (v != ValueNull.INSTANCE) {
return cursor;
}
}
return cursor;
}
TreeNode x = root, n;
while (x != null) {
n = x.right;
if (n == null) {
break;
}
x = n;
}
TreeCursor cursor = new TreeCursor(this, x, null, null);
if (x == null) {
return cursor;
}
// TODO optimization: this loops through NULL elements
do {
SearchRow row = cursor.getSearchRow();
if (row == null) {
break;
}
Value v = row.getValue(columnIds[0]);
if (v != ValueNull.INSTANCE) {
return cursor;
}
} while (cursor.previous());
return cursor;
}
@Test
public void testGetULong() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(Query.Type.UINT64).build())
.addFields(Field.newBuilder().setName("null").setType(Query.Type.UINT64).build())
.addRows(
Query.Row.newBuilder()
.addLengths("18446744073709551615".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("18446744073709551615")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(UnsignedLong.fromLongBits(-1), row.getULong("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getULong("null"));
Assert.assertTrue(row.wasNull());
}
}
/**
* Create a cursor reading data from timestamp onwards. If timestamp is before the first message
* then the cursor reads starting at the first message. If timestamp is past the last message then
* the cursor will return false until more messages appear in the file.
*/
@SuppressWarnings("StatementWithEmptyBody")
public MessageCursor cursorByTimestamp(long timestamp) throws IOException {
int i = findBucketByTimestamp(timestamp);
if (i < 0) return new Cursor(firstMessageId);
// the first message with timestamp >= the time we are looking for may be in a previous bucket
// because
// the bucket timestamp resolution is only ms so go back until we get a change in time .. that
// way we
// are sure to find it
Bucket b = getBucket(i);
for (; b.getTimestamp() == timestamp && i > 0; b = getBucket(--i)) ;
Cursor c = new Cursor(getBucket(i).getFirstMessageId());
for (; c.next(); ) { // skip messages until we get one >= timestamp
if (c.getTimestamp() >= timestamp) {
c.unget();
break;
}
}
return c;
}
@Test
public void testGetTime() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(Query.Type.TIME).build())
.addFields(Field.newBuilder().setName("null").setType(Query.Type.TIME).build())
.addRows(
Query.Row.newBuilder()
.addLengths("12:34:56".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("12:34:56")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(Time.valueOf("12:34:56"), row.getObject("col1"));
Assert.assertEquals(Time.valueOf("12:34:56"), row.getTime("col1"));
Assert.assertEquals(new Time(45296000L), row.getTime("col1", GMT));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getTime("null"));
Assert.assertTrue(row.wasNull());
}
}
@Test
public void testGetShort() throws Exception {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(Query.Type.YEAR).build())
.addFields(Field.newBuilder().setName("null").setType(Query.Type.YEAR).build())
.addRows(
Query.Row.newBuilder()
.addLengths("1234".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("1234")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals(1234, row.getShort("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(0, row.getShort("null"));
Assert.assertTrue(row.wasNull());
Assert.assertEquals(null, row.getObject("null", Short.class));
Assert.assertTrue(row.wasNull());
}
}
@Test
public void testGetString() throws Exception {
List<Query.Type> types = Arrays.asList(Query.Type.ENUM, Query.Type.SET);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("val123".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("val123")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals("val123", row.getString("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getString("null"));
Assert.assertTrue(row.wasNull());
}
}
}
private boolean step(Cursor cursor) {
return reverse ? cursor.previous() : cursor.next();
}
