/** Expire unused objects. */
public final class Expirer implements ThreadPoolRunnable {
private static com.sun.org.apache.commons.logging.Log log =
com.sun.org.apache.commons.logging.LogFactory.getLog(Expirer.class);
// We can use Event/Listener, but this is probably simpler
// and more efficient
public static interface ExpireCallback {
public void expired(TimeStamp o);
}
private int checkInterval = 60;
private Reaper reaper;
ExpireCallback expireCallback;
public Expirer() {}
// ------------------------------------------------------------- Properties
public int getCheckInterval() {
return checkInterval;
}
public void setCheckInterval(int checkInterval) {
this.checkInterval = checkInterval;
}
public void setExpireCallback(ExpireCallback cb) {
expireCallback = cb;
}
// -------------------- Managed objects --------------------
static final int INITIAL_SIZE = 8;
TimeStamp managedObjs[] = new TimeStamp[INITIAL_SIZE];
TimeStamp checkedObjs[] = new TimeStamp[INITIAL_SIZE];
int managedLen = managedObjs.length;
int managedCount = 0;
public void addManagedObject(TimeStamp ts) {
synchronized (managedObjs) {
if (managedCount >= managedLen) {
// What happens if expire is on the way ? Nothing,
// expire will do it's job on the old array ( GC magic )
// and the expired object will be marked as such
// Same thing would happen ( and did ) with Hashtable
TimeStamp newA[] = new TimeStamp[2 * managedLen];
System.arraycopy(managedObjs, 0, newA, 0, managedLen);
managedObjs = newA;
managedLen = 2 * managedLen;
}
managedObjs[managedCount] = ts;
managedCount++;
}
}
public void removeManagedObject(TimeStamp ts) {
for (int i = 0; i < managedCount; i++) {
if (ts == managedObjs[i]) {
synchronized (managedObjs) {
managedObjs[i] = managedObjs[managedCount - 1];
managedCount--;
}
return;
}
}
}
// --------------------------------------------------------- Public Methods
public void start() {
// Start the background reaper thread
if (reaper == null) {
reaper = new Reaper("Expirer");
reaper.addCallback(this, checkInterval * 1000);
}
reaper.startReaper();
}
public void stop() {
reaper.stopReaper();
}
// -------------------------------------------------------- Private Methods
// ThreadPoolRunnable impl
public Object[] getInitData() {
return null;
}
public void runIt(Object td[]) {
long timeNow = System.currentTimeMillis();
if (dL > 2) debug("Checking " + timeNow);
int checkedCount;
synchronized (managedObjs) {
checkedCount = managedCount;
if (checkedObjs.length < checkedCount) checkedObjs = new TimeStamp[managedLen];
System.arraycopy(managedObjs, 0, checkedObjs, 0, checkedCount);
}
for (int i = 0; i < checkedCount; i++) {
TimeStamp ts = checkedObjs[i];
checkedObjs[i] = null;
if (ts == null || !ts.isValid()) continue;
long maxInactiveInterval = ts.getMaxInactiveInterval();
if (dL > 3) debug("TS: " + maxInactiveInterval + " " + ts.getLastAccessedTime());
if (maxInactiveInterval < 0) continue;
long timeIdle = timeNow - ts.getLastAccessedTime();
if (timeIdle >= maxInactiveInterval) {
if (expireCallback != null) {
if (dL > 0) debug(ts + " " + timeIdle + " " + maxInactiveInterval);
expireCallback.expired(ts);
}
}
}
}
private static final int dL = 0;
private void debug(String s) {
if (log.isDebugEnabled()) log.debug("Expirer: " + s);
}
}
@Category(IntegrationTest.class)
public class TestTajoClient {
private static TajoTestingCluster cluster;
private static TajoConf conf;
private static TajoClient client;
private static Path testDir;
@BeforeClass
public static void setUp() throws Exception {
cluster = TpchTestBase.getInstance().getTestingCluster();
conf = cluster.getConfiguration();
client = new TajoClient(conf);
testDir = CommonTestingUtil.getTestDir();
}
private static Path writeTmpTable(String tableName) throws IOException {
Path tablePath = StorageUtil.concatPath(testDir, tableName);
BackendTestingUtil.writeTmpTable(conf, tablePath);
return tablePath;
}
@Test
public final void testUpdateQuery() throws IOException, ServiceException {
final String tableName = "testUpdateQuery";
Path tablePath = writeTmpTable(tableName);
assertFalse(client.existTable(tableName));
String sql =
"create external table "
+ tableName
+ " (deptname text, score integer) "
+ "using csv location '"
+ tablePath
+ "'";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
client.dropTable(tableName);
assertFalse(client.existTable(tableName));
}
@Test
public final void testCreateAndDropExternalTable()
throws IOException, ServiceException, SQLException {
final String tableName = "testCreateAndDropExternalTable";
Path tablePath = writeTmpTable(tableName);
LOG.error("Full path:" + tablePath.toUri().getRawPath());
FileSystem fs = tablePath.getFileSystem(conf);
assertTrue(fs.exists(tablePath));
assertFalse(client.existTable(tableName));
client.createExternalTable(
tableName, BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName));
client.dropTable(tableName);
assertFalse(client.existTable(tableName));
fs = tablePath.getFileSystem(conf);
assertTrue(fs.exists(tablePath));
}
@Test
public final void testCreateAndPurgeExternalTable()
throws IOException, ServiceException, SQLException {
final String tableName = "testCreateAndPurgeExternalTable";
Path tablePath = writeTmpTable(tableName);
LOG.error("Full path:" + tablePath.toUri().getRawPath());
FileSystem fs = tablePath.getFileSystem(conf);
assertTrue(fs.exists(tablePath));
assertFalse(client.existTable(tableName));
client.createExternalTable(
tableName, BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName));
client.dropTable(tableName, true);
assertFalse(client.existTable(tableName));
fs = tablePath.getFileSystem(conf);
assertFalse("Checking if table data exists", fs.exists(tablePath));
}
@Test
public final void testCreateAndDropExternalTableByExecuteQuery()
throws IOException, ServiceException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropExternalTableByExecuteQuery";
Path tablePath = writeTmpTable(tableName);
assertFalse(client.existTable(tableName));
String sql =
"create external table "
+ tableName
+ " (deptname text, score int4) "
+ "using csv location '"
+ tablePath
+ "'";
client.executeQueryAndGetResult(sql);
assertTrue(client.existTable(tableName));
client.updateQuery("drop table " + tableName);
assertFalse(client.existTable(tableName));
FileSystem localFS = FileSystem.getLocal(conf);
assertTrue(localFS.exists(tablePath));
}
@Test
public final void testCreateAndPurgeExternalTableByExecuteQuery()
throws IOException, ServiceException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndPurgeExternalTableByExecuteQuery";
Path tablePath = writeTmpTable(tableName);
assertFalse(client.existTable(tableName));
String sql =
"create external table "
+ tableName
+ " (deptname text, score int4) "
+ "using csv location '"
+ tablePath
+ "'";
client.executeQueryAndGetResult(sql);
assertTrue(client.existTable(tableName));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
FileSystem localFS = FileSystem.getLocal(conf);
assertFalse(localFS.exists(tablePath));
}
@Test
public final void testCreateAndDropTableByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTableByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName);
assertFalse(client.existTable(tableName));
assertTrue(hdfs.exists(tablePath));
}
@Test
public final void testCreateAndPurgeTableByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndPurgeTableByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
@Test
public final void testDDLByExecuteQuery() throws IOException, ServiceException {
final String tableName = "testDDLByExecuteQuery";
Path tablePath = writeTmpTable(tableName);
assertFalse(client.existTable(tableName));
String sql =
"create external table "
+ tableName
+ " (deptname text, score int4) "
+ "using csv location '"
+ tablePath
+ "'";
client.executeQueryAndGetResult(sql);
assertTrue(client.existTable(tableName));
}
@Test
public final void testGetTableList() throws IOException, ServiceException {
String tableName1 = "GetTableList1".toLowerCase();
String tableName2 = "GetTableList2".toLowerCase();
assertFalse(client.existTable(tableName1));
assertFalse(client.existTable(tableName2));
client.updateQuery("create table GetTableList1 (age int, name text);");
client.updateQuery("create table GetTableList2 (age int, name text);");
assertTrue(client.existTable(tableName1));
assertTrue(client.existTable(tableName2));
Set<String> tables = Sets.newHashSet(client.getTableList());
assertTrue(tables.contains(tableName1));
assertTrue(tables.contains(tableName2));
}
Log LOG = LogFactory.getLog(TestTajoClient.class);
@Test
public final void testGetTableDesc() throws IOException, ServiceException, SQLException {
final String tableName1 = "table3";
Path tablePath = writeTmpTable(tableName1);
LOG.error("Full path:" + tablePath.toUri().getRawPath());
FileSystem fs = tablePath.getFileSystem(conf);
assertTrue(fs.exists(tablePath));
assertNotNull(tablePath);
assertFalse(client.existTable(tableName1));
client.createExternalTable(
"table3", BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName1));
TableDesc desc = client.getTableDesc(tableName1);
assertNotNull(desc);
assertEquals(tableName1, desc.getName());
assertTrue(desc.getStats().getNumBytes() > 0);
}
// @Test
public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTablePartitionedHash1ByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += " PARTITION BY HASH (deptname)";
sql += " (PARTITION sub_part1, PARTITION sub_part2, PARTITION sub_part3)";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName);
assertFalse(client.existTable(tableName));
assertTrue(hdfs.exists(tablePath));
}
// @Test
public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndPurgeTablePartitionedHash1ByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += " PARTITION BY HASH (deptname)";
sql += " (PARTITION sub_part1, PARTITION sub_part2, PARTITION sub_part3)";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
// @Test
public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTablePartitionedHash2ByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += "PARTITION BY HASH (deptname)";
sql += "PARTITIONS 2";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
// @Test
public final void testCreateAndDropTablePartitionedListByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTablePartitionedListByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += "PARTITION BY LIST (deptname)";
sql += "( PARTITION sub_part1 VALUES('r&d', 'design'),";
sql += "PARTITION sub_part2 VALUES('sales', 'hr') )";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
// @Test
public final void testCreateAndDropTablePartitionedRangeByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTablePartitionedRangeByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += "PARTITION BY RANGE (score)";
sql += "( PARTITION sub_part1 VALUES LESS THAN (2),";
sql += "PARTITION sub_part2 VALUES LESS THAN (5),";
sql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
@Test
public final void testFailCreateTablePartitionedOtherExceptColumn()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testFailCreateTablePartitionedOtherExceptColumn";
assertFalse(client.existTable(tableName));
String rangeSql = "create table " + tableName + " (deptname text, score int4)";
rangeSql += "PARTITION BY RANGE (score)";
rangeSql += "( PARTITION sub_part1 VALUES LESS THAN (2),";
rangeSql += "PARTITION sub_part2 VALUES LESS THAN (5),";
rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )";
assertFalse(client.updateQuery(rangeSql));
String listSql = "create table " + tableName + " (deptname text, score int4)";
listSql += "PARTITION BY LIST (deptname)";
listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),";
listSql += "PARTITION sub_part2 VALUES('sales', 'hr') )";
assertFalse(client.updateQuery(listSql));
String hashSql = "create table " + tableName + " (deptname text, score int4)";
hashSql += "PARTITION BY HASH (deptname)";
hashSql += "PARTITIONS 2";
assertFalse(client.updateQuery(hashSql));
}
@Test
public final void testCreateAndDropTablePartitionedColumnByExecuteQuery()
throws IOException, ServiceException, SQLException {
TajoConf conf = cluster.getConfiguration();
final String tableName = "testCreateAndDropTablePartitionedColumnByExecuteQuery";
assertFalse(client.existTable(tableName));
String sql = "create table " + tableName + " (deptname text, score int4)";
sql += "PARTITION BY COLUMN (deptname text)";
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
Path tablePath = client.getTableDesc(tableName).getPath();
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
client.updateQuery("drop table " + tableName + " purge");
assertFalse(client.existTable(tableName));
assertFalse(hdfs.exists(tablePath));
}
@Test
public final void testGetFunctions() throws IOException, ServiceException, SQLException {
Collection<FunctionDesc> catalogFunctions = cluster.getMaster().getCatalog().getFunctions();
String functionName = "sum";
int numFunctions = 0;
for (FunctionDesc eachFunction : catalogFunctions) {
if (functionName.equals(eachFunction.getSignature())) {
numFunctions++;
}
}
List<CatalogProtos.FunctionDescProto> functions = client.getFunctions(functionName);
assertEquals(numFunctions, functions.size());
functions = client.getFunctions("notmatched");
assertEquals(0, functions.size());
functions = client.getFunctions(null);
assertEquals(catalogFunctions.size(), functions.size());
}
}
