private List<String> executeDDL(
String ddlFileName,
@Nullable String dataFileName,
boolean isLocalTable,
@Nullable String[] args)
throws Exception {
Path ddlFilePath = new Path(currentQueryPath, ddlFileName);
FileSystem fs = ddlFilePath.getFileSystem(conf);
assertTrue(ddlFilePath + " existence check", fs.exists(ddlFilePath));
String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
String dataFilePath = null;
if (dataFileName != null) {
dataFilePath = getDataSetFile(dataFileName).toString();
}
String compiled = compileTemplate(template, dataFilePath, args);
List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
List<String> createdTableNames = new ArrayList<String>();
for (ParsedResult parsedResult : parsedResults) {
// parse a statement
Expr expr = sqlParser.parse(parsedResult.getStatement());
assertNotNull(ddlFilePath + " cannot be parsed", expr);
if (expr.getType() == OpType.CreateTable) {
CreateTable createTable = (CreateTable) expr;
String tableName = createTable.getTableName();
assertTrue("Table creation is failed.", client.updateQuery(parsedResult.getStatement()));
TableDesc createdTable = client.getTableDesc(tableName);
String createdTableName = createdTable.getName();
assertTrue(
"table '" + createdTableName + "' creation check", client.existTable(createdTableName));
if (isLocalTable) {
createdTableGlobalSet.add(createdTableName);
createdTableNames.add(tableName);
}
} else if (expr.getType() == OpType.DropTable) {
DropTable dropTable = (DropTable) expr;
String tableName = dropTable.getTableName();
assertTrue(
"table '" + tableName + "' existence check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
assertTrue("table drop is failed.", client.updateQuery(parsedResult.getStatement()));
assertFalse(
"table '" + tableName + "' dropped check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
if (isLocalTable) {
createdTableGlobalSet.remove(tableName);
}
} else {
assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
}
}
return createdTableNames;
}
public void assertScriptFailure(String cmd) throws Exception {
Path resultFile = getResultFile(getMethodName() + ".result");
String expected = FileUtil.readTextFile(new File(resultFile.toUri()));
tajoCli.executeScript(cmd);
String consoleResult = new String(err.toByteArray());
assertEquals(expected, consoleResult);
}
/**
* Ensure that this buffer has enough remaining space to add the size. Creates and copies to a new
* buffer if necessary
*
* @param size Size to add
*/
public void ensureSize(int size) {
if (remain() - size < 0) {
if (!limitSpec.canIncrease(memorySize)) {
throw new RuntimeException("Cannot increase RowBlock anymore.");
}
int newBlockSize = limitSpec.increasedSize(memorySize);
resize(newBlockSize);
LOG.info(
"Increase DirectRowBlock to " + FileUtil.humanReadableByteCount(newBlockSize, false));
}
}
/**
* Execute a query contained in the given named file. This methods tries to find the given file
* within the directory src/test/resources/results/<i>ClassName</i>.
*
* @param queryFileName The file name to be used to execute a query.
* @return ResultSet of query execution.
*/
public ResultSet executeFile(String queryFileName) throws Exception {
Path queryFilePath = getQueryFilePath(queryFileName);
FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
assertTrue(queryFilePath.toString() + " existence check", fs.exists(queryFilePath));
List<ParsedResult> parsedResults =
SimpleParser.parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
if (parsedResults.size() > 1) {
assertNotNull("This script \"" + queryFileName + "\" includes two or more queries");
}
ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getStatement());
assertNotNull("Query succeeded test", result);
return result;
}
private void assertOutputResult(
String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
throws Exception {
FileSystem fs =
currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
if (paramKeys != null) {
for (int i = 0; i < paramKeys.length; i++) {
if (i < paramValues.length) {
expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
}
}
}
assertEquals(expectedResult.trim(), actual.trim());
}
private void verifyResult(String message, ResultSet res, Path resultFile)
throws SQLException, IOException {
String actualResult = resultSetToString(res);
String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
assertEquals(message, expectedResult.trim(), actualResult.trim());
}
public static String getResultText(Class clazz, String fileName) throws IOException {
FileSystem localFS = FileSystem.getLocal(new Configuration());
Path path = getResultPath(clazz, fileName);
Preconditions.checkState(localFS.exists(path) && localFS.isFile(path));
return FileUtil.readTextFile(new File(path.toUri()));
}
@Override
public Tuple next() throws IOException {
try {
Tuple tuple;
int partId;
long numRows = 0;
while (!context.isStopped() && (tuple = child.next()) != null) {
partId = partitioner.getPartition(tuple);
MemoryRowBlock rowBlock = partitionMemoryMap.get(partId);
if (rowBlock == null) {
rowBlock = new MemoryRowBlock(dataTypes, initialBufferSize, true, plan.getStorageType());
partitionMemoryMap.put(partId, rowBlock);
totalBufferCapacity += rowBlock.capacity();
}
RowWriter writer = rowBlock.getWriter();
long prevUsedMem = rowBlock.usedMem();
totalBufferCapacity -= rowBlock.capacity();
writer.addTuple(tuple);
numRows++;
totalBufferCapacity += rowBlock.capacity(); // calculate resizeable buffer capacity
usedBufferSize += (rowBlock.usedMem() - prevUsedMem);
// if total buffer capacity are required more than maxBufferSize,
// all partitions are flushed and the buffers are released
if (totalBufferCapacity > maxBufferSize) {
if (LOG.isDebugEnabled()) {
LOG.debug(
String.format(
"Too low buffer usage. threshold: %s, total capacity: %s, used: %s",
FileUtil.humanReadableByteCount(maxBufferSize, false),
FileUtil.humanReadableByteCount(totalBufferCapacity, false),
FileUtil.humanReadableByteCount(usedBufferSize, false)));
}
// flush and release buffer
flushBuffer(partitionMemoryMap, true);
writtenBytes += usedBufferSize;
totalBufferCapacity = usedBufferSize = 0;
} else if (usedBufferSize > bufferThreshold) {
// flush and reuse buffer
flushBuffer(partitionMemoryMap, false);
writtenBytes += usedBufferSize;
usedBufferSize = 0;
}
}
// flush remaining buffers
flushBuffer(partitionMemoryMap, true);
writtenBytes += usedBufferSize;
usedBufferSize = totalBufferCapacity = 0;
TableStats aggregated = (TableStats) child.getInputStats().clone();
aggregated.setNumBytes(writtenBytes);
aggregated.setNumRows(numRows);
context.setResultStats(aggregated);
return null;
} catch (RuntimeException e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
}
}
