/** Evaluate the expression and pop value from the stack */
Var evalPop(ParserRuleContext ctx) {
exec.visit(ctx);
if (!exec.stack.isEmpty()) {
return exec.stackPop();
}
return Var.Empty;
}
/**
* @param path path to device in question
* @return the free space (normalized) left on device accessible by path
*/
public static double freeSpaceN(String path) {
try {
if (System.getProperty("os.name").startsWith("Windows")) {
return -1;
}
Exec exec = new Exec();
String cmd[] = {"df", path};
int status = exec.command(cmd);
Process p = exec.getProcess();
if (status == 0) {
String buff = exec.getOutputGobbler().output();
int pctIdx = buff.lastIndexOf("%");
if (pctIdx > 3) {
int startIdx = pctIdx - 3;
int endIdx = pctIdx;
String pct = buff.substring(startIdx, endIdx);
return 1. - (Double.parseDouble(pct) / 100.);
} else {
return -1.;
}
} else {
System.out.println(exec.getErrorGobbler().output());
return -1.;
}
} catch (java.lang.StringIndexOutOfBoundsException ignored) {
} catch (Exception e) {
e.printStackTrace();
}
return -1.;
}
Var evalPop(ParserRuleContext ctx, long def) {
if (ctx != null) {
exec.visit(ctx);
return exec.stackPop();
}
return new Var(def);
}
/** Binary boolean expression in executable SQL statement */
public Integer execBoolBinarySql(HplsqlParser.Bool_expr_binaryContext ctx) {
StringBuilder sql = new StringBuilder();
sql.append(evalPop(ctx.expr(0)).toString());
sql.append(" " + exec.getFormattedText(ctx.bool_expr_binary_operator()) + " ");
sql.append(evalPop(ctx.expr(1)).toString());
exec.stackPush(sql);
return 0;
}
/** Single table name in FROM */
public Integer fromTable(HplsqlParser.From_table_name_clauseContext ctx) {
StringBuilder sql = new StringBuilder();
sql.append(evalPop(ctx.table_name()));
if (ctx.from_alias_clause() != null) {
sql.append(" ").append(exec.getText(ctx.from_alias_clause()));
}
exec.stackPush(sql);
return 0;
}
private void appendOperations(SyncmlBody syncmlBody) throws WindowsOperationException {
Get getElement = new Get();
List<Item> itemsGet = new ArrayList<Item>();
Exec execElement = new Exec();
List<Item> itemsExec = new ArrayList<Item>();
Atomic atomicElement = new Atomic();
List<Add> addsAtomic = new ArrayList<Add>();
if (operations != null) {
for (int x = 0; x < operations.size(); x++) {
Operation operation = operations.get(x);
Operation.Type type = operation.getType();
switch (type) {
case CONFIG:
List<Add> addConfig = appendAddConfiguration(operation);
for (Add addConfiguration : addConfig) {
addsAtomic.add(addConfiguration);
}
break;
case MESSAGE:;
break;
case INFO:
Item itemGet = appendGetInfo(operation);
itemsGet.add(itemGet);
break;
case COMMAND:
Item itemExec = appendExecInfo(operation);
itemsExec.add(itemExec);
break;
default:
throw new WindowsOperationException("Operation with no type found");
}
}
}
if (!itemsGet.isEmpty()) {
getElement.setCommandId(75);
getElement.setItems(itemsGet);
}
if (!itemsExec.isEmpty()) {
execElement.setCommandId(5);
execElement.setItems(itemsExec);
}
if (!addsAtomic.isEmpty()) {
atomicElement.setCommandId(300);
atomicElement.setAdds(addsAtomic);
}
syncmlBody.setGet(getElement);
syncmlBody.setExec(execElement);
}
/** Division operator */
public void operatorDiv(HplsqlParser.ExprContext ctx) {
Var v1 = evalPop(ctx.expr(0));
Var v2 = evalPop(ctx.expr(1));
if (v1.value == null || v2.value == null) {
evalNull();
} else if (v1.type == Type.BIGINT && v2.type == Type.BIGINT) {
exec.stackPush(new Var((Long) v1.value / (Long) v2.value));
} else {
exec.signal(Signal.Type.UNSUPPORTED_OPERATION, "Unsupported data types in division operator");
}
}
/** GET DIAGNOSTICS EXCEPTION statement */
public Integer getDiagnosticsException(HplsqlParser.Get_diag_stmt_exception_itemContext ctx) {
trace(ctx, "GET DIAGNOSTICS EXCEPTION");
Signal signal = exec.signalPeek();
if (signal == null || (signal != null && signal.type != Signal.Type.SQLEXCEPTION)) {
signal = exec.currentSignal;
}
if (signal != null) {
exec.setVariable(ctx.ident().getText(), signal.getValue());
}
return 0;
}
/** CLOSE cursor statement */
public Integer close(HplsqlParser.Close_stmtContext ctx) {
trace(ctx, "CLOSE");
String name = ctx.L_ID().toString();
Var var = exec.findVariable(name);
if (var != null && var.type == Type.CURSOR) {
exec.closeQuery((Query) var.value, exec.conf.defaultConnection);
exec.setSqlCode(0);
} else if (trace) {
trace(ctx, "Cursor not found: " + name);
}
return 0;
}
/** MERGE statement */
public Integer merge(HplsqlParser.Merge_stmtContext ctx) {
trace(ctx, "MERGE");
String sql = exec.getFormattedText(ctx);
trace(ctx, sql);
Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
if (query.error()) {
exec.signal(query);
return 1;
}
exec.setSqlSuccess();
exec.closeQuery(query, exec.conf.defaultConnection);
return 0;
}
// covtype.altered response column has this distribution:
// -1 20510
// 1 211840
// 2 283301
// 3 35754
// 4 2747
// 6 17367
// 10000 9493
@Test
public void testSplitCats() {
Frame cov = parse_test_file(Key.make("cov"), "smalldata/covtype/covtype.altered.gz");
System.out.println(cov.toString(0, 10));
Val v_ddply = Exec.exec("(ddply cov [54] nrow)");
System.out.println(v_ddply.toString());
((ValFrame) v_ddply)._fr.delete();
Val v_groupby = Exec.exec("(GB cov [54] nrow 54 \"all\")");
System.out.println(v_groupby.toString());
((ValFrame) v_groupby)._fr.delete();
cov.delete();
}
/** SQL INSERT SELECT statement */
public Integer insertSelect(HplsqlParser.Insert_stmtContext ctx) {
trace(ctx, "INSERT SELECT");
String table = evalPop(ctx.table_name()).toString();
String select = evalPop(ctx.select_stmt()).toString();
String sql = "INSERT INTO TABLE " + table + " " + select;
trace(ctx, sql);
Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
if (query.error()) {
exec.signal(query);
return 1;
}
exec.setSqlSuccess();
exec.closeQuery(query, exec.conf.defaultConnection);
return 0;
}
/** Check if an exception is raised or EXIT executed, and we should leave the block */
boolean canContinue(String label) {
Signal signal = exec.signalPeek();
if (signal != null && signal.type == Signal.Type.SQLEXCEPTION) {
return false;
}
signal = exec.signalPeek();
if (signal != null && signal.type == Signal.Type.LEAVE_LOOP) {
if (signal.value == null
|| signal.value.isEmpty()
|| (label != null && label.equalsIgnoreCase(signal.value))) {
exec.signalPop();
}
return false;
}
return true;
}
static Frame exec_str(String str, String id) {
Val val = Exec.exec(str);
switch (val.type()) {
case Val.FRM:
Frame fr = val.getFrame();
Key k = Key.make(id);
// Smart delete any prior top-level result
Iced i = DKV.getGet(k);
if (i instanceof Lockable) ((Lockable) i).delete();
else if (i instanceof Keyed) ((Keyed) i).remove();
else if (i != null)
throw new IllegalArgumentException("Attempting to overright an unexpected key");
DKV.put(fr = new Frame(k, fr._names, fr.vecs()));
System.out.println(fr);
checkSaneFrame();
return fr;
case Val.NUM:
System.out.println("num= " + val.getNum());
assert id == null;
checkSaneFrame();
return null;
case Val.STR:
System.out.println("str= " + val.getStr());
assert id == null;
checkSaneFrame();
return null;
default:
throw water.H2O.fail();
}
}
public void zoomIn() {
Dimension asz = this.getSize();
int maxzf = 3;
int coef = 1;
int r;
cmdline =
"/bin/sh get.sh "
+ j2kfilename
+ " "
+ iw
+ " "
+ ih
+ " "
+ rect.x
+ " "
+ rect.y
+ " "
+ rect.width
+ " "
+ rect.height;
Exec.execPrint(cmdline);
rect.x = rect.y = rect.width = rect.height = 0;
img = pgm.open("out.pgm");
iw = img.getWidth(this);
ih = img.getHeight(this);
bi = new BufferedImage(iw, ih, BufferedImage.TYPE_INT_RGB);
big = bi.createGraphics();
selected = 0;
fullRefresh = true;
repaint();
}
/** Comparison operator */
public void operatorCompare(
HplsqlParser.Bool_expr_binaryContext ctx, HplsqlParser.Bool_expr_binary_operatorContext op) {
Var v1 = evalPop(ctx.expr(0));
Var v2 = evalPop(ctx.expr(1));
int cmp = v1.compareTo(v2);
boolean bool = false;
if (op.T_GREATER() != null) {
if (cmp > 0) {
bool = true;
}
} else if (op.T_GREATEREQUAL() != null) {
if (cmp >= 0) {
bool = true;
}
}
if (op.T_LESS() != null) {
if (cmp < 0) {
bool = true;
}
} else if (op.T_LESSEQUAL() != null) {
if (cmp <= 0) {
bool = true;
}
}
exec.stackPush(bool);
}
/** FROM TABLE (VALUES ...) clause */
public Integer fromTableValues(HplsqlParser.From_table_values_clauseContext ctx) {
StringBuilder sql = new StringBuilder();
int rows = ctx.from_table_values_row().size();
sql.append("(");
for (int i = 0; i < rows; i++) {
int cols = ctx.from_table_values_row(i).expr().size();
int cols_as = ctx.from_alias_clause().L_ID().size();
sql.append("SELECT ");
for (int j = 0; j < cols; j++) {
sql.append(evalPop(ctx.from_table_values_row(i).expr(j)));
if (j < cols_as) {
sql.append(" AS ");
sql.append(ctx.from_alias_clause().L_ID(j));
}
if (j + 1 < cols) {
sql.append(", ");
}
}
sql.append(" FROM " + conf.dualTable);
if (i + 1 < rows) {
sql.append("\nUNION ALL\n");
}
}
sql.append(") ");
if (ctx.from_alias_clause() != null) {
sql.append(ctx.from_alias_clause().ident().getText());
}
exec.stackPush(sql);
return 0;
}
/** WHERE clause */
public Integer where(HplsqlParser.Where_clauseContext ctx) {
StringBuilder sql = new StringBuilder();
sql.append(ctx.T_WHERE().getText());
sql.append(" " + evalPop(ctx.bool_expr()));
exec.stackPush(sql);
return 0;
}
/** JOIN clause in FROM */
public Integer fromJoin(HplsqlParser.From_join_clauseContext ctx) {
StringBuilder sql = new StringBuilder();
if (ctx.T_COMMA() != null) {
sql.append(", ");
sql.append(evalPop(ctx.from_table_clause()));
} else if (ctx.from_join_type_clause() != null) {
sql.append(" ");
sql.append(exec.getText(ctx.from_join_type_clause()));
sql.append(" ");
sql.append(evalPop(ctx.from_table_clause()));
sql.append(" ");
sql.append(exec.getText(ctx, ctx.T_ON().getSymbol(), ctx.bool_expr().getStop()));
}
exec.stackPush(sql);
return 0;
}
public Integer subselect(HplsqlParser.Subselect_stmtContext ctx) {
StringBuilder sql = new StringBuilder();
if (ctx.T_SELECT() != null) {
sql.append(ctx.T_SELECT().getText());
}
sql.append(" " + evalPop(ctx.select_list()));
if (ctx.from_clause() != null) {
sql.append(" " + evalPop(ctx.from_clause()));
} else {
sql.append(" FROM " + conf.dualTable);
}
if (ctx.where_clause() != null) {
sql.append(" " + evalPop(ctx.where_clause()));
}
if (ctx.group_by_clause() != null) {
sql.append(" " + getText(ctx.group_by_clause()));
}
if (ctx.having_clause() != null) {
sql.append(" " + getText(ctx.having_clause()));
}
if (ctx.order_by_clause() != null) {
sql.append(" " + getText(ctx.order_by_clause()));
}
if (ctx.select_options() != null) {
sql.append(" " + evalPop(ctx.select_options()));
}
if (ctx.select_list().select_list_limit() != null) {
sql.append(" LIMIT " + evalPop(ctx.select_list().select_list_limit().expr()));
}
exec.stackPush(sql);
return 0;
}
/** USE statement */
public Integer use(HplsqlParser.Use_stmtContext ctx) {
if (trace) {
trace(ctx, "USE");
}
String sql = ctx.T_USE().toString() + " " + evalPop(ctx.expr()).toString();
if (trace) {
trace(ctx, "Query: " + sql);
}
Query query = exec.executeSql(ctx, sql, exec.conf.defaultConnection);
if (query.error()) {
exec.signal(query);
return 1;
}
exec.setSqlCode(0);
exec.closeQuery(query, exec.conf.defaultConnection);
return 0;
}
/** WHILE statement */
public Integer while_(HplsqlParser.While_stmtContext ctx) {
trace(ctx, "WHILE - ENTERED");
String label = exec.labelPop();
while (true) {
if (evalPop(ctx.bool_expr()).isTrue()) {
exec.enterScope(Scope.Type.LOOP);
visit(ctx.block());
exec.leaveScope();
if (canContinue(label)) {
continue;
}
}
break;
}
trace(ctx, "WHILE - LEFT");
return 0;
}
/** RESIGNAL statement */
public Integer resignal(HplsqlParser.Resignal_stmtContext ctx) {
trace(ctx, "RESIGNAL");
if (ctx.T_SQLSTATE() != null) {
String sqlstate = evalPop(ctx.expr(0)).toString();
String text = "";
if (ctx.T_MESSAGE_TEXT() != null) {
text = evalPop(ctx.expr(1)).toString();
}
SQLException exception = new SQLException(text, sqlstate, -1);
Signal signal = new Signal(Signal.Type.SQLEXCEPTION, text, exception);
exec.setSqlCode(exception);
exec.resignal(signal);
} else {
exec.resignal();
}
return 0;
}
/** RETURN statement */
public Integer return_(HplsqlParser.Return_stmtContext ctx) {
trace(ctx, "RETURN");
if (ctx.expr() != null) {
eval(ctx.expr());
}
exec.signal(Signal.Type.LEAVE_ROUTINE);
return 0;
}
/** Equality operator */
public void operatorEqual(HplsqlParser.Bool_expr_binaryContext ctx, boolean equal) {
Var v1 = evalPop(ctx.expr(0));
Var v2 = evalPop(ctx.expr(1));
boolean eq = v1.equals(v2);
if (!equal) {
eq = !eq;
}
exec.stackPush(eq);
}
@Test
public void par_update_get_compact() throws InterruptedException {
int scale = TT.scale();
if (scale == 0) return;
int threadNum = Math.min(4, scale * 4);
final long end = TT.nowPlusMinutes(10);
e = openEngine();
final BlockingQueue<Fun.Pair<Long, byte[]>> q = new ArrayBlockingQueue(threadNum * 10);
for (int i = 0; i < threadNum; i++) {
byte[] b = TT.randomByteArray(new Random().nextInt(10000));
long recid = e.put(b, BYTE_ARRAY_NOSIZE);
q.put(new Fun.Pair(recid, b));
}
final CountDownLatch l = new CountDownLatch(2);
Thread tt =
new Thread() {
@Override
public void run() {
try {
while (l.getCount() > 1) e.compact();
} finally {
l.countDown();
}
}
};
tt.setDaemon(true);
tt.run();
Exec.execNTimes(
threadNum,
new Callable() {
@Override
public Object call() throws Exception {
Random r = new Random();
while (System.currentTimeMillis() < end) {
Fun.Pair<Long, byte[]> t = q.take();
assertTrue(
Serializer.BYTE_ARRAY.equals(t.b, e.get(t.a, Serializer.BYTE_ARRAY_NOSIZE)));
int size = r.nextInt(1000);
if (r.nextInt(10) == 1) size = size * 100;
byte[] b = TT.randomByteArray(size);
e.update(t.a, b, Serializer.BYTE_ARRAY_NOSIZE);
q.put(new Fun.Pair<Long, byte[]>(t.a, b));
}
return null;
}
});
l.countDown();
l.await();
for (Fun.Pair<Long, byte[]> t : q) {
assertTrue(Serializer.BYTE_ARRAY.equals(t.b, e.get(t.a, Serializer.BYTE_ARRAY_NOSIZE)));
}
e.close();
}
@Override
public void execute() throws NullPointerException {
getProject().log(this, msg, Project.MSG_INFO);
PropertyHelper propertyHelper = PropertyHelper.getPropertyHelper(getProject());
if (qtjambiConfig == null) {
String thisQtjambiConfig = AntUtil.getPropertyAsString(propertyHelper, Constants.CONFIG);
if (thisQtjambiConfig != null) {
if (Constants.CONFIG_RELEASE.equals(thisQtjambiConfig)) qtjambiConfig = thisQtjambiConfig;
else if (Constants.CONFIG_DEBUG.equals(thisQtjambiConfig))
qtjambiConfig = thisQtjambiConfig;
else if (Constants.CONFIG_TEST.equals(thisQtjambiConfig)) qtjambiConfig = thisQtjambiConfig;
else
getProject()
.log(
this,
"WARNING: QTJAMBI_CONFIG will not be exported as value "
+ thisQtjambiConfig
+ " is not recognised (from "
+ Constants.CONFIG
+ ")",
Project.MSG_INFO);
if (thisQtjambiConfig != null)
getProject()
.log(
this,
"QTJAMBI_CONFIG will be exported as "
+ qtjambiConfig
+ " (from "
+ Constants.CONFIG
+ ")",
Project.MSG_INFO);
}
}
String proFile = "";
if (!pro.equals("")) proFile = Util.makeCanonical(pro).getAbsolutePath();
final List<String> command = new ArrayList<String>();
command.add(resolveExecutableAbsolutePath());
command.add(proFile);
List<String> arguments = parseArguments();
if (!arguments.isEmpty()) command.addAll(arguments);
List<String> parameters = parseParameters();
if (!parameters.isEmpty()) command.addAll(parameters);
File dirExecute = null;
if (dir != null) dirExecute = new File(dir);
String binpath = AntUtil.getPropertyAsString(propertyHelper, Constants.BINDIR);
Exec.execute(command, dirExecute, getProject(), binpath, null);
}
/** SQL INSERT VALUES statement */
public Integer insertValues(HplsqlParser.Insert_stmtContext ctx) {
trace(ctx, "INSERT VALUES");
String table = evalPop(ctx.table_name()).toString();
String conn = exec.getObjectConnection(ctx.table_name().getText());
Conn.Type type = exec.getConnectionType(conn);
StringBuilder sql = new StringBuilder();
if (type == Conn.Type.HIVE) {
sql.append("INSERT INTO TABLE " + table + " ");
if (conf.insertValues == Conf.InsertValues.NATIVE) {
sql.append("VALUES\n(");
}
} else {
sql.append("INSERT INTO " + table);
if (ctx.insert_stmt_cols() != null) {
sql.append(" " + exec.getFormattedText(ctx.insert_stmt_cols()));
}
sql.append(" VALUES\n(");
}
int rows = ctx.insert_stmt_rows().insert_stmt_row().size();
for (int i = 0; i < rows; i++) {
HplsqlParser.Insert_stmt_rowContext row = ctx.insert_stmt_rows().insert_stmt_row(i);
int cols = row.expr().size();
for (int j = 0; j < cols; j++) {
String value = evalPop(row.expr(j)).toSqlString();
if (j == 0 && type == Conn.Type.HIVE && conf.insertValues == Conf.InsertValues.SELECT) {
sql.append("SELECT ");
}
sql.append(value);
if (j + 1 != cols) {
sql.append(", ");
}
}
if (type != Conn.Type.HIVE || conf.insertValues == Conf.InsertValues.NATIVE) {
if (i + 1 == rows) {
sql.append(")");
} else {
sql.append("),\n(");
}
} else if (type == Conn.Type.HIVE && conf.insertValues == Conf.InsertValues.SELECT) {
sql.append(" FROM " + conf.dualTable);
if (i + 1 < rows) {
sql.append("\nUNION ALL\n");
}
}
}
if (trace) {
trace(ctx, sql.toString());
}
Query query = exec.executeSql(ctx, sql.toString(), conn);
if (query.error()) {
exec.signal(query);
return 1;
}
exec.setSqlSuccess();
exec.closeQuery(query, exec.conf.defaultConnection);
return 0;
}
/** FROM clause */
public Integer from(HplsqlParser.From_clauseContext ctx) {
StringBuilder sql = new StringBuilder();
sql.append(ctx.T_FROM().getText()).append(" ");
sql.append(evalPop(ctx.from_table_clause()));
int cnt = ctx.from_join_clause().size();
for (int i = 0; i < cnt; i++) {
sql.append(evalPop(ctx.from_join_clause(i)));
}
exec.stackPush(sql);
return 0;
}
@Test
public void testGroupbyTableSpeed() {
Frame ids = parse_test_file(Key.make("cov"), "smalldata/junit/id_cols.csv");
ids.replace(0, ids.anyVec().toCategoricalVec()).remove();
System.out.println(ids.toString(0, 10));
long start = System.currentTimeMillis();
Val v_gb = Exec.exec("(GB cov [0] nrow 0 \"all\")");
System.out.println("GB Time= " + (System.currentTimeMillis() - start) + "msec");
System.out.println(v_gb.toString());
((ValFrame) v_gb)._fr.delete();
long start2 = System.currentTimeMillis();
Val v_tb = Exec.exec("(table cov FALSE)");
System.out.println("Table Time= " + (System.currentTimeMillis() - start2) + "msec");
System.out.println(v_tb.toString());
((ValFrame) v_tb)._fr.delete();
ids.delete();
}
