@Override
public JsonIterator iter(final Context context) throws Exception {
final JsonString cmd = (JsonString) exprs[1].eval(context);
if (cmd == null) {
return JsonIterator.NULL;
}
iter = exprs[0].iter(context);
if (iter.isNull()) {
return JsonIterator.NULL;
}
proc = Runtime.getRuntime().exec(cmd.toString());
// TODO: add thread pool to context
InputThread inputThread = new InputThread();
ErrorThread errorThread = new ErrorThread();
final MutableJsonString str = new MutableJsonString();
try {
InputStream is = proc.getInputStream();
final BufferedReader reader = new BufferedReader(new InputStreamReader(is));
errorThread.start();
inputThread.start();
return new JsonIterator(str) {
@Override
public boolean moveNext() throws Exception {
try {
String s = reader.readLine();
if (s == null) {
reader.close();
int rc = proc.waitFor();
if (rc != 0) {
System.err.println("non-zero exit code from process [" + cmd + "]: " + rc);
}
return false;
}
str.setCopy(s);
return true; // currentValue == str
} catch (Throwable e) {
if (error == null) {
error = e;
}
proc.destroy();
if (error instanceof Exception) {
throw (Exception) error;
}
throw new UndeclaredThrowableException(error);
}
}
};
} catch (Throwable e) {
if (error == null) {
error = e;
}
proc.destroy();
if (error instanceof Exception) {
throw (Exception) error;
}
throw new UndeclaredThrowableException(error);
}
}
private void initParams(JsonRecord rec) {
for (Iterator iter = rec.iterator(); iter.hasNext(); ) {
Entry tmp = (Entry) iter.next();
if (tmp.getKey().equals(CMD)) {
cmd = (JsonValue) tmp.getValue();
} else if (tmp.getKey().equals(WRITEOPTS)) {
writeOpts = (JsonValue) tmp.getValue();
} else if (tmp.getKey().equals(READOPTS)) {
readOpts = (JsonValue) tmp.getValue();
}
}
if (cmd == null) throw new IllegalArgumentException("cmd cannot be null in externalfn()");
try {
if (writeOpts == null) {
String in = "{outoptions: {adapter: 'com.ibm.jaql.io.stream.StreamOutputAdapter',";
if (mode.equals(new JsonString("push")))
in += "format: 'com.ibm.jaql.io.stream.converter.ArgumentsOutputStream'}}";
else in += "format: 'com.ibm.jaql.io.stream.converter.LineTextOutputStream'}}";
writeOpts = new JsonParser().parse(in);
}
if (readOpts == null) {
String out =
"{inoptions: {adapter: 'com.ibm.jaql.io.stream.StreamInputAdapter',"
+ "format: 'com.ibm.jaql.io.stream.converter.LineTextInputStream'}}";
readOpts = new JsonParser().parse(out);
}
} catch (ParseException e) {
e.printStackTrace();
}
}
@Override
public int compare(Object o1, Object o2) {
JsonString name1;
if (o1 instanceof Field) {
name1 = ((Field) o1).getName();
} else {
name1 = (JsonString) o1;
}
JsonString name2;
if (o2 instanceof Field) {
name2 = ((Field) o2).getName();
} else {
name2 = (JsonString) o2;
}
return name1.compareTo(name2);
}
@Override
public JsonString eval(Context context) throws Exception {
JsonString inpath = (JsonString) exprs[0].eval(context);
String outpath;
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path path = new Path(inpath.toString());
if (path.isAbsolute()) {
String uri = fs.getUri().toString();
outpath = uri + inpath;
} else {
String home = fs.getHomeDirectory().toString();
outpath = home + "/" + inpath;
}
if (!fs.exists(new Path(outpath))) {
throw new IllegalArgumentException("The input path doesn't exist in HDFS");
}
return new JsonString(outpath);
}
/*
* (non-Javadoc)
*
* @see com.ibm.jaql.lang.expr.core.IterExpr#iter(com.ibm.jaql.lang.core.Context)
*/
public JsonArray eval(final Context context) throws Exception {
final JsonRegex regex = (JsonRegex) exprs[0].eval(context);
if (regex == null) {
return null;
}
JsonString text = (JsonString) exprs[1].eval(context);
if (text == null) {
return null;
}
final Matcher matcher = regex.takeMatcher();
matcher.reset(text.toString());
if (!matcher.find()) {
regex.returnMatcher(matcher);
return null;
}
int n = matcher.groupCount();
BufferedJsonArray arr = new BufferedJsonArray(n); // TODO: memory
for (int i = 0; i < n; i++) {
String s = matcher.group(i + 1);
arr.set(i, s == null ? null : new JsonString(s)); // TODO: memory
}
regex.returnMatcher(matcher);
return arr;
}
@Override
public void run() {
try {
OutputStream os = proc.getOutputStream();
// PrintStream out = new PrintStream(new BufferedOutputStream(os));
OutputStream out = new BufferedOutputStream(os);
for (JsonValue sv : iter) {
// TODO: force jstrings here? add i/o layer here? add serialize function that has i/o
// layer?
JsonString s = (JsonString) sv;
if (s != null) {
// out.println(s.toString()); // TODO: PrintStream swallows IOExceptions...
out.write(s.getInternalBytes(), s.bytesOffset(), s.bytesLength());
out.write('\n');
}
}
out.close();
} catch (Throwable e) {
if (error == null) {
error = e;
}
proc.destroy();
}
}
public static JsonValue eval(JsonValue value1, JsonValue value2, int op) {
// changes here should be reflected in getSchema() below
if (value1 == null || value2 == null) {
return null;
}
switch (value1.getType()) {
case LONG:
switch (value2.getType()) {
case LONG:
return longEval((JsonNumber) value1, (JsonNumber) value2, op);
case DOUBLE:
return doubleEval((JsonNumber) value1, (JsonNumber) value2, op);
case DECFLOAT:
return decimalEval((JsonNumber) value1, (JsonNumber) value2, op);
}
break;
case DOUBLE:
switch (value2.getType()) {
case LONG:
return doubleEval((JsonNumber) value1, (JsonNumber) value2, op);
case DOUBLE:
return doubleEval((JsonNumber) value1, (JsonNumber) value2, op);
case DECFLOAT:
return decimalEval((JsonNumber) value1, (JsonNumber) value2, op);
}
break;
case DECFLOAT:
switch (value2.getType()) {
case LONG:
return decimalEval((JsonNumber) value1, (JsonNumber) value2, op);
case DOUBLE:
return decimalEval((JsonNumber) value1, (JsonNumber) value2, op);
case DECFLOAT:
return decimalEval((JsonNumber) value1, (JsonNumber) value2, op);
}
break;
case STRING:
if (value2.getType() == JsonType.STRING && op == PLUS) {
JsonString text1 = (JsonString) value1;
JsonString text2 = (JsonString) value2;
MutableJsonString result = new MutableJsonString();
result.ensureCapacity(text1.bytesLength() + text2.bytesLength());
byte[] buf = result.get();
text1.writeBytes(buf, 0, text1.bytesLength());
text2.writeBytes(buf, text1.bytesLength(), text2.bytesLength());
result.set(buf, text1.bytesLength() + text2.bytesLength());
return result;
}
break;
}
// if we reach this point, we cannot apply the op
throw new RuntimeException(
"Operation "
+ OP_STR[op]
+ " not defined for input types "
+ value1.getType()
+ " and "
+ value2.getType());
}
@Override
public JsonIterator iter(Context context) throws Exception {
try {
initProcess(context);
if (mode.equals(new JsonString("push"))) {
args = exprs[0].eval(context);
writer.write(args);
writer.flush();
return reader;
} else if (mode.equals(new JsonString("streaming"))) {
data = exprs[0].iter(context);
return new ClosableJsonIterator() {
boolean firsttime = true;
InputHelper helper = new InputHelper();
@Override
public boolean moveNext() throws Exception {
if (firsttime) {
helper.start();
firsttime = false;
} else {
helper.stop = false;
}
if (reader.moveNext()) {
helper.stop = true;
currentValue = reader.current();
return true;
}
return false;
}
class InputHelper extends Thread {
public boolean stop = false;
public void run() {
try {
while (!stop) {
if (data.moveNext()) {
writer.write(data.current());
} else {
stop = true;
writer.close();
writer = null;
}
}
} catch (Throwable e) {
e.printStackTrace(); // TODO: move to log
if (error == null) {
error = e;
}
}
}
}
};
} else {
throw new RuntimeException("unsupported mode: " + mode);
}
} catch (Throwable e) {
if (error == null) {
error = e;
}
if (stdin != null) {
try {
stdin.close();
} catch (Throwable t) { // FIXME: More lost exceptions!
}
stdin = null;
}
if (process != null) {
try {
process.destroy();
} catch (Throwable t) {
}
process = null;
}
if (stdout != null) {
try {
stdout.close();
} catch (Throwable t) {
}
stdout = null;
}
if (error instanceof Exception) {
throw (Exception) error;
}
throw new UndeclaredThrowableException(error);
}
}