public boolean shutdown(int port, boolean ssl) {
try {
String protocol = "http" + (ssl ? "s" : "");
URL url = new URL(protocol, "127.0.0.1", port, "shutdown");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
conn.setRequestProperty("servicemanager", "shutdown");
conn.connect();
StringBuffer sb = new StringBuffer();
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
int n;
char[] cbuf = new char[1024];
while ((n = br.read(cbuf, 0, cbuf.length)) != -1) sb.append(cbuf, 0, n);
br.close();
String message = sb.toString().replace("<br>", "\n");
if (message.contains("Goodbye")) {
cp.appendln("Shutting down the server:");
String[] lines = message.split("\n");
for (String line : lines) {
cp.append("...");
cp.appendln(line);
}
return true;
}
} catch (Exception ex) {
}
cp.appendln("Unable to shutdown CTP");
return false;
}
private String replace(String string, Properties table) {
try {
Pattern pattern = Pattern.compile("\\$\\{\\w+\\}");
Matcher matcher = pattern.matcher(string);
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
String group = matcher.group();
String key = group.substring(2, group.length() - 1).trim();
String repl = table.getProperty(key);
if (repl == null) repl = matcher.quoteReplacement(group);
matcher.appendReplacement(sb, repl);
}
matcher.appendTail(sb);
return sb.toString();
} catch (Exception ex) {
return string;
}
}
/** Thanks to Teodor Danciu for this method (c) 2003 Teodor Danciu */
public static String replaceTabWithBlank(String source) {
String result = source;
if (source != null && source.length() > 0) {
StringBuffer sbuffer = new StringBuffer(source);
int offset = 0;
int pos = source.indexOf("\t", offset);
while (pos >= 0) {
sbuffer.setCharAt(pos, ' ');
offset = pos + 1;
pos = source.indexOf("\t", offset);
}
result = sbuffer.toString();
}
return result;
}
private boolean checkServer(int port, boolean ssl) {
try {
URL url = new URL("http://127.0.0.1:" + port);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
conn.connect();
int length = conn.getContentLength();
StringBuffer text = new StringBuffer();
InputStream is = conn.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
int size = 256;
char[] buf = new char[size];
int len;
while ((len = isr.read(buf, 0, size)) != -1) text.append(buf, 0, len);
isr.close();
if (programName.equals("ISN")) return !shutdown(port, ssl);
return true;
} catch (Exception ex) {
return false;
}
}
/**
* This method inserts a blank character between to consecutive newline characters if encoutered.
* Also appends a blank character at the beginning of the text, if the first character is a
* newline character and at the end of the text, if the last character is also a newline. This is
* useful when trying to layout the paragraphs. Thanks to Teodor Danciu for this this method (c)
* 2003 Teodor Danciu
*/
public static String treatNewLineChars(String source) {
String result = source;
if (source != null && source.length() > 0) {
StringBuffer sbuffer = new StringBuffer(source);
// insert a blank character between every two consecutives
// newline characters
int offset = source.length() - 1;
int pos = source.lastIndexOf("\n\n", offset);
while (pos >= 0 && offset > 0) {
sbuffer = sbuffer.insert(pos + 1, " ");
offset = pos - 1;
pos = source.lastIndexOf("\n\n", offset);
}
// append a blank character at the and of the text
// if the last character is a newline character
if (sbuffer.charAt(sbuffer.length() - 1) == '\n') {
sbuffer.append(' ');
}
// append a blank character at the begining of the text
// if the first character is a newline character
if (sbuffer.charAt(0) == '\n') {
sbuffer.insert(0, ' ');
}
result = sbuffer.toString();
}
// remove this if you want to treat the tab characters in a special way
result = replaceTabWithBlank(result);
return result;
}
// Recursively walk the tree and write the nodes to a StringWriter.
private static void renderNode(StringBuffer sb, Node node) {
if (node == null) {
sb.append("null");
return;
}
switch (node.getNodeType()) {
case Node.DOCUMENT_NODE:
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
Node root = ((Document) node).getDocumentElement();
renderNode(sb, root);
break;
case Node.ELEMENT_NODE:
String name = getNodeNameWithNamespace(node);
NamedNodeMap attributes = node.getAttributes();
if (attributes.getLength() == 0) {
sb.append("<" + name + ">");
} else {
sb.append("<" + name + " ");
int attrlen = attributes.getLength();
for (int i = 0; i < attrlen; i++) {
Node attr = attributes.item(i);
String attrName = getNodeNameWithNamespace(attr);
sb.append(attrName + "=\"" + escapeChars(attr.getNodeValue()));
if (i < attrlen - 1) sb.append("\" ");
else sb.append("\">");
}
}
NodeList children = node.getChildNodes();
if (children != null) {
for (int i = 0; i < children.getLength(); i++) {
renderNode(sb, children.item(i));
}
}
sb.append("</" + name + ">");
break;
case Node.TEXT_NODE:
sb.append(escapeChars(node.getNodeValue()));
break;
case Node.CDATA_SECTION_NODE:
sb.append("<![CDATA[" + node.getNodeValue() + "]]>");
break;
case Node.PROCESSING_INSTRUCTION_NODE:
sb.append("<?" + node.getNodeName() + " " + escapeChars(node.getNodeValue()) + "?>");
break;
case Node.ENTITY_REFERENCE_NODE:
sb.append("&" + node.getNodeName() + ";");
break;
case Node.DOCUMENT_TYPE_NODE:
// Ignore document type nodes
break;
case Node.COMMENT_NODE:
sb.append("<!--" + node.getNodeValue() + "-->");
break;
}
return;
}
private static String toString(Node node) {
StringBuffer sb = new StringBuffer();
renderNode(sb, node);
return sb.toString();
}
/**
* The Main-Class for the Pig Jar that will provide a shell and setup a classpath appropriate for
* executing Jar files.
*
* @param args -jar can be used to add additional jar files (colon separated). - will start a
* shell. -e will execute the rest of the command line as if it was input to the shell.
* @throws IOException
*/
public static void main(String args[]) {
int rc = 1;
Properties properties = new Properties();
PropertiesUtil.loadPropertiesFromFile(properties);
boolean verbose = false;
boolean gruntCalled = false;
String logFileName = null;
try {
BufferedReader pin = null;
boolean debug = false;
boolean dryrun = false;
ArrayList<String> params = new ArrayList<String>();
ArrayList<String> paramFiles = new ArrayList<String>();
HashSet<String> optimizerRules = new HashSet<String>();
CmdLineParser opts = new CmdLineParser(args);
opts.registerOpt('4', "log4jconf", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('b', "brief", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('c', "cluster", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('d', "debug", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('e', "execute", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('f', "file", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('h', "help", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('l', "logfile", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('m', "param_file", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('o', "hod", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('r', "dryrun", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('t', "optimizer_off", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('v', "verbose", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('w', "warning", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('x', "exectype", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('F', "stop_on_failure", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('M', "no_multiquery", CmdLineParser.ValueExpected.NOT_ACCEPTED);
ExecMode mode = ExecMode.UNKNOWN;
String file = null;
ExecType execType = ExecType.MAPREDUCE;
String execTypeString = properties.getProperty("exectype");
if (execTypeString != null && execTypeString.length() > 0) {
execType = PigServer.parseExecType(execTypeString);
}
String cluster = "local";
String clusterConfigured = properties.getProperty("cluster");
if (clusterConfigured != null && clusterConfigured.length() > 0) {
cluster = clusterConfigured;
}
// by default warning aggregation is on
properties.setProperty("aggregate.warning", "" + true);
// by default multiquery optimization is on
properties.setProperty("opt.multiquery", "" + true);
// by default we keep going on error on the backend
properties.setProperty("stop.on.failure", "" + false);
char opt;
while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
switch (opt) {
case '4':
String log4jconf = opts.getValStr();
if (log4jconf != null) {
properties.setProperty(LOG4J_CONF, log4jconf);
}
break;
case 'b':
properties.setProperty(BRIEF, "true");
break;
case 'c':
// Needed away to specify the cluster to run the MR job on
// Bug 831708 - fixed
String clusterParameter = opts.getValStr();
if (clusterParameter != null && clusterParameter.length() > 0) {
cluster = clusterParameter;
}
break;
case 'd':
String logLevel = opts.getValStr();
if (logLevel != null) {
properties.setProperty(DEBUG, logLevel);
}
debug = true;
break;
case 'e':
mode = ExecMode.STRING;
break;
case 'f':
mode = ExecMode.FILE;
file = opts.getValStr();
break;
case 'F':
properties.setProperty("stop.on.failure", "" + true);
break;
case 'h':
usage();
return;
case 'i':
System.out.println(getVersionString());
return;
case 'j':
String jarsString = opts.getValStr();
if (jarsString != null) {
properties.setProperty(JAR, jarsString);
}
break;
case 'l':
// call to method that validates the path to the log file
// and sets up the file to store the client side log file
String logFileParameter = opts.getValStr();
if (logFileParameter != null && logFileParameter.length() > 0) {
logFileName = validateLogFile(logFileParameter, null);
} else {
logFileName = validateLogFile(logFileName, null);
}
properties.setProperty("pig.logfile", logFileName);
break;
case 'm':
paramFiles.add(opts.getValStr());
break;
case 'M':
// turns off multiquery optimization
properties.setProperty("opt.multiquery", "" + false);
break;
case 'o':
// TODO sgroschupf using system properties is always a very bad idea
String gateway = System.getProperty("ssh.gateway");
if (gateway == null || gateway.length() == 0) {
properties.setProperty("hod.server", "local");
} else {
properties.setProperty("hod.server", System.getProperty("ssh.gateway"));
}
break;
case 'p':
String val = opts.getValStr();
params.add(opts.getValStr());
break;
case 'r':
// currently only used for parameter substitution
// will be extended in the future
dryrun = true;
break;
case 't':
optimizerRules.add(opts.getValStr());
break;
case 'v':
properties.setProperty(VERBOSE, "" + true);
verbose = true;
break;
case 'w':
properties.setProperty("aggregate.warning", "" + false);
break;
case 'x':
try {
execType = PigServer.parseExecType(opts.getValStr());
} catch (IOException e) {
throw new RuntimeException("ERROR: Unrecognized exectype.", e);
}
break;
default:
{
Character cc = new Character(opt);
throw new AssertionError("Unhandled option " + cc.toString());
}
}
}
// configure logging
configureLog4J(properties);
// create the context with the parameter
PigContext pigContext = new PigContext(execType, properties);
if (logFileName == null) {
logFileName = validateLogFile(null, null);
}
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (optimizerRules.size() > 0) {
pigContext
.getProperties()
.setProperty("pig.optimizer.rules", ObjectSerializer.serialize(optimizerRules));
}
LogicalPlanBuilder.classloader = pigContext.createCl(null);
// construct the parameter substitution preprocessor
Grunt grunt = null;
BufferedReader in;
String substFile = null;
switch (mode) {
case FILE:
{
// Run, using the provided file as a pig file
in = new BufferedReader(new FileReader(file));
// run parameter substitution preprocessor first
substFile = file + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, file);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(file).getName());
if (!debug) {
new File(substFile).deleteOnExit();
}
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
case STRING:
{
// Gather up all the remaining arguments into a string and pass them into
// grunt.
StringBuffer sb = new StringBuffer();
String remainders[] = opts.getRemainingArgs();
for (int i = 0; i < remainders.length; i++) {
if (i != 0) sb.append(' ');
sb.append(remainders[i]);
}
in = new BufferedReader(new StringReader(sb.toString()));
grunt = new Grunt(in, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
default:
break;
}
// If we're here, we don't know yet what they want. They may have just
// given us a jar to execute, they might have given us a pig script to
// execute, or they might have given us a dash (or nothing) which means to
// run grunt interactive.
String remainders[] = opts.getRemainingArgs();
if (remainders == null) {
// Interactive
mode = ExecMode.SHELL;
ConsoleReader reader = new ConsoleReader(System.in, new OutputStreamWriter(System.out));
reader.setDefaultPrompt("grunt> ");
final String HISTORYFILE = ".pig_history";
String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE;
reader.setHistory(new History(new File(historyFile)));
ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
grunt.setConsoleReader(reader);
gruntCalled = true;
grunt.run();
rc = 0;
return;
} else {
// They have a pig script they want us to run.
if (remainders.length > 1) {
throw new RuntimeException(
"You can only run one pig script " + "at a time from the command line.");
}
mode = ExecMode.FILE;
in = new BufferedReader(new FileReader(remainders[0]));
// run parameter substitution preprocessor first
substFile = remainders[0] + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, remainders[0]);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (!debug) {
new File(substFile).deleteOnExit();
}
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(remainders[0]).getName());
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int[] results = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
// Per Utkarsh and Chris invocation of jar file via pig depricated.
} catch (ParseException e) {
usage();
rc = 2;
} catch (NumberFormatException e) {
usage();
rc = 2;
} catch (PigException pe) {
if (pe.retriable()) {
rc = 1;
} else {
rc = 2;
}
if (!gruntCalled) {
LogUtils.writeLog(pe, logFileName, log, verbose);
}
} catch (Throwable e) {
rc = 2;
if (!gruntCalled) {
LogUtils.writeLog(e, logFileName, log, verbose);
}
} finally {
// clear temp files
FileLocalizer.deleteTempFiles();
PerformanceTimerFactory.getPerfTimerFactory().dumpTimers();
System.exit(rc);
}
}