/**
* Main Method
*
* @param args command line arguments
*/
public static void main(String[] args) {
// init logging
try {
Logging.init("lucane.log", "ALL");
} catch (IOException ioe) {
System.err.println("Unable to init logging, exiting.");
System.exit(1);
}
Server server = null;
ServerConfig config = null;
try {
config = new ServerConfig(CONFIG_FILE);
} catch (Exception e) {
Logging.getLogger().severe("Unable to read or parse the config file.");
e.printStackTrace();
System.exit(1);
}
// Server creation
server = new Server(config);
server.generateKeys();
Logging.getLogger().info("Server is ready.");
server.run();
}
/**
* Creates a new Server object.
*
* @param sqlDriver JDBC driver
* @param dbURL JDBC connection url
* @param dbLogin database login
* @param dbPasswd database password
*/
private Server(ServerConfig config) {
Server.instance = this;
this.connections = new ArrayList();
this.services = new ArrayList();
this.port = config.getPort();
this.socket = null;
this.dbLayer = null;
try {
dbLayer = DatabaseAbstractionLayer.createLayer(config);
Logging.getLogger().finer("dbLayer : " + dbLayer);
this.store = new Store(config);
} catch (Exception ex) {
Logging.getLogger().severe("#Err > Unable to connect to the database : " + ex.getMessage());
ex.printStackTrace();
System.exit(1);
}
try {
this.serverIp = InetAddress.getLocalHost().getHostAddress();
this.socket = new ServerSocket(this.port);
} catch (IOException e) {
Logging.getLogger().severe("#Err > Unable to listen on the port " + port + ".");
e.printStackTrace();
System.exit(1);
}
loadInternalServices();
}
// -creatediff -applydiff -debug -output file
public static void main(String[] args) throws IOException {
boolean diff = true;
boolean minimal = true;
String outputFile = "out.jardiff";
for (int counter = 0; counter < args.length; counter++) {
// for backward compatibilty with 1.0.1/1.0
if (args[counter].equals("-nonminimal") || args[counter].equals("-n")) {
minimal = false;
} else if (args[counter].equals("-creatediff") || args[counter].equals("-c")) {
diff = true;
} else if (args[counter].equals("-applydiff") || args[counter].equals("-a")) {
diff = false;
} else if (args[counter].equals("-debug") || args[counter].equals("-d")) {
_debug = true;
} else if (args[counter].equals("-output") || args[counter].equals("-o")) {
if (++counter < args.length) {
outputFile = args[counter];
}
} else if (args[counter].equals("-applydiff") || args[counter].equals("-a")) {
diff = false;
} else {
if ((counter + 2) != args.length) {
showHelp();
System.exit(0);
}
if (diff) {
try {
OutputStream os = new FileOutputStream(outputFile);
JarDiff.createPatch(args[counter], args[counter + 1], os, minimal);
os.close();
} catch (IOException ioe) {
try {
System.out.println(getResources().getString("jardiff.error.create") + " " + ioe);
} catch (MissingResourceException mre) {
}
}
} else {
try {
OutputStream os = new FileOutputStream(outputFile);
new JarDiffPatcher().applyPatch(null, args[counter], args[counter + 1], os);
os.close();
} catch (IOException ioe) {
try {
System.out.println(getResources().getString("jardiff.error.apply") + " " + ioe);
} catch (MissingResourceException mre) {
}
}
}
System.exit(0);
}
}
showHelp();
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: java -jar jar2jad [jarfile] [jadfile]");
System.exit(1);
}
if (!extractJarDir(args[0], args[1])) {
System.out.println("Error: Could not find manifest in Jar.");
System.exit(1);
} else {
System.out.println("Jad created.");
}
}
public static void ensureJavaVersion() {
Class<?> nio;
try {
nio = Util.class.getClassLoader().loadClass("java.nio.charset.StandardCharsets");
if (nio == null) {
System.out.println(
"Could not load NIO! Are we running on Java 7 or greater? Sorry, exiting...");
System.exit(1);
}
} catch (java.lang.ClassNotFoundException e) {
System.out.println(
"Could not load NIO! Are we running on Java 7 or greater? Sorry, exiting...");
System.exit(1);
}
}
/** Generates server's keys for signature */
public void generateKeys() {
Logging.getLogger().info("Generating keypair");
try {
String[] pair = KeyGenerator.generateKeyPair();
myConnectInfo =
new ConnectInfo("server", this.serverIp, this.serverIp, this.port, pair[1], "Server");
this.connections.add(myConnectInfo);
this.signer = new Signer(pair[0]);
} catch (SignatureException e) {
Logging.getLogger().severe("Unable to generate keypair : " + e);
System.exit(1);
}
}
public static void unzipInteralZip(
ClassLoader classLoader, String resourcePath, File libDir, boolean debug) {
if (debug) System.out.println("Extracting " + resourcePath);
libDir.mkdir();
URL resource = classLoader.getResource(resourcePath);
if (resource == null) {
System.err.println("Could not find the " + resourcePath + " on classpath!");
System.exit(1);
}
class PrintDot extends TimerTask {
public void run() {
System.out.print(".");
}
}
Timer timer = new Timer();
PrintDot task = new PrintDot();
timer.schedule(task, 0, 2000);
try {
BufferedInputStream bis = new BufferedInputStream(resource.openStream());
JarInputStream jis = new JarInputStream(bis);
JarEntry je = null;
while ((je = jis.getNextJarEntry()) != null) {
java.io.File f =
new java.io.File(libDir.toString() + java.io.File.separator + je.getName());
if (je.isDirectory()) {
f.mkdir();
continue;
}
File parentDir = new File(f.getParent());
if (!parentDir.exists()) {
parentDir.mkdir();
}
FileOutputStream fileOutStream = new FileOutputStream(f);
writeStreamTo(jis, fileOutStream, 8 * KB);
if (f.getPath().endsWith("pack.gz")) {
unpack(f);
fileOutStream.close();
f.delete();
}
}
} catch (Exception exc) {
task.cancel();
exc.printStackTrace();
}
task.cancel();
}
/**
* Class constructor; creates a new Installer object, displays a JFrame introducing the program,
* allows the user to select an install directory, and copies files from the jar into the
* directory.
*/
public Installer(String args[]) {
// Inputs are --install-dir INSTALL_DIR
for (int k = 0; k < args.length; k = k + 2) {
switch (args[k]) {
case "--install-dir":
directory = new File(args[k + 1]);
System.out.println(directory);
break;
case "--port":
port = Integer.parseInt(args[k + 1]);
break;
}
}
cp = new Stream();
// Find the installer program so we can get to the files.
installer = getInstallerProgramFile();
String name = installer.getName();
programName = (name.substring(0, name.indexOf("-"))).toUpperCase();
// Get the installation information
thisJava = System.getProperty("java.version");
thisJavaBits = System.getProperty("sun.arch.data.model") + " bits";
// Find the ImageIO Tools and get the version
String javaHome = System.getProperty("java.home");
File extDir = new File(javaHome);
extDir = new File(extDir, "lib");
extDir = new File(extDir, "ext");
File clib = getFile(extDir, "clibwrapper_jiio", ".jar");
File jai = getFile(extDir, "jai_imageio", ".jar");
imageIOTools = (clib != null) && clib.exists() && (jai != null) && jai.exists();
if (imageIOTools) {
Hashtable<String, String> jaiManifest = getManifestAttributes(jai);
imageIOVersion = jaiManifest.get("Implementation-Version");
}
// Get the CTP.jar parameters
Hashtable<String, String> manifest = getJarManifestAttributes("/CTP/libraries/CTP.jar");
programDate = manifest.get("Date");
buildJava = manifest.get("Java-Version");
// Get the util.jar parameters
Hashtable<String, String> utilManifest = getJarManifestAttributes("/CTP/libraries/util.jar");
utilJava = utilManifest.get("Java-Version");
// Get the MIRC.jar parameters (if the plugin is present)
Hashtable<String, String> mircManifest = getJarManifestAttributes("/CTP/libraries/MIRC.jar");
if (mircManifest != null) {
mircJava = mircManifest.get("Java-Version");
mircDate = mircManifest.get("Date");
mircVersion = mircManifest.get("Version");
}
// Set up the installation information for display
if (imageIOVersion.equals("")) {
imageIOVersion = "<b><font color=\"red\">not installed</font></b>";
} else if (imageIOVersion.startsWith("1.0")) {
imageIOVersion = "<b><font color=\"red\">" + imageIOVersion + "</font></b>";
}
if (thisJavaBits.startsWith("64")) {
thisJavaBits = "<b><font color=\"red\">" + thisJavaBits + "</font></b>";
}
boolean javaOK = (thisJava.compareTo(buildJava) >= 0);
javaOK &= (thisJava.compareTo(utilJava) >= 0);
javaOK &= (thisJava.compareTo(mircJava) >= 0);
if (!javaOK) {
thisJava = "<b><font color=\"red\">" + thisJava + "</font></b>";
}
if (directory == null) exit();
// Point to the parent of the directory in which to install the program.
// so the copy process works correctly for directory trees.
//
// If the user has selected a directory named "CTP",
// then assume that this is the directory in which
// to install the program.
//
// If the directory is not CTP, see if it is called "RSNA" and contains
// the Launcher program, in which case we can assume that it is an
// installation that was done with Bill Weadock's all-in-one installer for Windows.
//
// If neither of those cases is true, then this is already the parent of the
// directory in which to install the program
if (directory.getName().equals("CTP")) {
directory = directory.getParentFile();
} else if (directory.getName().equals("RSNA")
&& (new File(directory, "Launcher.jar")).exists()) {
suppressFirstPathElement = true;
}
// Cleanup old releases
cleanup(directory);
// Get a port for the server.
if (port < 0) {
if (checkServer(-port, false)) {
System.err.println(
"CTP appears to be running.\nPlease stop CTP and run the installer again.");
System.exit(0);
}
}
// Now install the files and report the results.
int count =
unpackZipFile(installer, "CTP", directory.getAbsolutePath(), suppressFirstPathElement);
if (count > 0) {
// Create the service installer batch files.
updateWindowsServiceInstaller();
updateLinuxServiceInstaller();
// If this was a new installation, set up the config file and set the port
installConfigFile(port);
// Make any necessary changes in the config file to reflect schema evolution
fixConfigSchema();
System.out.println("Installation complete.");
System.out.println(programName + " has been installed successfully.");
System.out.println(count + " files were installed.");
} else {
System.err.println("Installation failed.");
System.err.println(programName + " could not be fully installed.");
}
if (!programName.equals("ISN") && startRunner(new File(directory, "CTP"))) System.exit(0);
}
private static void exit() {
System.exit(0);
}
public static void main(String[] args) throws Exception {
System.out.println(
"Checking that all known MBeans that are "
+ "NotificationBroadcasters have sane "
+ "MBeanInfo.getNotifications()");
System.out.println("Checking platform MBeans...");
checkPlatformMBeans();
URL codeBase = ClassLoader.getSystemResource("javax/management/MBeanServer.class");
if (codeBase == null) {
throw new Exception("Could not determine codeBase for " + MBeanServer.class);
}
System.out.println();
System.out.println("Looking for standard MBeans...");
String[] classes = findStandardMBeans(codeBase);
System.out.println("Testing standard MBeans...");
for (int i = 0; i < classes.length; i++) {
String name = classes[i];
Class<?> c;
try {
c = Class.forName(name);
} catch (Throwable e) {
System.out.println(name + ": cannot load (not public?): " + e);
continue;
}
if (!NotificationBroadcaster.class.isAssignableFrom(c)) {
System.out.println(name + ": not a NotificationBroadcaster");
continue;
}
if (Modifier.isAbstract(c.getModifiers())) {
System.out.println(name + ": abstract class");
continue;
}
NotificationBroadcaster mbean;
Constructor<?> constr;
try {
constr = c.getConstructor();
} catch (Exception e) {
System.out.println(name + ": no public no-arg constructor: " + e);
continue;
}
try {
mbean = (NotificationBroadcaster) constr.newInstance();
} catch (Exception e) {
System.out.println(name + ": no-arg constructor failed: " + e);
continue;
}
check(mbean);
}
System.out.println();
System.out.println("Testing some explicit cases...");
check(new RelationService(false));
/*
We can't do this:
check(new RequiredModelMBean());
because the Model MBean spec more or less forces us to use the
names GENERIC and ATTRIBUTE_CHANGE for its standard notifs.
*/
checkRMIConnectorServer();
System.out.println();
if (!suspicious.isEmpty()) System.out.println("SUSPICIOUS CLASSES: " + suspicious);
if (failed.isEmpty()) System.out.println("TEST PASSED");
else {
System.out.println("TEST FAILED: " + failed);
System.exit(1);
}
}
/**
* The Main-Class for the Pig Jar that will provide a shell and setup a classpath appropriate for
* executing Jar files.
*
* @param args -jar can be used to add additional jar files (colon separated). - will start a
* shell. -e will execute the rest of the command line as if it was input to the shell.
* @throws IOException
*/
public static void main(String args[]) {
int rc = 1;
Properties properties = new Properties();
PropertiesUtil.loadPropertiesFromFile(properties);
boolean verbose = false;
boolean gruntCalled = false;
String logFileName = null;
try {
BufferedReader pin = null;
boolean debug = false;
boolean dryrun = false;
ArrayList<String> params = new ArrayList<String>();
ArrayList<String> paramFiles = new ArrayList<String>();
HashSet<String> optimizerRules = new HashSet<String>();
CmdLineParser opts = new CmdLineParser(args);
opts.registerOpt('4', "log4jconf", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('b', "brief", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('c', "cluster", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('d', "debug", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('e', "execute", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('f', "file", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('h', "help", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('l', "logfile", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('m', "param_file", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('o', "hod", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('r', "dryrun", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('t', "optimizer_off", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('v', "verbose", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('w', "warning", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('x', "exectype", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('F', "stop_on_failure", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('M', "no_multiquery", CmdLineParser.ValueExpected.NOT_ACCEPTED);
ExecMode mode = ExecMode.UNKNOWN;
String file = null;
ExecType execType = ExecType.MAPREDUCE;
String execTypeString = properties.getProperty("exectype");
if (execTypeString != null && execTypeString.length() > 0) {
execType = PigServer.parseExecType(execTypeString);
}
String cluster = "local";
String clusterConfigured = properties.getProperty("cluster");
if (clusterConfigured != null && clusterConfigured.length() > 0) {
cluster = clusterConfigured;
}
// by default warning aggregation is on
properties.setProperty("aggregate.warning", "" + true);
// by default multiquery optimization is on
properties.setProperty("opt.multiquery", "" + true);
// by default we keep going on error on the backend
properties.setProperty("stop.on.failure", "" + false);
char opt;
while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
switch (opt) {
case '4':
String log4jconf = opts.getValStr();
if (log4jconf != null) {
properties.setProperty(LOG4J_CONF, log4jconf);
}
break;
case 'b':
properties.setProperty(BRIEF, "true");
break;
case 'c':
// Needed away to specify the cluster to run the MR job on
// Bug 831708 - fixed
String clusterParameter = opts.getValStr();
if (clusterParameter != null && clusterParameter.length() > 0) {
cluster = clusterParameter;
}
break;
case 'd':
String logLevel = opts.getValStr();
if (logLevel != null) {
properties.setProperty(DEBUG, logLevel);
}
debug = true;
break;
case 'e':
mode = ExecMode.STRING;
break;
case 'f':
mode = ExecMode.FILE;
file = opts.getValStr();
break;
case 'F':
properties.setProperty("stop.on.failure", "" + true);
break;
case 'h':
usage();
return;
case 'i':
System.out.println(getVersionString());
return;
case 'j':
String jarsString = opts.getValStr();
if (jarsString != null) {
properties.setProperty(JAR, jarsString);
}
break;
case 'l':
// call to method that validates the path to the log file
// and sets up the file to store the client side log file
String logFileParameter = opts.getValStr();
if (logFileParameter != null && logFileParameter.length() > 0) {
logFileName = validateLogFile(logFileParameter, null);
} else {
logFileName = validateLogFile(logFileName, null);
}
properties.setProperty("pig.logfile", logFileName);
break;
case 'm':
paramFiles.add(opts.getValStr());
break;
case 'M':
// turns off multiquery optimization
properties.setProperty("opt.multiquery", "" + false);
break;
case 'o':
// TODO sgroschupf using system properties is always a very bad idea
String gateway = System.getProperty("ssh.gateway");
if (gateway == null || gateway.length() == 0) {
properties.setProperty("hod.server", "local");
} else {
properties.setProperty("hod.server", System.getProperty("ssh.gateway"));
}
break;
case 'p':
String val = opts.getValStr();
params.add(opts.getValStr());
break;
case 'r':
// currently only used for parameter substitution
// will be extended in the future
dryrun = true;
break;
case 't':
optimizerRules.add(opts.getValStr());
break;
case 'v':
properties.setProperty(VERBOSE, "" + true);
verbose = true;
break;
case 'w':
properties.setProperty("aggregate.warning", "" + false);
break;
case 'x':
try {
execType = PigServer.parseExecType(opts.getValStr());
} catch (IOException e) {
throw new RuntimeException("ERROR: Unrecognized exectype.", e);
}
break;
default:
{
Character cc = new Character(opt);
throw new AssertionError("Unhandled option " + cc.toString());
}
}
}
// configure logging
configureLog4J(properties);
// create the context with the parameter
PigContext pigContext = new PigContext(execType, properties);
if (logFileName == null) {
logFileName = validateLogFile(null, null);
}
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (optimizerRules.size() > 0) {
pigContext
.getProperties()
.setProperty("pig.optimizer.rules", ObjectSerializer.serialize(optimizerRules));
}
LogicalPlanBuilder.classloader = pigContext.createCl(null);
// construct the parameter substitution preprocessor
Grunt grunt = null;
BufferedReader in;
String substFile = null;
switch (mode) {
case FILE:
{
// Run, using the provided file as a pig file
in = new BufferedReader(new FileReader(file));
// run parameter substitution preprocessor first
substFile = file + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, file);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(file).getName());
if (!debug) {
new File(substFile).deleteOnExit();
}
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
case STRING:
{
// Gather up all the remaining arguments into a string and pass them into
// grunt.
StringBuffer sb = new StringBuffer();
String remainders[] = opts.getRemainingArgs();
for (int i = 0; i < remainders.length; i++) {
if (i != 0) sb.append(' ');
sb.append(remainders[i]);
}
in = new BufferedReader(new StringReader(sb.toString()));
grunt = new Grunt(in, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
default:
break;
}
// If we're here, we don't know yet what they want. They may have just
// given us a jar to execute, they might have given us a pig script to
// execute, or they might have given us a dash (or nothing) which means to
// run grunt interactive.
String remainders[] = opts.getRemainingArgs();
if (remainders == null) {
// Interactive
mode = ExecMode.SHELL;
ConsoleReader reader = new ConsoleReader(System.in, new OutputStreamWriter(System.out));
reader.setDefaultPrompt("grunt> ");
final String HISTORYFILE = ".pig_history";
String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE;
reader.setHistory(new History(new File(historyFile)));
ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
grunt.setConsoleReader(reader);
gruntCalled = true;
grunt.run();
rc = 0;
return;
} else {
// They have a pig script they want us to run.
if (remainders.length > 1) {
throw new RuntimeException(
"You can only run one pig script " + "at a time from the command line.");
}
mode = ExecMode.FILE;
in = new BufferedReader(new FileReader(remainders[0]));
// run parameter substitution preprocessor first
substFile = remainders[0] + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, remainders[0]);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (!debug) {
new File(substFile).deleteOnExit();
}
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(remainders[0]).getName());
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int[] results = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
// Per Utkarsh and Chris invocation of jar file via pig depricated.
} catch (ParseException e) {
usage();
rc = 2;
} catch (NumberFormatException e) {
usage();
rc = 2;
} catch (PigException pe) {
if (pe.retriable()) {
rc = 1;
} else {
rc = 2;
}
if (!gruntCalled) {
LogUtils.writeLog(pe, logFileName, log, verbose);
}
} catch (Throwable e) {
rc = 2;
if (!gruntCalled) {
LogUtils.writeLog(e, logFileName, log, verbose);
}
} finally {
// clear temp files
FileLocalizer.deleteTempFiles();
PerformanceTimerFactory.getPerfTimerFactory().dumpTimers();
System.exit(rc);
}
}
/** Main routine to start program. */
public static void main(String args[]) {
Main jartool = new Main(System.out, System.err, "jar");
System.exit(jartool.run(args) ? 0 : 1);
}
