private String copyFromLocal(FileSystem fs, Path basePath, String[] files) throws IOException {
StringBuilder csv = new StringBuilder(files.length * (basePath.toString().length() + 16));
for (String localFile : files) {
Path src = new Path(localFile);
String filename = src.getName();
Path dst = new Path(basePath, filename);
URI localFileURI = null;
try {
localFileURI = new URI(localFile);
} catch (URISyntaxException e) {
throw new IOException(e);
}
if (localFileURI.getScheme() == null || localFileURI.getScheme().startsWith("file")) {
LOG.info("Copy {} from local filesystem to {}", localFile, dst);
fs.copyFromLocalFile(false, true, src, dst);
} else {
LOG.info("Copy {} from DFS to {}", localFile, dst);
FileUtil.copy(fs, src, fs, dst, false, true, conf);
}
if (csv.length() > 0) {
csv.append(LIB_JARS_SEP);
}
csv.append(dst.toString());
}
return csv.toString();
}
/**
* Launch application for the dag represented by this client.
*
* @throws YarnException
* @throws IOException
*/
public void startApplication() throws YarnException, IOException {
Class<?>[] defaultClasses;
if (applicationType.equals(YARN_APPLICATION_TYPE)) {
// TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
defaultClasses = DATATORRENT_SECURITY_CLASSES;
} else {
defaultClasses = DATATORRENT_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
}
LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses);
if (resources != null) {
localJarFiles.addAll(resources);
}
YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
LOG.info(
"Got Cluster metric info from ASM"
+ ", numNodeManagers="
+ clusterMetrics.getNumNodeManagers());
// GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
// GetClusterNodesResponse clusterNodesResp =
// rmClient.clientRM.getClusterNodes(clusterNodesReq);
// LOG.info("Got Cluster node info from ASM");
// for (NodeReport node : clusterNodesResp.getNodeReports()) {
// LOG.info("Got node report from ASM for"
// + ", nodeId=" + node.getNodeId()
// + ", nodeAddress" + node.getHttpAddress()
// + ", nodeRackName" + node.getRackName()
// + ", nodeNumContainers" + node.getNumContainers()
// + ", nodeHealthStatus" + node.getHealthReport());
// }
List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
LOG.info(
"User ACL Info for Queue"
+ ", queueName="
+ aclInfo.getQueueName()
+ ", userAcl="
+ userAcl.name());
}
}
// Get a new application id
YarnClientApplication newApp = yarnClient.createApplication();
appId = newApp.getNewApplicationResponse().getApplicationId();
// Dump out information about cluster capability as seen by the resource manager
int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory();
LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
int amMemory = dag.getMasterMemoryMB();
if (amMemory > maxMem) {
LOG.info(
"AM memory specified above max threshold of cluster. Using max value."
+ ", specified="
+ amMemory
+ ", max="
+ maxMem);
amMemory = maxMem;
}
if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) {
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString());
}
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME));
appContext.setApplicationType(this.applicationType);
if (YARN_APPLICATION_TYPE.equals(this.applicationType)) {
// appContext.setMaxAppAttempts(1); // no retries until Stram is HA
}
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
// Setup security tokens
// If security is enabled get ResourceManager and NameNode delegation tokens.
// Set these tokens on the container so that they are sent as part of application submission.
// This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken
// is also used by ResourceManager to fetch the jars from HDFS and set them up for the
// application master launch.
if (UserGroupInformation.isSecurityEnabled()) {
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
}
// For now, only getting tokens for the default file-system.
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
if (tokens != null) {
for (Token<?> token : tokens) {
LOG.info("Got dt for " + fs.getUri() + "; " + token);
}
}
} finally {
fs.close();
}
addRMDelegationToken(tokenRenewer, credentials);
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
amContainer.setTokens(fsTokens);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
// copy required jar files to dfs, to be localized for containers
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
Path appsBasePath =
new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
Path appPath = new Path(appsBasePath, appId.toString());
String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {}));
LOG.info("libjars: {}", libJarsCsv);
dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, libJarsCsv, localResources, fs);
if (archives != null) {
String[] localFiles = archives.split(",");
String archivesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("archives: {}", archivesCsv);
dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.ARCHIVE, archivesCsv, localResources, fs);
}
if (files != null) {
String[] localFiles = files.split(",");
String filesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("files: {}", filesCsv);
dag.getAttributes().put(LogicalPlan.FILES, filesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, filesCsv, localResources, fs);
}
dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString());
if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) {
/* which would be the most likely case */
Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS);
// use conf client side to pickup any proxy settings from dt-site.xml
dag.setAttribute(
OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf));
}
if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) {
dag.setAttribute(
LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator());
}
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
Path log4jSrc = new Path(log4jPropFile);
Path log4jDst = new Path(appPath, "log4j.props");
fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
log4jRsrc.setType(LocalResourceType.FILE);
log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
log4jRsrc.setSize(log4jFileStatus.getLen());
localResources.put("log4j.properties", log4jRsrc);
}
if (originalAppId != null) {
Path origAppPath = new Path(appsBasePath, this.originalAppId);
LOG.info("Restart from {}", origAppPath);
copyInitialState(origAppPath);
}
// push logical plan to DFS location
Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME);
FSDataOutputStream outStream = fs.create(cfgDst, true);
LogicalPlan.write(this.dag, outStream);
outStream.close();
Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME);
outStream = fs.create(launchConfigDst, true);
conf.writeXml(outStream);
outStream.close();
FileStatus topologyFileStatus = fs.getFileStatus(cfgDst);
LocalResource topologyRsrc = Records.newRecord(LocalResource.class);
topologyRsrc.setType(LocalResourceType.FILE);
topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri()));
topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime());
topologyRsrc.setSize(topologyFileStatus.getLen());
localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc);
// Set local resource info into app master container launch context
amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
// amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
LOG.info("Set the environment for the application master");
Map<String, String> env = new HashMap<String, String>();
// Add application jar(s) location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar(s)
// including ${CLASSPATH} will duplicate the class path in app master, removing it for now
// StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
StringBuilder classPathEnv = new StringBuilder("./*");
String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
for (String c :
StringUtils.isBlank(classpath)
? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
: classpath.split(",")) {
if (c.equals("$HADOOP_CLIENT_CONF_DIR")) {
// SPOI-2501
continue;
}
classPathEnv.append(':');
classPathEnv.append(c.trim());
}
env.put("CLASSPATH", classPathEnv.toString());
// propagate to replace node managers user name (effective in non-secure mode)
env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());
amContainer.setEnvironment(env);
// Set the necessary command to execute the application master
ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30);
// Set java executable command
LOG.info("Setting up app master command");
vargs.add(javaCmd);
if (dag.isDebug()) {
vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n");
}
// Set Xmx based on am memory size
// default heap size 75% of total memory
vargs.add("-Xmx" + (amMemory * 3 / 4) + "m");
vargs.add("-XX:+HeapDumpOnOutOfMemoryError");
vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin");
vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
if (dag.isDebug()) {
vargs.add("-Dlog4j.debug=true");
}
String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
if (loggersLevel != null) {
vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
}
vargs.add(StreamingAppMaster.class.getName());
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
// Get final command
StringBuilder command = new StringBuilder(9 * vargs.size());
for (CharSequence str : vargs) {
command.append(str).append(" ");
}
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
amContainer.setCommands(commands);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(amMemory);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(queueName);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
String specStr =
Objects.toStringHelper("Submitting application: ")
.add("name", appContext.getApplicationName())
.add("queue", appContext.getQueue())
.add("user", UserGroupInformation.getLoginUser())
.add("resource", appContext.getResource())
.toString();
LOG.info(specStr);
if (dag.isDebug()) {
// LOG.info("Full submission context: " + appContext);
}
yarnClient.submitApplication(appContext);
} finally {
fs.close();
}
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
