public MiniSparkShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir)
throws IOException {
mr = new MiniSparkOnYARNCluster("sparkOnYarn");
conf.set("fs.defaultFS", nameNode);
conf.set(
"yarn.resourcemanager.scheduler.class",
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
// disable resource monitoring, although it should be off by default
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
configureImpersonation(conf);
mr.init(conf);
mr.start();
this.conf = mr.getConfig();
}
@Override
public void setupConfiguration(Configuration conf) {
Configuration config = mr.getConfig();
for (Map.Entry<String, String> pair : config) {
conf.set(pair.getKey(), pair.getValue());
}
Path jarPath = new Path("hdfs:///user/hive");
Path hdfsPath = new Path("hdfs:///user/");
try {
FileSystem fs = cluster.getFileSystem();
jarPath = fs.makeQualified(jarPath);
conf.set("hive.jar.directory", jarPath.toString());
fs.mkdirs(jarPath);
hdfsPath = fs.makeQualified(hdfsPath);
conf.set("hive.user.install.directory", hdfsPath.toString());
fs.mkdirs(hdfsPath);
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void shutdown() throws IOException {
mr.stop();
}