public HivePartitionCleanJob(
final JobContext jobContext,
final HiveProcesser p,
final ApplicationContext applicationContext)
throws Exception {
super(jobContext);
this.processer = p;
this.tables = p.getOutputTables();
this.keepDays = p.getKeepDays();
this.tableManager = new CliTableManager(ConfUtil.getWorkConf(jobContext));
}
private void buildHiveConf(String jobType) {
File dir = new File(jobContext.getWorkDir() + File.separator + "hive_conf");
if (!dir.exists()) {
dir.mkdirs();
}
Map<String, String> hive = new HashMap<String, String>();
for (String key : jobContext.getProperties().getAllProperties().keySet()) {
if (key.startsWith("hive-site.")) {
hive.put(key.substring("hive-site.".length()), jobContext.getProperties().getProperty(key));
}
}
if (jobType != null && jobType.equals("HiveJob")) {
Configuration hiveC = ConfUtil.getDefaultHiveSite();
for (String key : hive.keySet()) {
hiveC.set(key, hive.get(key));
}
try {
File xml = new File(dir.getAbsolutePath() + File.separator + "hive-site.xml");
if (xml.exists()) {
xml.delete();
}
xml.createNewFile();
hiveC.writeXml(new FileOutputStream(xml));
} catch (Exception e) {
log.error("create file hive-site.xml error", e);
}
}
String HIVE_CONF_DIR =
jobContext.getWorkDir()
+ File.separator
+ "hive_conf"
+ File.pathSeparator
+ ConfUtil.getHiveConfDir();
envMap.put("HIVE_CONF_DIR", HIVE_CONF_DIR);
}
private void buildHadoopConf(String jobType) {
File dir = new File(jobContext.getWorkDir() + File.separator + "hadoop_conf");
if (!dir.exists()) {
dir.mkdirs();
}
Map<String, String> core = new HashMap<String, String>();
Map<String, String> hdfs = new HashMap<String, String>();
Map<String, String> mapred = new HashMap<String, String>();
Map<String, String> yarn = new HashMap<String, String>();
for (String key : jobContext.getProperties().getAllProperties().keySet()) {
if (key.startsWith("core-site.")) {
core.put(key.substring("core-site.".length()), jobContext.getProperties().getProperty(key));
} else if (key.startsWith("hdfs-site.")) {
hdfs.put(key.substring("hdfs-site.".length()), jobContext.getProperties().getProperty(key));
} else if (key.startsWith("mapred-site.")) {
mapred.put(
key.substring("mapred-site.".length()), jobContext.getProperties().getProperty(key));
} else if (key.startsWith("yarn-site.")) {
yarn.put(key.substring("yarn-site.".length()), jobContext.getProperties().getProperty(key));
}
}
if (jobType != null && (jobType.equals("MapReduceJob") || jobType.equals("HiveJob"))) {
Configuration coreC = ConfUtil.getDefaultCoreSite();
for (String key : core.keySet()) {
coreC.set(key, core.get(key));
}
try {
File xml = new File(dir.getAbsolutePath() + File.separator + "core-site.xml");
if (xml.exists()) {
xml.delete();
}
xml.createNewFile();
coreC.writeXml(new FileOutputStream(xml));
} catch (Exception e) {
log.error("create file core-site.xml error", e);
}
Configuration hdfsC = ConfUtil.getDefaultHdfsSite();
for (String key : hdfs.keySet()) {
hdfsC.set(key, hdfs.get(key));
}
try {
File xml = new File(dir.getAbsolutePath() + File.separator + "hdfs-site.xml");
if (xml.exists()) {
xml.delete();
}
xml.createNewFile();
hdfsC.writeXml(new FileOutputStream(xml));
} catch (Exception e) {
log.error("create file hdfs-site.xml error", e);
}
Configuration mapredC = ConfUtil.getDefaultMapredSite();
for (String key : mapred.keySet()) {
mapredC.set(key, mapred.get(key));
}
try {
File xml = new File(dir.getAbsolutePath() + File.separator + "mapred-site.xml");
if (xml.exists()) {
xml.delete();
}
xml.createNewFile();
mapredC.writeXml(new FileOutputStream(xml));
} catch (Exception e) {
log.error("create file mapred-site.xml error", e);
}
Configuration yarnC = ConfUtil.getDefaultYarnSite();
for (String key : yarn.keySet()) {
yarnC.set(key, mapred.get(key));
}
try {
File xml = new File(dir.getAbsolutePath() + File.separator + "yarn-site.xml");
if (xml.exists()) {
xml.delete();
}
xml.createNewFile();
yarnC.writeXml(new FileOutputStream(xml));
} catch (Exception e) {
log.error("create file yarn-site.xml error", e);
}
}
// HADOOP_CONF_DIR添加2个路径，分别为 WorkDir/hadoop_conf 和 HADOOP_HOME/conf
String HADOOP_CONF_DIR =
jobContext.getWorkDir()
+ File.separator
+ "hadoop_conf"
+ File.pathSeparator
+ ConfUtil.getHadoopConfDir();
envMap.put("HADOOP_CONF_DIR", HADOOP_CONF_DIR);
}
