// 初始化环境
static {
// Properties props = getProperties("hadoop.properties");
conf = new Configuration();
conf.addResource(new Path("F:\\hdfs-site.xml"));
conf.addResource(new Path("F:\\hbase-site.xml"));
conf.set("fs.defaultFS", "hdfs://ju51nn");
try {
String ugi = "";
if (Strings.isNullOrEmpty(ugi)) {
fs = FileSystem.get(conf);
System.out.println("======" + fs);
} else {
UserGroupInformation.createProxyUser(ugi, UserGroupInformation.getLoginUser())
.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
fs = FileSystem.get(conf);
return null;
}
});
}
} catch (Exception e) {
LOG.error("初始化FileSytem对象异常: ", e.getMessage());
e.printStackTrace();
}
}
public static void main(String args[]) {
if (args.length == 0) {
System.out.println("JavaHBaseDistributedScan {master} {tableName}");
}
String master = args[0];
String tableName = args[1];
JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan");
jsc.addJar("SparkHBase.jar");
Configuration conf = HBaseConfiguration.create();
conf.addResource(new Path("/etc/hbase/conf/core-site.xml"));
conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml"));
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
Scan scan = new Scan();
scan.setCaching(100);
JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd =
hbaseContext.hbaseRDD(tableName, scan);
List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect();
results.size();
}
/**
* Returns a new Hadoop Configuration object using the path to the hadoop conf configured in the
* main configuration (flink-conf.yaml). This method is public because its being used in the
* HadoopDataSource.
*/
public static org.apache.hadoop.conf.Configuration getHadoopConfiguration() {
Configuration retConf = new org.apache.hadoop.conf.Configuration();
// We need to load both core-site.xml and hdfs-site.xml to determine the default fs path and
// the hdfs configuration
// Try to load HDFS configuration from Hadoop's own configuration files
// 1. approach: Flink configuration
final String hdfsDefaultPath =
GlobalConfiguration.getString(ConfigConstants.HDFS_DEFAULT_CONFIG, null);
if (hdfsDefaultPath != null) {
retConf.addResource(new org.apache.hadoop.fs.Path(hdfsDefaultPath));
} else {
LOG.debug("Cannot find hdfs-default configuration file");
}
final String hdfsSitePath =
GlobalConfiguration.getString(ConfigConstants.HDFS_SITE_CONFIG, null);
if (hdfsSitePath != null) {
retConf.addResource(new org.apache.hadoop.fs.Path(hdfsSitePath));
} else {
LOG.debug("Cannot find hdfs-site configuration file");
}
// 2. Approach environment variables
String[] possibleHadoopConfPaths = new String[4];
possibleHadoopConfPaths[0] =
GlobalConfiguration.getString(ConfigConstants.PATH_HADOOP_CONFIG, null);
possibleHadoopConfPaths[1] = System.getenv("HADOOP_CONF_DIR");
if (System.getenv("HADOOP_HOME") != null) {
possibleHadoopConfPaths[2] = System.getenv("HADOOP_HOME") + "/conf";
possibleHadoopConfPaths[3] = System.getenv("HADOOP_HOME") + "/etc/hadoop"; // hadoop 2.2
}
for (String possibleHadoopConfPath : possibleHadoopConfPaths) {
if (possibleHadoopConfPath != null) {
if (new File(possibleHadoopConfPath).exists()) {
if (new File(possibleHadoopConfPath + "/core-site.xml").exists()) {
retConf.addResource(
new org.apache.hadoop.fs.Path(possibleHadoopConfPath + "/core-site.xml"));
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding " + possibleHadoopConfPath + "/core-site.xml to hadoop configuration");
}
}
if (new File(possibleHadoopConfPath + "/hdfs-site.xml").exists()) {
retConf.addResource(
new org.apache.hadoop.fs.Path(possibleHadoopConfPath + "/hdfs-site.xml"));
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding " + possibleHadoopConfPath + "/hdfs-site.xml to hadoop configuration");
}
}
}
}
}
return retConf;
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
Configuration conf = new Configuration();
conf.addResource(new Path("src/conf/core-site.xml"));
conf.addResource(new Path("src/conf/mapred-site.xml"));
conf.reloadConfiguration();
@SuppressWarnings("deprecation")
Job job = new Job(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(Map.class);
job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
@Before
public void setUp() throws Exception {
server = new Server();
// Context scontext = new Context();
// scontext.setContextPath("/");
// scontext.setResourceBase(RES_DIR);
// // servlet handler?
// scontext.addServlet("JSP", "*.jsp",
// "org.apache.jasper.servlet.JspServlet");
// scontext.addHandler(new ResourceHandler());
Context root = new Context(server, "/", Context.SESSIONS);
root.setContextPath("/");
root.setResourceBase(RES_DIR);
ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
root.addServlet(sh, "*.jsp");
conf = new Configuration();
conf.addResource("nutch-default.xml");
conf.addResource("nutch-site-test.xml");
http = new Http();
http.setConf(conf);
}
public static void main(String args[]) {
if (args.length == 0) {
System.out.println("JavaHBaseBulkPutExample {master} {tableName} {columnFamily}");
}
String master = args[0];
String tableName = args[1];
String columnFamily = args[2];
JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkPutExample");
List<String> list = new ArrayList<String>();
list.add("1," + columnFamily + ",a,1");
list.add("2," + columnFamily + ",a,2");
list.add("3," + columnFamily + ",a,3");
list.add("4," + columnFamily + ",a,4");
list.add("5," + columnFamily + ",a,5");
JavaRDD<String> rdd = jsc.parallelize(list);
Configuration conf = HBaseConfiguration.create();
conf.addResource(new Path("/opt/hadoop-2.6.0/etc/hadoop/core-site.xml"));
conf.addResource(new Path("/opt/hbase/conf/hbase-site.xml"));
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
hbaseContext.bulkPut(rdd, tableName, new PutFunction(), true);
}
public static Configuration addHbaseResources(Configuration conf) {
conf.addResource("hbase-default.xml");
conf.addResource("hbase-site.xml");
checkDefaultsVersion(conf);
HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
return conf;
}
public HDFSFileClient() throws IOException {
conf = new Configuration();
// Get the requisite environment variable from the host system below.
String hadoop_home = System.getenv("HADOOP_HOME");
// Set the required information needed for the configuration object to work.
conf.addResource(new Path(hadoop_home + "/etc/hadoop/core-site.xml"));
conf.addResource(new Path(hadoop_home + "/etc/hadoop/hdfs-site.xml"));
conf.addResource(new Path(hadoop_home + "/etc/hadoop/mapred-site.xml"));
// Assign the file system object the configuration
fileSystem = FileSystem.get(conf);
}
public void initialise() throws IOException {
Configuration conf = HBaseConfiguration.create();
conf.addResource(new Path(clientConfig));
conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(user, keytabLocation);
System.out.println(conf.toString());
connection = ConnectionFactory.createConnection(conf);
}
public void readFile(String file) throws IOException {
Configuration conf = new Configuration();
conf.addResource(new Path("/opt/hadoop-0.20.0/conf/core-site.xml"));
FileSystem fileSystem = FileSystem.get(conf);
Path path = new Path(file);
if (!fileSystem.exists(path)) {
System.out.println("File " + file + " does not exists");
return;
}
FSDataInputStream in = fileSystem.open(path);
String filename = file.substring(file.lastIndexOf('/') + 1, file.length());
OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(filename)));
byte[] b = new byte[1024];
int numBytes = 0;
while ((numBytes = in.read(b)) > 0) {
out.write(b, 0, numBytes);
}
in.close();
out.close();
fileSystem.close();
}
public void run(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("MatMulMap2 <inPath> <outPath>");
System.exit(-1);
}
inPath = new Path(args[0]);
APath = new Path(inPath, "A");
BPath = new Path(inPath, "B");
outPath = new Path(args[1]);
conf = new Configuration();
conf.addResource("matmul-conf.xml");
FileSystem fs = FileSystem.get(conf);
long start, end;
// prepare
if ((conf.getBoolean("matmul.initialize", true)) || (!fs.exists(inPath))) {
MatMulMap2Prep prep = new MatMulMap2Prep();
prep.setPath(APath, BPath);
prep.run();
}
fs.delete(outPath);
start = System.currentTimeMillis();
conf.setLong("matmul.versionId", start);
waitForJobFinish(configStage());
end = System.currentTimeMillis();
System.out.println("===map2 experiment===<time>[MatMulMap2]: " + (end - start) + " ms");
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.addResource("blat-test-conf.xml");
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
/*
process arguments
*/
if (otherArgs.length != 2) {
System.err.println("Usage: blat <seqfilepath> <ecfilepath>");
System.exit(2);
}
Map<String, String> l = new HashMap<String, String>();
Set<String> r;
Text t = new Text();
FileInputStream fstream = new FileInputStream(otherArgs[0]);
FastaBlockLineReader in = new FastaBlockLineReader(fstream);
int bytes = in.readLine(t, l);
BlatCommand b = new BlatCommand();
r = b.exec(l, otherArgs[1], null);
System.out.println("matches = " + r);
}
@Override
public int run(String[] args) throws Exception {
CommandLine cmd = getCommand(args);
DistCpOptions options = getDistCpOptions(cmd);
Configuration conf = this.getConf();
// inject wf configs
Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));
LOG.info(confPath + " found conf ? " + confPath.getFileSystem(conf).exists(confPath));
conf.addResource(confPath);
String falconFeedStorageType = cmd.getOptionValue("falconFeedStorageType").trim();
Storage.TYPE feedStorageType = Storage.TYPE.valueOf(falconFeedStorageType);
DistCp distCp =
(feedStorageType == Storage.TYPE.FILESYSTEM)
? new CustomReplicator(conf, options)
: new DistCp(conf, options);
LOG.info("Started DistCp");
distCp.execute();
if (feedStorageType == Storage.TYPE.FILESYSTEM) {
executePostProcessing(options); // this only applies for FileSystem Storage.
}
LOG.info("Completed DistCp");
return 0;
}
public static void main(String[] args) throws Exception {
final Configuration configuration = HBaseConfiguration.create();
configuration.addResource("grade.xml");
String tables = configuration.get("hbase.cdn.tables");
if (Strings.isNullOrEmpty(tables)) {
return;
}
List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
List<String> results =
Lists.transform(
list,
new Function<String, String>() {
@Override
public String apply(@Nullable java.lang.String input) {
return String.format(
configuration.get("hdfs.directory.base.db"), new Date(), input);
}
});
String[] arrays =
new String[] {
Joiner.on(",").join(results),
String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
String.format(configuration.get("hdfs.directory.num.result"), new Date())
};
AbstractJob job = new TopNJob();
// job.setStart(true);
int i = ToolRunner.run(configuration, job, arrays);
System.exit(i);
}
public void check(String addr) throws IOException {
Configuration conf = new Configuration();
File fhdfs = new File("hdfs-site.xml");
InputStream inhdfs = new FileInputStream(fhdfs);
File fcore = new File("core-site.xml");
InputStream incore = new FileInputStream(fcore);
conf.addResource(inhdfs);
conf.addResource(incore);
InetSocketAddress inetSocketAddress = new InetSocketAddress(addr, 9000);
DFSClient client = new DFSClient(inetSocketAddress, conf);
if (client.getMissingBlocksCount() > 0) {
System.out.println("[ERROR] missing block count > 0");
this.alert("[" + addr + "-HDFS检查]", "missing block个数大于0");
} else {
System.out.println("[INFO] there is no missing block count in cluster.");
}
DatanodeInfo[] datanodeInfos = client.datanodeReport(HdfsConstants.DatanodeReportType.DEAD);
if (datanodeInfos.length > 0) {
String ips = "";
for (DatanodeInfo info : datanodeInfos) {
ips += info.getIpAddr() + ";";
}
System.out.println("[ERROR] dead node ips is : " + ips);
this.alert("[" + addr + "-HDFS检查]", "DEAD node个数大于0");
} else {
System.out.println("[INFO] dead node size is 0.");
}
org.apache.hadoop.fs.Path path = new Path(this.hdfsPath);
FileSystem fs = null;
FSDataOutputStream output = null;
try {
fs = path.getFileSystem(conf);
if (fs.exists(path)) {
System.out.println("[INFO] " + hdfsPath + " already exists.");
fs.delete(path, false);
}
output = fs.create(path);
for (String line : contents) {
output.write(line.getBytes("UTF-8"));
output.flush();
}
} catch (IOException e) {
e.printStackTrace();
System.out.println("[ERROR] write content failed.");
this.alert("[" + addr + "-HDFS检查]", "HDFS可用性异常，删除写入文件失败");
} finally {
try {
output.close();
System.out.println("[INFO] delete and write file success.");
} catch (IOException e) {
e.printStackTrace();
}
}
client.close();
}
private static Map<String, Set<Path>> getPermMap(Configuration conf) {
String permLoc = conf.get("hdfsproxy.user.permissions.file.location", "user-permissions.xml");
if (conf.getResource(permLoc) == null) {
LOG.warn("HdfsProxy user permissions file not found");
return null;
}
Configuration permConf = new Configuration(false);
permConf.addResource(permLoc);
Map<String, Set<Path>> map = new HashMap<String, Set<Path>>();
for (Map.Entry<String, String> e : permConf) {
String k = e.getKey();
String v = e.getValue();
if (k != null && k.length() != 0 && v != null && v.length() != 0) {
Set<Path> pathSet = new HashSet<Path>();
String[] paths = v.split(",\\s*");
for (String p : paths) {
if (p.length() != 0) {
pathSet.add(new Path(p));
}
}
map.put(k, pathSet);
}
}
return map;
}
private static Map<String, Set<BigInteger>> getCertsMap(Configuration conf) {
String certsLoc = conf.get("hdfsproxy.user.certs.file.location", "user-certs.xml");
if (conf.getResource(certsLoc) == null) {
LOG.warn("HdfsProxy user certs file not found");
return null;
}
Configuration certsConf = new Configuration(false);
certsConf.addResource(certsLoc);
Map<String, Set<BigInteger>> map = new HashMap<String, Set<BigInteger>>();
for (Map.Entry<String, String> e : certsConf) {
String k = e.getKey();
String v = e.getValue().trim();
if (k != null && k.length() != 0 && v != null && v.length() != 0) {
Set<BigInteger> numSet = new HashSet<BigInteger>();
String[] serialnumbers = v.split("\\s*,\\s*");
for (String num : serialnumbers) {
if (num.length() != 0) {
numSet.add(new BigInteger(num, 16));
}
}
map.put(k, numSet);
}
}
return map;
}
static {
Configuration conf = new Configuration(false);
conf.addResource("hdfsproxy-default.xml");
Map<String, Set<Path>> pMap = getPermMap(conf);
permsMap = pMap != null ? pMap : new HashMap<String, Set<Path>>();
Map<String, Set<BigInteger>> cMap = getCertsMap(conf);
certsMap = cMap != null ? cMap : new HashMap<String, Set<BigInteger>>();
}
/**
* Intersect llap-daemon-site.xml configuration properties against an existing Configuration
* object, while resolving any ${} parameters that might be present.
*
* @param raw
* @return configuration object which is a slice of configured
*/
public static Configuration resolve(Configuration configured, String first, String... resources) {
Configuration defaults = new Configuration(false);
defaults.addResource(first);
for (String resource : resources) {
defaults.addResource(resource);
}
Configuration slice = new Configuration(false);
// for everything in defaults, slice out those from the configured
for (Map.Entry<String, String> kv : defaults) {
slice.set(kv.getKey(), configured.get(kv.getKey()));
}
return slice;
}
public HdfsFrameworkConfig() {
// The path is configurable via the mesos.conf.path system property
// so it can be changed when starting up the scheduler via bash
Properties props = System.getProperties();
Path configPath = new Path(props.getProperty("mesos.conf.path", "etc/hadoop/mesos-site.xml"));
Configuration configuration = new Configuration();
configuration.addResource(configPath);
setConf(configuration);
}
private synchronized Configuration getConfiguration(Configuration conf, String confFileName)
throws YarnException, IOException {
InputStream confFileInputStream =
this.rmContext.getConfigurationProvider().getConfigurationInputStream(conf, confFileName);
if (confFileInputStream != null) {
conf.addResource(confFileInputStream);
}
return conf;
}
// 上传本地文件
public void test() throws IOException, URISyntaxException, InterruptedException {
Configuration conf = new Configuration();
conf.addResource("core-site.xml");
conf.addResource("hdfs-site.xml");
FileSystem fs = FileSystem.get(conf);
List<String> ls = listFiles(fs, "hdfs://elkcluster/tmp/");
for (String str : ls) System.out.println(str);
}
/**
* Load the Configuration file with the values from the command line and config files, and place
* stuff in the DistrubutedCacheService as needed
*
* @param conf The Configuration to populate
* @param args The command line arguments
* @throws Exception
*/
public static void initalizeConf(Configuration conf, String[] args) throws Exception {
// Parse the arguments and make sure the required args are there
final CommandLine commandLine;
final Options options = constructGnuOptions();
try {
commandLine = new GnuParser().parse(options, args);
} catch (MissingOptionException ex) {
HelpFormatter help = new HelpFormatter();
help.printHelp("hadoop jar <jarFile> " + FrameworkDriver.class.getCanonicalName(), options);
return;
} catch (Exception ex) {
ex.printStackTrace();
return;
}
final String userConfFilePath = commandLine.getOptionValue("amino_config_file_path", "");
final String aminoDefaultConfigPath = commandLine.getOptionValue("amino_default_config_path");
final String baseDir = commandLine.getOptionValue("base_dir");
stopOnFirstPhase = commandLine.hasOption("stop");
// Set the base dir config value if it was provided.
if (StringUtils.isNotEmpty(baseDir)) {
conf.set(AminoConfiguration.BASE_DIR, baseDir);
}
conf.set(AminoConfiguration.DEFAULT_CONFIGURATION_PATH_KEY, aminoDefaultConfigPath);
// create a single DistributedCacheService so that multiple cache entries are deduped.
// Cache files are added after each config is loaded in case the the property value changes.
final DistributedCacheService distributedCacheService = new DistributedCacheService();
// 1. load AminoDefaults
AminoConfiguration.loadAndMergeWithDefault(conf, true);
distributedCacheService.addFilesToDistributedCache(conf);
// 2. load user config files, allowing them to overwrite
if (!StringUtils.isEmpty(userConfFilePath)) {
for (String path : getUserConfigFiles(userConfFilePath)) {
Configuration userConf = new Configuration(false);
logger.info("Grabbing configuration information from: " + path);
userConf.addResource(new FileInputStream(path));
HadoopConfigurationUtils.mergeConfs(conf, userConf);
}
}
distributedCacheService.addFilesToDistributedCache(conf);
// 3. load command line arguments as properties, allowing them to overwrite
final Properties propertyOverrides = commandLine.getOptionProperties("property_override");
for (Object key : propertyOverrides.keySet()) {
conf.set((String) key, (String) propertyOverrides.get(key));
}
distributedCacheService.addFilesToDistributedCache(conf);
}
public void setUp(boolean redirection) throws Exception {
conf = new Configuration();
conf.addResource("nutch-default.xml");
conf.addResource("nutch-site-test.xml");
http = new Http();
http.setConf(conf);
server = new Server();
if (redirection) {
root = new Context(server, "/redirection", Context.SESSIONS);
root.setAttribute("newContextURL", "/redirect");
} else {
root = new Context(server, "/", Context.SESSIONS);
}
ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
root.addServlet(sh, "*.jsp");
root.setResourceBase(RES_DIR);
}
/**
* Loads properties from distcp-default.xml into configuration object
*
* @return Configuration which includes properties from distcp-default.xml
*/
private static Configuration getDefaultConf() {
Configuration config = new Configuration();
// Propagate properties related to delegation tokens.
String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
if (tokenFile != null) {
config.set("mapreduce.job.credentials.binary", tokenFile);
}
config.addResource(DISTCP_DEFAULT_XML);
return config;
}
/**
* Sets up the default Configuration.
*
* @return The default Configuration
*/
public static Configuration setupConf() {
Configuration conf = new Configuration();
// Get HDFS configuration parameters from Hadoop XML files
conf.addResource(new Path("/Applications/hadoop-2.2.0/etc/hadoop/core-site.xml"));
conf.addResource(new Path("/Applications/hadoop-2.2.0/etc/hadoop/hdfs-site.xml"));
// Output a CSV file
conf.set("mapreduce.output.textoutputformat.separator", SEPARATOR);
return conf;
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.addResource(new Path("/home/impadmin/hadoop-2.2.0/etc/hadoop/core-site.xml"));
Job job = Job.getInstance(conf);
job.setJarByClass(MovieLensSix.class);
job.setMapperClass(TokenizerMapper.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FloatWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public static Configuration getDefaultMapredSite() {
try {
File f = new File("/etc/hadoop/conf/mapred-site.xml");
if (f.exists()) {
Configuration conf = new Configuration(false);
conf.addResource(f.toURI().toURL());
return conf;
}
} catch (IOException e) {
log.error("load $HADOOP_CONF_DIR/mapred-site.xml error", e);
}
return null;
}
public static Configuration getDefaultYarnSite() {
try {
File f = new File(getHadoopConfDir() + File.separator + "yarn-site.xml");
if (f.exists()) {
Configuration conf = new Configuration(false);
conf.addResource(f.toURI().toURL());
return conf;
}
} catch (IOException e) {
log.error("load $HADOOP_CONF_DIR/yarn-site.xml", e);
}
return null;
}
@Test
public void testApplication() throws IOException, Exception {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
lma.prepareDAG(new Application(), conf);
LocalMode.Controller lc = lma.getController();
lc.run(10000); // runs for 10 seconds and quits
} catch (ConstraintViolationException e) {
Assert.fail("constraint violations: " + e.getConstraintViolations());
}
}
