Ejemplo n.º 1
0
  // 初始化环境
  static {
    // Properties props = getProperties("hadoop.properties");

    conf = new Configuration();
    conf.addResource(new Path("F:\\hdfs-site.xml"));
    conf.addResource(new Path("F:\\hbase-site.xml"));
    conf.set("fs.defaultFS", "hdfs://ju51nn");
    try {
      String ugi = "";
      if (Strings.isNullOrEmpty(ugi)) {
        fs = FileSystem.get(conf);
        System.out.println("======" + fs);
      } else {
        UserGroupInformation.createProxyUser(ugi, UserGroupInformation.getLoginUser())
            .doAs(
                new PrivilegedExceptionAction<Void>() {
                  @Override
                  public Void run() throws Exception {
                    fs = FileSystem.get(conf);
                    return null;
                  }
                });
      }
    } catch (Exception e) {
      LOG.error("初始化FileSytem对象异常: ", e.getMessage());
      e.printStackTrace();
    }
  }
  public static void main(String args[]) {
    if (args.length == 0) {
      System.out.println("JavaHBaseDistributedScan  {master} {tableName}");
    }

    String master = args[0];
    String tableName = args[1];

    JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan");
    jsc.addJar("SparkHBase.jar");

    Configuration conf = HBaseConfiguration.create();
    conf.addResource(new Path("/etc/hbase/conf/core-site.xml"));
    conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml"));

    JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);

    Scan scan = new Scan();
    scan.setCaching(100);

    JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd =
        hbaseContext.hbaseRDD(tableName, scan);

    List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect();

    results.size();
  }
Ejemplo n.º 3
0
  /**
   * Returns a new Hadoop Configuration object using the path to the hadoop conf configured in the
   * main configuration (flink-conf.yaml). This method is public because its being used in the
   * HadoopDataSource.
   */
  public static org.apache.hadoop.conf.Configuration getHadoopConfiguration() {
    Configuration retConf = new org.apache.hadoop.conf.Configuration();

    // We need to load both core-site.xml and hdfs-site.xml to determine the default fs path and
    // the hdfs configuration
    // Try to load HDFS configuration from Hadoop's own configuration files
    // 1. approach: Flink configuration
    final String hdfsDefaultPath =
        GlobalConfiguration.getString(ConfigConstants.HDFS_DEFAULT_CONFIG, null);
    if (hdfsDefaultPath != null) {
      retConf.addResource(new org.apache.hadoop.fs.Path(hdfsDefaultPath));
    } else {
      LOG.debug("Cannot find hdfs-default configuration file");
    }

    final String hdfsSitePath =
        GlobalConfiguration.getString(ConfigConstants.HDFS_SITE_CONFIG, null);
    if (hdfsSitePath != null) {
      retConf.addResource(new org.apache.hadoop.fs.Path(hdfsSitePath));
    } else {
      LOG.debug("Cannot find hdfs-site configuration file");
    }

    // 2. Approach environment variables
    String[] possibleHadoopConfPaths = new String[4];
    possibleHadoopConfPaths[0] =
        GlobalConfiguration.getString(ConfigConstants.PATH_HADOOP_CONFIG, null);
    possibleHadoopConfPaths[1] = System.getenv("HADOOP_CONF_DIR");

    if (System.getenv("HADOOP_HOME") != null) {
      possibleHadoopConfPaths[2] = System.getenv("HADOOP_HOME") + "/conf";
      possibleHadoopConfPaths[3] = System.getenv("HADOOP_HOME") + "/etc/hadoop"; // hadoop 2.2
    }

    for (String possibleHadoopConfPath : possibleHadoopConfPaths) {
      if (possibleHadoopConfPath != null) {
        if (new File(possibleHadoopConfPath).exists()) {
          if (new File(possibleHadoopConfPath + "/core-site.xml").exists()) {
            retConf.addResource(
                new org.apache.hadoop.fs.Path(possibleHadoopConfPath + "/core-site.xml"));

            if (LOG.isDebugEnabled()) {
              LOG.debug(
                  "Adding " + possibleHadoopConfPath + "/core-site.xml to hadoop configuration");
            }
          }
          if (new File(possibleHadoopConfPath + "/hdfs-site.xml").exists()) {
            retConf.addResource(
                new org.apache.hadoop.fs.Path(possibleHadoopConfPath + "/hdfs-site.xml"));

            if (LOG.isDebugEnabled()) {
              LOG.debug(
                  "Adding " + possibleHadoopConfPath + "/hdfs-site.xml to hadoop configuration");
            }
          }
        }
      }
    }
    return retConf;
  }
Ejemplo n.º 4
0
  public static void main(String[] args) throws Exception {

    if (args.length != 2) {
      System.err.println("Usage: wordcount <in> <out>");
      System.exit(2);
    }

    Configuration conf = new Configuration();

    conf.addResource(new Path("src/conf/core-site.xml"));
    conf.addResource(new Path("src/conf/mapred-site.xml"));
    conf.reloadConfiguration();

    @SuppressWarnings("deprecation")
    Job job = new Job(conf, "word count");
    job.setJarByClass(WordCount.class);

    job.setMapperClass(Map.class);
    job.setCombinerClass(Reduce.class);
    job.setReducerClass(Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
  @Before
  public void setUp() throws Exception {

    server = new Server();

    //		Context scontext = new Context();
    //		scontext.setContextPath("/");
    //		scontext.setResourceBase(RES_DIR);
    //		// servlet handler?
    //		scontext.addServlet("JSP", "*.jsp",
    //				"org.apache.jasper.servlet.JspServlet");
    //		scontext.addHandler(new ResourceHandler());

    Context root = new Context(server, "/", Context.SESSIONS);
    root.setContextPath("/");
    root.setResourceBase(RES_DIR);
    ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
    root.addServlet(sh, "*.jsp");

    conf = new Configuration();
    conf.addResource("nutch-default.xml");
    conf.addResource("nutch-site-test.xml");

    http = new Http();
    http.setConf(conf);
  }
  public static void main(String args[]) {
    if (args.length == 0) {
      System.out.println("JavaHBaseBulkPutExample  {master} {tableName} {columnFamily}");
    }

    String master = args[0];
    String tableName = args[1];
    String columnFamily = args[2];

    JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkPutExample");

    List<String> list = new ArrayList<String>();
    list.add("1," + columnFamily + ",a,1");
    list.add("2," + columnFamily + ",a,2");
    list.add("3," + columnFamily + ",a,3");
    list.add("4," + columnFamily + ",a,4");
    list.add("5," + columnFamily + ",a,5");
    JavaRDD<String> rdd = jsc.parallelize(list);

    Configuration conf = HBaseConfiguration.create();
    conf.addResource(new Path("/opt/hadoop-2.6.0/etc/hadoop/core-site.xml"));
    conf.addResource(new Path("/opt/hbase/conf/hbase-site.xml"));

    JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);

    hbaseContext.bulkPut(rdd, tableName, new PutFunction(), true);
  }
Ejemplo n.º 7
0
  public static Configuration addHbaseResources(Configuration conf) {
    conf.addResource("hbase-default.xml");
    conf.addResource("hbase-site.xml");

    checkDefaultsVersion(conf);
    HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
    return conf;
  }
Ejemplo n.º 8
0
  public HDFSFileClient() throws IOException {
    conf = new Configuration();
    // Get the requisite environment variable from the host system below.
    String hadoop_home = System.getenv("HADOOP_HOME");
    // Set the required information needed for the configuration object to work.
    conf.addResource(new Path(hadoop_home + "/etc/hadoop/core-site.xml"));
    conf.addResource(new Path(hadoop_home + "/etc/hadoop/hdfs-site.xml"));
    conf.addResource(new Path(hadoop_home + "/etc/hadoop/mapred-site.xml"));

    // Assign the file system object the configuration
    fileSystem = FileSystem.get(conf);
  }
Ejemplo n.º 9
0
  public void initialise() throws IOException {
    Configuration conf = HBaseConfiguration.create();
    conf.addResource(new Path(clientConfig));
    conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));

    UserGroupInformation.setConfiguration(conf);
    UserGroupInformation.loginUserFromKeytab(user, keytabLocation);

    System.out.println(conf.toString());

    connection = ConnectionFactory.createConnection(conf);
  }
Ejemplo n.º 10
0
  public void readFile(String file) throws IOException {
    Configuration conf = new Configuration();
    conf.addResource(new Path("/opt/hadoop-0.20.0/conf/core-site.xml"));

    FileSystem fileSystem = FileSystem.get(conf);

    Path path = new Path(file);
    if (!fileSystem.exists(path)) {
      System.out.println("File " + file + " does not exists");
      return;
    }

    FSDataInputStream in = fileSystem.open(path);

    String filename = file.substring(file.lastIndexOf('/') + 1, file.length());

    OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(filename)));

    byte[] b = new byte[1024];
    int numBytes = 0;
    while ((numBytes = in.read(b)) > 0) {
      out.write(b, 0, numBytes);
    }

    in.close();
    out.close();
    fileSystem.close();
  }
Ejemplo n.º 11
0
  public void run(String[] args) throws Exception {
    if (args.length != 2) {
      System.out.println("MatMulMap2 <inPath> <outPath>");
      System.exit(-1);
    }
    inPath = new Path(args[0]);
    APath = new Path(inPath, "A");
    BPath = new Path(inPath, "B");
    outPath = new Path(args[1]);
    conf = new Configuration();
    conf.addResource("matmul-conf.xml");
    FileSystem fs = FileSystem.get(conf);
    long start, end;

    // prepare
    if ((conf.getBoolean("matmul.initialize", true)) || (!fs.exists(inPath))) {
      MatMulMap2Prep prep = new MatMulMap2Prep();
      prep.setPath(APath, BPath);
      prep.run();
    }

    fs.delete(outPath);

    start = System.currentTimeMillis();
    conf.setLong("matmul.versionId", start);
    waitForJobFinish(configStage());
    end = System.currentTimeMillis();

    System.out.println("===map2 experiment===<time>[MatMulMap2]: " + (end - start) + " ms");
  }
Ejemplo n.º 12
0
  public static void main(String[] args) throws Exception {

    Configuration conf = new Configuration();

    conf.addResource("blat-test-conf.xml");
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

    /*
    process arguments
     */

    if (otherArgs.length != 2) {
      System.err.println("Usage: blat <seqfilepath> <ecfilepath>");
      System.exit(2);
    }

    Map<String, String> l = new HashMap<String, String>();
    Set<String> r;

    Text t = new Text();
    FileInputStream fstream = new FileInputStream(otherArgs[0]);
    FastaBlockLineReader in = new FastaBlockLineReader(fstream);
    int bytes = in.readLine(t, l);

    BlatCommand b = new BlatCommand();
    r = b.exec(l, otherArgs[1], null);

    System.out.println("matches = " + r);
  }
Ejemplo n.º 13
0
  @Override
  public int run(String[] args) throws Exception {
    CommandLine cmd = getCommand(args);
    DistCpOptions options = getDistCpOptions(cmd);

    Configuration conf = this.getConf();
    // inject wf configs
    Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));

    LOG.info(confPath + " found conf ? " + confPath.getFileSystem(conf).exists(confPath));
    conf.addResource(confPath);

    String falconFeedStorageType = cmd.getOptionValue("falconFeedStorageType").trim();
    Storage.TYPE feedStorageType = Storage.TYPE.valueOf(falconFeedStorageType);

    DistCp distCp =
        (feedStorageType == Storage.TYPE.FILESYSTEM)
            ? new CustomReplicator(conf, options)
            : new DistCp(conf, options);
    LOG.info("Started DistCp");
    distCp.execute();

    if (feedStorageType == Storage.TYPE.FILESYSTEM) {
      executePostProcessing(options); // this only applies for FileSystem Storage.
    }

    LOG.info("Completed DistCp");
    return 0;
  }
Ejemplo n.º 14
0
  public static void main(String[] args) throws Exception {
    final Configuration configuration = HBaseConfiguration.create();
    configuration.addResource("grade.xml");
    String tables = configuration.get("hbase.cdn.tables");
    if (Strings.isNullOrEmpty(tables)) {
      return;
    }
    List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
    List<String> results =
        Lists.transform(
            list,
            new Function<String, String>() {
              @Override
              public String apply(@Nullable java.lang.String input) {
                return String.format(
                    configuration.get("hdfs.directory.base.db"), new Date(), input);
              }
            });

    String[] arrays =
        new String[] {
          Joiner.on(",").join(results),
          String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
          String.format(configuration.get("hdfs.directory.num.result"), new Date())
        };
    AbstractJob job = new TopNJob();
    //        job.setStart(true);
    int i = ToolRunner.run(configuration, job, arrays);
    System.exit(i);
  }
Ejemplo n.º 15
0
  public void check(String addr) throws IOException {
    Configuration conf = new Configuration();
    File fhdfs = new File("hdfs-site.xml");
    InputStream inhdfs = new FileInputStream(fhdfs);
    File fcore = new File("core-site.xml");
    InputStream incore = new FileInputStream(fcore);
    conf.addResource(inhdfs);
    conf.addResource(incore);

    InetSocketAddress inetSocketAddress = new InetSocketAddress(addr, 9000);
    DFSClient client = new DFSClient(inetSocketAddress, conf);

    if (client.getMissingBlocksCount() > 0) {
      System.out.println("[ERROR] missing block count > 0");
      this.alert("[" + addr + "-HDFS检查]", "missing block个数大于0");
    } else {
      System.out.println("[INFO] there is no missing block count in cluster.");
    }

    DatanodeInfo[] datanodeInfos = client.datanodeReport(HdfsConstants.DatanodeReportType.DEAD);
    if (datanodeInfos.length > 0) {
      String ips = "";
      for (DatanodeInfo info : datanodeInfos) {
        ips += info.getIpAddr() + ";";
      }
      System.out.println("[ERROR] dead node ips is : " + ips);
      this.alert("[" + addr + "-HDFS检查]", "DEAD node个数大于0");
    } else {
      System.out.println("[INFO] dead node size is 0.");
    }

    org.apache.hadoop.fs.Path path = new Path(this.hdfsPath);
    FileSystem fs = null;
    FSDataOutputStream output = null;
    try {
      fs = path.getFileSystem(conf);
      if (fs.exists(path)) {
        System.out.println("[INFO] " + hdfsPath + " already exists.");
        fs.delete(path, false);
      }

      output = fs.create(path);
      for (String line : contents) {
        output.write(line.getBytes("UTF-8"));
        output.flush();
      }
    } catch (IOException e) {
      e.printStackTrace();
      System.out.println("[ERROR] write content failed.");
      this.alert("[" + addr + "-HDFS检查]", "HDFS可用性异常,删除写入文件失败");
    } finally {
      try {
        output.close();
        System.out.println("[INFO] delete and write file success.");
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
    client.close();
  }
Ejemplo n.º 16
0
 private static Map<String, Set<Path>> getPermMap(Configuration conf) {
   String permLoc = conf.get("hdfsproxy.user.permissions.file.location", "user-permissions.xml");
   if (conf.getResource(permLoc) == null) {
     LOG.warn("HdfsProxy user permissions file not found");
     return null;
   }
   Configuration permConf = new Configuration(false);
   permConf.addResource(permLoc);
   Map<String, Set<Path>> map = new HashMap<String, Set<Path>>();
   for (Map.Entry<String, String> e : permConf) {
     String k = e.getKey();
     String v = e.getValue();
     if (k != null && k.length() != 0 && v != null && v.length() != 0) {
       Set<Path> pathSet = new HashSet<Path>();
       String[] paths = v.split(",\\s*");
       for (String p : paths) {
         if (p.length() != 0) {
           pathSet.add(new Path(p));
         }
       }
       map.put(k, pathSet);
     }
   }
   return map;
 }
Ejemplo n.º 17
0
 private static Map<String, Set<BigInteger>> getCertsMap(Configuration conf) {
   String certsLoc = conf.get("hdfsproxy.user.certs.file.location", "user-certs.xml");
   if (conf.getResource(certsLoc) == null) {
     LOG.warn("HdfsProxy user certs file not found");
     return null;
   }
   Configuration certsConf = new Configuration(false);
   certsConf.addResource(certsLoc);
   Map<String, Set<BigInteger>> map = new HashMap<String, Set<BigInteger>>();
   for (Map.Entry<String, String> e : certsConf) {
     String k = e.getKey();
     String v = e.getValue().trim();
     if (k != null && k.length() != 0 && v != null && v.length() != 0) {
       Set<BigInteger> numSet = new HashSet<BigInteger>();
       String[] serialnumbers = v.split("\\s*,\\s*");
       for (String num : serialnumbers) {
         if (num.length() != 0) {
           numSet.add(new BigInteger(num, 16));
         }
       }
       map.put(k, numSet);
     }
   }
   return map;
 }
Ejemplo n.º 18
0
 static {
   Configuration conf = new Configuration(false);
   conf.addResource("hdfsproxy-default.xml");
   Map<String, Set<Path>> pMap = getPermMap(conf);
   permsMap = pMap != null ? pMap : new HashMap<String, Set<Path>>();
   Map<String, Set<BigInteger>> cMap = getCertsMap(conf);
   certsMap = cMap != null ? cMap : new HashMap<String, Set<BigInteger>>();
 }
Ejemplo n.º 19
0
  /**
   * Intersect llap-daemon-site.xml configuration properties against an existing Configuration
   * object, while resolving any ${} parameters that might be present.
   *
   * @param raw
   * @return configuration object which is a slice of configured
   */
  public static Configuration resolve(Configuration configured, String first, String... resources) {
    Configuration defaults = new Configuration(false);

    defaults.addResource(first);

    for (String resource : resources) {
      defaults.addResource(resource);
    }

    Configuration slice = new Configuration(false);
    // for everything in defaults, slice out those from the configured
    for (Map.Entry<String, String> kv : defaults) {
      slice.set(kv.getKey(), configured.get(kv.getKey()));
    }

    return slice;
  }
Ejemplo n.º 20
0
 public HdfsFrameworkConfig() {
   // The path is configurable via the mesos.conf.path system property
   // so it can be changed when starting up the scheduler via bash
   Properties props = System.getProperties();
   Path configPath = new Path(props.getProperty("mesos.conf.path", "etc/hadoop/mesos-site.xml"));
   Configuration configuration = new Configuration();
   configuration.addResource(configPath);
   setConf(configuration);
 }
Ejemplo n.º 21
0
 private synchronized Configuration getConfiguration(Configuration conf, String confFileName)
     throws YarnException, IOException {
   InputStream confFileInputStream =
       this.rmContext.getConfigurationProvider().getConfigurationInputStream(conf, confFileName);
   if (confFileInputStream != null) {
     conf.addResource(confFileInputStream);
   }
   return conf;
 }
Ejemplo n.º 22
0
  // 上传本地文件
  public void test() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.addResource("core-site.xml");
    conf.addResource("hdfs-site.xml");
    FileSystem fs = FileSystem.get(conf);

    List<String> ls = listFiles(fs, "hdfs://elkcluster/tmp/");
    for (String str : ls) System.out.println(str);
  }
Ejemplo n.º 23
0
  /**
   * Load the Configuration file with the values from the command line and config files, and place
   * stuff in the DistrubutedCacheService as needed
   *
   * @param conf The Configuration to populate
   * @param args The command line arguments
   * @throws Exception
   */
  public static void initalizeConf(Configuration conf, String[] args) throws Exception {
    // Parse the arguments and make sure the required args are there
    final CommandLine commandLine;
    final Options options = constructGnuOptions();
    try {
      commandLine = new GnuParser().parse(options, args);
    } catch (MissingOptionException ex) {
      HelpFormatter help = new HelpFormatter();
      help.printHelp("hadoop jar <jarFile> " + FrameworkDriver.class.getCanonicalName(), options);
      return;
    } catch (Exception ex) {
      ex.printStackTrace();
      return;
    }

    final String userConfFilePath = commandLine.getOptionValue("amino_config_file_path", "");
    final String aminoDefaultConfigPath = commandLine.getOptionValue("amino_default_config_path");
    final String baseDir = commandLine.getOptionValue("base_dir");

    stopOnFirstPhase = commandLine.hasOption("stop");

    // Set the base dir config value if it was provided.
    if (StringUtils.isNotEmpty(baseDir)) {
      conf.set(AminoConfiguration.BASE_DIR, baseDir);
    }

    conf.set(AminoConfiguration.DEFAULT_CONFIGURATION_PATH_KEY, aminoDefaultConfigPath);

    // create a single DistributedCacheService so that multiple cache entries are deduped.
    // Cache files are added after each config is loaded in case the the property value changes.
    final DistributedCacheService distributedCacheService = new DistributedCacheService();

    // 1. load AminoDefaults
    AminoConfiguration.loadAndMergeWithDefault(conf, true);
    distributedCacheService.addFilesToDistributedCache(conf);

    // 2. load user config files, allowing them to overwrite
    if (!StringUtils.isEmpty(userConfFilePath)) {
      for (String path : getUserConfigFiles(userConfFilePath)) {
        Configuration userConf = new Configuration(false);
        logger.info("Grabbing configuration information from: " + path);
        userConf.addResource(new FileInputStream(path));
        HadoopConfigurationUtils.mergeConfs(conf, userConf);
      }
    }
    distributedCacheService.addFilesToDistributedCache(conf);

    // 3. load command line arguments as properties, allowing them to overwrite
    final Properties propertyOverrides = commandLine.getOptionProperties("property_override");
    for (Object key : propertyOverrides.keySet()) {
      conf.set((String) key, (String) propertyOverrides.get(key));
    }
    distributedCacheService.addFilesToDistributedCache(conf);
  }
Ejemplo n.º 24
0
  public void setUp(boolean redirection) throws Exception {
    conf = new Configuration();
    conf.addResource("nutch-default.xml");
    conf.addResource("nutch-site-test.xml");

    http = new Http();
    http.setConf(conf);

    server = new Server();

    if (redirection) {
      root = new Context(server, "/redirection", Context.SESSIONS);
      root.setAttribute("newContextURL", "/redirect");
    } else {
      root = new Context(server, "/", Context.SESSIONS);
    }

    ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
    root.addServlet(sh, "*.jsp");
    root.setResourceBase(RES_DIR);
  }
Ejemplo n.º 25
0
  /**
   * Loads properties from distcp-default.xml into configuration object
   *
   * @return Configuration which includes properties from distcp-default.xml
   */
  private static Configuration getDefaultConf() {
    Configuration config = new Configuration();

    // Propagate properties related to delegation tokens.
    String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
    if (tokenFile != null) {
      config.set("mapreduce.job.credentials.binary", tokenFile);
    }

    config.addResource(DISTCP_DEFAULT_XML);
    return config;
  }
Ejemplo n.º 26
0
  /**
   * Sets up the default Configuration.
   *
   * @return The default Configuration
   */
  public static Configuration setupConf() {
    Configuration conf = new Configuration();

    // Get HDFS configuration parameters from Hadoop XML files
    conf.addResource(new Path("/Applications/hadoop-2.2.0/etc/hadoop/core-site.xml"));
    conf.addResource(new Path("/Applications/hadoop-2.2.0/etc/hadoop/hdfs-site.xml"));

    // Output a CSV file
    conf.set("mapreduce.output.textoutputformat.separator", SEPARATOR);

    return conf;
  }
Ejemplo n.º 27
0
 public static void main(String[] args) throws Exception {
   Configuration conf = new Configuration();
   conf.addResource(new Path("/home/impadmin/hadoop-2.2.0/etc/hadoop/core-site.xml"));
   Job job = Job.getInstance(conf);
   job.setJarByClass(MovieLensSix.class);
   job.setMapperClass(TokenizerMapper.class);
   job.setReducerClass(IntSumReducer.class);
   job.setOutputKeyClass(Text.class);
   job.setOutputValueClass(FloatWritable.class);
   FileInputFormat.addInputPath(job, new Path(args[0]));
   FileOutputFormat.setOutputPath(job, new Path(args[1]));
   System.exit(job.waitForCompletion(true) ? 0 : 1);
 }
Ejemplo n.º 28
0
 public static Configuration getDefaultMapredSite() {
   try {
     File f = new File("/etc/hadoop/conf/mapred-site.xml");
     if (f.exists()) {
       Configuration conf = new Configuration(false);
       conf.addResource(f.toURI().toURL());
       return conf;
     }
   } catch (IOException e) {
     log.error("load $HADOOP_CONF_DIR/mapred-site.xml error", e);
   }
   return null;
 }
Ejemplo n.º 29
0
 public static Configuration getDefaultYarnSite() {
   try {
     File f = new File(getHadoopConfDir() + File.separator + "yarn-site.xml");
     if (f.exists()) {
       Configuration conf = new Configuration(false);
       conf.addResource(f.toURI().toURL());
       return conf;
     }
   } catch (IOException e) {
     log.error("load $HADOOP_CONF_DIR/yarn-site.xml", e);
   }
   return null;
 }
 @Test
 public void testApplication() throws IOException, Exception {
   try {
     LocalMode lma = LocalMode.newInstance();
     Configuration conf = new Configuration(false);
     conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
     lma.prepareDAG(new Application(), conf);
     LocalMode.Controller lc = lma.getController();
     lc.run(10000); // runs for 10 seconds and quits
   } catch (ConstraintViolationException e) {
     Assert.fail("constraint violations: " + e.getConstraintViolations());
   }
 }