static { try { // call newInstance() instead of using a shared instance from a cache // to avoid accidentally having it closed by someone else FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(CONF), CONF); if (!(fs instanceof DistributedFileSystem)) { String error = "Cannot connect to HDFS. " + CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY + "(" + CONF.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) + ")" + " might be set incorrectly"; throw new RuntimeException(error); } DFS = (DistributedFileSystem) fs; } catch (IOException e) { throw new RuntimeException("couldn't retrieve FileSystem:\n" + e.getMessage(), e); } SUPPORTS_VOLUME_ID = CONF.getBoolean( DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); }
@Test public void testMRMaxLine() throws Exception { final int MAXPOS = 1024 * 1024; final int MAXLINE = 10 * 1024; final int BUF = 64 * 1024; final InputStream infNull = new InputStream() { int position = 0; final int MAXPOSBUF = 1024 * 1024 + BUF; // max LRR pos + LineReader buf @Override public int read() { ++position; return 0; } @Override public int read(byte[] b) { assertTrue("Read too many bytes from the stream", position < MAXPOSBUF); Arrays.fill(b, (byte) 0); position += b.length; return b.length; } }; final LongWritable key = new LongWritable(); final Text val = new Text(); LOG.info("Reading a line from /dev/null"); final Configuration conf = new Configuration(false); conf.setInt(org.apache.hadoop.mapreduce.lib.input.LineRecordReader.MAX_LINE_LENGTH, MAXLINE); conf.setInt("io.file.buffer.size", BUF); // used by LRR final LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf); assertFalse("Read a line from null", lrr.next(key, val)); }
public static void main(String[] args) throws Exception { final Configuration configuration = HBaseConfiguration.create(); configuration.addResource("grade.xml"); String tables = configuration.get("hbase.cdn.tables"); if (Strings.isNullOrEmpty(tables)) { return; } List<String> list = Lists.newArrayList(Splitter.on(",").split(tables)); List<String> results = Lists.transform( list, new Function<String, String>() { @Override public String apply(@Nullable java.lang.String input) { return String.format( configuration.get("hdfs.directory.base.db"), new Date(), input); } }); String[] arrays = new String[] { Joiner.on(",").join(results), String.format(configuration.get("hdfs.directory.num.middle"), new Date()), String.format(configuration.get("hdfs.directory.num.result"), new Date()) }; AbstractJob job = new TopNJob(); // job.setStart(true); int i = ToolRunner.run(configuration, job, arrays); System.exit(i); }
/** * Add a {@link Path} to the list of inputs for the map-reduce job. * * @param job The {@link Job} to modify * @param path {@link Path} to be added to the list of inputs for the map-reduce job. */ public static void addInputPath(Job job, Path path) throws IOException { Configuration conf = job.getConfiguration(); path = path.getFileSystem(conf).makeQualified(path); String dirStr = StringUtils.escapeString(path.toString()); String dirs = conf.get(INPUT_DIR); conf.set(INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr); }
@Test public void testNonDefaultFS() throws IOException { FileSystem fs = cluster.getFileSystem(); Configuration conf = fs.getConf(); conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString()); TestTrash.trashNonDefaultFS(conf); }
private static void dumpConfig(Configuration conf, StringBuilder sb) { Iterator<Map.Entry<String, String>> configIter = conf.iterator(); List<Map.Entry<String, String>> configVals = new ArrayList<>(); while (configIter.hasNext()) { configVals.add(configIter.next()); } Collections.sort( configVals, new Comparator<Map.Entry<String, String>>() { @Override public int compare(Map.Entry<String, String> ent, Map.Entry<String, String> ent2) { return ent.getKey().compareTo(ent2.getKey()); } }); for (Map.Entry<String, String> entry : configVals) { // use get() to make sure variable substitution works if (entry.getKey().toLowerCase().contains("path")) { StringTokenizer st = new StringTokenizer(conf.get(entry.getKey()), File.pathSeparator); sb.append(entry.getKey()).append("=\n"); while (st.hasMoreTokens()) { sb.append(" ").append(st.nextToken()).append(File.pathSeparator).append('\n'); } } else { sb.append(entry.getKey()).append('=').append(conf.get(entry.getKey())).append('\n'); } } }
private static Map<String, Set<Path>> getPermMap(Configuration conf) { String permLoc = conf.get("hdfsproxy.user.permissions.file.location", "user-permissions.xml"); if (conf.getResource(permLoc) == null) { LOG.warn("HdfsProxy user permissions file not found"); return null; } Configuration permConf = new Configuration(false); permConf.addResource(permLoc); Map<String, Set<Path>> map = new HashMap<String, Set<Path>>(); for (Map.Entry<String, String> e : permConf) { String k = e.getKey(); String v = e.getValue(); if (k != null && k.length() != 0 && v != null && v.length() != 0) { Set<Path> pathSet = new HashSet<Path>(); String[] paths = v.split(",\\s*"); for (String p : paths) { if (p.length() != 0) { pathSet.add(new Path(p)); } } map.put(k, pathSet); } } return map; }
/** * Remove the @Ignore to try out timeout and retry asettings * * @throws IOException */ @Ignore @Test public void testTimeoutAndRetries() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; LOG.info("Start"); try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } LOG.info("Stop"); assertTrue(t != null); }
/** * Test that operation timeout prevails over rpc default timeout and retries, etc. * * @throws IOException */ @Test public void testRocTimeout() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); int pause = 10; localConfig.setInt("hbase.client.pause", pause); localConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10); // Set the operation timeout to be < the pause. Expectation is that after first pause, we will // fail out of the rpc because the rpc timeout will have been set to the operation tiemout // and it has expired. Otherwise, if this functionality is broke, all retries will be run -- // all ten of them -- and we'll get the RetriesExhaustedException exception. localConfig.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, pause - 1); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } assertTrue(t != null); }
@Override protected void setup(Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); a = Double.parseDouble(conf.get("noise_value_0")); // 平滑因子 b = Double.parseDouble(conf.get("noise_value_1")); // 平滑因子 };
@Test public void testConnectionPing() throws Exception { Configuration conf = new Configuration(); int pingInterval = 50; conf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true); conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval); final Server server = new RPC.Builder(conf) .setProtocol(TestProtocol.class) .setInstance(new TestImpl()) .setBindAddress(ADDRESS) .setPort(0) .setNumHandlers(5) .setVerbose(true) .build(); server.start(); final TestProtocol proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, server.getListenerAddress(), conf); try { // this call will throw exception if server couldn't decode the ping proxy.sleep(pingInterval * 4); } finally { if (proxy != null) RPC.stopProxy(proxy); } }
@Test public void testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set( YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); // upload default configurations uploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.init(configuration); rm.start(); } catch (Exception ex) { fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); uploadConfiguration(coreConf, "core-site.xml"); rm.adminService.refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest.newInstance()); Assert.assertTrue(ProxyUsers.getProxyGroups().get("hadoop.proxyuser.test.groups").size() == 1); Assert.assertTrue( ProxyUsers.getProxyGroups().get("hadoop.proxyuser.test.groups").contains("test_groups")); Assert.assertTrue(ProxyUsers.getProxyHosts().get("hadoop.proxyuser.test.hosts").size() == 1); Assert.assertTrue( ProxyUsers.getProxyHosts().get("hadoop.proxyuser.test.hosts").contains("test_hosts")); }
@Test(timeout = 20000) public void testAppSubmissionWithInvalidDelegationToken() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); MockRM rm = new MockRM(conf); ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1), "BOGUS"); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); fail("Error was excepted."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Bad header found in token storage")); } }
/** Test that all open files are closed when client dies abnormally. */ public void testDFSClientDeath() throws IOException { Configuration conf = new Configuration(); System.out.println("Testing adbornal client death."); if (simulatedStorage) { conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); } MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fs = cluster.getFileSystem(); DistributedFileSystem dfs = (DistributedFileSystem) fs; DFSClient dfsclient = dfs.dfs; try { // create a new file in home directory. Do not close it. // Path file1 = new Path("/clienttest.dat"); FSDataOutputStream stm = createFile(fs, file1, 1); System.out.println("Created file clienttest.dat"); // write to file writeFile(stm); // close the dfsclient before closing the output stream. // This should close all existing file. dfsclient.close(); // reopen file system and verify that file exists. assertTrue( file1 + " does not exist.", AppendTestUtil.createHdfsWithDifferentUsername(conf).exists(file1)); } finally { cluster.shutdown(); } }
@Override public void setup(Reducer<IntWritable, Text, NullWritable, NullWritable>.Context context) { Configuration conf = context.getConfiguration(); FileSystem fs; try { fs = FileSystem.get(conf); } catch (Exception e) { throw new RuntimeException("Error opening the FileSystem!"); } RetrievalEnvironment env = null; try { env = new RetrievalEnvironment(conf.get(Constants.IndexPath), fs); } catch (IOException e) { throw new RuntimeException("Unable to create RetrievalEnvironment!"); } collectionDocumentCount = env.readCollectionDocumentCount(); try { out = fs.create(new Path(env.getTermDocVectorsForwardIndex()), true); out.writeInt(env.readDocnoOffset()); out.writeInt(collectionDocumentCount); } catch (Exception e) { throw new RuntimeException("Error in creating files!"); } }
public int run(String[] args) throws Exception { Configuration conf = getConf(); GenericOptionsParser gop = new GenericOptionsParser(conf, args); conf = gop.getConfiguration(); Job job = new Job(conf, conf.get("job_name")); FileInputFormat.addInputPaths(job, conf.get("input_dir")); Path output = new Path(conf.get("output_dir")); FileOutputFormat.setOutputPath(job, output); output.getFileSystem(conf).delete(output, true); job.setJarByClass(BrowerLogFormatMR.class); job.setMapperClass(BrowerLogFormatMapper.class); job.setReducerClass(BrowerLogFormatReducer.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(1); int code = job.waitForCompletion(true) ? 0 : 1; return code; }
@Override public void configure(Configuration conf, FileSystem fs) { // read stopwords from file (stopwords will be empty set if file does not exist or is empty) String stopwordsFile = conf.get(Constants.StopwordList); stopwords = readInput(fs, stopwordsFile); String stemmedStopwordsFile = conf.get(Constants.StemmedStopwordList); stemmedStopwords = readInput(fs, stemmedStopwordsFile); isStopwordRemoval = !stopwords.isEmpty(); isStemming = conf.getBoolean(Constants.Stemming, true); VocabularyWritable vocab; try { vocab = (VocabularyWritable) HadoopAlign.loadVocab(new Path(conf.get(Constants.CollectionVocab)), fs); setVocab(vocab); } catch (Exception e) { LOG.warn("No vocabulary provided to tokenizer."); vocab = null; } LOG.warn( "Stemming is " + isStemming + "; Stopword removal is " + isStopwordRemoval + "; number of stopwords: " + stopwords.size() + "; stemmed: " + stemmedStopwords.size()); }
@Override public synchronized void serviceInit(Configuration conf) { this.maxTaskFailuresPerNode = conf.getInt( TezConfiguration.TEZ_MAX_TASK_FAILURES_PER_NODE, TezConfiguration.TEZ_MAX_TASK_FAILURES_PER_NODE_DEFAULT); this.nodeBlacklistingEnabled = conf.getBoolean( TezConfiguration.TEZ_NODE_BLACKLISTING_ENABLED, TezConfiguration.TEZ_NODE_BLACKLISTING_ENABLED_DEFAULT); this.blacklistDisablePercent = conf.getInt( TezConfiguration.TEZ_NODE_BLACKLISTING_IGNORE_THRESHOLD, TezConfiguration.TEZ_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT); LOG.info( "blacklistDisablePercent is " + blacklistDisablePercent + ", blacklistingEnabled: " + nodeBlacklistingEnabled + ", maxTaskFailuresPerNode: " + maxTaskFailuresPerNode); if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) { throw new TezUncheckedException( "Invalid blacklistDisablePercent: " + blacklistDisablePercent + ". Should be an integer between 0 and 100 or -1 to disabled"); } }
private static Map<String, Set<BigInteger>> getCertsMap(Configuration conf) { String certsLoc = conf.get("hdfsproxy.user.certs.file.location", "user-certs.xml"); if (conf.getResource(certsLoc) == null) { LOG.warn("HdfsProxy user certs file not found"); return null; } Configuration certsConf = new Configuration(false); certsConf.addResource(certsLoc); Map<String, Set<BigInteger>> map = new HashMap<String, Set<BigInteger>>(); for (Map.Entry<String, String> e : certsConf) { String k = e.getKey(); String v = e.getValue().trim(); if (k != null && k.length() != 0 && v != null && v.length() != 0) { Set<BigInteger> numSet = new HashSet<BigInteger>(); String[] serialnumbers = v.split("\\s*,\\s*"); for (String num : serialnumbers) { if (num.length() != 0) { numSet.add(new BigInteger(num, 16)); } } map.put(k, numSet); } } return map; }
private LogisticRegressionModel instantiateSparkModel() { Configuration conf = new Configuration(); conf.set("fs.defaultFS", topologyConfig.getProperty("hdfs.url")); double[] sparkModelInfo = null; try { sparkModelInfo = getSparkModelInfoFromHDFS( new Path(topologyConfig.getProperty("hdfs.url") + "/tmp/sparkML_weights"), conf); } catch (Exception e) { LOG.error("Couldn't instantiate Spark model in prediction bolt: " + e.getMessage()); e.printStackTrace(); throw new RuntimeException(e); } // all numbers besides the last value are the weights double[] weights = Arrays.copyOfRange(sparkModelInfo, 0, sparkModelInfo.length - 1); // the last number in the array is the intercept double intercept = sparkModelInfo[sparkModelInfo.length - 1]; org.apache.spark.mllib.linalg.Vector weightsV = (Vectors.dense(weights)); return new LogisticRegressionModel(weightsV, intercept); }
/** * Get a PathFilter instance of the filter set for the input paths. * * @return the PathFilter instance set for the job, NULL if none has been set. */ public static PathFilter getInputPathFilter(JobContext context) { Configuration conf = context.getConfiguration(); Class<?> filterClass = conf.getClass(PATHFILTER_CLASS, null, PathFilter.class); return (filterClass != null) ? (PathFilter) ReflectionUtils.newInstance(filterClass, conf) : null; }
@VisibleForTesting void initConfig() throws IOException { this.cgroupPrefix = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn"); this.cgroupMount = conf.getBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, false); this.cgroupMountPath = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null); this.deleteCgroupTimeout = conf.getLong( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT); // remove extra /'s at end or start of cgroupPrefix if (cgroupPrefix.charAt(0) == '/') { cgroupPrefix = cgroupPrefix.substring(1); } this.strictResourceUsageMode = conf.getBoolean( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); int len = cgroupPrefix.length(); if (cgroupPrefix.charAt(len - 1) == '/') { cgroupPrefix = cgroupPrefix.substring(0, len - 1); } }
@Test public void testJSONBulkImporter() throws Exception { // Prepare input file: File inputFile = File.createTempFile("TestJSONImportInput", ".txt", getLocalTempDir()); TestingResources.writeTextFile( inputFile, TestingResources.get(BulkImporterTestUtils.JSON_IMPORT_DATA)); Configuration conf = getConf(); conf.set( DescribedInputTextBulkImporter.CONF_FILE, BulkImporterTestUtils.localResource(BulkImporterTestUtils.FOO_IMPORT_DESCRIPTOR)); // Run the bulk-import: final KijiMapReduceJob job = KijiBulkImportJobBuilder.create() .withConf(conf) .withBulkImporter(JSONBulkImporter.class) .withInput(MapReduceJobInputs.newTextMapReduceJobInput(new Path(inputFile.toString()))) .withOutput(new DirectKijiTableMapReduceJobOutput(mTable.getURI())) .build(); assertTrue(job.run()); final Counters counters = job.getHadoopJob().getCounters(); assertEquals( 3, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_PROCESSED).getValue()); assertEquals( 1, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_INCOMPLETE).getValue()); assertEquals( 0, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_REJECTED).getValue()); // Validate output: final KijiRowScanner scanner = mReader.getScanner(KijiDataRequest.create("info")); BulkImporterTestUtils.validateImportedRows(scanner, false); scanner.close(); }
@Test public void testPopulateConfiguration() { String[] args = new String[] { "-inputPath", "input", "-url", "jdbc:mysql://localhost:3306/hiho", "-userName", "root", "-password", "newpwd", "-querySuffix", "mrTest fields terminated by ','" }; ExportToMySQLDB exportToMySQLDB = new ExportToMySQLDB(); Configuration conf = new Configuration(); exportToMySQLDB.populateConfiguration(args, conf); assertEquals("jdbc:mysql://localhost:3306/hiho", conf.get(DBConfiguration.URL_PROPERTY)); assertEquals("root", conf.get(DBConfiguration.USERNAME_PROPERTY)); assertEquals("newpwd", conf.get(DBConfiguration.PASSWORD_PROPERTY)); assertEquals("mrTest fields terminated by ','", conf.get(HIHOConf.LOAD_QUERY_SUFFIX)); }
@Override protected void configureJob(Job job) throws IOException { Configuration conf = job.getConfiguration(); job.setJarByClass(PartialBuilder.class); FileInputFormat.setInputPaths(job, getDataPath()); FileOutputFormat.setOutputPath(job, getOutputPath(conf)); job.setOutputKeyClass(TreeID.class); job.setOutputValueClass(MapredOutput.class); job.setMapperClass(Step1Mapper.class); job.setNumReduceTasks(0); // no reducers job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); // For this implementation to work, mapred.map.tasks needs to be set to the actual // number of mappers Hadoop will use: TextInputFormat inputFormat = new TextInputFormat(); List<?> splits = inputFormat.getSplits(job); if (splits == null || splits.isEmpty()) { log.warn("Unable to compute number of splits?"); } else { int numSplits = splits.size(); log.info("Setting mapred.map.tasks = {}", numSplits); conf.setInt("mapred.map.tasks", numSplits); } }
@Override protected synchronized void startInternal() throws Exception { // create filesystem only now, as part of service-start. By this time, RM is // authenticated with kerberos so we are good to create a file-system // handle. fsConf = new Configuration(getConfig()); fsConf.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = fsConf.get( YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); fsConf.set("dfs.client.retry.policy.spec", retryPolicy); String scheme = fsWorkingPath.toUri().getScheme(); if (scheme == null) { scheme = FileSystem.getDefaultUri(fsConf).getScheme(); } if (scheme != null) { String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); fsConf.setBoolean(disableCacheName, true); } fs = fsWorkingPath.getFileSystem(fsConf); mkdirsWithRetries(rmDTSecretManagerRoot); mkdirsWithRetries(rmAppRoot); mkdirsWithRetries(amrmTokenSecretManagerRoot); mkdirsWithRetries(reservationRoot); }
@Override public int run(String[] strings) throws Exception { Configuration configuration = getConf(); configuration.setLong("mapred.min.split.size", 512 * 1024 * 1024L); Job numJob = new Job(configuration, "calculate film program seed num job "); Path[] paths = getPaths(strings[0].split(",")); HadoopUtils.deleteIfExist(strings[1]); MapReduceUtils.initMapperJob( NumCountMapper.class, Text.class, Text.class, this.getClass(), numJob, paths); // TableMapReduceUtil.initTableReducerJob(strings[1], NumCountReducer.class, numJob); MapReduceUtils.initReducerJob(new Path(strings[1]), NumCountReducer.class, numJob); numJob.waitForCompletion(true); Job programeSets = new Job(configuration, "calculate program set num job"); HadoopUtils.deleteIfExist(strings[2]); MapReduceUtils.initMapperJob( NumProgramSetsMapper.class, Text.class, Text.class, this.getClass(), programeSets, new Path(strings[1])); programeSets.setCombinerClass(NumProgramSetCombiner.class); MapReduceUtils.initReducerJob(new Path(strings[2]), NumProgramSetsReducer.class, programeSets); return programeSets.waitForCompletion(true) ? 0 : 1; // return 0; }
/** v1 service implementation to submit a job, either workflow or coordinator */ @Override protected JSONObject submitJob(HttpServletRequest request, Configuration conf) throws XServletException, IOException { JSONObject json = null; String jobType = request.getParameter(RestConstants.JOBTYPE_PARAM); if (jobType == null) { String wfPath = conf.get(OozieClient.APP_PATH); String coordPath = conf.get(OozieClient.COORDINATOR_APP_PATH); String bundlePath = conf.get(OozieClient.BUNDLE_APP_PATH); ServletUtilities.ValidateAppPath(wfPath, coordPath, bundlePath); if (wfPath != null) { json = submitWorkflowJob(request, conf); } else if (coordPath != null) { json = submitCoordinatorJob(request, conf); } else { json = submitBundleJob(request, conf); } } else { // This is a http submission job if (jobType.equals("pig") || jobType.equals("mapreduce")) { json = submitHttpJob(request, conf, jobType); } else { throw new XServletException( HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303, RestConstants.JOBTYPE_PARAM, jobType); } } return json; }
static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy( DatanodeID datanodeid, Configuration conf, int socketTimeout, boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException { final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname); InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr); if (LOG.isDebugEnabled()) { LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); UserGroupInformation ticket = UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString()); ticket.addToken(locatedBlock.getBlockToken()); return createClientDatanodeProtocolProxy( addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout); }
@Test public void testGetTokensForViewFS() throws IOException, URISyntaxException { Configuration conf = new Configuration(jConf); FileSystem dfs = dfsCluster.getFileSystem(); String serviceName = dfs.getCanonicalServiceName(); Path p1 = new Path("/mount1"); Path p2 = new Path("/mount2"); p1 = dfs.makeQualified(p1); p2 = dfs.makeQualified(p2); conf.set("fs.viewfs.mounttable.default.link./dir1", p1.toString()); conf.set("fs.viewfs.mounttable.default.link./dir2", p2.toString()); Credentials credentials = new Credentials(); Path lp1 = new Path("viewfs:///dir1"); Path lp2 = new Path("viewfs:///dir2"); Path[] paths = new Path[2]; paths[0] = lp1; paths[1] = lp2; TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); Collection<Token<? extends TokenIdentifier>> tns = credentials.getAllTokens(); assertEquals("number of tokens is not 1", 1, tns.size()); boolean found = false; for (Token<? extends TokenIdentifier> tt : tns) { System.out.println("token=" + tt); if (tt.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND) && tt.getService().equals(new Text(serviceName))) { found = true; } assertTrue("didn't find token for [" + lp1 + ", " + lp2 + "]", found); } }