/** Initialize SecondaryNameNode. */ private void initialize(Configuration conf) throws IOException { // initiate Java VM metrics JvmMetrics.init("SecondaryNameNode", conf.get("session.id")); // Create connection to the namenode. shouldRun = true; nameNodeAddr = NameNode.getAddress(conf); this.conf = conf; this.namenode = (NamenodeProtocol) RPC.waitForProxy( NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf); // initialize checkpoint directories fsName = getInfoServer(); checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointImage = new CheckpointStorage(conf); checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs); // Initialize other scheduling parameters from the configuration checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600); checkpointSize = conf.getLong("fs.checkpoint.size", 4194304); // initialize the webserver for uploading files. String infoAddr = NetUtils.getServerAddress( conf, "dfs.secondary.info.bindAddress", "dfs.secondary.info.port", "dfs.secondary.http.address"); InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr); infoBindAddress = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf); infoServer.setAttribute("name.system.image", checkpointImage); this.infoServer.setAttribute("name.conf", conf); infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class); infoServer.start(); // The web-server port can be ephemeral... ensure we have the correct info infoPort = infoServer.getPort(); conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort); LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort); LOG.warn( "Checkpoint Period :" + checkpointPeriod + " secs " + "(" + checkpointPeriod / 60 + " min)"); LOG.warn( "Log Size Trigger :" + checkpointSize + " bytes " + "(" + checkpointSize / 1024 + " KB)"); }
@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException { this.master = master; this.rootDir = master.getMasterFileSystem().getRootDir(); checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT); long timeoutMillis = Math.max( conf.getLong( SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY, SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT), conf.getLong( SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS, SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME)); int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT); // setup the default procedure coordinator String name = master.getServerName().toString(); ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs( master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); this.executorService = master.getExecutorService(); resetTempDir(); }
protected void initialize(Configuration conf) throws IOException { InetSocketAddress socAddr = UtilizationCollector.getAddress(conf); int handlerCount = conf.getInt("mapred.resourceutilization.handler.count", 10); // create rpc server this.server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf); // The rpc-server port can be ephemeral... ensure we have the correct info this.serverAddress = this.server.getListenerAddress(); LOG.info("Collector up at: " + this.serverAddress); // start RPC server this.server.start(); // How long does the TaskTracker reports expire timeLimit = conf.getLong("mapred.resourceutilization.timelimit", DEFAULT_TIME_LIMIT); // How long do we consider a job is finished after it stops stopTimeLimit = conf.getLong("mapred.resourceutilization.stoptimelimit", DEFAULT_STOP_TIME_LIMIT); // How often do we aggregate the reports aggregatePeriod = conf.getLong("mapred.resourceutilization.aggregateperiod", DEFAULT_AGGREGATE_SLEEP_TIME); // Start the daemon thread to aggregate the TaskTracker reports this.aggregateDaemon = new Daemon(new AggregateRun()); this.aggregateDaemon.start(); }
@Override public void populateDAG(DAG dag, Configuration conf) { String lPhoneRange = conf.get(PHONE_RANGE_PROP, null); if (lPhoneRange != null) { String[] tokens = lPhoneRange.split("-"); if (tokens.length != 2) { throw new IllegalArgumentException("Invalid range: " + lPhoneRange); } this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1])); } LOG.debug("Phone range {}", this.phoneRange); RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class); phones.setMinvalue(this.phoneRange.getMinimum()); phones.setMaxvalue(this.phoneRange.getMaximum()); PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class); dag.setAttribute( movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>()); StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>(); partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000)); partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000)); partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000)); dag.setAttribute( movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] {partitioner})); dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner); // generate seed numbers Random random = new Random(); int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum(); int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10); for (int i = phonesToDisplay; i-- > 0; ) { int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1); LOG.info("seed no: " + phoneNo); movementGen.phoneRegister.add(phoneNo); } // done generating data LOG.info("Finished generating seed data."); String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS); URI uri = URI.create("ws://" + gatewayAddress + "/pubsub"); PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>()); wsOut.setUri(uri); PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>()); wsIn.setUri(uri); // default partitioning: first connected stream to movementGen will be partitioned dag.addStream("Phone-Data", phones.integer_data, movementGen.data); dag.addStream("Results", movementGen.locationQueryResult, wsOut.input); dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery); }
HealthMonitor(Configuration conf, HAServiceTarget target) { this.targetToMonitor = target; this.conf = conf; this.sleepAfterDisconnectMillis = conf.getLong(HA_HM_SLEEP_AFTER_DISCONNECT_KEY, HA_HM_SLEEP_AFTER_DISCONNECT_DEFAULT); this.checkIntervalMillis = conf.getLong(HA_HM_CHECK_INTERVAL_KEY, HA_HM_CHECK_INTERVAL_DEFAULT); this.connectRetryInterval = conf.getLong(HA_HM_CONNECT_RETRY_INTERVAL_KEY, HA_HM_CONNECT_RETRY_INTERVAL_DEFAULT); this.rpcTimeout = conf.getInt(HA_HM_RPC_TIMEOUT_KEY, HA_HM_RPC_TIMEOUT_DEFAULT); this.daemon = new MonitorDaemon(); }
private void startTrashEmptier(final Configuration conf) throws IOException { long trashInterval = conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); if (trashInterval == 0) { return; } else if (trashInterval < 0) { throw new IOException( "Cannot start tresh emptier with negative interval." + " Set " + FS_TRASH_INTERVAL_KEY + " to a positive value."); } // This may be called from the transitionToActive code path, in which // case the current user is the administrator, not the NN. The trash // emptier needs to run as the NN. See HDFS-3972. FileSystem fs = SecurityUtil.doAsLoginUser( new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return FileSystem.get(conf); } }); this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier"); this.emptier.setDaemon(true); this.emptier.start(); }
protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); this.reduceSleepCount = conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount); this.reduceSleepDuration = reduceSleepCount == 0 ? 0 : conf.getLong(REDUCE_SLEEP_TIME, 100) / reduceSleepCount; vertexName = conf.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME); }
InputErrorTracker(Configuration conf) { // default threshold : 0.01% errorThreshold = conf.getFloat(BAD_RECORD_THRESHOLD_CONF_KEY, 0.0001f); minErrors = conf.getLong(BAD_RECORD_MIN_COUNT_CONF_KEY, 2); numRecords = 0; numErrors = 0; }
/** * @param conn The HBase connection. * @param conf The HBase configuration * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops for * each region server before dropping the request. */ public HTableMultiplexer( Connection conn, Configuration conf, int perRegionServerBufferQueueSize) { this.conn = (ClusterConnection) conn; this.pool = HTable.getDefaultExecutor(conf); // how many times we could try in total, one more than retry number this.maxAttempts = conf.getInt( HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER) + 1; this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize; this.maxKeyValueSize = HTable.getMaxKeyValueSize(conf); this.flushPeriod = conf.getLong(TABLE_MULTIPLEXER_FLUSH_PERIOD_MS, 100); int initThreads = conf.getInt(TABLE_MULTIPLEXER_INIT_THREADS, 10); this.executor = Executors.newScheduledThreadPool( initThreads, new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("HTableFlushWorker-%d") .build()); this.workerConf = HBaseConfiguration.create(conf); // We do not do the retry because we need to reassign puts to different queues if regions are // moved. this.workerConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); }
@VisibleForTesting void initConfig() throws IOException { this.cgroupPrefix = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn"); this.cgroupMount = conf.getBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, false); this.cgroupMountPath = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null); this.deleteCgroupTimeout = conf.getLong( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT); // remove extra /'s at end or start of cgroupPrefix if (cgroupPrefix.charAt(0) == '/') { cgroupPrefix = cgroupPrefix.substring(1); } this.strictResourceUsageMode = conf.getBoolean( YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE); int len = cgroupPrefix.length(); if (cgroupPrefix.charAt(len - 1) == '/') { cgroupPrefix = cgroupPrefix.substring(0, len - 1); } }
/** * Returns a Thread pool for the RPC's to region replicas. Similar to Connection's thread pool. */ private ExecutorService getDefaultThreadPool(Configuration conf) { int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256); int coreThreads = conf.getInt("hbase.region.replica.replication.threads.core", 16); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } if (coreThreads == 0) { coreThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>( maxThreads * conf.getInt( HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS)); ThreadPoolExecutor tpe = new ThreadPoolExecutor( coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(this.getClass().getSimpleName() + "-rpc-shared-")); tpe.allowCoreThreadTimeOut(true); return tpe; }
/** * A dead server's wals have been split so that all the regions used to be open on it can be * safely assigned now. Mark them assignable. */ public synchronized void logSplit(final ServerName serverName) { for (Iterator<Map.Entry<String, ServerName>> it = lastAssignments.entrySet().iterator(); it.hasNext(); ) { Map.Entry<String, ServerName> e = it.next(); if (e.getValue().equals(serverName)) { it.remove(); } } long now = System.currentTimeMillis(); if (LOG.isDebugEnabled()) { LOG.debug("Adding to log splitting servers " + serverName); } processedServers.put(serverName, Long.valueOf(now)); Configuration conf = server.getConfiguration(); long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME); // Doesn't have to be very accurate about the clean up time if (now > lastProcessedServerCleanTime + obsoleteTime) { lastProcessedServerCleanTime = now; long cutoff = now - obsoleteTime; for (Iterator<Map.Entry<ServerName, Long>> it = processedServers.entrySet().iterator(); it.hasNext(); ) { Map.Entry<ServerName, Long> e = it.next(); if (e.getValue().longValue() < cutoff) { if (LOG.isDebugEnabled()) { LOG.debug("Removed from log splitting servers " + e.getKey()); } it.remove(); } } } }
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); long totalLimit = conf.getLong(GeneratorJob.GENERATOR_TOP_N, Long.MAX_VALUE); if (totalLimit == Long.MAX_VALUE) { limit = Long.MAX_VALUE; } else { limit = totalLimit / context.getNumReduceTasks(); } maxCount = conf.getLong(GeneratorJob.GENERATOR_MAX_COUNT, -2); batchId = new Utf8(conf.get(GeneratorJob.BATCH_ID)); String countMode = conf.get(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_HOST); if (countMode.equals(GeneratorJob.GENERATOR_COUNT_VALUE_DOMAIN)) { byDomain = true; } }
/** Start the JobTracker process, listen on the indicated port */ JobTracker(Configuration conf) throws IOException { // // Grab some static constants // maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2); RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000); RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000); TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f); PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f); MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks; // This is a directory of temporary submission files. We delete it // on startup, and can delete any files that we're done with this.conf = conf; JobConf jobConf = new JobConf(conf); this.systemDir = jobConf.getSystemDir(); this.fs = FileSystem.get(conf); FileUtil.fullyDelete(fs, systemDir); fs.mkdirs(systemDir); // Same with 'localDir' except it's always on the local disk. jobConf.deleteLocalFiles(SUBDIR); // Set ports, start RPC servers, etc. InetSocketAddress addr = getAddress(conf); this.localMachine = addr.getHostName(); this.port = addr.getPort(); this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf); this.interTrackerServer.start(); Properties p = System.getProperties(); for (Iterator it = p.keySet().iterator(); it.hasNext(); ) { String key = (String) it.next(); String val = (String) p.getProperty(key); LOG.info("Property '" + key + "' is " + val); } this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030); this.infoServer = new JobTrackerInfoServer(this, infoPort); this.infoServer.start(); this.startTime = System.currentTimeMillis(); new Thread(this.expireTrackers).start(); new Thread(this.retireJobs).start(); new Thread(this.initJobs).start(); }
public void setConf(Configuration conf) { super.setConf(conf); if (conf == null) return; if (conf.getBoolean("segment.merger.filter", false)) filters = new URLFilters(conf); sliceSize = conf.getLong("segment.merger.slice", -1); if ((sliceSize > 0) && (LOG.isInfoEnabled())) { LOG.info("Slice size: " + sliceSize + " URLs."); } }
/** * Reads a positive long integer from a configuration. * * @param Configuration conf configuration objects * @param String propertyName name of the property * @return time */ long getTimeProperty(Configuration conf, String propertyName, long defaultValue) throws IllegalArgumentException { // possible improvement: change date format to human readable ? long time = conf.getLong(propertyName, defaultValue); if (time <= 0) { throw new IllegalArgumentException(propertyName + "time must be positive: " + time); } return time; }
protected AbstractColumnFamilyRecordWriter(Configuration conf) { this.conf = conf; this.ringCache = new RingCache(conf); this.queueSize = conf.getInt( AbstractColumnFamilyOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(AbstractColumnFamilyOutputFormat.BATCH_THRESHOLD, 32); consistencyLevel = ConsistencyLevel.valueOf(ConfigHelper.getWriteConsistencyLevel(conf)); }
public TimelineMetricClusterAggregatorHourly( PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) { super(hBaseAccessor, metricsConf); String checkpointDir = metricsConf.get(TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION); checkpointLocation = FilenameUtils.concat(checkpointDir, CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE); sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong(CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l)); checkpointCutOffIntervalMillis = SECONDS.toMillis( metricsConf.getLong(CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_INTERVAL, 7200l)); checkpointCutOffMultiplier = metricsConf.getInt(CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2); }
/** * Create a new ClientScanner for the specified table Note that the passed {@link Scan}'s start * row maybe changed changed. * * @param conf The {@link Configuration} to use. * @param scan {@link Scan} to use in this scanner * @param tableName The table that we wish to scan * @param connection Connection identifying the cluster * @throws IOException */ public ClientScanner( final Configuration conf, final Scan scan, final TableName tableName, ClusterConnection connection, RpcRetryingCallerFactory rpcFactory, RpcControllerFactory controllerFactory, ExecutorService pool, int primaryOperationTimeout) throws IOException { if (LOG.isTraceEnabled()) { LOG.trace( "Scan table=" + tableName + ", startRow=" + Bytes.toStringBinary(scan.getStartRow())); } this.scan = scan; this.tableName = tableName; this.lastNext = System.currentTimeMillis(); this.connection = connection; this.pool = pool; this.primaryOperationTimeout = primaryOperationTimeout; this.retries = conf.getInt( HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); if (scan.getMaxResultSize() > 0) { this.maxScannerResultSize = scan.getMaxResultSize(); } else { this.maxScannerResultSize = conf.getLong( HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); } this.scannerTimeout = HBaseConfiguration.getInt( conf, HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD); // check if application wants to collect scan metrics initScanMetrics(scan); // Use the caching from the Scan. If not set, use the default cache setting for this table. if (this.scan.getCaching() > 0) { this.caching = this.scan.getCaching(); } else { this.caching = conf.getInt( HConstants.HBASE_CLIENT_SCANNER_CACHING, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING); } this.caller = rpcFactory.<Result[]>newCaller(); this.rpcControllerFactory = controllerFactory; this.conf = conf; initializeScannerInConstruction(); }
@Override protected void serviceInit(Configuration conf) throws Exception { this.conf = conf; int serialNumberLowDigits = 3; serialNumberFormat = ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d"); long maxFSWaitTime = conf.getLong( JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME); createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime); this.aclsMgr = new JobACLsManager(conf); maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE); jobListCache = createJobListCache(); serialNumberIndex = new SerialNumberIndex( conf.getInt( JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE, JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE)); int numMoveThreads = conf.getInt( JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT, JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT); ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build(); moveToDoneExecutor = new ThreadPoolExecutor( numMoveThreads, numMoveThreads, 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf); super.serviceInit(conf); }
/** * Confirm ImportTsv via data in online table. * * @param dataAvailable */ private static void validateTable( Configuration conf, TableName tableName, String family, int valueMultiplier, boolean dataAvailable) throws IOException { LOG.debug("Validating table."); Connection connection = ConnectionFactory.createConnection(conf); Table table = connection.getTable(tableName); boolean verified = false; long pause = conf.getLong("hbase.client.pause", 5 * 1000); int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); for (int i = 0; i < numRetries; i++) { try { Scan scan = new Scan(); // Scan entire family. scan.addFamily(Bytes.toBytes(family)); if (dataAvailable) { ResultScanner resScanner = table.getScanner(scan); for (Result res : resScanner) { LOG.debug("Getting results " + res.size()); assertTrue(res.size() == 2); List<Cell> kvs = res.listCells(); assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY"))); assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY"))); assertTrue( CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier))); assertTrue( CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier))); // Only one result set is expected, so let it loop. verified = true; } } else { ResultScanner resScanner = table.getScanner(scan); Result[] next = resScanner.next(2); assertEquals(0, next.length); verified = true; } break; } catch (NullPointerException e) { // If here, a cell was empty. Presume its because updates came in // after the scanner had been opened. Wait a while and retry. } try { Thread.sleep(pause); } catch (InterruptedException e) { // continue } } table.close(); connection.close(); assertTrue(verified); }
@SuppressWarnings("deprecation") @Override protected void serviceInit(Configuration conf) throws Exception { asyncApiPollIntervalMillis = conf.getLong( YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS, YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS); asyncApiPollTimeoutMillis = conf.getLong( YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS); submitPollIntervalMillis = asyncApiPollIntervalMillis; if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS) != null) { submitPollIntervalMillis = conf.getLong( YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS); } if (conf.getBoolean( YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { historyServiceEnabled = true; historyClient = AHSClient.createAHSClient(); historyClient.init(conf); } if (conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineServiceEnabled = true; timelineClient = createTimelineClient(); timelineClient.init(conf); timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); timelineService = TimelineUtils.buildTimelineTokenService(conf); } timelineServiceBestEffort = conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT); super.serviceInit(conf); }
/** * Write random bytes in the distributed cache files that will be used by all simulated jobs of * current gridmix run, if files are to be generated. Do this as part of the MapReduce job {@link * GenerateDistCacheData#JOB_NAME} * * @see org.apache.hadoop.mapred.gridmix.GenerateDistCacheData */ protected void writeDistCacheData(Configuration conf) throws IOException, InterruptedException { int fileCount = conf.getInt(GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_COUNT, -1); if (fileCount > 0) { // generate distributed cache files final GridmixJob genDistCacheData = new GenerateDistCacheData(conf); LOG.info( "Generating distributed cache data of size " + conf.getLong(GenerateDistCacheData.GRIDMIX_DISTCACHE_BYTE_COUNT, -1)); launchGridmixJob(genDistCacheData); } }
public static void iterativePageRank(Configuration conf) throws IOException, InterruptedException, ClassNotFoundException { String initialVector = conf.get("initialRankVectorPath"); String currentVector = conf.get("currentRankVectorPath"); String finalVector = conf.get("finalRankVectorPath"); /*here the testing system will seach for the final rank vector*/ Double epsilon = conf.getDouble("epsilon", 0.1); Double beta = conf.getDouble("beta", 0.8); // Launch remove dead ends jobs RemoveDeadends.job(conf); // Create initial vector Long nNodes = conf.getLong("numNodes", 1); createInitialRankVector(initialVector, conf.getLong("numNodes", 1)); // Create stochastic matrix GraphToMatrix.job(conf); boolean converg = false; // multiplication M * r MatrixVectorMult.job(conf); avoidSpiderTraps(currentVector, nNodes, beta); while (!converg) { FileUtils.deleteQuietly(new File(initialVector)); FileUtils.moveFile( new File(currentVector + "/part-r-00000"), new File(initialVector + "/part-r-00000")); // multiplication M * r MatrixVectorMult.job(conf); avoidSpiderTraps(currentVector, nNodes, beta); converg = checkConvergence(initialVector, currentVector, epsilon); } FileUtils.copyDirectory(new File(currentVector), new File(finalVector)); }
@Override public void initialize(URI uri, Configuration conf) throws IOException { checkNotNull(uri, "uri is null"); checkNotNull(conf, "conf is null"); super.initialize(uri, conf); setConf(conf); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDirectory = new Path("/").makeQualified(this.uri, new Path("/")); HiveClientConfig defaults = new HiveClientConfig(); this.stagingDirectory = new File(conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString())); this.maxClientRetries = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()); this.maxBackoffTime = Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString())); this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString())); int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); Duration socketTimeout = Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString())); int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections()); long minFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes()); long minPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes()); ClientConfiguration configuration = new ClientConfiguration(); configuration.setMaxErrorRetry(maxErrorRetries); configuration.setProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP); configuration.setConnectionTimeout(Ints.checkedCast(connectTimeout.toMillis())); configuration.setSocketTimeout(Ints.checkedCast(socketTimeout.toMillis())); configuration.setMaxConnections(maxConnections); this.s3 = new AmazonS3Client(getAwsCredentials(uri, conf), configuration); transferConfig.setMultipartUploadThreshold(minFileSize); transferConfig.setMinimumUploadPartSize(minPartSize); }
/** * Creates a WARC file, and opens it for writing. If a file with the same name already exists, it * is *overwritten*. Note that this is different behaviour from the other constructor. Yes, this * sucks. It will probably change in a future version. * * @param conf The Hadoop configuration. * @param codec If null, the file is uncompressed. If non-null, this compression codec will be * used. The codec's default file extension is appended to the filename. * @param workOutputPath The directory and filename prefix to which the data should be written. We * append a segment number and filename extensions to it. * @param progress An object used by the mapred API for tracking a task's progress. * @throws IOException */ public WARCFileWriter( Configuration conf, CompressionCodec codec, Path workOutputPath, Progressable progress) throws IOException { this.conf = conf; this.codec = codec; this.workOutputPath = workOutputPath; this.progress = progress; this.extensionFormat = ".seg-%05d.attempt-%05d.warc" + (codec == null ? "" : codec.getDefaultExtension()); this.maxSegmentSize = conf.getLong("warc.output.segment.size", DEFAULT_MAX_SEGMENT_SIZE); createSegment(); }
private void loadMaxRssMemoryConfig(Configuration conf) { long reservedRssMemoryMB = conf.getLong( TaskMemoryManagerThread.TT_RESERVED_PHYSICAL_MEMORY_MB, JobConf.DISABLED_MEMORY_LIMIT); if (reservedRssMemoryMB == JobConf.DISABLED_MEMORY_LIMIT) { reservedRssMemory = JobConf.DISABLED_MEMORY_LIMIT; maxRssMemoryAllowedForAllTasks = JobConf.DISABLED_MEMORY_LIMIT; } else { reservedRssMemory = reservedRssMemoryMB * 1024 * 1024L; maxRssMemoryAllowedForAllTasks = taskTracker.getTotalPhysicalMemoryOnTT() - reservedRssMemory; } }
FakeServer( final Configuration c, final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids) { this.meta = meta; this.sequenceids = sequenceids; // Pause to simulate the server taking time applying the edits. This will drive up the // number of threads used over in client. this.multiPause = c.getLong("hbase.test.multi.pause.when.done", 0); this.tooManyMultiRequests = c.getInt("hbase.test.multi.too.many", 3); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { // 初始化函数 // System.out.println("initialize"); // Logger.getLogger("KirchhoffMigration").log(Level.INFO, // "enter initialize()"); FileSplit inputsplit = (FileSplit) split; Configuration conf = context.getConfiguration(); LENGTH = conf.getLong("FileSplitLength", 0); SPILL = LENGTH / conf.getInt("SplitPerMap", 1); // LENGTH = 8; // SPILL = 8; assert (LENGTH >= SPILL); // System.out.println("length:" + LENGTH); // System.out.println("spill:" + SPILL); String filename = inputsplit.getPath().toString(); // System.out.println("filename:" + filename); // String buf = filename.substring(filename.lastIndexOf("fcxy") + 4, // filename.lastIndexOf(".")); // int count = Integer.parseInt(buf); // System.out.println(filename); // start = inputsplit.getStart(); // 得到此分片开始位置 start = inputsplit.getStart(); shotNum += start * 8 / Float.SIZE; long offset = LENGTH >= inputsplit.getLength() ? inputsplit.getLength() : LENGTH; end = start + offset; // 结束此分片位置 // System.out.println("inputSplitLength:" + split.getLength()); // System.out.println("end:" + end); // start = inputsplit.getStart(); // 得到此分片开始位置 // end = start + inputsplit.getLength();// 结束此分片位置 // System.out.println("start:" + start + " ,end:" + end); final Path file = inputsplit.getPath(); // System.out.println(file.toString()); // 打开文件 FileSystem fs = file.getFileSystem(context.getConfiguration()); fileIn = fs.open(inputsplit.getPath()); // 关键位置2 // 将文件指针移动到当前分片,因为每次默认打开文件时,`其指针指向开头 fileIn.seek(start); // in = new LineReader(fileIn, context.getConfiguration()); // if (start != 0) { // System.out.println("not the first split"); // // 关键解决位置1 // // // 如果这不是第一个分片,那么假设第一个分片是0——4,那么,第4个位置已经被读取,则需要跳过4,否则会产生读入错误,因为你回头又去读之前读过的地方 // start += (end - pos + 1); // } pos = start; }
ManyServersManyRegionsConnection( Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed, pool, user); int serverCount = conf.getInt("hbase.test.servers", 10); this.serversByClient = new HashMap<ServerName, ClientService.BlockingInterface>(serverCount); this.meta = makeMeta( Bytes.toBytes(conf.get("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE))), conf.getInt("hbase.test.regions", 100), conf.getLong("hbase.test.namespace.span", 1000), serverCount); this.conf = conf; }