/** Initialize SecondaryNameNode. */
private void initialize(Configuration conf) throws IOException {
// initiate Java VM metrics
JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.getAddress(conf);
this.conf = conf;
this.namenode =
(NamenodeProtocol)
RPC.waitForProxy(
NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);
// initialize checkpoint directories
fsName = getInfoServer();
checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointImage = new CheckpointStorage(conf);
checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);
// Initialize other scheduling parameters from the configuration
checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);
// initialize the webserver for uploading files.
String infoAddr =
NetUtils.getServerAddress(
conf,
"dfs.secondary.info.bindAddress",
"dfs.secondary.info.port",
"dfs.secondary.http.address");
InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
infoBindAddress = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
infoServer.setAttribute("name.system.image", checkpointImage);
this.infoServer.setAttribute("name.conf", conf);
infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
infoServer.start();
// The web-server port can be ephemeral... ensure we have the correct info
infoPort = infoServer.getPort();
conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
LOG.warn(
"Checkpoint Period :"
+ checkpointPeriod
+ " secs "
+ "("
+ checkpointPeriod / 60
+ " min)");
LOG.warn(
"Log Size Trigger :"
+ checkpointSize
+ " bytes "
+ "("
+ checkpointSize / 1024
+ " KB)");
}
@Override
public void initialize(MasterServices master, MetricsMaster metricsMaster)
throws KeeperException, IOException, UnsupportedOperationException {
this.master = master;
this.rootDir = master.getMasterFileSystem().getRootDir();
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
// get the configuration for the coordinator
Configuration conf = master.getConfiguration();
long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
long timeoutMillis =
Math.max(
conf.getLong(
SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY,
SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT),
conf.getLong(
SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME));
int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
// setup the default procedure coordinator
String name = master.getServerName().toString();
ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
ProcedureCoordinatorRpcs comms =
new ZKProcedureCoordinatorRpcs(
master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
this.executorService = master.getExecutorService();
resetTempDir();
}
protected void initialize(Configuration conf) throws IOException {
InetSocketAddress socAddr = UtilizationCollector.getAddress(conf);
int handlerCount = conf.getInt("mapred.resourceutilization.handler.count", 10);
// create rpc server
this.server =
RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf);
// The rpc-server port can be ephemeral... ensure we have the correct info
this.serverAddress = this.server.getListenerAddress();
LOG.info("Collector up at: " + this.serverAddress);
// start RPC server
this.server.start();
// How long does the TaskTracker reports expire
timeLimit = conf.getLong("mapred.resourceutilization.timelimit", DEFAULT_TIME_LIMIT);
// How long do we consider a job is finished after it stops
stopTimeLimit =
conf.getLong("mapred.resourceutilization.stoptimelimit", DEFAULT_STOP_TIME_LIMIT);
// How often do we aggregate the reports
aggregatePeriod =
conf.getLong("mapred.resourceutilization.aggregateperiod", DEFAULT_AGGREGATE_SLEEP_TIME);
// Start the daemon thread to aggregate the TaskTracker reports
this.aggregateDaemon = new Daemon(new AggregateRun());
this.aggregateDaemon.start();
}
@Override
public void populateDAG(DAG dag, Configuration conf) {
String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
if (lPhoneRange != null) {
String[] tokens = lPhoneRange.split("-");
if (tokens.length != 2) {
throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
}
this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
}
LOG.debug("Phone range {}", this.phoneRange);
RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
phones.setMinvalue(this.phoneRange.getMinimum());
phones.setMaxvalue(this.phoneRange.getMaximum());
PhoneMovementGenerator movementGen =
dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
dag.setAttribute(
movementGen,
OperatorContext.COUNTERS_AGGREGATOR,
new BasicCounters.LongAggregator<MutableLong>());
StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner =
new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
dag.setAttribute(
movementGen,
OperatorContext.STATS_LISTENERS,
Arrays.asList(new StatsListener[] {partitioner}));
dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
// generate seed numbers
Random random = new Random();
int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
for (int i = phonesToDisplay; i-- > 0; ) {
int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
LOG.info("seed no: " + phoneNo);
movementGen.phoneRegister.add(phoneNo);
}
// done generating data
LOG.info("Finished generating seed data.");
String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
PubSubWebSocketOutputOperator<Object> wsOut =
dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
wsOut.setUri(uri);
PubSubWebSocketInputOperator<Map<String, String>> wsIn =
dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
wsIn.setUri(uri);
// default partitioning: first connected stream to movementGen will be partitioned
dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
}
HealthMonitor(Configuration conf, HAServiceTarget target) {
this.targetToMonitor = target;
this.conf = conf;
this.sleepAfterDisconnectMillis =
conf.getLong(HA_HM_SLEEP_AFTER_DISCONNECT_KEY, HA_HM_SLEEP_AFTER_DISCONNECT_DEFAULT);
this.checkIntervalMillis = conf.getLong(HA_HM_CHECK_INTERVAL_KEY, HA_HM_CHECK_INTERVAL_DEFAULT);
this.connectRetryInterval =
conf.getLong(HA_HM_CONNECT_RETRY_INTERVAL_KEY, HA_HM_CONNECT_RETRY_INTERVAL_DEFAULT);
this.rpcTimeout = conf.getInt(HA_HM_RPC_TIMEOUT_KEY, HA_HM_RPC_TIMEOUT_DEFAULT);
this.daemon = new MonitorDaemon();
}
private void startTrashEmptier(final Configuration conf) throws IOException {
long trashInterval = conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT);
if (trashInterval == 0) {
return;
} else if (trashInterval < 0) {
throw new IOException(
"Cannot start tresh emptier with negative interval."
+ " Set "
+ FS_TRASH_INTERVAL_KEY
+ " to a positive value.");
}
// This may be called from the transitionToActive code path, in which
// case the current user is the administrator, not the NN. The trash
// emptier needs to run as the NN. See HDFS-3972.
FileSystem fs =
SecurityUtil.doAsLoginUser(
new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws IOException {
return FileSystem.get(conf);
}
});
this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier");
this.emptier.setDaemon(true);
this.emptier.start();
}
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
this.reduceSleepCount = conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
this.reduceSleepDuration =
reduceSleepCount == 0 ? 0 : conf.getLong(REDUCE_SLEEP_TIME, 100) / reduceSleepCount;
vertexName = conf.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
}
InputErrorTracker(Configuration conf) {
// default threshold : 0.01%
errorThreshold = conf.getFloat(BAD_RECORD_THRESHOLD_CONF_KEY, 0.0001f);
minErrors = conf.getLong(BAD_RECORD_MIN_COUNT_CONF_KEY, 2);
numRecords = 0;
numErrors = 0;
}
/**
* @param conn The HBase connection.
* @param conf The HBase configuration
* @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops for
* each region server before dropping the request.
*/
public HTableMultiplexer(
Connection conn, Configuration conf, int perRegionServerBufferQueueSize) {
this.conn = (ClusterConnection) conn;
this.pool = HTable.getDefaultExecutor(conf);
// how many times we could try in total, one more than retry number
this.maxAttempts =
conf.getInt(
HConstants.HBASE_CLIENT_RETRIES_NUMBER,
HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER)
+ 1;
this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize;
this.maxKeyValueSize = HTable.getMaxKeyValueSize(conf);
this.flushPeriod = conf.getLong(TABLE_MULTIPLEXER_FLUSH_PERIOD_MS, 100);
int initThreads = conf.getInt(TABLE_MULTIPLEXER_INIT_THREADS, 10);
this.executor =
Executors.newScheduledThreadPool(
initThreads,
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("HTableFlushWorker-%d")
.build());
this.workerConf = HBaseConfiguration.create(conf);
// We do not do the retry because we need to reassign puts to different queues if regions are
// moved.
this.workerConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);
}
@VisibleForTesting
void initConfig() throws IOException {
this.cgroupPrefix =
conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn");
this.cgroupMount = conf.getBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.cgroupMountPath = conf.get(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, null);
this.deleteCgroupTimeout =
conf.getLong(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT);
// remove extra /'s at end or start of cgroupPrefix
if (cgroupPrefix.charAt(0) == '/') {
cgroupPrefix = cgroupPrefix.substring(1);
}
this.strictResourceUsageMode =
conf.getBoolean(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE);
int len = cgroupPrefix.length();
if (cgroupPrefix.charAt(len - 1) == '/') {
cgroupPrefix = cgroupPrefix.substring(0, len - 1);
}
}
/**
* Returns a Thread pool for the RPC's to region replicas. Similar to Connection's thread pool.
*/
private ExecutorService getDefaultThreadPool(Configuration conf) {
int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256);
int coreThreads = conf.getInt("hbase.region.replica.replication.threads.core", 16);
if (maxThreads == 0) {
maxThreads = Runtime.getRuntime().availableProcessors() * 8;
}
if (coreThreads == 0) {
coreThreads = Runtime.getRuntime().availableProcessors() * 8;
}
long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60);
LinkedBlockingQueue<Runnable> workQueue =
new LinkedBlockingQueue<Runnable>(
maxThreads
* conf.getInt(
HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
ThreadPoolExecutor tpe =
new ThreadPoolExecutor(
coreThreads,
maxThreads,
keepAliveTime,
TimeUnit.SECONDS,
workQueue,
Threads.newDaemonThreadFactory(this.getClass().getSimpleName() + "-rpc-shared-"));
tpe.allowCoreThreadTimeOut(true);
return tpe;
}
/**
* A dead server's wals have been split so that all the regions used to be open on it can be
* safely assigned now. Mark them assignable.
*/
public synchronized void logSplit(final ServerName serverName) {
for (Iterator<Map.Entry<String, ServerName>> it = lastAssignments.entrySet().iterator();
it.hasNext(); ) {
Map.Entry<String, ServerName> e = it.next();
if (e.getValue().equals(serverName)) {
it.remove();
}
}
long now = System.currentTimeMillis();
if (LOG.isDebugEnabled()) {
LOG.debug("Adding to log splitting servers " + serverName);
}
processedServers.put(serverName, Long.valueOf(now));
Configuration conf = server.getConfiguration();
long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME);
// Doesn't have to be very accurate about the clean up time
if (now > lastProcessedServerCleanTime + obsoleteTime) {
lastProcessedServerCleanTime = now;
long cutoff = now - obsoleteTime;
for (Iterator<Map.Entry<ServerName, Long>> it = processedServers.entrySet().iterator();
it.hasNext(); ) {
Map.Entry<ServerName, Long> e = it.next();
if (e.getValue().longValue() < cutoff) {
if (LOG.isDebugEnabled()) {
LOG.debug("Removed from log splitting servers " + e.getKey());
}
it.remove();
}
}
}
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
long totalLimit = conf.getLong(GeneratorJob.GENERATOR_TOP_N, Long.MAX_VALUE);
if (totalLimit == Long.MAX_VALUE) {
limit = Long.MAX_VALUE;
} else {
limit = totalLimit / context.getNumReduceTasks();
}
maxCount = conf.getLong(GeneratorJob.GENERATOR_MAX_COUNT, -2);
batchId = new Utf8(conf.get(GeneratorJob.BATCH_ID));
String countMode =
conf.get(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_HOST);
if (countMode.equals(GeneratorJob.GENERATOR_COUNT_VALUE_DOMAIN)) {
byDomain = true;
}
}
/** Start the JobTracker process, listen on the indicated port */
JobTracker(Configuration conf) throws IOException {
//
// Grab some static constants
//
maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
// This is a directory of temporary submission files. We delete it
// on startup, and can delete any files that we're done with
this.conf = conf;
JobConf jobConf = new JobConf(conf);
this.systemDir = jobConf.getSystemDir();
this.fs = FileSystem.get(conf);
FileUtil.fullyDelete(fs, systemDir);
fs.mkdirs(systemDir);
// Same with 'localDir' except it's always on the local disk.
jobConf.deleteLocalFiles(SUBDIR);
// Set ports, start RPC servers, etc.
InetSocketAddress addr = getAddress(conf);
this.localMachine = addr.getHostName();
this.port = addr.getPort();
this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
this.interTrackerServer.start();
Properties p = System.getProperties();
for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
String key = (String) it.next();
String val = (String) p.getProperty(key);
LOG.info("Property '" + key + "' is " + val);
}
this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();
this.startTime = System.currentTimeMillis();
new Thread(this.expireTrackers).start();
new Thread(this.retireJobs).start();
new Thread(this.initJobs).start();
}
public void setConf(Configuration conf) {
super.setConf(conf);
if (conf == null) return;
if (conf.getBoolean("segment.merger.filter", false)) filters = new URLFilters(conf);
sliceSize = conf.getLong("segment.merger.slice", -1);
if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
LOG.info("Slice size: " + sliceSize + " URLs.");
}
}
/**
* Reads a positive long integer from a configuration.
*
* @param Configuration conf configuration objects
* @param String propertyName name of the property
* @return time
*/
long getTimeProperty(Configuration conf, String propertyName, long defaultValue)
throws IllegalArgumentException {
// possible improvement: change date format to human readable ?
long time = conf.getLong(propertyName, defaultValue);
if (time <= 0) {
throw new IllegalArgumentException(propertyName + "time must be positive: " + time);
}
return time;
}
protected AbstractColumnFamilyRecordWriter(Configuration conf) {
this.conf = conf;
this.ringCache = new RingCache(conf);
this.queueSize =
conf.getInt(
AbstractColumnFamilyOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors());
batchThreshold = conf.getLong(AbstractColumnFamilyOutputFormat.BATCH_THRESHOLD, 32);
consistencyLevel = ConsistencyLevel.valueOf(ConfigHelper.getWriteConsistencyLevel(conf));
}
public TimelineMetricClusterAggregatorHourly(
PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) {
super(hBaseAccessor, metricsConf);
String checkpointDir =
metricsConf.get(TIMELINE_METRICS_AGGREGATOR_CHECKPOINT_DIR, DEFAULT_CHECKPOINT_LOCATION);
checkpointLocation =
FilenameUtils.concat(checkpointDir, CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE);
sleepIntervalMillis =
SECONDS.toMillis(metricsConf.getLong(CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l));
checkpointCutOffIntervalMillis =
SECONDS.toMillis(
metricsConf.getLong(CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_INTERVAL, 7200l));
checkpointCutOffMultiplier =
metricsConf.getInt(CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2);
}
/**
* Create a new ClientScanner for the specified table Note that the passed {@link Scan}'s start
* row maybe changed changed.
*
* @param conf The {@link Configuration} to use.
* @param scan {@link Scan} to use in this scanner
* @param tableName The table that we wish to scan
* @param connection Connection identifying the cluster
* @throws IOException
*/
public ClientScanner(
final Configuration conf,
final Scan scan,
final TableName tableName,
ClusterConnection connection,
RpcRetryingCallerFactory rpcFactory,
RpcControllerFactory controllerFactory,
ExecutorService pool,
int primaryOperationTimeout)
throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace(
"Scan table=" + tableName + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
}
this.scan = scan;
this.tableName = tableName;
this.lastNext = System.currentTimeMillis();
this.connection = connection;
this.pool = pool;
this.primaryOperationTimeout = primaryOperationTimeout;
this.retries =
conf.getInt(
HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
if (scan.getMaxResultSize() > 0) {
this.maxScannerResultSize = scan.getMaxResultSize();
} else {
this.maxScannerResultSize =
conf.getLong(
HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
}
this.scannerTimeout =
HBaseConfiguration.getInt(
conf,
HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
// check if application wants to collect scan metrics
initScanMetrics(scan);
// Use the caching from the Scan. If not set, use the default cache setting for this table.
if (this.scan.getCaching() > 0) {
this.caching = this.scan.getCaching();
} else {
this.caching =
conf.getInt(
HConstants.HBASE_CLIENT_SCANNER_CACHING,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
}
this.caller = rpcFactory.<Result[]>newCaller();
this.rpcControllerFactory = controllerFactory;
this.conf = conf;
initializeScannerInConstruction();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
int serialNumberLowDigits = 3;
serialNumberFormat =
("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
long maxFSWaitTime =
conf.getLong(
JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge =
conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
jobListCache = createJobListCache();
serialNumberIndex =
new SerialNumberIndex(
conf.getInt(
JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));
int numMoveThreads =
conf.getInt(
JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf =
new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
moveToDoneExecutor =
new ThreadPoolExecutor(
numMoveThreads,
numMoveThreads,
1,
TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
super.serviceInit(conf);
}
/**
* Confirm ImportTsv via data in online table.
*
* @param dataAvailable
*/
private static void validateTable(
Configuration conf,
TableName tableName,
String family,
int valueMultiplier,
boolean dataAvailable)
throws IOException {
LOG.debug("Validating table.");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName);
boolean verified = false;
long pause = conf.getLong("hbase.client.pause", 5 * 1000);
int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
for (int i = 0; i < numRetries; i++) {
try {
Scan scan = new Scan();
// Scan entire family.
scan.addFamily(Bytes.toBytes(family));
if (dataAvailable) {
ResultScanner resScanner = table.getScanner(scan);
for (Result res : resScanner) {
LOG.debug("Getting results " + res.size());
assertTrue(res.size() == 2);
List<Cell> kvs = res.listCells();
assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
assertTrue(
CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
assertTrue(
CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
// Only one result set is expected, so let it loop.
verified = true;
}
} else {
ResultScanner resScanner = table.getScanner(scan);
Result[] next = resScanner.next(2);
assertEquals(0, next.length);
verified = true;
}
break;
} catch (NullPointerException e) {
// If here, a cell was empty. Presume its because updates came in
// after the scanner had been opened. Wait a while and retry.
}
try {
Thread.sleep(pause);
} catch (InterruptedException e) {
// continue
}
}
table.close();
connection.close();
assertTrue(verified);
}
@SuppressWarnings("deprecation")
@Override
protected void serviceInit(Configuration conf) throws Exception {
asyncApiPollIntervalMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
asyncApiPollTimeoutMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS);
submitPollIntervalMillis = asyncApiPollIntervalMillis;
if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS) != null) {
submitPollIntervalMillis =
conf.getLong(
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
}
if (conf.getBoolean(
YarnConfiguration.APPLICATION_HISTORY_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
historyServiceEnabled = true;
historyClient = AHSClient.createAHSClient();
historyClient.init(conf);
}
if (conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
timelineServiceEnabled = true;
timelineClient = createTimelineClient();
timelineClient.init(conf);
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
timelineService = TimelineUtils.buildTimelineTokenService(conf);
}
timelineServiceBestEffort =
conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
super.serviceInit(conf);
}
/**
* Write random bytes in the distributed cache files that will be used by all simulated jobs of
* current gridmix run, if files are to be generated. Do this as part of the MapReduce job {@link
* GenerateDistCacheData#JOB_NAME}
*
* @see org.apache.hadoop.mapred.gridmix.GenerateDistCacheData
*/
protected void writeDistCacheData(Configuration conf) throws IOException, InterruptedException {
int fileCount = conf.getInt(GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_COUNT, -1);
if (fileCount > 0) { // generate distributed cache files
final GridmixJob genDistCacheData = new GenerateDistCacheData(conf);
LOG.info(
"Generating distributed cache data of size "
+ conf.getLong(GenerateDistCacheData.GRIDMIX_DISTCACHE_BYTE_COUNT, -1));
launchGridmixJob(genDistCacheData);
}
}
public static void iterativePageRank(Configuration conf)
throws IOException, InterruptedException, ClassNotFoundException {
String initialVector = conf.get("initialRankVectorPath");
String currentVector = conf.get("currentRankVectorPath");
String finalVector = conf.get("finalRankVectorPath");
/*here the testing system will seach for the final rank vector*/
Double epsilon = conf.getDouble("epsilon", 0.1);
Double beta = conf.getDouble("beta", 0.8);
// Launch remove dead ends jobs
RemoveDeadends.job(conf);
// Create initial vector
Long nNodes = conf.getLong("numNodes", 1);
createInitialRankVector(initialVector, conf.getLong("numNodes", 1));
// Create stochastic matrix
GraphToMatrix.job(conf);
boolean converg = false;
// multiplication M * r
MatrixVectorMult.job(conf);
avoidSpiderTraps(currentVector, nNodes, beta);
while (!converg) {
FileUtils.deleteQuietly(new File(initialVector));
FileUtils.moveFile(
new File(currentVector + "/part-r-00000"), new File(initialVector + "/part-r-00000"));
// multiplication M * r
MatrixVectorMult.job(conf);
avoidSpiderTraps(currentVector, nNodes, beta);
converg = checkConvergence(initialVector, currentVector, epsilon);
}
FileUtils.copyDirectory(new File(currentVector), new File(finalVector));
}
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
checkNotNull(uri, "uri is null");
checkNotNull(conf, "conf is null");
super.initialize(uri, conf);
setConf(conf);
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.workingDirectory = new Path("/").makeQualified(this.uri, new Path("/"));
HiveClientConfig defaults = new HiveClientConfig();
this.stagingDirectory =
new File(conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString()));
this.maxClientRetries = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries());
this.maxBackoffTime =
Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString()));
this.maxRetryTime =
Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString()));
int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
Duration connectTimeout =
Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
Duration socketTimeout =
Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections());
long minFileSize =
conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes());
long minPartSize =
conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes());
ClientConfiguration configuration = new ClientConfiguration();
configuration.setMaxErrorRetry(maxErrorRetries);
configuration.setProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP);
configuration.setConnectionTimeout(Ints.checkedCast(connectTimeout.toMillis()));
configuration.setSocketTimeout(Ints.checkedCast(socketTimeout.toMillis()));
configuration.setMaxConnections(maxConnections);
this.s3 = new AmazonS3Client(getAwsCredentials(uri, conf), configuration);
transferConfig.setMultipartUploadThreshold(minFileSize);
transferConfig.setMinimumUploadPartSize(minPartSize);
}
/**
* Creates a WARC file, and opens it for writing. If a file with the same name already exists, it
* is *overwritten*. Note that this is different behaviour from the other constructor. Yes, this
* sucks. It will probably change in a future version.
*
* @param conf The Hadoop configuration.
* @param codec If null, the file is uncompressed. If non-null, this compression codec will be
* used. The codec's default file extension is appended to the filename.
* @param workOutputPath The directory and filename prefix to which the data should be written. We
* append a segment number and filename extensions to it.
* @param progress An object used by the mapred API for tracking a task's progress.
* @throws IOException
*/
public WARCFileWriter(
Configuration conf, CompressionCodec codec, Path workOutputPath, Progressable progress)
throws IOException {
this.conf = conf;
this.codec = codec;
this.workOutputPath = workOutputPath;
this.progress = progress;
this.extensionFormat =
".seg-%05d.attempt-%05d.warc" + (codec == null ? "" : codec.getDefaultExtension());
this.maxSegmentSize = conf.getLong("warc.output.segment.size", DEFAULT_MAX_SEGMENT_SIZE);
createSegment();
}
private void loadMaxRssMemoryConfig(Configuration conf) {
long reservedRssMemoryMB =
conf.getLong(
TaskMemoryManagerThread.TT_RESERVED_PHYSICAL_MEMORY_MB, JobConf.DISABLED_MEMORY_LIMIT);
if (reservedRssMemoryMB == JobConf.DISABLED_MEMORY_LIMIT) {
reservedRssMemory = JobConf.DISABLED_MEMORY_LIMIT;
maxRssMemoryAllowedForAllTasks = JobConf.DISABLED_MEMORY_LIMIT;
} else {
reservedRssMemory = reservedRssMemoryMB * 1024 * 1024L;
maxRssMemoryAllowedForAllTasks = taskTracker.getTotalPhysicalMemoryOnTT() - reservedRssMemory;
}
}
FakeServer(
final Configuration c,
final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
final AtomicLong sequenceids) {
this.meta = meta;
this.sequenceids = sequenceids;
// Pause to simulate the server taking time applying the edits. This will drive up the
// number of threads used over in client.
this.multiPause = c.getLong("hbase.test.multi.pause.when.done", 0);
this.tooManyMultiRequests = c.getInt("hbase.test.multi.too.many", 3);
}
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// 初始化函数
// System.out.println("initialize");
// Logger.getLogger("KirchhoffMigration").log(Level.INFO,
// "enter initialize()");
FileSplit inputsplit = (FileSplit) split;
Configuration conf = context.getConfiguration();
LENGTH = conf.getLong("FileSplitLength", 0);
SPILL = LENGTH / conf.getInt("SplitPerMap", 1);
// LENGTH = 8;
// SPILL = 8;
assert (LENGTH >= SPILL);
// System.out.println("length:" + LENGTH);
// System.out.println("spill:" + SPILL);
String filename = inputsplit.getPath().toString();
// System.out.println("filename:" + filename);
// String buf = filename.substring(filename.lastIndexOf("fcxy") + 4,
// filename.lastIndexOf("."));
// int count = Integer.parseInt(buf);
// System.out.println(filename);
// start = inputsplit.getStart(); // 得到此分片开始位置
start = inputsplit.getStart();
shotNum += start * 8 / Float.SIZE;
long offset = LENGTH >= inputsplit.getLength() ? inputsplit.getLength() : LENGTH;
end = start + offset; // 结束此分片位置
// System.out.println("inputSplitLength:" + split.getLength());
// System.out.println("end:" + end);
// start = inputsplit.getStart(); // 得到此分片开始位置
// end = start + inputsplit.getLength();// 结束此分片位置
// System.out.println("start:" + start + " ,end:" + end);
final Path file = inputsplit.getPath();
// System.out.println(file.toString());
// 打开文件
FileSystem fs = file.getFileSystem(context.getConfiguration());
fileIn = fs.open(inputsplit.getPath());
// 关键位置2
// 将文件指针移动到当前分片，因为每次默认打开文件时，`其指针指向开头
fileIn.seek(start);
// in = new LineReader(fileIn, context.getConfiguration());
// if (start != 0) {
// System.out.println("not the first split");
// // 关键解决位置1
// //
// 如果这不是第一个分片，那么假设第一个分片是0——4，那么，第4个位置已经被读取，则需要跳过4，否则会产生读入错误，因为你回头又去读之前读过的地方
// start += (end - pos + 1);
// }
pos = start;
}
ManyServersManyRegionsConnection(
Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException {
super(conf, managed, pool, user);
int serverCount = conf.getInt("hbase.test.servers", 10);
this.serversByClient = new HashMap<ServerName, ClientService.BlockingInterface>(serverCount);
this.meta =
makeMeta(
Bytes.toBytes(conf.get("hbase.test.tablename", Bytes.toString(BIG_USER_TABLE))),
conf.getInt("hbase.test.regions", 100),
conf.getLong("hbase.test.namespace.span", 1000),
serverCount);
this.conf = conf;
}
