void setConfig(Configuration config) {
log.debug("config: " + config);
proxyHost = config.get(PARAM_PROXY_HOST);
proxyPort = config.getInt(PARAM_PROXY_PORT, DEFAULT_PROXY_PORT);
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
String http_proxy = System.getenv("http_proxy");
if (!StringUtil.isNullString(http_proxy)) {
try {
HostPortParser hpp = new HostPortParser(http_proxy);
proxyHost = hpp.getHost();
proxyPort = hpp.getPort();
} catch (HostPortParser.InvalidSpec e) {
log.warning("Can't parse http_proxy environment var, ignoring: " + http_proxy + ": " + e);
}
}
}
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
proxyHost = null;
} else {
log.info("Proxying through " + proxyHost + ":" + proxyPort);
}
userAgent = config.get(PARAM_USER_AGENT);
if (StringUtil.isNullString(userAgent)) {
userAgent = null;
} else {
log.debug("Setting User-Agent to " + userAgent);
}
}
/** Start the JobTracker process, listen on the indicated port */
JobTracker(Configuration conf) throws IOException {
//
// Grab some static constants
//
maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
// This is a directory of temporary submission files. We delete it
// on startup, and can delete any files that we're done with
this.conf = conf;
JobConf jobConf = new JobConf(conf);
this.systemDir = jobConf.getSystemDir();
this.fs = FileSystem.get(conf);
FileUtil.fullyDelete(fs, systemDir);
fs.mkdirs(systemDir);
// Same with 'localDir' except it's always on the local disk.
jobConf.deleteLocalFiles(SUBDIR);
// Set ports, start RPC servers, etc.
InetSocketAddress addr = getAddress(conf);
this.localMachine = addr.getHostName();
this.port = addr.getPort();
this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
this.interTrackerServer.start();
Properties p = System.getProperties();
for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
String key = (String) it.next();
String val = (String) p.getProperty(key);
LOG.info("Property '" + key + "' is " + val);
}
this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();
this.startTime = System.currentTimeMillis();
new Thread(this.expireTrackers).start();
new Thread(this.retireJobs).start();
new Thread(this.initJobs).start();
}
/** An FSDataset has a directory where it loads its data files. */
public FSDataset(DataStorage storage, Configuration conf) throws IOException {
this.maxBlocksPerDir = conf.getInt("dfs.datanode.numblocks", 64);
FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()];
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(), conf);
}
volumes = new FSVolumeSet(volArray);
volumeMap = new HashMap<Block, DatanodeBlockInfo>();
volumes.getVolumeMap(volumeMap);
registerMBean(storage.getStorageID());
}
private boolean startCrawl(ArchivalUnit au, boolean force, boolean deep)
throws CrawlManagerImpl.NotEligibleException {
CrawlManagerImpl cmi = (CrawlManagerImpl) crawlMgr;
if (force) {
RateLimiter limit = cmi.getNewContentRateLimiter(au);
if (!limit.isEventOk()) {
limit.unevent();
}
}
cmi.checkEligibleToQueueNewContentCrawl(au);
String delayMsg = "";
String deepMsg = "";
try {
cmi.checkEligibleForNewContentCrawl(au);
} catch (CrawlManagerImpl.NotEligibleException e) {
delayMsg = ", Start delayed due to: " + e.getMessage();
}
Configuration config = ConfigManager.getCurrentConfig();
int pri = config.getInt(PARAM_CRAWL_PRIORITY, DEFAULT_CRAWL_PRIORITY);
CrawlReq req;
try {
req = new CrawlReq(au);
req.setPriority(pri);
if (deep) {
int d = Integer.parseInt(formDepth);
if (d < 0) {
errMsg = "Illegal refetch depth: " + d;
return false;
}
req.setRefetchDepth(d);
deepMsg = "Deep (" + req.getRefetchDepth() + ") ";
}
} catch (NumberFormatException e) {
errMsg = "Illegal refetch depth: " + formDepth;
return false;
} catch (RuntimeException e) {
log.error("Couldn't create CrawlReq: " + au, e);
errMsg = "Couldn't create CrawlReq: " + e.toString();
return false;
}
cmi.startNewContentCrawl(req, null);
statusMsg = deepMsg + "Crawl requested for " + au.getName() + delayMsg;
return true;
}
public VehiclesConfiguration(Configuration cfg, File dataFolder) {
this.dataFolder = dataFolder;
matBoostMax = Material.getMaterial(cfg.getInt("max-boost-block", 41));
matBoost25x = Material.getMaterial(cfg.getInt("25x-boost-block", 14));
matSlow50x = Material.getMaterial(cfg.getInt("50x-slow-block", 88));
matSlow20x = Material.getMaterial(cfg.getInt("20x-slow-block", 13));
matReverse = Material.getMaterial(cfg.getInt("reverse-block", 35));
matStation = Material.getMaterial(cfg.getInt("station-block", 49));
matSorter = Material.getMaterial(cfg.getInt("sort-block", 87));
matEjector = Material.getMaterial(cfg.getInt("eject-block", 42));
matDeposit = Material.getMaterial(cfg.getInt("deposit-block", 15));
matTeleport = Material.getMaterial(cfg.getInt("teleport-block", 89));
matDispenser =
Material.getMaterial(54); // this can't be configurable because we need it to be a chest!
minecartSlowWhenEmpty = cfg.getBoolean("minecart-slow-when-empty", true);
minecartMaxSpeedModifier = cfg.getDouble("minecart-max-speed-modifier", 1);
}
public void setConf(Configuration conf) {
this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
this.conf = conf;
}
/** Called by org.lockss.config.MiscConfig */
public static void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
if (diffs.contains(PREFIX)) {
maxErrors = config.getInt(PARAM_MAX_ERRORS, DEFAULT_MAX_ERRORS);
}
}
