public void testHexValues() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.hex1", "0x10");
appendProperty("test.hex2", "0xF");
appendProperty("test.hex3", "-0x10");
// Invalid?
appendProperty("test.hex4", "-0x10xyz");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(16, conf.getInt("test.hex1", 0));
assertEquals(16, conf.getLong("test.hex1", 0));
assertEquals(15, conf.getInt("test.hex2", 0));
assertEquals(15, conf.getLong("test.hex2", 0));
assertEquals(-16, conf.getInt("test.hex3", 0));
assertEquals(-16, conf.getLong("test.hex3", 0));
try {
conf.getLong("test.hex4", 0);
fail("Property had invalid long value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
try {
conf.getInt("test.hex4", 0);
fail("Property had invalid int value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
public void testIntegerValues() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.int1", "20");
appendProperty("test.int2", "020");
appendProperty("test.int3", "-20");
appendProperty("test.int4", " -20 ");
appendProperty("test.int5", " -20xyz ");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(20, conf.getInt("test.int1", 0));
assertEquals(20, conf.getLong("test.int1", 0));
assertEquals(20, conf.getLongBytes("test.int1", 0));
assertEquals(20, conf.getInt("test.int2", 0));
assertEquals(20, conf.getLong("test.int2", 0));
assertEquals(20, conf.getLongBytes("test.int2", 0));
assertEquals(-20, conf.getInt("test.int3", 0));
assertEquals(-20, conf.getLong("test.int3", 0));
assertEquals(-20, conf.getLongBytes("test.int3", 0));
assertEquals(-20, conf.getInt("test.int4", 0));
assertEquals(-20, conf.getLong("test.int4", 0));
assertEquals(-20, conf.getLongBytes("test.int4", 0));
try {
conf.getInt("test.int5", 0);
fail("Property had invalid int value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
public void run() {
setName("Bukkitmanager Automessage");
while (run) {
if (config.getInt("Automessage.Interval") == 0) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
if (config.getDebug()) e.printStackTrace();
}
continue;
}
for (int i = 0; i < config.getInt("Automessage.Interval"); i++) {
try {
if (!run) {
return;
}
Thread.sleep(1000);
} catch (InterruptedException e) {
if (config.getDebug()) e.printStackTrace();
}
}
plugin
.getServer()
.getScheduler()
.runTaskAsynchronously(
plugin,
new Runnable() {
public void run() {
msgReader.sendMsg();
}
});
}
}
public boolean createShield(Location location, Player player) {
// Creates a new shield.
Configuration config = this.plugin.getConfiguration();
// Make sure the shield is placable at this position.
for (Shield intersect : this.getIntersects(location, config.getInt("shield.create.radius"))) {
if (!intersect.isOwner(player)) {
return false;
}
}
if (!config.getBoolean("mechanics.blocks-above-crystal") && this.hasBlocksAbove(location)) {
return false;
}
// Create shield.
Shield shield =
new Shield(
location,
this.plugin,
config.getInt("shield.create.radius") * config.getInt("shield.health-multipler"),
config.getInt("shield.create.radius"),
false);
shield.addOwner(player.getName());
this.shields.add(shield);
for (Location claimedLocation : this.getLocations(shield.getLocation(), shield.getRadius())) {
this.plugin.getWorldHealer().removeLocation(claimedLocation);
}
return true;
}
void setConfig(Configuration config) {
log.debug("config: " + config);
proxyHost = config.get(PARAM_PROXY_HOST);
proxyPort = config.getInt(PARAM_PROXY_PORT, DEFAULT_PROXY_PORT);
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
String http_proxy = System.getenv("http_proxy");
if (!StringUtil.isNullString(http_proxy)) {
try {
HostPortParser hpp = new HostPortParser(http_proxy);
proxyHost = hpp.getHost();
proxyPort = hpp.getPort();
} catch (HostPortParser.InvalidSpec e) {
log.warning("Can't parse http_proxy environment var, ignoring: " + http_proxy + ": " + e);
}
}
}
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
proxyHost = null;
} else {
log.info("Proxying through " + proxyHost + ":" + proxyPort);
}
userAgent = config.get(PARAM_USER_AGENT);
if (StringUtil.isNullString(userAgent)) {
userAgent = null;
} else {
log.debug("Setting User-Agent to " + userAgent);
}
}
public void testVariableSubstitution() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
declareProperty("my.int", "${intvar}", "42");
declareProperty("intvar", "42", "42");
declareProperty("my.base", "/tmp/${user.name}", UNSPEC);
declareProperty("my.file", "hello", "hello");
declareProperty("my.suffix", ".txt", ".txt");
declareProperty("my.relfile", "${my.file}${my.suffix}", "hello.txt");
declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", UNSPEC);
// check that undefined variables are returned as-is
declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
for (Prop p : props) {
System.out.println("p=" + p.name);
String gotVal = conf.get(p.name);
String gotRawVal = conf.getRaw(p.name);
assertEq(p.val, gotRawVal);
if (p.expectEval == UNSPEC) {
// expansion is system-dependent (uses System properties)
// can't do exact match so just check that all variables got expanded
assertTrue(gotVal != null && -1 == gotVal.indexOf("${"));
} else {
assertEq(p.expectEval, gotVal);
}
}
// check that expansion also occurs for getInt()
assertTrue(conf.getInt("intvar", -1) == 42);
assertTrue(conf.getInt("my.int", -1) == 42);
}
// Called by RegistryPlugin iff any config below RegistryPlugin.PREFIX
// has changed
protected void setConfig(
Configuration config, Configuration prevConfig, Configuration.Differences changedKeys) {
m_maxRefetchDepth =
config.getInt(
NewContentCrawler.PARAM_MAX_CRAWL_DEPTH, NewContentCrawler.DEFAULT_MAX_CRAWL_DEPTH);
fetchRateLimiter = recomputeFetchRateLimiter(fetchRateLimiter);
enablePolls = config.getBoolean(PARAM_ENABLE_REGISTRY_POLLS, DEFAULT_ENABLE_REGISTRY_POLLS);
}
/** Start the JobTracker process, listen on the indicated port */
JobTracker(Configuration conf) throws IOException {
//
// Grab some static constants
//
maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
// This is a directory of temporary submission files. We delete it
// on startup, and can delete any files that we're done with
this.conf = conf;
JobConf jobConf = new JobConf(conf);
this.systemDir = jobConf.getSystemDir();
this.fs = FileSystem.get(conf);
FileUtil.fullyDelete(fs, systemDir);
fs.mkdirs(systemDir);
// Same with 'localDir' except it's always on the local disk.
jobConf.deleteLocalFiles(SUBDIR);
// Set ports, start RPC servers, etc.
InetSocketAddress addr = getAddress(conf);
this.localMachine = addr.getHostName();
this.port = addr.getPort();
this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
this.interTrackerServer.start();
Properties p = System.getProperties();
for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
String key = (String) it.next();
String val = (String) p.getProperty(key);
LOG.info("Property '" + key + "' is " + val);
}
this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();
this.startTime = System.currentTimeMillis();
new Thread(this.expireTrackers).start();
new Thread(this.retireJobs).start();
new Thread(this.initJobs).start();
}
/** An FSDataset has a directory where it loads its data files. */
public FSDataset(DataStorage storage, Configuration conf) throws IOException {
this.maxBlocksPerDir = conf.getInt("dfs.datanode.numblocks", 64);
FSVolume[] volArray = new FSVolume[storage.getNumStorageDirs()];
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
volArray[idx] = new FSVolume(storage.getStorageDir(idx).getCurrentDir(), conf);
}
volumes = new FSVolumeSet(volArray);
volumeMap = new HashMap<Block, DatanodeBlockInfo>();
volumes.getVolumeMap(volumeMap);
registerMBean(storage.getStorageID());
}
private boolean startCrawl(ArchivalUnit au, boolean force, boolean deep)
throws CrawlManagerImpl.NotEligibleException {
CrawlManagerImpl cmi = (CrawlManagerImpl) crawlMgr;
if (force) {
RateLimiter limit = cmi.getNewContentRateLimiter(au);
if (!limit.isEventOk()) {
limit.unevent();
}
}
cmi.checkEligibleToQueueNewContentCrawl(au);
String delayMsg = "";
String deepMsg = "";
try {
cmi.checkEligibleForNewContentCrawl(au);
} catch (CrawlManagerImpl.NotEligibleException e) {
delayMsg = ", Start delayed due to: " + e.getMessage();
}
Configuration config = ConfigManager.getCurrentConfig();
int pri = config.getInt(PARAM_CRAWL_PRIORITY, DEFAULT_CRAWL_PRIORITY);
CrawlReq req;
try {
req = new CrawlReq(au);
req.setPriority(pri);
if (deep) {
int d = Integer.parseInt(formDepth);
if (d < 0) {
errMsg = "Illegal refetch depth: " + d;
return false;
}
req.setRefetchDepth(d);
deepMsg = "Deep (" + req.getRefetchDepth() + ") ";
}
} catch (NumberFormatException e) {
errMsg = "Illegal refetch depth: " + formDepth;
return false;
} catch (RuntimeException e) {
log.error("Couldn't create CrawlReq: " + au, e);
errMsg = "Couldn't create CrawlReq: " + e.toString();
return false;
}
cmi.startNewContentCrawl(req, null);
statusMsg = deepMsg + "Crawl requested for " + au.getName() + delayMsg;
return true;
}
public void setConf(Configuration conf) {
this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
this.conf = conf;
}
/** Called by org.lockss.config.MiscConfig */
public static void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
if (diffs.contains(PREFIX)) {
maxErrors = config.getInt(PARAM_MAX_ERRORS, DEFAULT_MAX_ERRORS);
}
}
public void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
// Build list of repositories from list of disk (fs) paths). Needs to
// be generalized if ever another repository implementation.
if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
List lst = new ArrayList();
String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
List paths = StringUtil.breakAt(dspace, ';');
if (paths != null) {
for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
lst.add("local:" + (String) iter.next());
}
}
repoList = lst;
}
if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
paramNodeCacheSize =
config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
LockssRepository repo = (LockssRepository) iter.next();
if (repo instanceof LockssRepositoryImpl) {
LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
repoImpl.setNodeCacheSize(paramNodeCacheSize);
}
}
}
if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
paramSuspectVersionsCacheSize =
config.getInt(
PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
}
if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
paramIsGlobalNodeCache =
config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
if (paramIsGlobalNodeCache) {
paramGlobalNodeCacheSize =
config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
log.debug("global node cache size: " + paramGlobalNodeCacheSize);
globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
}
}
if (changedKeys.contains(DISK_PREFIX)) {
int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
double minPer =
config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
}
if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
}
if (changedKeys.contains(PREFIX)) {
maxUnusedDirSearch =
config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
isStatefulUnusedDirSearch =
config.getBoolean(
PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
enableLongComponents =
config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
enableLongComponentsCompatibility =
config.getBoolean(
PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
checkUnnormalized =
(CheckUnnormalizedMode)
config.getEnum(
CheckUnnormalizedMode.class,
PARAM_CHECK_UNNORMALIZED,
DEFAULT_CHECK_UNNORMALIZED);
}
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
