private void assertEqualMessages(V3LcapMessage a, V3LcapMessage b) throws Exception {
assertTrue(a.getOriginatorId() == b.getOriginatorId());
assertEquals(a.getOpcode(), b.getOpcode());
assertEquals(a.getTargetUrl(), b.getTargetUrl());
assertEquals(a.getArchivalId(), b.getArchivalId());
assertEquals(a.getProtocolVersion(), b.getProtocolVersion());
assertEquals(a.getPollerNonce(), b.getPollerNonce());
assertEquals(a.getVoterNonce(), b.getVoterNonce());
assertEquals(a.getVoterNonce2(), b.getVoterNonce2());
assertEquals(a.getPluginVersion(), b.getPluginVersion());
assertEquals(a.getHashAlgorithm(), b.getHashAlgorithm());
assertEquals(a.isVoteComplete(), b.isVoteComplete());
assertEquals(a.getRepairDataLength(), b.getRepairDataLength());
assertEquals(a.getLastVoteBlockURL(), b.getLastVoteBlockURL());
assertIsomorphic(a.getNominees(), b.getNominees());
List aBlocks = new ArrayList();
List bBlocks = new ArrayList();
for (VoteBlocksIterator iter = a.getVoteBlockIterator(); iter.hasNext(); ) {
aBlocks.add(iter.next());
}
for (VoteBlocksIterator iter = b.getVoteBlockIterator(); iter.hasNext(); ) {
bBlocks.add(iter.next());
}
assertTrue(aBlocks.equals(bBlocks));
// TODO: Figure out how to test time.
}
public void setRepoMap(Map<String, PlatformUtil.DF> repoMap) {
List repos = new ArrayList();
this.repoMap = repoMap;
for (String repo : repoMap.keySet()) {
repos.add(repo);
}
setRepos(repos);
}
public void emitMetadata(ArticleFiles af, ArticleMetadata md) {
if (log.isDebug3()) log.debug3("emit(" + af + ", " + md + ")");
if (md != null) {
log.debug3("add " + md + " to amlist");
amlst.add(md);
}
;
}
// make a Schedule with one chunk per task
Schedule sched(List tasks) {
List events = new ArrayList();
for (Iterator iter = tasks.iterator(); iter.hasNext(); ) {
Object obj = iter.next();
if (obj instanceof Schedule.Event) {
events.add(obj);
} else {
SchedulableTask task = (SchedulableTask) obj;
if (task.isBackgroundTask()) {
events.add(bEvent((BackgroundTask) task, Schedule.EventType.START));
} else {
events.add(chunk((StepTask) task));
}
}
}
Schedule s = new Schedule(events);
return s;
}
List<String> auUrls(ArchivalUnit au) {
List<String> res = new ArrayList<String>();
for (Iterator iter = au.getAuCachedUrlSet().contentHashIterator(); iter.hasNext(); ) {
CachedUrlSetNode cusn = (CachedUrlSetNode) iter.next();
if (cusn.hasContent()) {
res.add(cusn.getUrl());
}
}
return res;
}
private List getSummaryInfo(CrawlerStatus status) {
List res = new ArrayList();
StatusTable.SummaryInfo statusSi =
new StatusTable.SummaryInfo(
"Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
ArchivalUnit au = status.getAu();
if (au != null) {
AuState aus = AuUtil.getAuState(au);
if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
}
}
res.add(statusSi);
String sources = StringUtil.separatedString(status.getSources());
res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
String startUrls = StringUtil.separatedString(status.getStartUrls());
res.add(
new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
return res;
}
public void testRequestMessageCreation() throws Exception {
V3LcapMessage reqMsg =
new V3LcapMessage(
"ArchivalID_2",
"key",
"Plug42",
m_testBytes,
m_testBytes,
V3LcapMessage.MSG_REPAIR_REQ,
987654321,
m_testID,
tempDir,
theDaemon);
reqMsg.setTargetUrl("http://foo.com/");
for (Iterator ix = m_testVoteBlocks.iterator(); ix.hasNext(); ) {
reqMsg.addVoteBlock((VoteBlock) ix.next());
}
assertEquals(3, reqMsg.getProtocolVersion());
assertEquals("Plug42", reqMsg.getPluginVersion());
assertTrue(m_testID == reqMsg.getOriginatorId());
assertEquals(V3LcapMessage.MSG_REPAIR_REQ, reqMsg.getOpcode());
assertEquals("ArchivalID_2", reqMsg.getArchivalId());
assertEquals("http://foo.com/", reqMsg.getTargetUrl());
assertEquals(m_testBytes, reqMsg.getPollerNonce());
assertEquals(m_testBytes, reqMsg.getVoterNonce());
assertEquals(null, reqMsg.getVoterNonce2());
List aBlocks = new ArrayList();
List bBlocks = new ArrayList();
for (VoteBlocksIterator iter = m_testMsg.getVoteBlockIterator(); iter.hasNext(); ) {
aBlocks.add(iter.next());
}
for (VoteBlocksIterator iter = reqMsg.getVoteBlockIterator(); iter.hasNext(); ) {
bBlocks.add(iter.next());
}
assertEquals(aBlocks, bBlocks);
// Actual size of test vote blocks is unpredictable
assertTrue(reqMsg.getEstimatedEncodedLength() > V3LcapMessage.EST_ENCODED_HEADER_LENGTH);
}
/** iterate over the mime-types makeRow for each */
private List getRows(CrawlerStatus status, String key) {
Collection mimeTypes = status.getMimeTypes();
List rows = new ArrayList();
if (mimeTypes != null) {
String mimeType;
for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) {
mimeType = (String) it.next();
rows.add(makeRow(status, mimeType, key));
}
}
return rows;
}
public void testStoreNodeState() throws Exception {
TimeBase.setSimulated(100);
CachedUrlSet mcus =
new MockCachedUrlSet(mau, new RangeCachedUrlSetSpec("http://www.example.com"));
CrawlState crawl = new CrawlState(1, 2, 123);
List polls = new ArrayList(2);
PollState poll1 = new PollState(1, "sdf", "jkl", 2, 123, Deadline.at(456), false);
PollState poll2 = new PollState(2, "abc", "def", 3, 321, Deadline.at(654), false);
polls.add(poll1);
polls.add(poll2);
NodeState nodeState = new NodeStateImpl(mcus, 123321, crawl, polls, repository);
((NodeStateImpl) nodeState).setState(NodeState.DAMAGE_AT_OR_BELOW);
repository.storeNodeState(nodeState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath =
LockssRepositoryImpl.mapUrlToFileLocation(
filePath, "http://www.example.com/" + HistoryRepositoryImpl.NODE_FILE_NAME);
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
nodeState = null;
nodeState = repository.loadNodeState(mcus);
assertSame(mcus, nodeState.getCachedUrlSet());
assertEquals(123321, nodeState.getAverageHashDuration());
assertEquals(1, nodeState.getCrawlState().getType());
assertEquals(2, nodeState.getCrawlState().getStatus());
assertEquals(123, nodeState.getCrawlState().getStartTime());
assertEquals(NodeState.DAMAGE_AT_OR_BELOW, nodeState.getState());
Iterator pollIt = nodeState.getActivePolls();
assertTrue(pollIt.hasNext());
PollState loadedPoll = (PollState) pollIt.next();
assertEquals(1, loadedPoll.getType());
assertEquals("sdf", loadedPoll.getLwrBound());
assertEquals("jkl", loadedPoll.getUprBound());
assertEquals(2, loadedPoll.getStatus());
assertEquals(123, loadedPoll.getStartTime());
assertEquals(456, loadedPoll.getDeadline().getExpirationTime());
assertTrue(pollIt.hasNext());
loadedPoll = (PollState) pollIt.next();
assertEquals(2, loadedPoll.getType());
assertEquals("abc", loadedPoll.getLwrBound());
assertEquals("def", loadedPoll.getUprBound());
assertEquals(3, loadedPoll.getStatus());
assertEquals(321, loadedPoll.getStartTime());
assertEquals(654, loadedPoll.getDeadline().getExpirationTime());
assertFalse(pollIt.hasNext());
TimeBase.setReal();
}
private PollHistoryBean createPollHistoryBean(int voteCount) throws Exception {
PollState state = new PollState(1, "lwr", "upr", 2, 5, null, false);
List votes = new ArrayList(voteCount);
for (int ii = 0; ii < voteCount; ii++) {
VoteBean bean = new VoteBean();
bean.setId(idKey);
bean.setAgreeState(true);
bean.setChallengeString("1234");
bean.setHashString("2345");
bean.setVerifierString("3456");
votes.add(bean.getVote());
}
return new PollHistoryBean(new PollHistory(state, 0, votes));
}
public List findExistingRepositoriesFor(String auid) {
List res = null;
for (Iterator iter = getRepositoryList().iterator(); iter.hasNext(); ) {
String repoName = (String) iter.next();
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
if (LockssRepositoryImpl.doesAuDirExist(auid, path)) {
if (res == null) {
res = new ArrayList();
}
res.add(repoName);
}
}
return res == null ? Collections.EMPTY_LIST : res;
}
private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
throws FileNotFoundException {
String first = null;
String next = extMapName;
List<String> urls = new ArrayList<String>();
ExternalizableMap res = null;
while (next != null) {
// convert the plugin class name to an xml file name
String mapFile = next.replace('.', '/') + MAP_SUFFIX;
URL url = loader.getResource(mapFile);
if (url != null && urls.contains(url.toString())) {
throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
}
// load into map
ExternalizableMap oneMap = new ExternalizableMap();
oneMap.loadMapFromResource(mapFile, loader);
urls.add(url.toString());
// apply overrides one plugin at a time in inheritance chain
processOverrides(oneMap);
if (res == null) {
res = oneMap;
} else {
for (Map.Entry ent : oneMap.entrySet()) {
String key = (String) ent.getKey();
Object val = ent.getValue();
if (!res.containsKey(key)) {
res.setMapElement(key, val);
}
}
}
if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
next = oneMap.getString(KEY_PLUGIN_PARENT);
} else {
next = null;
}
}
loadedFromUrls = urls;
return res;
}
public void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
// Build list of repositories from list of disk (fs) paths). Needs to
// be generalized if ever another repository implementation.
if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
List lst = new ArrayList();
String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
List paths = StringUtil.breakAt(dspace, ';');
if (paths != null) {
for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
lst.add("local:" + (String) iter.next());
}
}
repoList = lst;
}
if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
paramNodeCacheSize =
config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
LockssRepository repo = (LockssRepository) iter.next();
if (repo instanceof LockssRepositoryImpl) {
LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
repoImpl.setNodeCacheSize(paramNodeCacheSize);
}
}
}
if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
paramSuspectVersionsCacheSize =
config.getInt(
PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
}
if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
paramIsGlobalNodeCache =
config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
if (paramIsGlobalNodeCache) {
paramGlobalNodeCacheSize =
config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
log.debug("global node cache size: " + paramGlobalNodeCacheSize);
globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
}
}
if (changedKeys.contains(DISK_PREFIX)) {
int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
double minPer =
config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
}
if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
}
if (changedKeys.contains(PREFIX)) {
maxUnusedDirSearch =
config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
isStatefulUnusedDirSearch =
config.getBoolean(
PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
enableLongComponents =
config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
enableLongComponentsCompatibility =
config.getBoolean(
PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
checkUnnormalized =
(CheckUnnormalizedMode)
config.getEnum(
CheckUnnormalizedMode.class,
PARAM_CHECK_UNNORMALIZED,
DEFAULT_CHECK_UNNORMALIZED);
}
}
private void addIfNonZero(List res, String head, int val) {
if (val != 0) {
res.add(new StatusTable.SummaryInfo(head, ColumnDescriptor.TYPE_INT, new Long(val)));
}
}
protected void recordExportFile(File file) {
if (fileSet.add(file)) {
fileList.add(file);
}
}
protected void recordError(String msg) {
log.error(msg);
errors.add(msg);
}
protected void recordError(String msg, Throwable t) {
log.error(msg, t);
errors.add(msg + ": " + t.toString());
}
void doSizeCalc(RepositoryNode node) {
TimerUtil.guaranteedSleep(10);
nodes.add(node);
sem.give();
}