private void initTestPolls() throws Exception {
testV1polls = new V1Poll[testV1msg.length];
for (int i = 0; i < testV1polls.length; i++) {
log.debug3("initTestPolls: V1 " + i);
BasePoll p = pollmanager.makePoll(testV1msg[i]);
assertNotNull(p);
assertNotNull(p.getMessage());
log.debug("initTestPolls: V1 " + i + " returns " + p);
assertTrue(p instanceof V1Poll);
switch (i) {
case 0:
assertTrue(p instanceof V1NamePoll);
break;
case 1:
assertTrue(p instanceof V1ContentPoll);
break;
case 2:
assertTrue(p instanceof V1VerifyPoll);
break;
}
testV1polls[i] = (V1Poll) p;
assertNotNull(testV1polls[i]);
log.debug3("initTestPolls: " + i + " " + p.toString());
}
}
public String getDefaultArticleMimeType() {
String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
log.debug3("DefaultArticleMimeType " + ret);
if (ret == null) {
ret = super.getDefaultArticleMimeType();
log.debug3("DefaultArticleMimeType from super " + ret);
}
return ret;
}
/** test for method scheduleVote(..) */
public void testScheduleVote() {
V1Poll p = testV1polls[1];
assertTrue(p instanceof V1ContentPoll);
log.debug3("testScheduleVote 1");
p.scheduleVote();
log.debug3("testScheduleVote 2");
assertNotNull(p.m_voteTime);
assertTrue(p.m_voteTime.getRemainingTime() < p.m_deadline.getRemainingTime());
log.debug3("at end of testScheduleVote");
}
/**
* Finds the directory for this AU. If none found in the map, designates a new dir for it.
*
* @param auid AU id representing the au
* @param repoRoot path to the root of the repository
* @return the dir String
*/
static String getAuDir(String auid, String repoRoot, boolean create) {
String repoCachePath = extendCacheLocation(repoRoot);
LocalRepository localRepo = getLocalRepository(repoRoot);
synchronized (localRepo) {
Map aumap = localRepo.getAuMap();
String auPathSlash = (String) aumap.get(auid);
if (auPathSlash != null) {
return auPathSlash;
}
if (!create) {
return null;
}
logger.debug3("Creating new au directory for '" + auid + "'.");
String auDir = localRepo.getPrevAuDir();
for (int cnt = RepositoryManager.getMaxUnusedDirSearch(); cnt > 0; cnt--) {
// loop through looking for an available dir
auDir = getNextDirName(auDir);
File testDir = new File(repoCachePath, auDir);
if (logger.isDebug3()) logger.debug3("Probe for unused: " + testDir);
if (!testDir.exists()) {
if (RepositoryManager.isStatefulUnusedDirSearch()) {
localRepo.setPrevAuDir(auDir);
}
String auPath = testDir.toString();
logger.debug3("New au directory: " + auPath);
auPathSlash = auPath + File.separator;
// write the new au property file to the new dir
// XXX this data should be backed up elsewhere to avoid single-point
// corruption
Properties idProps = new Properties();
idProps.setProperty(AU_ID_PROP, auid);
saveAuIdProperties(auPath, idProps);
aumap.put(auid, auPathSlash);
return auPathSlash;
} else {
if (logger.isDebug3()) {
logger.debug3("Existing directory found at '" + auDir + "'. Checking next...");
}
}
}
}
throw new RuntimeException(
"Can't find unused repository dir after "
+ RepositoryManager.getMaxUnusedDirSearch()
+ " tries in "
+ repoCachePath);
}
/**
* Checks the consistency of the node, and continues with its children if it's consistent.
*
* @param node RepositoryNodeImpl the node to check
*/
private void recurseConsistencyCheck(RepositoryNodeImpl node) {
logger.debug2("Checking node '" + node.getNodeUrl() + "'...");
// check consistency at each node
// correct/deactivate as necessary
// 'checkNodeConsistency()' will repair if possible
if (node.checkNodeConsistency()) {
logger.debug3("Node consistent; recursing on children...");
List children = node.getNodeList(null, false);
Iterator iter = children.iterator();
while (iter.hasNext()) {
RepositoryNodeImpl child = (RepositoryNodeImpl) iter.next();
recurseConsistencyCheck(child);
}
} else {
logger.debug3("Node inconsistent; deactivating...");
deactivateInconsistentNode(node);
}
}
@Override
public ArticleMetadata extract(MetadataTarget target, CachedUrl cu) throws IOException {
log.debug3("Metadata - cachedurl cu:" + cu.getUrl());
ArticleMetadata am = super.extract(target, cu);
am.cook(tagMap);
return am;
} // extract
/**
* Return the auid -> au-subdir-path mapping. Enumerating the directories if necessary to
* initialize the map
*/
Map getAuMap() {
if (auMap == null) {
logger.debug3("Loading name map for '" + repoCacheFile + "'.");
auMap = new HashMap();
if (!repoCacheFile.exists()) {
logger.debug3("Creating cache dir:" + repoCacheFile + "'.");
if (!repoCacheFile.mkdirs()) {
logger.critical("Couldn't create directory, check owner/permissions: " + repoCacheFile);
// return empty map
return auMap;
}
} else {
// read each dir's property file and store mapping auid -> dir
File[] auDirs = repoCacheFile.listFiles();
for (int ii = 0; ii < auDirs.length; ii++) {
String dirName = auDirs[ii].getName();
// if (dirName.compareTo(lastPluginDir) == 1) {
// // adjust the 'lastPluginDir' upwards if necessary
// lastPluginDir = dirName;
// }
String path = auDirs[ii].getAbsolutePath();
Properties idProps = getAuIdProperties(path);
if (idProps != null) {
String auid = idProps.getProperty(AU_ID_PROP);
StringBuilder sb = new StringBuilder(path.length() + File.separator.length());
sb.append(path);
sb.append(File.separator);
auMap.put(auid, sb.toString());
logger.debug3("Mapping to: " + auMap.get(auid) + ": " + auid);
} else {
logger.debug3("Not mapping " + path + ", no auid file.");
}
}
}
}
return auMap;
}
/** test for method checkVote(..) */
public void testCheckVote() throws Exception {
V1LcapMessage msg = null;
log.debug3("starting testCheeckVote");
msg =
V1LcapMessage.makeReplyMsg(
testV1polls[0].getMessage(),
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
null,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID);
log.debug3("testCheeckVote 2");
V1Poll p = null;
p = createCompletedPoll(theDaemon, testau, msg, 8, 2, pollmanager);
assertTrue(p instanceof V1NamePoll);
log.debug3("testCheeckVote 3");
assertNotNull(p);
PeerIdentity id = msg.getOriginatorId();
assertNotNull(id);
assertNotNull(p.m_tally);
int rep = p.m_tally.wtAgree + idmgr.getReputation(id);
// good vote check
p.checkVote(msg.getHashed(), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(2, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtAgree);
rep = p.m_tally.wtDisagree + idmgr.getReputation(id);
// bad vote check
p.checkVote(ByteArray.makeRandomBytes(20), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(3, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtDisagree);
}
public MultiPartRequest getMultiPartRequest(int maxLen)
throws FormDataTooLongException, IOException {
if (req.getContentType() == null || !req.getContentType().startsWith("multipart/form-data")) {
return null;
}
if (req.getContentLength() > maxLen) {
throw new FormDataTooLongException(req.getContentLength() + " bytes, " + maxLen + " allowed");
}
MultiPartRequest multi = new MultiPartRequest(req);
if (log.isDebug2()) {
String[] parts = multi.getPartNames();
log.debug3("Multipart request, " + parts.length + " parts");
if (log.isDebug3()) {
for (int p = 0; p < parts.length; p++) {
String name = parts[p];
String cont = multi.getString(parts[p]);
log.debug3(name + ": " + cont);
}
}
}
multiReq = multi;
return multi;
}
/**
* Refill the buffer from the specified reader
*
* @param reader reader from which to refill the charBuffer
* @return true if the reader has reached eof
* @throws IllegalArgumentException if called with a null reader
*/
public boolean refillBuffer(Reader reader) throws IOException {
if (reader == null) {
throw new IllegalArgumentException("Called with null reader");
}
if (isTrace) {
logger.debug3("Refilling buffer");
}
int maxRead;
while ((maxRead = capacity - size) > 0) {
// max chars to add to the end of array
int maxEnd = (maxRead <= (capacity - tail) ? maxRead : (capacity - tail));
// max chars to add to the beginning of array
int maxStart = maxRead - maxEnd;
if (maxStart > 0) {
// We have room at the beginning and end. Using a temporary array
// seems to be cheaper than calling read() twice
if (preBuffer == null) {
preBuffer = new char[capacity];
}
int charsRead = reader.read(preBuffer, 0, maxRead);
if (charsRead == -1) {
return true;
}
try {
add(preBuffer, 0, charsRead);
} catch (CharRing.RingFullException e) {
logger.error("Overfilled a CharRing", e);
throw new IOException("Overfilled a CharRing");
}
} else {
// Adding only to the middle or end, read directly into char buffer
int charsRead = reader.read(chars, tail, maxEnd);
if (charsRead == -1) {
return true;
}
tail = (tail + charsRead) % capacity;
size += charsRead;
if (charsRead < maxEnd) {
continue;
}
}
}
return false;
}
static void saveAuIdProperties(String location, Properties props) {
// XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
// single-point corruption
File propDir = new File(location);
if (!propDir.exists()) {
logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
propDir.mkdirs();
}
File propFile = new File(propDir, AU_ID_FILE);
try {
logger.debug3("Saving au id properties at '" + location + "'.");
OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
props.store(os, "ArchivalUnit id info");
os.close();
propFile.setReadOnly();
} catch (IOException ioe) {
logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
}
}
protected CIProperties makeCIProperties(ArchiveRecordHeader elementHeader) throws IOException {
CIProperties ret = new CIProperties();
Set elementHeaderFieldKeys = elementHeader.getHeaderFieldKeys();
for (Iterator i = elementHeaderFieldKeys.iterator(); i.hasNext(); ) {
String key = (String) i.next();
try {
Object valueObject = elementHeader.getHeaderValue(key);
if (valueObject == null) {
logger.warning("Ignoring null value for key '" + key + "'.");
} else {
String value = valueObject.toString();
logger.debug3(key + ": " + value);
ret.put(key, value);
}
} catch (ClassCastException ex) {
logger.error("makeCIProperties: " + key + " threw ", ex);
throw new CacheException.ExploderException(ex);
}
}
return (ret);
}
private void setMdTypeFact(Map factClassMap, String mdType, String factName) {
log.debug3("Metadata type: " + mdType + " factory " + factName);
FileMetadataExtractorFactory fact =
(FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class);
factClassMap.put(mdType, fact);
}
public static V1Poll createCompletedPoll(
LockssDaemon daemon,
ArchivalUnit au,
V1LcapMessage testmsg,
int numAgree,
int numDisagree,
PollManager pollmanager)
throws Exception {
log.debug(
"createCompletedPoll: au: "
+ au.toString()
+ " peer "
+ testmsg.getOriginatorId()
+ " votes "
+ numAgree
+ "/"
+ numDisagree);
CachedUrlSetSpec cusSpec = null;
if ((testmsg.getLwrBound() != null)
&& (testmsg.getLwrBound().equals(PollSpec.SINGLE_NODE_LWRBOUND))) {
cusSpec = new SingleNodeCachedUrlSetSpec(testmsg.getTargetUrl());
} else {
cusSpec =
new RangeCachedUrlSetSpec(
testmsg.getTargetUrl(), testmsg.getLwrBound(), testmsg.getUprBound());
}
CachedUrlSet cus = au.makeCachedUrlSet(cusSpec);
PollSpec spec = new PollSpec(cus, Poll.V1_CONTENT_POLL);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
V1Poll p = null;
if (testmsg.isContentPoll()) {
p =
new V1ContentPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isNamePoll()) {
p =
new V1NamePoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isVerifyPoll()) {
p =
new V1VerifyPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm(),
testmsg.getVerifier());
}
assertNotNull(p);
p.setMessage(testmsg);
p.m_tally.quorum = numAgree + numDisagree;
p.m_tally.numAgree = numAgree;
p.m_tally.numDisagree = numDisagree;
p.m_tally.wtAgree = 2000;
p.m_tally.wtDisagree = 200;
p.m_tally.localEntries = makeEntries(1, 3);
p.m_tally.votedEntries = makeEntries(1, 5);
p.m_tally.votedEntries.remove(1);
p.m_pollstate = V1Poll.PS_COMPLETE;
p.m_callerID = testmsg.getOriginatorId();
log.debug3("poll " + p.toString());
p.m_tally.tallyVotes();
return p;
}
protected ArchiveReader wrapStream(String url, InputStream arcStream) throws IOException {
ArchiveReader ret = null;
logger.debug3("Getting an ArchiveReader");
ret = ArchiveReaderFactory.get(url, arcStream, true);
return (ret);
}
/** Explode the archive into its constituent elements */
public void explode() throws CacheException {
CachedUrl cachedUrl = null;
int goodEntries = 0;
int badEntries = 0;
int ignoredEntries = 0;
int entriesBetweenSleep = 0;
ArchiveReader arcReader = null;
logger.info(
(storeArchive ? "Storing" : "Fetching") + " WARC file: " + origUrl + " will explode");
try {
if (storeArchive) {
UrlCacher uc = au.makeUrlCacher(new UrlData(arcStream, arcProps, fetchUrl));
BitSet bs = new BitSet();
bs.set(UrlCacher.DONT_CLOSE_INPUT_STREAM_FLAG);
uc.setFetchFlags(bs);
uc.storeContent();
archiveData.resetInputStream();
arcStream = archiveData.input;
}
// Wrap it in an ArchiveReader
logger.debug3("About to wrap stream");
arcReader = wrapStream(fetchUrl, arcStream);
logger.debug3("wrapStream() returns " + (arcReader == null ? "null" : "non-null"));
// Explode it
if (arcReader == null) {
throw new CacheException.ExploderException("no WarcReader for " + origUrl);
}
ArchivalUnit au = crawlFacade.getAu();
Set stemSet = new HashSet();
logger.debug("Exploding " + fetchUrl);
// Iterate through the elements in the WARC file, except the first
Iterator i = arcReader.iterator();
// Skip first record
for (i.next(); i.hasNext(); ) {
// XXX probably not necessary
helper.pokeWDog();
if ((++entriesBetweenSleep % sleepAfter) == 0) {
long pauseTime =
CurrentConfig.getTimeIntervalParam(PARAM_RETRY_PAUSE, DEFAULT_RETRY_PAUSE);
Deadline pause = Deadline.in(pauseTime);
logger.debug3("Sleeping for " + StringUtil.timeIntervalToString(pauseTime));
while (!pause.expired()) {
try {
pause.sleep();
} catch (InterruptedException ie) {
// no action
}
}
}
ArchiveRecord element = (ArchiveRecord) i.next();
// Each element is a URL to be cached in a suitable AU
ArchiveRecordHeader elementHeader = element.getHeader();
String elementUrl = elementHeader.getUrl();
String elementMimeType = elementHeader.getMimetype();
long elementLength = elementHeader.getLength();
logger.debug2("WARC url " + elementUrl + " mime " + elementMimeType);
if (elementUrl.startsWith("http:")) {
ArchiveEntry ae =
new ArchiveEntry(
elementUrl,
elementLength,
0, // XXX need to convert getDate string to long
element, // ArchiveRecord extends InputStream
this,
fetchUrl);
ae.setHeaderFields(makeCIProperties(elementHeader));
long bytesStored = elementLength;
logger.debug3("ArchiveEntry: " + ae.getName() + " bytes " + bytesStored);
try {
helper.process(ae);
} catch (PluginException ex) {
throw new CacheException.ExploderException("helper.process() threw", ex);
}
if (ae.getBaseUrl() != null) {
if (ae.getRestOfUrl() != null && ae.getHeaderFields() != null) {
storeEntry(ae);
handleAddText(ae);
goodEntries++;
crawlFacade.getCrawlerStatus().addContentBytesFetched(bytesStored);
} else {
ignoredEntries++;
}
} else {
badEntries++;
logger.debug2("Can't map " + elementUrl + " from " + archiveUrl);
}
}
}
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
} finally {
if (arcReader != null)
try {
arcReader.close();
arcReader = null;
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
}
if (cachedUrl != null) {
cachedUrl.release();
}
IOUtil.safeClose(arcStream);
}
if (badEntries == 0 && goodEntries > 0) {
// Make it look like a new crawl finished on each AU to which
// URLs were added.
for (Iterator it = touchedAus.iterator(); it.hasNext(); ) {
ArchivalUnit au = (ArchivalUnit) it.next();
logger.debug3(archiveUrl + " touching " + au.toString());
AuUtil.getDaemon(au).getNodeManager(au).newContentCrawlFinished();
}
} else {
ArchivalUnit au = crawlFacade.getAu();
String msg = archiveUrl + ": " + badEntries + "/" + goodEntries + " bad entries";
throw new CacheException.UnretryableException(msg);
}
}
/** Common request handling. */
public void service(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
resetState();
boolean success = false;
HttpSession session = req.getSession(false);
try {
this.req = req;
this.resp = resp;
if (log.isDebug()) {
logParams();
}
resp.setContentType("text/html");
if (!mayPageBeCached()) {
resp.setHeader("pragma", "no-cache");
resp.setHeader("Cache-control", "no-cache");
}
reqURL = new URL(UrlUtil.getRequestURL(req));
clientAddr = getLocalIPAddr();
// check that current user has permission to run this servlet
if (!isServletAllowed(myServletDescr())) {
displayWarningInLieuOfPage("You are not authorized to use " + myServletDescr().heading);
return;
}
// check whether servlet is disabled
String reason = ServletUtil.servletDisabledReason(myServletDescr().getServletName());
if (reason != null) {
displayWarningInLieuOfPage("This function is disabled. " + reason);
return;
}
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, getHeading());
String reqHost = req.getRemoteHost();
String forw = req.getHeader(HttpFields.__XForwardedFor);
if (!StringUtil.isNullString(forw)) {
reqHost += " (proxies for " + forw + ")";
}
session.setAttribute(SESSION_KEY_REQUEST_HOST, reqHost);
}
lockssHandleRequest();
success = (errMsg == null);
} catch (ServletException e) {
log.error("Servlet threw", e);
throw e;
} catch (IOException e) {
log.error("Servlet threw", e);
throw e;
} catch (RuntimeException e) {
log.error("Servlet threw", e);
throw e;
} finally {
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, null);
session.setAttribute(LockssFormAuthenticator.__J_AUTH_ACTIVITY, TimeBase.nowMs());
}
if ("please".equalsIgnoreCase(req.getHeader("X-Lockss-Result"))) {
log.debug3("X-Lockss-Result: " + (success ? "Ok" : "Fail"));
resp.setHeader("X-Lockss-Result", success ? "Ok" : "Fail");
}
resetMyLocals();
resetLocals();
}
}
