/** Create LockssKeystore from a config subtree */
LockssKeyStore createLockssKeyStore(Configuration config) {
log.debug2("Creating LockssKeyStore from config: " + config);
String name = config.get(KEYSTORE_PARAM_NAME);
LockssKeyStore lk = new LockssKeyStore(name);
String file = config.get(KEYSTORE_PARAM_FILE);
String resource = config.get(KEYSTORE_PARAM_RESOURCE);
String url = config.get(KEYSTORE_PARAM_URL);
if (!StringUtil.isNullString(file)) {
lk.setLocation(file, LocationType.File);
} else if (!StringUtil.isNullString(resource)) {
lk.setLocation(resource, LocationType.Resource);
} else if (!StringUtil.isNullString(url)) {
lk.setLocation(url, LocationType.Url);
}
lk.setType(config.get(KEYSTORE_PARAM_TYPE, defaultKeyStoreType));
lk.setProvider(config.get(KEYSTORE_PARAM_PROVIDER, defaultKeyStoreProvider));
lk.setPassword(config.get(KEYSTORE_PARAM_PASSWORD));
lk.setKeyPassword(config.get(KEYSTORE_PARAM_KEY_PASSWORD));
lk.setKeyPasswordFile(config.get(KEYSTORE_PARAM_KEY_PASSWORD_FILE));
lk.setMayCreate(config.getBoolean(KEYSTORE_PARAM_CREATE, DEFAULT_CREATE));
return lk;
}
void setConfig(Configuration config) {
log.debug("config: " + config);
proxyHost = config.get(PARAM_PROXY_HOST);
proxyPort = config.getInt(PARAM_PROXY_PORT, DEFAULT_PROXY_PORT);
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
String http_proxy = System.getenv("http_proxy");
if (!StringUtil.isNullString(http_proxy)) {
try {
HostPortParser hpp = new HostPortParser(http_proxy);
proxyHost = hpp.getHost();
proxyPort = hpp.getPort();
} catch (HostPortParser.InvalidSpec e) {
log.warning("Can't parse http_proxy environment var, ignoring: " + http_proxy + ": " + e);
}
}
}
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
proxyHost = null;
} else {
log.info("Proxying through " + proxyHost + ":" + proxyPort);
}
userAgent = config.get(PARAM_USER_AGENT);
if (StringUtil.isNullString(userAgent)) {
userAgent = null;
} else {
log.debug("Setting User-Agent to " + userAgent);
}
}
public int match(String url) {
if (StringUtil.equalStringsIgnoreCase(url, m_registryUrl)
|| StringUtil.endsWithIgnoreCase(url, ".jar")) {
return CrawlRule.INCLUDE;
} else {
return CrawlRule.EXCLUDE;
}
}
/**
* Did the client provide the minimal parameters required?
*
* @return true If so
*/
private boolean verifyMinimumParameters() {
int count = 0;
if (!StringUtil.isNullString(getParameter(AP_E_PUBLICATION))) count++;
if (!StringUtil.isNullString(getParameter(AP_E_CLASSNAME))) count++;
if (!StringUtil.isNullString(getParameter(AP_E_PLUGIN))) count++;
return (count > 0);
}
/** Concatenate params for URL string */
static String concatParams(String p1, String p2) {
if (StringUtil.isNullString(p1)) {
return p2;
}
if (StringUtil.isNullString(p2)) {
return p1;
}
return p1 + "&" + p2;
}
// Break the line at commas, return a map of the resulting strings
// broken at equals sign. (<i>Ie</i>, name value pairs.)
Map getRow(String line) {
Map map = new HashMap();
for (Iterator iter = StringUtil.breakAt(line, ',').iterator(); iter.hasNext(); ) {
String item = (String) iter.next();
List pair = StringUtil.breakAt(item, '=');
map.put(pair.get(0), pair.get(1));
}
return map;
}
public void testFilterE() throws Exception {
InputStream in;
// all these should match, once filtered, the string HtmlHashEFiltered
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashE), ENC);
String filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashF), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashG), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashH), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashI), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashJ), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashK), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashL), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashO), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashEFiltered, filtStr);
}
public void testFiltering() throws IOException {
Reader readerA;
Reader readerB;
readerA = rule.createFilteredReader(new StringReader(inst1));
readerB = rule.createFilteredReader(new StringReader(inst2));
assertEquals(StringUtil.fromReader(readerA), StringUtil.fromReader(readerB));
readerA = rule.createFilteredReader(new StringReader(inst1));
readerB = rule.createFilteredReader(new StringReader(inst3));
assertEquals(StringUtil.fromReader(readerA), StringUtil.fromReader(readerB));
}
public void testFilterViewedBy() throws Exception {
InputStream in;
String filtStr = null;
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashN), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashNFiltered, filtStr);
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashP), ENC);
filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashPFiltered, filtStr);
}
private void doSleep() throws IOException {
String timestr = getParameter(KEY_TIME);
try {
long time = StringUtil.parseTimeInterval(timestr);
Deadline.in(time).sleep();
statusMsg = "Slept for " + StringUtil.timeIntervalToString(time);
} catch (NumberFormatException e) {
errMsg = "Illegal duration: " + e;
} catch (InterruptedException e) {
errMsg = "Interrupted: " + e;
}
}
/** Concatenate params for URL string */
String concatParams(Properties props) {
if (props == null) {
return null;
}
java.util.List list = new ArrayList();
for (Iterator iter = props.keySet().iterator(); iter.hasNext(); ) {
String key = (String) iter.next();
String val = props.getProperty(key);
if (!StringUtil.isNullString(val)) {
list.add(key + "=" + urlEncode(val));
}
}
return StringUtil.separatedString(list, "&");
}
protected Properties filterResponseProps(Properties props) {
Properties res = new Properties();
for (Map.Entry ent : props.entrySet()) {
String key = (String) ent.getKey();
if (StringUtil.startsWithIgnoreCase(key, "x-lockss")
|| StringUtil.startsWithIgnoreCase(key, "x_lockss")
|| key.equalsIgnoreCase("org.lockss.version.number")) {
continue;
}
// We've lost the original case - capitalize them the way most people
// expect
res.put(StringUtil.titleCase(key, '-'), (String) ent.getValue());
}
return res;
}
/**
* A target system is required to create an AU - was it provided?
*
* @return true If at least one target was specified
*/
private boolean verifyTarget() {
if (!isCreateCommand()) {
return true;
}
return !StringUtil.isNullString(getParameter(AP_E_TARGET));
}
public void testFilterD() throws Exception {
InputStream in;
in = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashD), ENC);
String filtStr = StringUtil.fromInputStream(in);
assertEquals(HtmlHashDFiltered, filtStr);
}
/**
* Escapes instances of File.separator from the query. These are safe from filename overlap, but
* can't convert into extended paths and directories.
*
* @param query the query
* @return the escaped query
*/
static String escapeQuery(String query) {
if (query.indexOf(File.separator) >= 0) {
return StringUtil.replaceString(query, File.separator, ESCAPE_STR + ENCODED_SEPARATOR_CHAR);
} else {
return query;
}
}
protected void logParams() {
Enumeration en = req.getParameterNames();
while (en.hasMoreElements()) {
String name = (String) en.nextElement();
String vals[];
String dispval;
if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
dispval = req.getParameter(name).length() == 0 ? "" : "********";
} else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
dispval = StringUtil.separatedString(vals, ", ");
} else {
dispval = req.getParameter(name);
}
log.debug(name + " = " + dispval);
}
}
public void testFilterB() throws Exception {
InputStream inB;
inB = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashB), ENC);
String filtStrB = StringUtil.fromInputStream(inB);
assertEquals(HtmlHashBFiltered, filtStrB);
}
/**
* Decide whether the current title record should be displayed, based on whether it is a duplicate
* of the previous record, given the combination of output fields included in the ordering. If
* range fields are included in the output, or if identifying properties change between titles,
* the title will be shown.
*
* <p>If there are neither range fields nor id fields in the output, we can't tell if it is a
* duplicate so we show it anyway. Output with no id fields is pretty meaningless though.
*
* <p>Note that the method also sets the last output title if the response is true. Thus calling
* this method multiple times on the same title will give false after the first (unless there are
* no identifying fields).
*
* @param currentTitle the current title
* @return whether to show currentTitle
*/
public boolean isTitleForOutput(KbartTitle currentTitle) {
// The approach is to trueify this variable. If it becomes true, at the
// end of the method the lastOutputTitle is set before the result is
// returned. Do not return early!
boolean isOutput = false;
// Show title if it has no range or id fields by which we can decide duplicates
if (!rangeFieldsIncludedInDisplay && !idFieldsIncludedInDisplay) {
isOutput = true;
}
// Show the title if it is the first or the output includes range fields
if (lastOutputTitle == null || rangeFieldsIncludedInDisplay) {
isOutput = true;
} else if (!isOutput) { // don't do this check if we've already trueified the var
// At this point there are no range fields and this is not the first title
// Show the title if any visible idField differs between titles
for (Field f : idFields) {
if (visibleColumnOrdering.getFields().contains(f)
&& !lastOutputTitle.getField(f).equals(currentTitle.getField(f))) {
isOutput = true;
break;
}
}
}
// Finally, we refuse to output the title if it has no id and
// excludeNoIdTitles is true.
if (excludeNoIdTitles && StringUtil.isNullString(currentTitle.getField(Field.TITLE_ID)))
isOutput = false;
// Record the previous title
if (isOutput) lastOutputTitle = currentTitle;
return isOutput;
}
/**
* Return the archive file type corresponding to the filename extension in the URL, or null if
* none.
*/
public String getFromUrl(String url) throws MalformedURLException {
if (StringUtil.endsWithIgnoreCase(url, ".tar.gz")) {
return getExtMimeMap().get(".tar.gz");
}
String ext = UrlUtil.getFileExtension(url).toLowerCase();
return getExtMimeMap().get("." + ext);
}
/**
* Escapes instances of the ESCAPE_CHAR from the path. This avoids name conflicts with the
* repository files, such as '#nodestate.xml'.
*
* @param path the path
* @return the escaped path
*/
static String escapePath(String path) {
// XXX escaping disabled because of URL encoding
if (false && path.indexOf(ESCAPE_CHAR) >= 0) {
return StringUtil.replaceString(path, ESCAPE_STR, ESCAPE_STR + ESCAPE_STR);
} else {
return path;
}
}
/**
* Add javascript to page. Normally adds a link to the script file, but can be told to include the
* script directly in the page, to accomodate unit testing of individual servlets, when other
* fetches won't work.
*/
protected void addJavaScript(Composite comp) {
String include = (String) context.getAttribute(ATTR_INCLUDE_SCRIPT);
if (StringUtil.isNullString(include)) {
linkToJavaScript(comp);
} else {
includeJavaScript0(comp);
}
}
// Don't put the 2nd string through the filter - use it as a constant
private void assertFilterToString(String orgString, String finalString) throws Exception {
InputStream inA =
fact.createFilteredInputStream(
mau, new StringInputStream(orgString), Constants.DEFAULT_ENCODING);
String filtered = StringUtil.fromInputStream(inA);
assertEquals(filtered, finalString, filtered);
}
private static void doFilterTest(
ArchivalUnit au, FilterFactory fact, String nameToHash, String expectedStr)
throws PluginException, IOException {
InputStream actIn;
actIn =
fact.createFilteredInputStream(
au, new StringInputStream(nameToHash), Constants.DEFAULT_ENCODING);
assertEquals(expectedStr, StringUtil.fromInputStream(actIn));
}
protected void assertEqualTables(Object[][] a1, List lines) {
assertEquals("numrows", a1.length, lines.size() - NUM_HEADER_LINES);
for (int irow = 0; irow <= a1.length - 1; irow++) {
Object expRow[] = a1[irow];
List row = StringUtil.breakAt((String) lines.get(irow + NUM_HEADER_LINES), ',');
assertEquals("numcols", expRow.length, row.size());
assertEquals(("row " + irow), SetUtil.fromArray(expRow), new HashSet(row));
}
}
public void testRisFilter() throws Exception {
InputStream actIn =
fact.createFilteredInputStream(
mau, new StringInputStream(moreRisData), Constants.DEFAULT_ENCODING);
String test = StringUtil.fromInputStream(actIn);
// log.info("filtered input: "+test);
// log.info("should match: "+moreRisDataFiltered);
assertEquals(moreRisDataFiltered, test);
}
public void testFilterA() throws Exception {
InputStream inA;
// viewed-by test
inA = fact.createFilteredInputStream(mau, new StringInputStream(HtmlHashA), ENC);
String filtStrA = StringUtil.fromInputStream(inA);
assertEquals(HtmlHashAFiltered, filtStrA);
}
/**
* Create a response that maps the event to the named new state.
*
* @param event the pattern event against which incoming events are matched
* @param newState name of state to transition to if matching event is signalled.
*/
public PsmResponse(PsmEvent event, String newState) {
if (event == null) {
throw new PsmException.IllegalStateMachine("event is null");
}
if (StringUtil.isNullString(newState)) {
throw new PsmException.IllegalStateMachine("newState is null string");
}
this.event = event;
this.newState = newState;
}
private List getSummaryInfo(CrawlerStatus status) {
List res = new ArrayList();
StatusTable.SummaryInfo statusSi =
new StatusTable.SummaryInfo(
"Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
ArchivalUnit au = status.getAu();
if (au != null) {
AuState aus = AuUtil.getAuState(au);
if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
}
}
res.add(statusSi);
String sources = StringUtil.separatedString(status.getSources());
res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
String startUrls = StringUtil.separatedString(status.getStartUrls());
res.add(
new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
return res;
}
/**
* mapUrlToFileLocation() is the method used to resolve urls into file names. This maps a given
* url to a file location, using the au top directory as the base. It creates directories which
* mirror the html string, so 'http://www.journal.org/issue1/index.html' would be cached in the
* file: <rootLocation>/www.journal.org/http/issue1/index.html
*
* @param rootLocation the top directory for ArchivalUnit this URL is in
* @param urlStr the url to translate
* @return the url file location
* @throws java.net.MalformedURLException
*/
public static String mapUrlToFileLocation(String rootLocation, String urlStr)
throws MalformedURLException {
int totalLength = rootLocation.length() + urlStr.length();
URL url = new URL(urlStr);
StringBuilder buffer = new StringBuilder(totalLength);
buffer.append(rootLocation);
if (!rootLocation.endsWith(File.separator)) {
buffer.append(File.separator);
}
buffer.append(url.getHost().toLowerCase());
int port = url.getPort();
if (port != -1) {
buffer.append(PORT_SEPARATOR);
buffer.append(port);
}
buffer.append(File.separator);
buffer.append(url.getProtocol());
if (RepositoryManager.isEnableLongComponents()) {
String escapedPath =
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator));
String query = url.getQuery();
if (query != null) {
escapedPath = escapedPath + "?" + escapeQuery(query);
}
String encodedPath = RepositoryNodeImpl.encodeUrl(escapedPath);
// encodeUrl strips leading / from path
buffer.append(File.separator);
buffer.append(encodedPath);
} else {
buffer.append(
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator)));
String query = url.getQuery();
if (query != null) {
buffer.append("?");
buffer.append(escapeQuery(query));
}
}
return buffer.toString();
}
ArchivalUnit getAu() {
if (StringUtil.isNullString(formAuid)) {
errMsg = "Select an AU";
return null;
}
ArchivalUnit au = pluginMgr.getAuFromId(formAuid);
if (au == null) {
errMsg = "No such AU. Select an AU";
return null;
}
return au;
}
