/** @param args */
private static void registerStudents() throws Exception {
Scanner keyboard = new Scanner(System.in);
Class.forName("com.mysql.jdbc.Driver");
Properties config = DBUtil.getConfigProperties();
Connection conn = DBUtil.connectToDatabase(config, "cloudcoder.db");
List<Course> courses =
DBUtil.getAllModelObjects(
conn,
Course.SCHEMA,
new IFactory<Course>() {
@Override
public Course create() {
return new Course();
}
});
Course c =
ConfigurationUtil.choose(
keyboard, "For which course would you like to register students?", courses);
// TODO: look up the term for each course
String filename =
ConfigurationUtil.ask(
keyboard,
"Enter the name of the file containing a tab-separated list student registration entries in this format: \n"
+ "username\tfirstname\tlastname\temail\tpassword\tsection\n"
+ "Usernames in the datbase will be re-used, but the names/email/password will not be updated,"
+ "and users will not be registered for a course if they are already registered");
int num =
ConfigurationUtil.registerStudentsForCourseId(
new FileInputStream(filename), c.getId(), conn);
System.out.println("Registered " + num + " students for " + c.getName());
}
public void testGetMissingFileAction() throws Exception {
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(true);
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(false);
ConfigurationUtil.setFromArgs(ServeContent.PARAM_MISSING_FILE_ACTION, "Redirect");
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.Redirect, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(true);
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(false);
ConfigurationUtil.setFromArgs(ServeContent.PARAM_MISSING_FILE_ACTION, "AlwaysRedirect");
assertEquals(MissingFileAction.AlwaysRedirect, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.AlwaysRedirect, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.AlwaysRedirect, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.AlwaysRedirect, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(true);
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.HostAuIndex, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(false);
ConfigurationUtil.setFromArgs(ServeContent.PARAM_MISSING_FILE_ACTION, "Error_404");
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.Unknown));
sc.setNeverProxy(true);
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.KnownDown));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.RecentlyDown));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.NoContent));
assertEquals(MissingFileAction.Error_404, sc.getMissingFileAction(PubState.Unknown));
}
// one task to drop, two failed schedule tries
public void testAddToScheduleFailNoCleanup() {
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched = sched(ListUtil.list(t1));
fact.setResult(sched);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
assertFalse(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
fact.createArgs);
assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
assertEquals(sched, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(1, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
}
public void setUp() throws Exception {
super.setUp();
tempDirPath = getTempDir().getAbsolutePath() + File.separator;
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.getHashService();
MockSystemMetrics metrics = new MyMockSystemMetrics();
metrics.initService(theDaemon);
theDaemon.setSystemMetrics(metrics);
theDaemon.setDaemonInited(true);
Properties props = new Properties();
props.setProperty(SystemMetrics.PARAM_HASH_TEST_DURATION, "1000");
props.setProperty(SystemMetrics.PARAM_HASH_TEST_BYTE_STEP, "1024");
props.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
ConfigurationUtil.setCurrentConfigFromProps(props);
pluginMgr = theDaemon.getPluginManager();
pluginMgr.startService();
theDaemon.getHashService().startService();
metrics.startService();
metrics.setHashSpeed(100);
simPlugin = PluginTestUtil.findPlugin(SimulatedPlugin.class);
}
public void setUp() throws Exception {
super.setUp();
theDaemon = getMockLockssDaemon();
tempDir = getTempDir();
String tempDirPath = tempDir.getAbsolutePath();
System.setProperty("java.io.tmpdir", tempDirPath);
Properties p = new Properties();
p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb");
p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1");
p.setProperty(V3LcapMessage.PARAM_REPAIR_DATA_THRESHOLD, "4096");
ConfigurationUtil.setCurrentConfigFromProps(p);
IdentityManager idmgr = theDaemon.getIdentityManager();
idmgr.startService();
mPollMgr = new MockPollManager();
theDaemon.setPollManager(mPollMgr);
try {
m_testID = idmgr.stringToPeerIdentity("127.0.0.1");
} catch (IOException ex) {
fail("can't open test host 127.0.0.1: " + ex);
}
m_repairProps = new CIProperties();
m_repairProps.setProperty("key1", "val1");
m_repairProps.setProperty("key2", "val2");
m_repairProps.setProperty("key3", "val3");
m_testVoteBlocks = V3TestUtils.makeVoteBlockList(10);
m_testMsg = this.makeTestVoteMessage(m_testVoteBlocks);
}
/**
* Loads the properties in the system property file associated with the framework installation
* into <tt>System.setProperty()</tt>. These properties are not directly used by the framework in
* anyway. By default, the system property file is located in the <tt>conf/</tt> directory and is
* called "<tt>system.properties</tt>".
*/
protected void loadSystemProperties(JsonValue configuration, URI projectDirectory) {
JsonValue systemProperties = configuration.get(SYSTEM_PROPERTIES_PROP);
if (systemProperties.isMap()) {
for (Map.Entry<String, Object> entry : systemProperties.copy().asMap().entrySet()) {
// The user.dir MUST not be overwritten!!!
if (entry.getValue() instanceof String && !"user.dir".equals(entry.getKey())) {
System.setProperty(entry.getKey(), (String) entry.getValue());
}
}
} else {
Properties props =
loadPropertyFile(
projectDirectory,
systemProperties
.expect(String.class)
.defaultTo(SYSTEM_PROPERTIES_FILE_VALUE)
.asString());
if (props == null) return;
// Perform variable substitution on specified properties.
for (Enumeration e = props.propertyNames(); e.hasMoreElements(); ) {
String name = (String) e.nextElement();
if (!"user.dir".equals(name)) {
Object newValue = ConfigurationUtil.substVars(props.getProperty(name), propertyAccessor);
if (newValue instanceof String) {
System.setProperty(name, (String) newValue);
}
}
}
}
}
/**
* Make a basic TaylorAndFrancis test AU to which URLs can be added.
*
* @return a basic Ingenta test AU
* @throws ConfigurationException if can't set configuration
*/
MockArchivalUnit makeAu() throws ConfigurationException {
MockArchivalUnit mau = new MockArchivalUnit();
Configuration config = ConfigurationUtil.fromArgs("base_url", "http://www.xyz.com/");
mau.setConfiguration(config);
mau.setUrlStems(ListUtil.list("http://www.xyz.com/"));
return mau;
}
/**
* Checks if is resonse size within limit.
*
* @param maxResultSize the max result size
* @return true, if is resonse size within limit
*/
public boolean isResonseSizeWithinLimit(long maxResultSize) {
// System.out.println("isResonseSizeWithinLimit() : getResponseSize() < (input|default result
// size - maximum packet size ) ="+
// getResponseSize()+ " < " + ( maxResultSize
// -ConfigurationUtil.getMaxRowSize()));
return getResponseSize() < (maxResultSize - ConfigurationUtil.getMaxRowSize());
}
private void setEnabled(boolean enabled, int port) {
ConfigurationUtil.addFromArgs(
IcpManager.PARAM_ICP_ENABLED,
Boolean.toString(enabled),
IcpManager.PARAM_ICP_PORT,
Integer.toString(enabled ? port : BAD_PORT));
}
public void testSkipsScriptTagsSpansRing() throws IOException {
Properties p = new Properties();
p.setProperty(GoslingHtmlLinkExtractor.PARAM_BUFFER_CAPACITY, "90");
ConfigurationUtil.setCurrentConfigFromProps(p);
extractor = new GoslingHtmlLinkExtractor();
doScriptSkipTest("<script>", "</script>");
}
public static void configHistoryParams(String rootLocation) throws IOException {
ConfigurationUtil.addFromArgs(
HistoryRepositoryImpl.PARAM_HISTORY_LOCATION,
rootLocation,
LockssRepositoryImpl.PARAM_CACHE_LOCATION,
rootLocation,
IdentityManager.PARAM_LOCAL_IP,
"127.0.0.7");
}
public void testGetRepositoryList() throws Exception {
assertEmpty(mgr.getRepositoryList());
String tempDirPath = setUpDiskSpace();
assertEquals(ListUtil.list("local:" + tempDirPath), mgr.getRepositoryList());
String tempdir2 = getTempDir().getAbsolutePath() + File.separator;
ConfigurationUtil.setFromArgs(
"org.lockss.platform.diskSpacePaths", tempdir2 + ";" + tempDirPath);
assertEquals(
ListUtil.list("local:" + tempdir2, "local:" + tempDirPath), mgr.getRepositoryList());
}
@Override
public RecordReader<LongWritable, MapWritable> getRecordReader(
InputSplit split, JobConf conf, Reporter reporter) throws IOException {
List<Integer> readColIDs = getReadColumnIDs(conf);
boolean addAll = (readColIDs.size() == 0);
String columnString = conf.get(ConfigurationUtil.COLUMN_MAPPING);
if (StringUtils.isBlank(columnString)) {
throw new IOException("no column mapping found!");
}
String[] columns = ConfigurationUtil.getAllColumns(columnString);
if (readColIDs.size() > columns.length) {
throw new IOException("read column count larger than that in column mapping string!");
}
String[] cols;
if (addAll) {
cols = columns;
} else {
cols = new String[readColIDs.size()];
for (int i = 0; i < cols.length; i++) {
cols[i] = columns[readColIDs.get(i)];
}
}
String filterExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
if (filterExprSerialized != null) {
ExprNodeDesc filterExpr = Utilities.deserializeExpression(filterExprSerialized, conf);
/*String columnNameProperty = conf.get(
org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS);
System.err.println("======list columns:" + columnNameProperty);*/
dumpFilterExpr(filterExpr);
// TODO:
}
return new SolrReader(
ConfigurationUtil.getUrl(conf),
(SolrSplit) split,
cols,
ConfigurationUtil.getNumInputBufferRows(conf));
}
// only one try, because no tasks to drop
public void testAddToScheduleFailNothingToDrop() {
ConfigurationUtil.addFromArgs(TaskRunner.PARAM_DROP_TASK_MAX, "2");
fact.setResult(null);
StepTask t1 = task(100, 200, 50);
assertFalse(tr.addToSchedule(t1));
assertEmpty(tr.getAcceptedTasks());
assertEquals(1, fact.createArgs.size());
assertForegroundStat(1, TaskRunner.STAT_REFUSED);
assertForegroundStat(0, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
}
protected ArchivalUnit createAu() throws ArchivalUnit.ConfigurationException {
return PluginTestUtil.createAndStartAu(
PLUGIN_NAME,
ConfigurationUtil.fromArgs(
"base_url",
"http://pediatrics.aappublications.org/",
"volume_name",
"52",
"journal_issn",
"1098-4275"));
}
public void setUp() throws Exception {
super.setUp();
TimeBase.setSimulated();
ConfigurationUtil.setFromArgs(SortScheduler.PARAM_OVERHEAD_LOAD, "0");
removedChunks = new ArrayList();
removedTasks = new ArrayList();
fact = new SchedFact(null);
tr = new MyMockTaskRunner(fact);
tr.initService(getMockLockssDaemon());
tr.startService();
}
public void testConfig() throws Exception {
MyMockLockssRepositoryImpl repo1 = makeRepo("foo");
assertEquals(RepositoryManager.DEFAULT_MAX_PER_AU_CACHE_SIZE, repo1.nodeCacheSize);
ConfigurationUtil.setFromArgs(RepositoryManager.PARAM_MAX_PER_AU_CACHE_SIZE, "4");
MyMockLockssRepositoryImpl repo2 = makeRepo("bar");
assertEquals(4, repo1.nodeCacheSize);
assertEquals(4, repo2.nodeCacheSize);
repo1.cnt = 0;
ConfigurationUtil.setFromArgs(RepositoryManager.PARAM_MAX_PER_AU_CACHE_SIZE, "37");
assertEquals(37, repo1.nodeCacheSize);
assertEquals(37, repo2.nodeCacheSize);
assertEquals(1, repo1.cnt);
// ensure setNodeCacheSize doesn't get called if param doesn't change
ConfigurationUtil.setFromArgs(
RepositoryManager.PARAM_MAX_PER_AU_CACHE_SIZE, "37", "org.lockss.somethingElse", "bar");
assertEquals(1, repo1.cnt);
PlatformUtil.DF warn = mgr.getDiskWarnThreshold();
PlatformUtil.DF full = mgr.getDiskFullThreshold();
assertEquals(5000 * 1024, warn.getAvail());
assertEquals(0.98, warn.getPercent(), .00001);
assertEquals(100 * 1024, full.getAvail());
assertEquals(0.99, full.getPercent(), .00001);
Properties p = new Properties();
p.put(RepositoryManager.PARAM_DISK_WARN_FRRE_MB, "17");
p.put(RepositoryManager.PARAM_DISK_WARN_FRRE_PERCENT, "20");
p.put(RepositoryManager.PARAM_DISK_FULL_FRRE_MB, "7");
p.put(RepositoryManager.PARAM_DISK_FULL_FRRE_PERCENT, "10");
ConfigurationUtil.setCurrentConfigFromProps(p);
warn = mgr.getDiskWarnThreshold();
full = mgr.getDiskFullThreshold();
assertEquals(17 * 1024, warn.getAvail());
assertEquals(0.80, warn.getPercent(), .00001);
assertEquals(7 * 1024, full.getAvail());
assertEquals(0.90, full.getPercent(), .00001);
}
protected ArchivalUnit createTarAu() throws ArchivalUnit.ConfigurationException {
// in this directory this is file "test_elsevierdtd5.tdb" but it becomes xml
try {
ConfigurationUtil.addFromUrl(getResource("test_elsevierdtd5.xml"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Tdb tdb = ConfigManager.getCurrentConfig().getTdb();
TdbAu tdbau1 = tdb.getTdbAusLikeName("Elsevier Source Content 2014").get(0);
assertNotNull("Didn't find named TdbAu", tdbau1);
return PluginTestUtil.createAndStartAu(tdbau1);
}
public void setUp() throws Exception {
super.setUp();
String tempDirPath = getTempDir().getAbsolutePath() + File.separator;
ConfigurationUtil.setFromArgs(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.setDaemonInited(true);
theDaemon.getPluginManager().startService();
theDaemon.getCrawlManager();
sau = PluginTestUtil.createAndStartSimAu(simAuConfig(tempDirPath));
nau = PluginTestUtil.createAndStartAu(PLUGIN_NAME, natureAuConfig());
}
/* Inherit documentation */
public void setUp() throws Exception {
super.setUp();
ConfigurationUtil.addFromArgs(
"org.lockss.log.IcpManager.level",
"debug3",
"org.lockss.log.IcpSocketImpl.level",
"debug3");
setEnabled(false, BAD_PORT);
setConfigCalled = false;
mockLockssDaemon = getMockLockssDaemon();
testableIcpManager = new TestableIcpManager();
mockLockssDaemon.setIcpManager(testableIcpManager);
testableIcpManager.initService(mockLockssDaemon);
mockLockssDaemon.setDaemonInited(true);
testableIcpManager.startService();
}
public void testSkipsScriptTagsWhiteSpace() throws IOException {
Properties p = new Properties();
p.setProperty(GoslingHtmlLinkExtractor.PARAM_BUFFER_CAPACITY, "90");
ConfigurationUtil.setCurrentConfigFromProps(p);
extractor = new GoslingHtmlLinkExtractor();
for (int ix = 1; ix < 200; ix += 5) {
String whiteSpace = mkStr(' ', ix);
doScriptSkipTest("<script" + whiteSpace + ">", "</script>", "Failed during iteration " + ix);
doScriptSkipTest("<" + whiteSpace + "script>", "</script>", "Failed during iteration " + ix);
doScriptSkipTest("<script>", "<" + whiteSpace + "/script>", "Failed during iteration " + ix);
doScriptSkipTest(
"<script" + whiteSpace + "blah=blah>", "</script>", "Failed during iteration " + ix);
// doScriptSkipTest("<script>", "</script"+whiteSpace+">",
// "Failed during iteration "+ix);
}
}
private MySearchForTestsTask findTestsWithProgress(
final FindCallback callback, final TestClassFilter classFilter) {
if (isSyncSearch()) {
THashSet<PsiClass> classes = new THashSet<PsiClass>();
boolean isJUnit4 = ConfigurationUtil.findAllTestClasses(classFilter, classes);
callback.found(classes, isJUnit4);
return null;
}
final THashSet<PsiClass> classes = new THashSet<PsiClass>();
final boolean[] isJunit4 = new boolean[1];
final MySearchForTestsTask task =
new MySearchForTestsTask(classFilter, isJunit4, classes, callback);
mySearchForTestsIndicator = new BackgroundableProcessIndicator(task);
ProgressManager.getInstance()
.runProcessWithProgressAsynchronously(task, mySearchForTestsIndicator);
return task;
}
// one task to drop, succeeds after dropping it
public void testAddToScheduleOkAfterDrops() {
log.debug("testAddToScheduleOkAfterDrops()");
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched1 = sched(ListUtil.list(t1));
Schedule sched2 = sched(ListUtil.list(t2));
fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
TimeBase.step(101);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
assertTrue(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(
ListUtil.list(t1),
ListUtil.list(t1, t2),
ListUtil.list(t1),
Collections.EMPTY_SET,
ListUtil.list(t1, t2)),
fact.createArgs);
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
assertTrue(t1.isDropped());
assertEquals(sched2, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks()));
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(1, TaskRunner.STAT_DROPPED);
t2.cancel();
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(0, TaskRunner.STAT_WAITING);
assertForegroundStat(1, TaskRunner.STAT_DROPPED);
assertForegroundStat(1, TaskRunner.STAT_CANCELLED);
}
public void testParseJSIfConf() throws IOException {
Properties p = new Properties();
p.setProperty(GoslingHtmlLinkExtractor.PARAM_PARSE_JS, "true");
ConfigurationUtil.setCurrentConfigFromProps(p);
extractor = new GoslingHtmlLinkExtractor();
String url = "http://www.example.com/link3.html";
String url2 = "http://www.example.com/link2.html";
String url3 = "http://www.example.com/link1.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href = javascript:newWindow('http://www.example.com/link3.html')</a>"
+ "<a href = javascript:popup('http://www.example.com/link2.html')</a>"
+ "<img src = javascript:popup('"
+ url3
+ "') </img>";
assertEquals(SetUtil.set(url, url2, url3), parseSingleSource(source));
}
private void initRequiredServices() {
theDaemon = getMockLockssDaemon();
pollmanager = new LocalPollManager();
pollmanager.initService(theDaemon);
theDaemon.setPollManager(pollmanager);
theDaemon.getPluginManager();
testau = PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
PluginTestUtil.registerArchivalUnit(testau);
String tempDirPath = null;
try {
tempDirPath = getTempDir().getAbsolutePath() + File.separator;
} catch (IOException ex) {
fail("unable to create a temporary directory");
}
Properties p = new Properties();
p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb");
p.setProperty(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1");
p.setProperty(ConfigManager.PARAM_NEW_SCHEDULER, "false");
// XXX we need to disable verification of votes because the
// voter isn't really there
p.setProperty(V1Poll.PARAM_AGREE_VERIFY, "0");
p.setProperty(V1Poll.PARAM_DISAGREE_VERIFY, "0");
ConfigurationUtil.setCurrentConfigFromProps(p);
idmgr = theDaemon.getIdentityManager();
idmgr.startService();
// theDaemon.getSchedService().startService();
theDaemon.getHashService().startService();
theDaemon.getDatagramRouterManager().startService();
theDaemon.getRouterManager().startService();
theDaemon.getSystemMetrics().startService();
theDaemon.getActivityRegulator(testau).startService();
theDaemon.setNodeManager(new MockNodeManager(), testau);
pollmanager.startService();
}
public void setUp() throws Exception {
super.setUp();
setUpDiskSpace(); // you need this to have startService work properly...
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.setDaemonInited(true);
theDaemon.getPluginManager().startService();
theDaemon.getCrawlManager();
// in this directory this is file "test_baseatypon.tdb" but it becomes xml
ConfigurationUtil.addFromUrl(getResource("test_baseatypon.xml"));
Tdb tdb = ConfigManager.getCurrentConfig().getTdb();
TdbAu tdbau1 = tdb.getTdbAusLikeName(goodJournal + " Volume " + goodVolume).get(0);
assertNotNull("Didn't find named TdbAu", tdbau1);
bau1 = PluginTestUtil.createAndStartAu(tdbau1);
assertNotNull(bau1);
TypedEntryMap auConfig = bau1.getProperties();
assertEquals(BASE_URL, auConfig.getString(BASE_URL_KEY));
}
@Override
public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
LOG.debug("Entry SdbSerDe::initialize");
final String columnString = tbl.getProperty(ConfigurationUtil.COLUMN_MAPPING);
if (StringUtils.isBlank(columnString)) {
throw new SerDeException("No column mapping found, use " + ConfigurationUtil.COLUMN_MAPPING);
}
final String[] columnNamesArray = ConfigurationUtil.getAllColumns(columnString);
fieldCount = columnNamesArray.length;
columnNames = new ArrayList<String>(columnNamesArray.length);
columnNames.addAll(Arrays.asList(columnNamesArray));
serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, getClass().getName());
LOG.debug(
"EscapeChar:"
+ serdeParams.getEscapeChar()
+ "getSeparators:"
+ new String(serdeParams.getSeparators()));
byte[] sparator = new byte[1];
sparator[0] = '|';
objectInspector =
LazyFactory.createLazyStructInspector(
serdeParams.getColumnNames(),
serdeParams.getColumnTypes(),
sparator,
serdeParams.getNullSequence(),
serdeParams.isLastColumnTakesRest(),
serdeParams.isEscaped(),
serdeParams.getEscapeChar());
row = new LazyStruct((LazySimpleStructObjectInspector) objectInspector);
LOG.debug("Exit SdbSerDe::initialize");
}
// one task not ready to start yet, so not droppable
public void testAddToScheduleFailNoDroppable() {
log.debug("testAddToScheduleOkAfterDrops()");
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched1 = sched(ListUtil.list(t1));
Schedule sched2 = sched(ListUtil.list(t2));
fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertFalse(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
fact.createArgs);
assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
assertFalse(t1.isDropped());
assertEquals(sched1, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertEmpty(SetUtil.theSet(tr.getOverrunTasks()));
}
protected List<BundleHandler> listBundleHandlers(BundleContext context)
throws MalformedURLException {
JsonValue bundle = getLauncherConfiguration().get("bundle");
BundleHandlerBuilder defaultBuilder = BundleHandlerBuilder.newBuilder(bundle.get("default"));
List<BundleHandler> result = new ArrayList<BundleHandler>();
URI installDirectory = getInstallURI();
for (JsonValue container : bundle.get("containers")) {
BundleHandlerBuilder innerBuilder =
BundleHandlerBuilder.newBuilder(container, defaultBuilder);
String location = container.get("location").required().asString();
if (location.toLowerCase().endsWith(".zip")) {
File inputFile = getFileForPath(location, installDirectory);
for (URL url :
ConfigurationUtil.getZipFileListing(
inputFile.toURI().toURL(),
container.get("includes").asList(String.class),
container.get("excludes").asList(String.class))) {
result.add(innerBuilder.build(url));
}
} else if (location.toLowerCase().endsWith(".jar")) {
File inputFile = getFileForPath(location, installDirectory);
result.add(innerBuilder.build(inputFile.toURI().toURL()));
} else {
DirectoryScanner scanner = new DirectoryScanner();
scanner.setBasedir(getFileForPath(location, installDirectory));
if (container.isDefined("includes")) {
List<String> includes = container.get("includes").asList(String.class);
scanner.setIncludes(includes.toArray(new String[includes.size()]));
}
if (container.isDefined("excludes")) {
List<String> includes = container.get("excludes").asList(String.class);
scanner.setExcludes(includes.toArray(new String[includes.size()]));
}
scanner.scan();
for (String bundleLocation : scanner.getIncludedFiles()) {
BundleHandler newHandler =
innerBuilder.build(
scanner
.getBasedir()
.toURI()
.resolve(bundleLocation.replaceAll("\\\\", "/"))
.toURL());
for (BundleHandler handler : result) {
if (newHandler.getBundleUrl().equals(handler.getBundleUrl())) {
if (newHandler.getActions().equals(handler.getActions())
&& newHandler.getStartLevel() == newHandler.getStartLevel()) {
// Do not duplicate
newHandler = null;
break;
} else {
StringBuilder sb = new StringBuilder("Controversial provisioning between ");
sb.append(handler).append(" and ").append(newHandler);
throw new IllegalArgumentException(sb.toString());
}
}
}
if (null != newHandler) {
result.add(newHandler);
}
}
}
}
return result;
}
@Override
public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException {
return SolrSplit.getSplits(conf, ConfigurationUtil.getUrl(conf), numSplits);
}
