protected void localSetUp() throws Exception {
super.localSetUp();
IThreadContext tc = ThreadContextFactory.make();
IDBInterface database =
DBInterfaceFactory.make(
tc,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
IConnectorManager mgr = ConnectorManagerFactory.make(tc);
IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
IJobManager jobManager = JobManagerFactory.make(tc);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);
// Deregistration should be done in a transaction
database.beginTransaction();
try {
int i;
String[] connectorClasses = getConnectorClasses();
String[] connectorNames = getConnectorNames();
i = 0;
while (i < connectorClasses.length) {
// First, register connector
mgr.registerConnector(connectorNames[i], connectorClasses[i]);
// Then, signal to all jobs that might depend on this connector that they can switch state
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
// For each connection name, modify the jobs to note that the connector is now installed
jobManager.noteConnectorRegistration(connectionNames);
i++;
}
String[] authorityClasses = getAuthorityClasses();
String[] authorityNames = getAuthorityNames();
i = 0;
while (i < authorityClasses.length) {
authMgr.registerConnector(authorityNames[i], authorityClasses[i]);
i++;
}
String[] outputClasses = getOutputClasses();
String[] outputNames = getOutputNames();
i = 0;
while (i < outputClasses.length) {
// First, register connector
outputMgr.registerConnector(outputNames[i], outputClasses[i]);
// Then, signal to all jobs that might depend on this connector that they can switch state
// Find the connection names that come with this class
String[] connectionNames = outputConnManager.findConnectionsForConnector(outputClasses[i]);
// For all connection names, notify all agents of the registration
AgentManagerFactory.noteOutputConnectorRegistration(tc, connectionNames);
i++;
}
} catch (ManifoldCFException e) {
database.signalRollback();
throw e;
} catch (Error e) {
database.signalRollback();
throw e;
} finally {
database.endTransaction();
}
}
protected void localCleanUp() throws Exception {
IThreadContext tc = ThreadContextFactory.make();
Exception currentException = null;
// First, tear down all jobs, connections, authority connections, and output connections.
try {
IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(tc);
IAuthorityConnectionManager authConnMgr = AuthorityConnectionManagerFactory.make(tc);
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
// Now, get a list of the repository connections
IRepositoryConnection[] connections = connMgr.getAllConnections();
int i = 0;
while (i < connections.length) {
connMgr.delete(connections[i++].getName());
}
// Get a list of authority connections
IAuthorityConnection[] authorities = authConnMgr.getAllConnections();
i = 0;
while (i < authorities.length) {
authConnMgr.delete(authorities[i++].getName());
}
// Finally, get rid of output connections
IOutputConnection[] outputs = outputMgr.getAllConnections();
i = 0;
while (i < outputs.length) {
outputMgr.delete(outputs[i++].getName());
}
} catch (Exception e) {
currentException = e;
}
try {
IDBInterface database =
DBInterfaceFactory.make(
tc,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
IConnectorManager mgr = ConnectorManagerFactory.make(tc);
IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);
IJobManager jobManager = JobManagerFactory.make(tc);
IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
// Deregistration should be done in a transaction
database.beginTransaction();
try {
int i;
String[] connectorClasses = getConnectorClasses();
i = 0;
while (i < connectorClasses.length) {
// Find the connection names that come with this class
String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
// For each connection name, modify the jobs to note that the connector is no longer
// installed
jobManager.noteConnectorDeregistration(connectionNames);
// Now that all jobs have been placed into an appropriate state, actually do the
// deregistration itself.
mgr.unregisterConnector(connectorClasses[i]);
i++;
}
String[] authorityClasses = getAuthorityClasses();
i = 0;
while (i < authorityClasses.length) {
authMgr.unregisterConnector(authorityClasses[i]);
i++;
}
String[] outputClasses = getOutputClasses();
i = 0;
while (i < outputClasses.length) {
// Find the connection names that come with this class
String[] connectionNames =
outputConnManager.findConnectionsForConnector(outputClasses[i]);
// For all connection names, notify all agents of the deregistration
AgentManagerFactory.noteOutputConnectorDeregistration(tc, connectionNames);
// Now that all jobs have been placed into an appropriate state, actually do the
// deregistration itself.
outputMgr.unregisterConnector(outputClasses[i]);
i++;
}
} catch (ManifoldCFException e) {
database.signalRollback();
throw e;
} catch (Error e) {
database.signalRollback();
throw e;
} finally {
database.endTransaction();
}
} catch (Exception e) {
if (currentException != null) currentException = e;
}
try {
super.localCleanUp();
} catch (Exception e) {
if (currentException != null) currentException = e;
}
if (currentException != null) throw currentException;
}
public void executeTest() throws Exception {
// Hey, we were able to install the file system connector etc.
// Now, create a local test job and run it.
IThreadContext tc = ThreadContextFactory.make();
// Create a basic file system connection, and save it.
IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection conn = mgr.create();
conn.setName("File Connection");
conn.setDescription("File Connection");
conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
conn.setMaxConnections(100);
// Now, save
mgr.save(conn);
// Create a basic null output connection, and save it.
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
IOutputConnection outputConn = outputMgr.create();
outputConn.setName("Null Connection");
outputConn.setDescription("Null Connection");
outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
outputConn.setMaxConnections(100);
// Now, save
outputMgr.save(outputConn);
// Create a job.
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription job = jobManager.createJob();
job.setDescription("Test Job");
job.setConnectionName("File Connection");
job.setOutputConnectionName("Null Connection");
job.setType(job.TYPE_SPECIFIED);
job.setStartMethod(job.START_DISABLE);
job.setHopcountMode(job.HOPCOUNT_ACCURATE);
// Now, set up the document specification.
DocumentSpecification ds = job.getSpecification();
// Crawl everything underneath the 'testdata' area
File testDataFile = new File("testdata").getCanonicalFile();
if (!testDataFile.exists())
throw new ManifoldCFException(
"Test data area not found! Looking in " + testDataFile.toString());
if (!testDataFile.isDirectory())
throw new ManifoldCFException(
"Test data area not a directory! Looking in " + testDataFile.toString());
SpecificationNode sn = new SpecificationNode("startpoint");
sn.setAttribute("path", testDataFile.toString());
SpecificationNode n = new SpecificationNode("include");
n.setAttribute("type", "file");
n.setAttribute("match", "*");
sn.addChild(sn.getChildCount(), n);
n = new SpecificationNode("include");
n.setAttribute("type", "directory");
n.setAttribute("match", "*");
sn.addChild(sn.getChildCount(), n);
ds.addChild(ds.getChildCount(), sn);
// Set up the output specification.
OutputSpecification os = job.getOutputSpecification();
// Null output connections have no output specification, so this is a no-op.
// Save the job.
jobManager.save(job);
// Create the test data files.
FileHelper.createFile(new File("testdata/test1.txt"), "This is a test file");
FileHelper.createFile(new File("testdata/test2.txt"), "This is another test file");
FileHelper.createDirectory(new File("testdata/testdir"));
FileHelper.createFile(new File("testdata/testdir/test3.txt"), "This is yet another test file");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID());
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
// Check to be sure we actually processed the right number of documents.
JobStatus status = jobManager.getStatus(job.getID());
// The test data area has 3 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 5)
throw new ManifoldCFException(
"Wrong number of documents processed - expected 5, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Add a file and recrawl using minimal crawl
FileHelper.createFile(new File("testdata/testdir/test4.txt"), "Added file");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after add - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Change a file, and recrawl, once again using minimal
FileHelper.changeFile(new File("testdata/test1.txt"), "Modified contents");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after change - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// We also need to make sure the new document was indexed. Have to think about how to do this
// though.
// MHL
// Delete a file, and recrawl
FileHelper.removeFile(new File("testdata/test2.txt"));
// Do a minimal recrawl first; the delete should not be picked up.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after delete with minimal crawl - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Now, do a complete crawl - the delete should be found now.
jobManager.manualStart(job.getID());
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
// Check to be sure we actually processed the right number of documents.
status = jobManager.getStatus(job.getID());
// The test data area has 3 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 5)
throw new ManifoldCFException(
"Wrong number of documents processed after delete - expected 5, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Now, delete the job.
jobManager.deleteJob(job.getID());
instance.waitJobDeletedNative(jobManager, job.getID(), 120000L);
// Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
// lying around.
}