/** Export configuration */
public void exportConfiguration(java.io.OutputStream os)
throws java.io.IOException, ManifoldCFException {
// Write a version indicator
ManifoldCF.writeDword(os, 1);
// Get the authority list
IRepositoryConnection[] list = getAllConnections();
// Write the number of authorities
ManifoldCF.writeDword(os, list.length);
// Loop through the list and write the individual repository connection info
int i = 0;
while (i < list.length) {
IRepositoryConnection conn = list[i++];
ManifoldCF.writeString(os, conn.getName());
ManifoldCF.writeString(os, conn.getDescription());
ManifoldCF.writeString(os, conn.getClassName());
ManifoldCF.writeString(os, conn.getConfigParams().toXML());
ManifoldCF.writeString(os, conn.getACLAuthority());
ManifoldCF.writeDword(os, conn.getMaxConnections());
String[] throttles = conn.getThrottles();
ManifoldCF.writeDword(os, throttles.length);
int j = 0;
while (j < throttles.length) {
String throttleName = throttles[j++];
ManifoldCF.writeString(os, throttleName);
ManifoldCF.writeString(os, conn.getThrottleDescription(throttleName));
ManifoldCF.writefloat(os, conn.getThrottleValue(throttleName));
}
}
}
/** Import configuration */
public void importConfiguration(java.io.InputStream is)
throws java.io.IOException, ManifoldCFException {
int version = ManifoldCF.readDword(is);
if (version != 1)
throw new java.io.IOException(
"Unknown repository connection configuration version: " + Integer.toString(version));
int count = ManifoldCF.readDword(is);
int i = 0;
while (i < count) {
IRepositoryConnection conn = create();
conn.setName(ManifoldCF.readString(is));
conn.setDescription(ManifoldCF.readString(is));
conn.setClassName(ManifoldCF.readString(is));
conn.getConfigParams().fromXML(ManifoldCF.readString(is));
conn.setACLAuthority(ManifoldCF.readString(is));
conn.setMaxConnections(ManifoldCF.readDword(is));
int throttleCount = ManifoldCF.readDword(is);
int j = 0;
while (j < throttleCount) {
String throttleName = ManifoldCF.readString(is);
conn.addThrottleValue(throttleName, ManifoldCF.readString(is), ManifoldCF.readfloat(is));
j++;
}
// Attempt to save this connection
save(conn);
i++;
}
}
public void executeTest() throws Exception {
// Hey, we were able to install the file system connector etc.
// Now, create a local test job and run it.
IThreadContext tc = ThreadContextFactory.make();
// Create a basic file system connection, and save it.
IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
IRepositoryConnection conn = mgr.create();
conn.setName("File Connection");
conn.setDescription("File Connection");
conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
conn.setMaxConnections(100);
// Now, save
mgr.save(conn);
// Create a basic null output connection, and save it.
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
IOutputConnection outputConn = outputMgr.create();
outputConn.setName("Null Connection");
outputConn.setDescription("Null Connection");
outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
outputConn.setMaxConnections(100);
// Now, save
outputMgr.save(outputConn);
// Create a job.
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription job = jobManager.createJob();
job.setDescription("Test Job");
job.setConnectionName("File Connection");
job.setOutputConnectionName("Null Connection");
job.setType(job.TYPE_SPECIFIED);
job.setStartMethod(job.START_DISABLE);
job.setHopcountMode(job.HOPCOUNT_ACCURATE);
// Now, set up the document specification.
DocumentSpecification ds = job.getSpecification();
// Crawl everything underneath the 'testdata' area
File testDataFile = new File("testdata").getCanonicalFile();
if (!testDataFile.exists())
throw new ManifoldCFException(
"Test data area not found! Looking in " + testDataFile.toString());
if (!testDataFile.isDirectory())
throw new ManifoldCFException(
"Test data area not a directory! Looking in " + testDataFile.toString());
SpecificationNode sn = new SpecificationNode("startpoint");
sn.setAttribute("path", testDataFile.toString());
SpecificationNode n = new SpecificationNode("include");
n.setAttribute("type", "file");
n.setAttribute("match", "*");
sn.addChild(sn.getChildCount(), n);
n = new SpecificationNode("include");
n.setAttribute("type", "directory");
n.setAttribute("match", "*");
sn.addChild(sn.getChildCount(), n);
ds.addChild(ds.getChildCount(), sn);
// Set up the output specification.
OutputSpecification os = job.getOutputSpecification();
// Null output connections have no output specification, so this is a no-op.
// Save the job.
jobManager.save(job);
// Create the test data files.
FileHelper.createFile(new File("testdata/test1.txt"), "This is a test file");
FileHelper.createFile(new File("testdata/test2.txt"), "This is another test file");
FileHelper.createDirectory(new File("testdata/testdir"));
FileHelper.createFile(new File("testdata/testdir/test3.txt"), "This is yet another test file");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID());
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
// Check to be sure we actually processed the right number of documents.
JobStatus status = jobManager.getStatus(job.getID());
// The test data area has 3 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 5)
throw new ManifoldCFException(
"Wrong number of documents processed - expected 5, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Add a file and recrawl using minimal crawl
FileHelper.createFile(new File("testdata/testdir/test4.txt"), "Added file");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after add - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Change a file, and recrawl, once again using minimal
FileHelper.changeFile(new File("testdata/test1.txt"), "Modified contents");
// Now, start the job, and wait until it completes.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after change - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// We also need to make sure the new document was indexed. Have to think about how to do this
// though.
// MHL
// Delete a file, and recrawl
FileHelper.removeFile(new File("testdata/test2.txt"));
// Do a minimal recrawl first; the delete should not be picked up.
jobManager.manualStart(job.getID(), true);
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
status = jobManager.getStatus(job.getID());
// The test data area has 4 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException(
"Wrong number of documents processed after delete with minimal crawl - expected 6, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Now, do a complete crawl - the delete should be found now.
jobManager.manualStart(job.getID());
instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);
// Check to be sure we actually processed the right number of documents.
status = jobManager.getStatus(job.getID());
// The test data area has 3 documents and one directory, and we have to count the root directory
// too.
if (status.getDocumentsProcessed() != 5)
throw new ManifoldCFException(
"Wrong number of documents processed after delete - expected 5, saw "
+ new Long(status.getDocumentsProcessed()).toString());
// Now, delete the job.
jobManager.deleteJob(job.getID());
instance.waitJobDeletedNative(jobManager, job.getID(), 120000L);
// Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
// lying around.
}
/**
* Save a repository connection object.
*
* @param object is the object to save.
* @return true if the object was created, false otherwise.
*/
public boolean save(IRepositoryConnection object) throws ManifoldCFException {
StringSetBuffer ssb = new StringSetBuffer();
ssb.add(getRepositoryConnectionsKey());
ssb.add(getRepositoryConnectionKey(object.getName()));
StringSet cacheKeys = new StringSet(ssb);
while (true) {
// Catch deadlock condition
long sleepAmt = 0L;
try {
ICacheHandle ch = cacheManager.enterCache(null, cacheKeys, getTransactionID());
try {
beginTransaction();
try {
// performLock();
// Notify of a change to the configuration
ManifoldCF.noteConfigurationChange();
boolean isNew = object.getIsNew();
// See whether the instance exists
ArrayList params = new ArrayList();
String query =
buildConjunctionClause(
params,
new ClauseDescription[] {new UnitaryClause(nameField, object.getName())});
IResultSet set =
performQuery(
"SELECT * FROM " + getTableName() + " WHERE " + query + " FOR UPDATE",
params,
null,
null);
HashMap values = new HashMap();
values.put(descriptionField, object.getDescription());
values.put(classNameField, object.getClassName());
values.put(groupNameField, object.getACLAuthority());
values.put(maxCountField, new Long((long) object.getMaxConnections()));
String configXML = object.getConfigParams().toXML();
values.put(configField, configXML);
boolean notificationNeeded = false;
boolean isCreated;
if (set.getRowCount() > 0) {
// If the object is supposedly new, it is bad that we found one that already exists.
if (isNew)
throw new ManifoldCFException(
"Repository connection '" + object.getName() + "' already exists");
isCreated = false;
IResultRow row = set.getRow(0);
String oldXML = (String) row.getValue(configField);
if (oldXML == null || !oldXML.equals(configXML)) notificationNeeded = true;
// Update
params.clear();
query =
buildConjunctionClause(
params,
new ClauseDescription[] {new UnitaryClause(nameField, object.getName())});
performUpdate(values, " WHERE " + query, params, null);
throttleSpecManager.deleteRows(object.getName());
} else {
// If the object is not supposed to be new, it is bad that we did not find one.
if (!isNew)
throw new ManifoldCFException(
"Repository connection '" + object.getName() + "' no longer exists");
isCreated = true;
// Insert
values.put(nameField, object.getName());
// We only need the general key because this is new.
performInsert(values, null);
}
// Write secondary table stuff
throttleSpecManager.writeRows(object.getName(), object);
// If notification required, do it.
if (notificationNeeded) {
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.noteConnectionChange(object.getName());
}
cacheManager.invalidateKeys(ch);
return isCreated;
} catch (ManifoldCFException e) {
signalRollback();
throw e;
} catch (Error e) {
signalRollback();
throw e;
} finally {
endTransaction();
}
} finally {
cacheManager.leaveCache(ch);
}
} catch (ManifoldCFException e) {
// Is this a deadlock exception? If so, we want to try again.
if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT) throw e;
sleepAmt = getSleepAmt();
} finally {
sleepFor(sleepAmt);
}
}
}