/**
* Create a bin manager handle.
*
* @param threadContext is the thread context.
* @return the handle.
*/
public static IBinManager make(IThreadContext threadContext) throws ManifoldCFException {
Object o = threadContext.get(binManagerName);
if (o == null || !(o instanceof IBinManager)) {
IDBInterface database =
DBInterfaceFactory.make(
threadContext,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
o = new org.apache.manifoldcf.crawler.bins.BinManager(database);
threadContext.save(binManagerName, o);
}
return (IBinManager) o;
}
public void run() {
resetManager.registerMe();
try {
// Create a thread context object.
IThreadContext threadContext = ThreadContextFactory.make();
IJobManager jobManager = JobManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionMgr =
RepositoryConnectionManagerFactory.make(threadContext);
IDBInterface database =
DBInterfaceFactory.make(
threadContext,
ManifoldCF.getMasterDatabaseName(),
ManifoldCF.getMasterDatabaseUsername(),
ManifoldCF.getMasterDatabasePassword());
// Loop
while (true) {
// Do another try/catch around everything in the loop
try {
// Before we begin, conditionally reset
resetManager.waitForReset(threadContext);
// Accumulate the wait before doing the next check.
// We start with 10 seconds, which is the maximum. If there's a service request
// that's faster than that, we'll adjust the time downward.
long waitTime = 10000L;
if (Logging.threads.isDebugEnabled()) Logging.threads.debug("Checking for deleting jobs");
// See if there are any starting jobs.
// Note: Since this following call changes the job state, we must be careful to reset it
// on any kind of failure.
JobDeleteRecord[] deleteJobs = jobManager.getJobsReadyForDelete(processID);
try {
if (deleteJobs.length == 0) {
ManifoldCF.sleep(waitTime);
continue;
}
if (Logging.threads.isDebugEnabled())
Logging.threads.debug(
"Found " + Integer.toString(deleteJobs.length) + " jobs ready to be deleted");
long currentTime = System.currentTimeMillis();
// Loop through jobs
int i = 0;
while (i < deleteJobs.length) {
JobDeleteRecord jsr = deleteJobs[i++];
Long jobID = jsr.getJobID();
try {
jobManager.prepareDeleteScan(jobID);
// Start deleting this job!
jobManager.noteJobDeleteStarted(jobID, currentTime);
jsr.noteStarted();
} catch (ManifoldCFException e) {
if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
throw new InterruptedException();
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR) throw e;
// We cannot abort the delete startup, but if we fall through, we'll put the job
// back into
// the state whence it came. So, fall through.
Logging.threads.error("Exception tossed: " + e.getMessage(), e);
}
}
} finally {
// Clean up all jobs that did not start
ManifoldCFException exception = null;
int i = 0;
while (i < deleteJobs.length) {
JobDeleteRecord jsr = deleteJobs[i++];
if (!jsr.wasStarted()) {
// Clean up from failed start.
try {
jobManager.resetStartDeleteJob(jsr.getJobID());
} catch (ManifoldCFException e) {
exception = e;
}
}
}
if (exception != null) throw exception;
}
// Sleep for the retry interval.
ManifoldCF.sleep(waitTime);
} catch (ManifoldCFException e) {
if (e.getErrorCode() == ManifoldCFException.INTERRUPTED) break;
if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR) {
resetManager.noteEvent();
Logging.threads.error(
"Start delete thread aborting and restarting due to database connection reset: "
+ e.getMessage(),
e);
try {
// Give the database a chance to catch up/wake up
ManifoldCF.sleep(10000L);
} catch (InterruptedException se) {
break;
}
continue;
}
// Log it, but keep the thread alive
Logging.threads.error("Exception tossed: " + e.getMessage(), e);
if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR) {
// Shut the whole system down!
System.exit(1);
}
} catch (InterruptedException e) {
// We're supposed to quit
break;
} catch (OutOfMemoryError e) {
System.err.println("agents process ran out of memory - shutting down");
e.printStackTrace(System.err);
System.exit(-200);
} catch (Throwable e) {
// A more severe error - but stay alive
Logging.threads.fatal("Error tossed: " + e.getMessage(), e);
}
}
} catch (Throwable e) {
// Severe error on initialization
System.err.println("agents process could not start - shutting down");
Logging.threads.fatal("StartDeleteThread initialization error tossed: " + e.getMessage(), e);
System.exit(-300);
}
}
public static void main(String[] args) {
if (args.length != 13) {
System.err.println(
"Usage: DefineJob <description> <connection_name> <output_name> <type> <start_method> <hopcount_method> <recrawl_interval> <expiration_interval> <reseed_interval> <job_priority> <hop_filters> <filespec_xml> <outputspec_xml>");
System.err.println("<type> is one of: continuous or specified");
System.err.println("<start_method> is one of: windowbegin, windowinside, disable");
System.err.println("<hopcount_method> is one of: accurate, nodelete, neverdelete");
System.err.println("<recrawl_interval> is the default document recrawl interval in minutes");
System.err.println(
"<expiration_interval> is the default document expiration interval in minutes");
System.err.println("<reseed_interval> is the default document reseed interval in minutes");
System.err.println("<job_priority> is the job priority (and integer between 0 and 10)");
System.err.println(
"<hop_filters> is a comma-separated list of tuples, of the form 'linktype=maxhops'");
System.err.println(
"<filespec_xml> is the document specification XML, its form dependent on the connection type");
System.err.println(
"<outputspec_xml> is the output specification XML, its form dependent on the output connection type");
System.exit(-1);
}
String description = args[0];
String connectionName = args[1];
String outputConnectionName = args[2];
String typeString = args[3];
String startString = args[4];
String hopcountString = args[5];
String recrawlInterval = args[6];
String expirationInterval = args[7];
String reseedInterval = args[8];
String jobPriority = args[9];
String hopFilters = args[10];
String filespecXML = args[11];
String outputspecXML = args[12];
try {
IThreadContext tc = ThreadContextFactory.make();
ManifoldCF.initializeEnvironment(tc);
IJobManager jobManager = JobManagerFactory.make(tc);
IJobDescription desc = jobManager.createJob();
desc.setDescription(description);
desc.setConnectionName(connectionName);
desc.setOutputConnectionName(outputConnectionName);
if (typeString.equals("continuous")) desc.setType(IJobDescription.TYPE_CONTINUOUS);
else if (typeString.equals("specified")) desc.setType(IJobDescription.TYPE_SPECIFIED);
else throw new ManifoldCFException("Unknown type: '" + typeString + "'");
if (startString.equals("windowbegin")) desc.setStartMethod(IJobDescription.START_WINDOWBEGIN);
else if (startString.equals("windowinside"))
desc.setStartMethod(IJobDescription.START_WINDOWINSIDE);
else if (startString.equals("disable")) desc.setStartMethod(IJobDescription.START_DISABLE);
else throw new ManifoldCFException("Unknown start method: '" + startString + "'");
if (hopcountString.equals("accurate"))
desc.setHopcountMode(IJobDescription.HOPCOUNT_ACCURATE);
else if (hopcountString.equals("nodelete"))
desc.setHopcountMode(IJobDescription.HOPCOUNT_NODELETE);
else if (hopcountString.equals("neverdelete"))
desc.setHopcountMode(IJobDescription.HOPCOUNT_NEVERDELETE);
else throw new ManifoldCFException("Unknown hopcount mode: '" + hopcountString + "'");
if (recrawlInterval.length() > 0) desc.setInterval(new Long(recrawlInterval));
if (expirationInterval.length() > 0) desc.setExpiration(new Long(expirationInterval));
if (reseedInterval.length() > 0) desc.setReseedInterval(new Long(reseedInterval));
desc.setPriority(Integer.parseInt(jobPriority));
String[] hopFilterSet = hopFilters.split(",");
int i = 0;
while (i < hopFilterSet.length) {
String hopFilter = hopFilterSet[i++];
if (hopFilter != null && hopFilter.length() > 0) {
String[] stuff = hopFilter.trim().split("=");
if (stuff != null && stuff.length == 2)
desc.addHopCountFilter(stuff[0], ((stuff[1].length() > 0) ? new Long(stuff[1]) : null));
}
}
desc.getSpecification().fromXML(filespecXML);
if (outputspecXML.length() > 0) desc.getOutputSpecification().fromXML(outputspecXML);
// Now, save
jobManager.save(desc);
System.out.print(desc.getID().toString());
} catch (Exception e) {
e.printStackTrace();
System.exit(-2);
}
}
