Пример #1
0
  public void run() {
    resetManager.registerMe();

    try {
      // Create a thread context object.
      IThreadContext threadContext = ThreadContextFactory.make();
      IJobManager jobManager = JobManagerFactory.make(threadContext);
      IRepositoryConnectionManager connectionMgr =
          RepositoryConnectionManagerFactory.make(threadContext);

      IDBInterface database =
          DBInterfaceFactory.make(
              threadContext,
              ManifoldCF.getMasterDatabaseName(),
              ManifoldCF.getMasterDatabaseUsername(),
              ManifoldCF.getMasterDatabasePassword());

      // Loop
      while (true) {
        // Do another try/catch around everything in the loop
        try {
          // Before we begin, conditionally reset
          resetManager.waitForReset(threadContext);

          // Accumulate the wait before doing the next check.
          // We start with 10 seconds, which is the maximum.  If there's a service request
          // that's faster than that, we'll adjust the time downward.
          long waitTime = 10000L;

          if (Logging.threads.isDebugEnabled()) Logging.threads.debug("Checking for deleting jobs");

          // See if there are any starting jobs.
          // Note: Since this following call changes the job state, we must be careful to reset it
          // on any kind of failure.
          JobDeleteRecord[] deleteJobs = jobManager.getJobsReadyForDelete(processID);
          try {

            if (deleteJobs.length == 0) {
              ManifoldCF.sleep(waitTime);
              continue;
            }

            if (Logging.threads.isDebugEnabled())
              Logging.threads.debug(
                  "Found " + Integer.toString(deleteJobs.length) + " jobs ready to be deleted");

            long currentTime = System.currentTimeMillis();

            // Loop through jobs
            int i = 0;
            while (i < deleteJobs.length) {
              JobDeleteRecord jsr = deleteJobs[i++];
              Long jobID = jsr.getJobID();
              try {
                jobManager.prepareDeleteScan(jobID);
                // Start deleting this job!
                jobManager.noteJobDeleteStarted(jobID, currentTime);
                jsr.noteStarted();
              } catch (ManifoldCFException e) {
                if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
                  throw new InterruptedException();
                if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR) throw e;
                // We cannot abort the delete startup, but if we fall through, we'll put the job
                // back into
                // the state whence it came.  So, fall through.
                Logging.threads.error("Exception tossed: " + e.getMessage(), e);
              }
            }
          } finally {
            // Clean up all jobs that did not start
            ManifoldCFException exception = null;
            int i = 0;
            while (i < deleteJobs.length) {
              JobDeleteRecord jsr = deleteJobs[i++];
              if (!jsr.wasStarted()) {
                // Clean up from failed start.
                try {
                  jobManager.resetStartDeleteJob(jsr.getJobID());
                } catch (ManifoldCFException e) {
                  exception = e;
                }
              }
            }
            if (exception != null) throw exception;
          }

          // Sleep for the retry interval.
          ManifoldCF.sleep(waitTime);
        } catch (ManifoldCFException e) {
          if (e.getErrorCode() == ManifoldCFException.INTERRUPTED) break;

          if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR) {
            resetManager.noteEvent();

            Logging.threads.error(
                "Start delete thread aborting and restarting due to database connection reset: "
                    + e.getMessage(),
                e);
            try {
              // Give the database a chance to catch up/wake up
              ManifoldCF.sleep(10000L);
            } catch (InterruptedException se) {
              break;
            }
            continue;
          }

          // Log it, but keep the thread alive
          Logging.threads.error("Exception tossed: " + e.getMessage(), e);

          if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR) {
            // Shut the whole system down!
            System.exit(1);
          }

        } catch (InterruptedException e) {
          // We're supposed to quit
          break;
        } catch (OutOfMemoryError e) {
          System.err.println("agents process ran out of memory - shutting down");
          e.printStackTrace(System.err);
          System.exit(-200);
        } catch (Throwable e) {
          // A more severe error - but stay alive
          Logging.threads.fatal("Error tossed: " + e.getMessage(), e);
        }
      }
    } catch (Throwable e) {
      // Severe error on initialization
      System.err.println("agents process could not start - shutting down");
      Logging.threads.fatal("StartDeleteThread initialization error tossed: " + e.getMessage(), e);
      System.exit(-300);
    }
  }
Пример #2
0
  public static void main(String[] args) {
    if (args.length != 13) {
      System.err.println(
          "Usage: DefineJob <description> <connection_name> <output_name> <type> <start_method> <hopcount_method> <recrawl_interval> <expiration_interval> <reseed_interval> <job_priority> <hop_filters> <filespec_xml> <outputspec_xml>");
      System.err.println("<type> is one of: continuous or specified");
      System.err.println("<start_method> is one of: windowbegin, windowinside, disable");
      System.err.println("<hopcount_method> is one of: accurate, nodelete, neverdelete");
      System.err.println("<recrawl_interval> is the default document recrawl interval in minutes");
      System.err.println(
          "<expiration_interval> is the default document expiration interval in minutes");
      System.err.println("<reseed_interval> is the default document reseed interval in minutes");
      System.err.println("<job_priority> is the job priority (and integer between 0 and 10)");
      System.err.println(
          "<hop_filters> is a comma-separated list of tuples, of the form 'linktype=maxhops'");
      System.err.println(
          "<filespec_xml> is the document specification XML, its form dependent on the connection type");
      System.err.println(
          "<outputspec_xml> is the output specification XML, its form dependent on the output connection type");
      System.exit(-1);
    }

    String description = args[0];
    String connectionName = args[1];
    String outputConnectionName = args[2];
    String typeString = args[3];
    String startString = args[4];
    String hopcountString = args[5];
    String recrawlInterval = args[6];
    String expirationInterval = args[7];
    String reseedInterval = args[8];
    String jobPriority = args[9];
    String hopFilters = args[10];
    String filespecXML = args[11];
    String outputspecXML = args[12];

    try {
      IThreadContext tc = ThreadContextFactory.make();
      ManifoldCF.initializeEnvironment(tc);
      IJobManager jobManager = JobManagerFactory.make(tc);
      IJobDescription desc = jobManager.createJob();

      desc.setDescription(description);
      desc.setConnectionName(connectionName);
      desc.setOutputConnectionName(outputConnectionName);

      if (typeString.equals("continuous")) desc.setType(IJobDescription.TYPE_CONTINUOUS);
      else if (typeString.equals("specified")) desc.setType(IJobDescription.TYPE_SPECIFIED);
      else throw new ManifoldCFException("Unknown type: '" + typeString + "'");
      if (startString.equals("windowbegin")) desc.setStartMethod(IJobDescription.START_WINDOWBEGIN);
      else if (startString.equals("windowinside"))
        desc.setStartMethod(IJobDescription.START_WINDOWINSIDE);
      else if (startString.equals("disable")) desc.setStartMethod(IJobDescription.START_DISABLE);
      else throw new ManifoldCFException("Unknown start method: '" + startString + "'");

      if (hopcountString.equals("accurate"))
        desc.setHopcountMode(IJobDescription.HOPCOUNT_ACCURATE);
      else if (hopcountString.equals("nodelete"))
        desc.setHopcountMode(IJobDescription.HOPCOUNT_NODELETE);
      else if (hopcountString.equals("neverdelete"))
        desc.setHopcountMode(IJobDescription.HOPCOUNT_NEVERDELETE);
      else throw new ManifoldCFException("Unknown hopcount mode: '" + hopcountString + "'");

      if (recrawlInterval.length() > 0) desc.setInterval(new Long(recrawlInterval));
      if (expirationInterval.length() > 0) desc.setExpiration(new Long(expirationInterval));
      if (reseedInterval.length() > 0) desc.setReseedInterval(new Long(reseedInterval));
      desc.setPriority(Integer.parseInt(jobPriority));

      String[] hopFilterSet = hopFilters.split(",");
      int i = 0;
      while (i < hopFilterSet.length) {
        String hopFilter = hopFilterSet[i++];
        if (hopFilter != null && hopFilter.length() > 0) {
          String[] stuff = hopFilter.trim().split("=");
          if (stuff != null && stuff.length == 2)
            desc.addHopCountFilter(stuff[0], ((stuff[1].length() > 0) ? new Long(stuff[1]) : null));
        }
      }

      desc.getSpecification().fromXML(filespecXML);
      if (outputspecXML.length() > 0) desc.getOutputSpecification().fromXML(outputspecXML);

      // Now, save
      jobManager.save(desc);

      System.out.print(desc.getID().toString());
    } catch (Exception e) {
      e.printStackTrace();
      System.exit(-2);
    }
  }