Exemplo n.º 1
0
  protected void localSetUp() throws Exception {

    super.localSetUp();

    IThreadContext tc = ThreadContextFactory.make();

    IDBInterface database =
        DBInterfaceFactory.make(
            tc,
            ManifoldCF.getMasterDatabaseName(),
            ManifoldCF.getMasterDatabaseUsername(),
            ManifoldCF.getMasterDatabasePassword());

    IConnectorManager mgr = ConnectorManagerFactory.make(tc);
    IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
    IJobManager jobManager = JobManagerFactory.make(tc);
    IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
    IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
    IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);

    // Deregistration should be done in a transaction
    database.beginTransaction();
    try {
      int i;

      String[] connectorClasses = getConnectorClasses();
      String[] connectorNames = getConnectorNames();

      i = 0;
      while (i < connectorClasses.length) {
        // First, register connector
        mgr.registerConnector(connectorNames[i], connectorClasses[i]);
        // Then, signal to all jobs that might depend on this connector that they can switch state
        // Find the connection names that come with this class
        String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
        // For each connection name, modify the jobs to note that the connector is now installed
        jobManager.noteConnectorRegistration(connectionNames);
        i++;
      }

      String[] authorityClasses = getAuthorityClasses();
      String[] authorityNames = getAuthorityNames();

      i = 0;
      while (i < authorityClasses.length) {
        authMgr.registerConnector(authorityNames[i], authorityClasses[i]);
        i++;
      }

      String[] outputClasses = getOutputClasses();
      String[] outputNames = getOutputNames();

      i = 0;
      while (i < outputClasses.length) {
        // First, register connector
        outputMgr.registerConnector(outputNames[i], outputClasses[i]);
        // Then, signal to all jobs that might depend on this connector that they can switch state
        // Find the connection names that come with this class
        String[] connectionNames = outputConnManager.findConnectionsForConnector(outputClasses[i]);
        // For all connection names, notify all agents of the registration
        AgentManagerFactory.noteOutputConnectorRegistration(tc, connectionNames);
        i++;
      }

    } catch (ManifoldCFException e) {
      database.signalRollback();
      throw e;
    } catch (Error e) {
      database.signalRollback();
      throw e;
    } finally {
      database.endTransaction();
    }
  }
Exemplo n.º 2
0
  protected void localCleanUp() throws Exception {
    IThreadContext tc = ThreadContextFactory.make();

    Exception currentException = null;
    // First, tear down all jobs, connections, authority connections, and output connections.
    try {
      IRepositoryConnectionManager connMgr = RepositoryConnectionManagerFactory.make(tc);
      IAuthorityConnectionManager authConnMgr = AuthorityConnectionManagerFactory.make(tc);
      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);

      // Now, get a list of the repository connections
      IRepositoryConnection[] connections = connMgr.getAllConnections();
      int i = 0;
      while (i < connections.length) {
        connMgr.delete(connections[i++].getName());
      }

      // Get a list of authority connections
      IAuthorityConnection[] authorities = authConnMgr.getAllConnections();
      i = 0;
      while (i < authorities.length) {
        authConnMgr.delete(authorities[i++].getName());
      }

      // Finally, get rid of output connections
      IOutputConnection[] outputs = outputMgr.getAllConnections();
      i = 0;
      while (i < outputs.length) {
        outputMgr.delete(outputs[i++].getName());
      }

    } catch (Exception e) {
      currentException = e;
    }
    try {
      IDBInterface database =
          DBInterfaceFactory.make(
              tc,
              ManifoldCF.getMasterDatabaseName(),
              ManifoldCF.getMasterDatabaseUsername(),
              ManifoldCF.getMasterDatabasePassword());

      IConnectorManager mgr = ConnectorManagerFactory.make(tc);
      IAuthorityConnectorManager authMgr = AuthorityConnectorManagerFactory.make(tc);
      IOutputConnectorManager outputMgr = OutputConnectorManagerFactory.make(tc);
      IOutputConnectionManager outputConnManager = OutputConnectionManagerFactory.make(tc);
      IJobManager jobManager = JobManagerFactory.make(tc);
      IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);

      // Deregistration should be done in a transaction
      database.beginTransaction();
      try {
        int i;

        String[] connectorClasses = getConnectorClasses();

        i = 0;
        while (i < connectorClasses.length) {
          // Find the connection names that come with this class
          String[] connectionNames = connManager.findConnectionsForConnector(connectorClasses[i]);
          // For each connection name, modify the jobs to note that the connector is no longer
          // installed
          jobManager.noteConnectorDeregistration(connectionNames);
          // Now that all jobs have been placed into an appropriate state, actually do the
          // deregistration itself.
          mgr.unregisterConnector(connectorClasses[i]);
          i++;
        }

        String[] authorityClasses = getAuthorityClasses();

        i = 0;
        while (i < authorityClasses.length) {
          authMgr.unregisterConnector(authorityClasses[i]);
          i++;
        }

        String[] outputClasses = getOutputClasses();

        i = 0;
        while (i < outputClasses.length) {
          // Find the connection names that come with this class
          String[] connectionNames =
              outputConnManager.findConnectionsForConnector(outputClasses[i]);
          // For all connection names, notify all agents of the deregistration
          AgentManagerFactory.noteOutputConnectorDeregistration(tc, connectionNames);
          // Now that all jobs have been placed into an appropriate state, actually do the
          // deregistration itself.
          outputMgr.unregisterConnector(outputClasses[i]);
          i++;
        }

      } catch (ManifoldCFException e) {
        database.signalRollback();
        throw e;
      } catch (Error e) {
        database.signalRollback();
        throw e;
      } finally {
        database.endTransaction();
      }
    } catch (Exception e) {
      if (currentException != null) currentException = e;
    }
    try {
      super.localCleanUp();
    } catch (Exception e) {
      if (currentException != null) currentException = e;
    }
    if (currentException != null) throw currentException;
  }
Exemplo n.º 3
0
  public void executeTest() throws Exception {
    // Hey, we were able to install the file system connector etc.
    // Now, create a local test job and run it.
    IThreadContext tc = ThreadContextFactory.make();

    // Create a basic file system connection, and save it.
    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
    IRepositoryConnection conn = mgr.create();
    conn.setName("File Connection");
    conn.setDescription("File Connection");
    conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
    conn.setMaxConnections(100);
    // Now, save
    mgr.save(conn);

    // Create a basic null output connection, and save it.
    IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
    IOutputConnection outputConn = outputMgr.create();
    outputConn.setName("Null Connection");
    outputConn.setDescription("Null Connection");
    outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
    outputConn.setMaxConnections(100);
    // Now, save
    outputMgr.save(outputConn);

    // Create a job.
    IJobManager jobManager = JobManagerFactory.make(tc);
    IJobDescription job = jobManager.createJob();
    job.setDescription("Test Job");
    job.setConnectionName("File Connection");
    job.setOutputConnectionName("Null Connection");
    job.setType(job.TYPE_SPECIFIED);
    job.setStartMethod(job.START_DISABLE);
    job.setHopcountMode(job.HOPCOUNT_ACCURATE);

    // Now, set up the document specification.
    DocumentSpecification ds = job.getSpecification();
    // Crawl everything underneath the 'testdata' area
    File testDataFile = new File("testdata").getCanonicalFile();
    if (!testDataFile.exists())
      throw new ManifoldCFException(
          "Test data area not found!  Looking in " + testDataFile.toString());
    if (!testDataFile.isDirectory())
      throw new ManifoldCFException(
          "Test data area not a directory!  Looking in " + testDataFile.toString());
    SpecificationNode sn = new SpecificationNode("startpoint");
    sn.setAttribute("path", testDataFile.toString());
    SpecificationNode n = new SpecificationNode("include");
    n.setAttribute("type", "file");
    n.setAttribute("match", "*");
    sn.addChild(sn.getChildCount(), n);
    n = new SpecificationNode("include");
    n.setAttribute("type", "directory");
    n.setAttribute("match", "*");
    sn.addChild(sn.getChildCount(), n);
    ds.addChild(ds.getChildCount(), sn);

    // Set up the output specification.
    OutputSpecification os = job.getOutputSpecification();
    // Null output connections have no output specification, so this is a no-op.

    // Save the job.
    jobManager.save(job);

    // Create the test data files.
    FileHelper.createFile(new File("testdata/test1.txt"), "This is a test file");
    FileHelper.createFile(new File("testdata/test2.txt"), "This is another test file");
    FileHelper.createDirectory(new File("testdata/testdir"));
    FileHelper.createFile(new File("testdata/testdir/test3.txt"), "This is yet another test file");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID());
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    // Check to be sure we actually processed the right number of documents.
    JobStatus status = jobManager.getStatus(job.getID());
    // The test data area has 3 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 5)
      throw new ManifoldCFException(
          "Wrong number of documents processed - expected 5, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Add a file and recrawl using minimal crawl
    FileHelper.createFile(new File("testdata/testdir/test4.txt"), "Added file");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after add - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Change a file, and recrawl, once again using minimal
    FileHelper.changeFile(new File("testdata/test1.txt"), "Modified contents");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after change - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());
    // We also need to make sure the new document was indexed.  Have to think about how to do this
    // though.
    // MHL

    // Delete a file, and recrawl
    FileHelper.removeFile(new File("testdata/test2.txt"));

    // Do a minimal recrawl first; the delete should not be picked up.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after delete with minimal crawl - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Now, do a complete crawl - the delete should be found now.
    jobManager.manualStart(job.getID());
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    // Check to be sure we actually processed the right number of documents.
    status = jobManager.getStatus(job.getID());
    // The test data area has 3 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 5)
      throw new ManifoldCFException(
          "Wrong number of documents processed after delete - expected 5, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Now, delete the job.
    jobManager.deleteJob(job.getID());
    instance.waitJobDeletedNative(jobManager, job.getID(), 120000L);

    // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
    // lying around.
  }