/** Export configuration */
 public void exportConfiguration(java.io.OutputStream os)
     throws java.io.IOException, ManifoldCFException {
   // Write a version indicator
   ManifoldCF.writeDword(os, 1);
   // Get the authority list
   IRepositoryConnection[] list = getAllConnections();
   // Write the number of authorities
   ManifoldCF.writeDword(os, list.length);
   // Loop through the list and write the individual repository connection info
   int i = 0;
   while (i < list.length) {
     IRepositoryConnection conn = list[i++];
     ManifoldCF.writeString(os, conn.getName());
     ManifoldCF.writeString(os, conn.getDescription());
     ManifoldCF.writeString(os, conn.getClassName());
     ManifoldCF.writeString(os, conn.getConfigParams().toXML());
     ManifoldCF.writeString(os, conn.getACLAuthority());
     ManifoldCF.writeDword(os, conn.getMaxConnections());
     String[] throttles = conn.getThrottles();
     ManifoldCF.writeDword(os, throttles.length);
     int j = 0;
     while (j < throttles.length) {
       String throttleName = throttles[j++];
       ManifoldCF.writeString(os, throttleName);
       ManifoldCF.writeString(os, conn.getThrottleDescription(throttleName));
       ManifoldCF.writefloat(os, conn.getThrottleValue(throttleName));
     }
   }
 }
 /** Import configuration */
 public void importConfiguration(java.io.InputStream is)
     throws java.io.IOException, ManifoldCFException {
   int version = ManifoldCF.readDword(is);
   if (version != 1)
     throw new java.io.IOException(
         "Unknown repository connection configuration version: " + Integer.toString(version));
   int count = ManifoldCF.readDword(is);
   int i = 0;
   while (i < count) {
     IRepositoryConnection conn = create();
     conn.setName(ManifoldCF.readString(is));
     conn.setDescription(ManifoldCF.readString(is));
     conn.setClassName(ManifoldCF.readString(is));
     conn.getConfigParams().fromXML(ManifoldCF.readString(is));
     conn.setACLAuthority(ManifoldCF.readString(is));
     conn.setMaxConnections(ManifoldCF.readDword(is));
     int throttleCount = ManifoldCF.readDword(is);
     int j = 0;
     while (j < throttleCount) {
       String throttleName = ManifoldCF.readString(is);
       conn.addThrottleValue(throttleName, ManifoldCF.readString(is), ManifoldCF.readfloat(is));
       j++;
     }
     // Attempt to save this connection
     save(conn);
     i++;
   }
 }
Exemple #3
0
  public void executeTest() throws Exception {
    // Hey, we were able to install the file system connector etc.
    // Now, create a local test job and run it.
    IThreadContext tc = ThreadContextFactory.make();

    // Create a basic file system connection, and save it.
    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
    IRepositoryConnection conn = mgr.create();
    conn.setName("File Connection");
    conn.setDescription("File Connection");
    conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
    conn.setMaxConnections(100);
    // Now, save
    mgr.save(conn);

    // Create a basic null output connection, and save it.
    IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
    IOutputConnection outputConn = outputMgr.create();
    outputConn.setName("Null Connection");
    outputConn.setDescription("Null Connection");
    outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
    outputConn.setMaxConnections(100);
    // Now, save
    outputMgr.save(outputConn);

    // Create a job.
    IJobManager jobManager = JobManagerFactory.make(tc);
    IJobDescription job = jobManager.createJob();
    job.setDescription("Test Job");
    job.setConnectionName("File Connection");
    job.setOutputConnectionName("Null Connection");
    job.setType(job.TYPE_SPECIFIED);
    job.setStartMethod(job.START_DISABLE);
    job.setHopcountMode(job.HOPCOUNT_ACCURATE);

    // Now, set up the document specification.
    DocumentSpecification ds = job.getSpecification();
    // Crawl everything underneath the 'testdata' area
    File testDataFile = new File("testdata").getCanonicalFile();
    if (!testDataFile.exists())
      throw new ManifoldCFException(
          "Test data area not found!  Looking in " + testDataFile.toString());
    if (!testDataFile.isDirectory())
      throw new ManifoldCFException(
          "Test data area not a directory!  Looking in " + testDataFile.toString());
    SpecificationNode sn = new SpecificationNode("startpoint");
    sn.setAttribute("path", testDataFile.toString());
    SpecificationNode n = new SpecificationNode("include");
    n.setAttribute("type", "file");
    n.setAttribute("match", "*");
    sn.addChild(sn.getChildCount(), n);
    n = new SpecificationNode("include");
    n.setAttribute("type", "directory");
    n.setAttribute("match", "*");
    sn.addChild(sn.getChildCount(), n);
    ds.addChild(ds.getChildCount(), sn);

    // Set up the output specification.
    OutputSpecification os = job.getOutputSpecification();
    // Null output connections have no output specification, so this is a no-op.

    // Save the job.
    jobManager.save(job);

    // Create the test data files.
    FileHelper.createFile(new File("testdata/test1.txt"), "This is a test file");
    FileHelper.createFile(new File("testdata/test2.txt"), "This is another test file");
    FileHelper.createDirectory(new File("testdata/testdir"));
    FileHelper.createFile(new File("testdata/testdir/test3.txt"), "This is yet another test file");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID());
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    // Check to be sure we actually processed the right number of documents.
    JobStatus status = jobManager.getStatus(job.getID());
    // The test data area has 3 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 5)
      throw new ManifoldCFException(
          "Wrong number of documents processed - expected 5, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Add a file and recrawl using minimal crawl
    FileHelper.createFile(new File("testdata/testdir/test4.txt"), "Added file");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after add - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Change a file, and recrawl, once again using minimal
    FileHelper.changeFile(new File("testdata/test1.txt"), "Modified contents");

    // Now, start the job, and wait until it completes.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after change - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());
    // We also need to make sure the new document was indexed.  Have to think about how to do this
    // though.
    // MHL

    // Delete a file, and recrawl
    FileHelper.removeFile(new File("testdata/test2.txt"));

    // Do a minimal recrawl first; the delete should not be picked up.
    jobManager.manualStart(job.getID(), true);
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    status = jobManager.getStatus(job.getID());
    // The test data area has 4 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 6)
      throw new ManifoldCFException(
          "Wrong number of documents processed after delete with minimal crawl - expected 6, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Now, do a complete crawl - the delete should be found now.
    jobManager.manualStart(job.getID());
    instance.waitJobInactiveNative(jobManager, job.getID(), 120000L);

    // Check to be sure we actually processed the right number of documents.
    status = jobManager.getStatus(job.getID());
    // The test data area has 3 documents and one directory, and we have to count the root directory
    // too.
    if (status.getDocumentsProcessed() != 5)
      throw new ManifoldCFException(
          "Wrong number of documents processed after delete - expected 5, saw "
              + new Long(status.getDocumentsProcessed()).toString());

    // Now, delete the job.
    jobManager.deleteJob(job.getID());
    instance.waitJobDeletedNative(jobManager, job.getID(), 120000L);

    // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
    // lying around.
  }
  /**
   * Save a repository connection object.
   *
   * @param object is the object to save.
   * @return true if the object was created, false otherwise.
   */
  public boolean save(IRepositoryConnection object) throws ManifoldCFException {
    StringSetBuffer ssb = new StringSetBuffer();
    ssb.add(getRepositoryConnectionsKey());
    ssb.add(getRepositoryConnectionKey(object.getName()));
    StringSet cacheKeys = new StringSet(ssb);
    while (true) {
      // Catch deadlock condition
      long sleepAmt = 0L;
      try {
        ICacheHandle ch = cacheManager.enterCache(null, cacheKeys, getTransactionID());
        try {
          beginTransaction();
          try {
            // performLock();
            // Notify of a change to the configuration
            ManifoldCF.noteConfigurationChange();
            boolean isNew = object.getIsNew();
            // See whether the instance exists
            ArrayList params = new ArrayList();
            String query =
                buildConjunctionClause(
                    params,
                    new ClauseDescription[] {new UnitaryClause(nameField, object.getName())});
            IResultSet set =
                performQuery(
                    "SELECT * FROM " + getTableName() + " WHERE " + query + " FOR UPDATE",
                    params,
                    null,
                    null);
            HashMap values = new HashMap();
            values.put(descriptionField, object.getDescription());
            values.put(classNameField, object.getClassName());
            values.put(groupNameField, object.getACLAuthority());
            values.put(maxCountField, new Long((long) object.getMaxConnections()));
            String configXML = object.getConfigParams().toXML();
            values.put(configField, configXML);
            boolean notificationNeeded = false;
            boolean isCreated;

            if (set.getRowCount() > 0) {
              // If the object is supposedly new, it is bad that we found one that already exists.
              if (isNew)
                throw new ManifoldCFException(
                    "Repository connection '" + object.getName() + "' already exists");
              isCreated = false;
              IResultRow row = set.getRow(0);
              String oldXML = (String) row.getValue(configField);
              if (oldXML == null || !oldXML.equals(configXML)) notificationNeeded = true;

              // Update
              params.clear();
              query =
                  buildConjunctionClause(
                      params,
                      new ClauseDescription[] {new UnitaryClause(nameField, object.getName())});
              performUpdate(values, " WHERE " + query, params, null);
              throttleSpecManager.deleteRows(object.getName());
            } else {
              // If the object is not supposed to be new, it is bad that we did not find one.
              if (!isNew)
                throw new ManifoldCFException(
                    "Repository connection '" + object.getName() + "' no longer exists");
              isCreated = true;
              // Insert
              values.put(nameField, object.getName());
              // We only need the general key because this is new.
              performInsert(values, null);
            }

            // Write secondary table stuff
            throttleSpecManager.writeRows(object.getName(), object);

            // If notification required, do it.
            if (notificationNeeded) {
              IJobManager jobManager = JobManagerFactory.make(threadContext);
              jobManager.noteConnectionChange(object.getName());
            }

            cacheManager.invalidateKeys(ch);
            return isCreated;
          } catch (ManifoldCFException e) {
            signalRollback();
            throw e;
          } catch (Error e) {
            signalRollback();
            throw e;
          } finally {
            endTransaction();
          }
        } finally {
          cacheManager.leaveCache(ch);
        }
      } catch (ManifoldCFException e) {
        // Is this a deadlock exception?  If so, we want to try again.
        if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT) throw e;
        sleepAmt = getSleepAmt();
      } finally {
        sleepFor(sleepAmt);
      }
    }
  }