/**
   * Create bitarchive with access denied to location of admin data verify that exceptions are
   * thrown
   */
  @Test
  public void testAccessDenied() {
    // Make sure archive exists
    assertTrue("Inaccessible archive dir must exist", NOACCESS_ARCHIVE_DIR.exists());

    if (NOACCESS_ARCHIVE_DIR.canWrite()) {
      NOACCESS_ARCHIVE_DIR.setReadOnly();
    }

    // and that admin file is inaccessible
    assertFalse(
        "Must not be able to write to inaccessible admin file", NOACCESS_ARCHIVE_DIR.canWrite());

    try {
      Settings.set(
          ArchiveSettings.BITARCHIVE_SERVER_FILEDIR, NOACCESS_ARCHIVE_DIR.getAbsolutePath());
      Bitarchive ba = Bitarchive.getInstance();
      ba.close();
      fail("Accessing read-only archive should throw exception"); // do not come here
    } catch (PermissionDenied e) {
      // Expected case
      StringAsserts.assertStringContains(
          "Should mention noaccess dir", "noaccess/filedir", e.getMessage());
    }
  }
  /** FIXME Fails in Hudson */
  public void failingTestExecute() {
    File arcFile = new File(TestInfo.BATCH_DIR, "MimeUrlSearch.jar");
    assertTrue(arcFile.isFile());

    Settings.set(
        "settings.common.batch.batchjobs.batchjob.class",
        "dk.netarkivet.common.utils.batch.UrlSearch");
    Settings.set("settings.common.batch.batchjobs.batchjob.jarfile", arcFile.getAbsolutePath());

    MockHttpServletRequest request =
        new MockHttpServletRequest() {
          @Override
          public Locale getLocale() {
            return new Locale("en");
          }

          @Override
          public int getRemotePort() {
            return 0; // To change body of implemented methods use File | Settings | File Templates.
          }

          @Override
          public String getLocalName() {
            return null; // To change body of implemented methods use File | Settings | File
                         // Templates.
          }

          @Override
          public String getLocalAddr() {
            return null; // To change body of implemented methods use File | Settings | File
                         // Templates.
          }

          @Override
          public int getLocalPort() {
            return 0; // To change body of implemented methods use File | Settings | File Templates.
          }
        };
    request.setupAddParameter(Constants.FILETYPE_PARAMETER, BatchFileType.Metadata.toString());
    request.setupAddParameter(Constants.JOB_ID_PARAMETER, "1234567890");
    request.setupAddParameter(
        Constants.BATCHJOB_PARAMETER, "dk.netarkivet.common.utils.batch.UrlSearch");
    request.setupAddParameter(Constants.REPLICA_PARAMETER, "BarOne");
    request.setupAddParameter("arg1", "DUMMY-ARG");
    request.setupAddParameter("arg2", "url");
    request.setupAddParameter("arg3", "mimetype");

    Locale l = new Locale("en");
    JspWriterMockup out = new JspWriterMockup();

    PageContext context = new WebinterfaceTestCase.TestPageContext(request, out, l);
    BatchGUI.execute(context);
  }
 @Before
 public void setUp() {
   rs.setUp();
   mjms.setUp();
   listener = new CreateIndexListener();
   JMSConnectionFactory.getInstance().setListener(Channels.getTheIndexServer(), listener);
   Settings.set(
       CommonSettings.REMOTE_FILE_CLASS, "dk.netarkivet.common.distribute.NullRemoteFile");
   Settings.set(CommonSettings.CACHE_DIR, TestInfo.CACHE_DIR.getPath());
   mtf.setUp();
   pss.setUp();
   pse.setUp();
 }
  public void setUp() throws Exception {
    ChannelsTester.resetChannels();
    rs.setUp();
    mtf.setUp();
    utrf.setUp();

    JMSConnectionMockupMQ.useJMSConnectionMockupMQ();

    DatabaseTestUtils.takeDatabase(TestInfo.DATABASE_FILE, TestInfo.DATABASE_DIR);

    // define the settings for accessing the database
    Settings.set(ArchiveSettings.BASEURL_ARCREPOSITORY_ADMIN_DATABASE, TestInfo.DATABASE_URL);
    Settings.set(ArchiveSettings.MACHINE_ARCREPOSITORY_ADMIN_DATABASE, "");
    Settings.set(ArchiveSettings.PORT_ARCREPOSITORY_ADMIN_DATABASE, "");
    Settings.set(ArchiveSettings.DIR_ARCREPOSITORY_ADMIN_DATABASE, "");

    Settings.set(CommonSettings.NOTIFICATIONS_CLASS, PrintNotifications.class.getName());
  }
  /**
   * Calls the Unix sort command with the options <code>$filesNames -o
   * $outputfile -T WaybackSettings#WAYBACK_AGGREGATOR_TEMP_DIR.
   *
   * Sets the LC_ALL environment variable before making the call.
   *
   * @param files The files to merge and sort
   * @param outputFile The resulting sorted file
   * @param additionalArgs A list af extra arguments, which (if different from
   *                       null) are added to the sort call.<p> Note: If any
   *                       of the args contain a whitespace the call will
   *                       fail.
   */
  private void processFiles(File[] files, File outputFile, List<String> additionalArgs) {
    if (files.length == 0) {
      // Empty file list will cause sort to wait for further input,
      // and the call will therefore never return
      return;
    }

    Process p = null;

    try {
      List<String> inputFileList = new LinkedList<String>();
      for (int i = 0; i < files.length; i++) {
        if (files[i].exists() && files[i].isFile()) {
          inputFileList.add(files[i].getCanonicalPath());
        } else {
          log.warn(
              "File "
                  + files[i]
                  + " doesn't exist or isn't a regular file, "
                  + "dropping from list of files to "
                  + "sort and merge");
        }
      }
      List<String> cmd = new LinkedList<String>();
      // Prepare to run the unix sort command, see sort manual page for
      // details
      cmd.add("sort");
      cmd.addAll(inputFileList);
      cmd.add("-o");
      cmd.add(outputFile.getCanonicalPath());
      cmd.add("-T");
      cmd.add(Settings.get(WaybackSettings.WAYBACK_AGGREGATOR_TEMP_DIR));
      if (additionalArgs != null && !additionalArgs.isEmpty()) {
        for (String argument : additionalArgs) {
          ArgumentNotValid.checkTrue(
              argument.indexOf(' ') == -1,
              "The argument '" + argument + "' contains spaces, this isn't allowed ");
        }
        cmd.addAll(additionalArgs);
      }
      ProcessBuilder pb = new ProcessBuilder(cmd);
      // Reset all locale definitions
      pb.environment().put("LC_ALL", "C");
      // Run the command in the user.dir directory
      pb.directory(new File(System.getProperty("user.dir")));
      p = pb.start();
      p.waitFor();
      if (p.exitValue() != 0) {
        log.error("Failed to sort index files, sort exited with " + "return code " + p.exitValue());
      }
    } catch (Exception e) {
      log.error("Failed to aggregate indexes ", e);
    }
  }
 @Before
 public void setUp() {
   rs.setUp();
   ulrf.setUp();
   mjms.setUp();
   mtf.setUp();
   pss.setUp();
   pse.setUp();
   marc.setUp();
   Settings.set(CommonSettings.NOTIFICATIONS_CLASS, RememberNotifications.class.getName());
 }
 /** Create bitarchive from scratch, no admin data and log files exists */
 @Test
 public void testFromScratch() {
   LogbackRecorder lr = LogbackRecorder.startRecorder();
   assertFalse("No bitarchive should exist before creating it", NEW_ARCHIVE_DIR.exists());
   // Create new test archive and close it
   Settings.set(ArchiveSettings.BITARCHIVE_SERVER_FILEDIR, NEW_ARCHIVE_DIR.getAbsolutePath());
   Bitarchive ba = Bitarchive.getInstance();
   ba.close();
   // verify that the directory, admin and log files are created
   assertTrue("The archive dir should exist after creation", NEW_ARCHIVE_DIR.exists());
   assertTrue("Log file should exist after creation", !lr.isEmpty());
   lr.stopRecorder();
 }
  @Before
  public void setUp() throws Exception {
    rs.setUp();
    mtf.setUp();
    JMSConnectionMockupMQ.useJMSConnectionMockupMQ();

    Settings.set(ArchiveSettings.DIRS_ARCREPOSITORY_ADMIN, TestInfo.WORKING_DIR.getAbsolutePath());

    if (!Replica.isKnownReplicaId("TWO") || !Replica.isKnownReplicaId("ONE")) {
      List<String> knownIds = new ArrayList<String>();

      fail(
          "These tests assume, that ONE and TWO are known replica ids. Only known replicas are: "
              + StringUtils.conjoin(", ", knownIds.toArray(Replica.getKnownIds())));
    }
    // super.setUp();
  }
public class IndexerTestCase {

  private String oldClient = System.getProperty(CommonSettings.ARC_REPOSITORY_CLIENT);
  private String oldFileDir = System.getProperty("settings.common.arcrepositoryClient.fileDir");
  protected static File tempdir = new File(Settings.get(WaybackSettings.WAYBACK_INDEX_TEMPDIR));

  ReloadSettings rs = new ReloadSettings();

  @Before
  public void setUp() {
    rs.setUp();
    System.setProperty(WaybackSettings.HIBERNATE_HBM2DDL_AUTO, "create-drop");
    HibernateUtil.getSession().getSessionFactory().close();
    FileUtils.removeRecursively(TestInfo.WORKING_DIR);
    TestFileUtils.copyDirectoryNonCVS(TestInfo.ORIGINALS_DIR, TestInfo.WORKING_DIR);
    System.setProperty(
        CommonSettings.ARC_REPOSITORY_CLIENT,
        "dk.netarkivet.common.distribute.arcrepository.LocalArcRepositoryClient");
    System.setProperty(
        "settings.common.arcrepositoryClient.fileDir", TestInfo.FILE_DIR.getAbsolutePath());
    System.setProperty(
        CommonSettings.REMOTE_FILE_CLASS, "dk.netarkivet.common.distribute.TestRemoteFile");
    assertTrue(
        ArcRepositoryClientFactory.getPreservationInstance() instanceof LocalArcRepositoryClient);
  }

  @After
  public void tearDown() {
    HibernateUtil.getSession().getSessionFactory().close();
    FileUtils.removeRecursively(TestInfo.WORKING_DIR);
    if (oldClient != null) {
      System.setProperty(CommonSettings.ARC_REPOSITORY_CLIENT, oldClient);
    } else {
      System.setProperty(CommonSettings.ARC_REPOSITORY_CLIENT, "");
    }
    if (oldFileDir != null) {
      System.setProperty("settings.common.arcrepositoryClient.fileDir", oldFileDir);
    } else {
      System.setProperty("settings.common.arcrepositoryClient.fileDir", "");
    }
    rs.tearDown();
  }
}
  public void setUp() {
    mtf.setUp();
    rs.setUp();

    Settings.set(CommonSettings.BATCHJOBS_BASEDIR, TestInfo.BATCH_DIR.getAbsolutePath());
  }
/**
 * Batchjob that extracts lines from a crawl log matching a regular expression The batch job should
 * be restricted to run on metadata files for a specific job only, using the {@link
 * #processOnlyFilesMatching(String)} construct.
 */
@SuppressWarnings({"serial"})
public class CrawlLogLinesMatchingRegexp extends ArchiveBatchJob {

  /** The logger. */
  // private final Log log = LogFactory.getLog(getClass().getName());
  private static final Logger log = LoggerFactory.getLogger(CrawlLogLinesMatchingRegexp.class);

  /** Metadata URL for crawl logs. */
  private static final String SETUP_URL_FORMAT =
      String.format(
          "metadata://%s/crawl/logs/crawl.log", Settings.get(CommonSettings.ORGANIZATION));

  /** The regular expression to match in the crawl.log line. */
  private final String regexp;

  /**
   * Initialise the batch job.
   *
   * @param regexp The regexp to match in the crawl.log lines.
   */
  public CrawlLogLinesMatchingRegexp(String regexp) {
    ArgumentNotValid.checkNotNullOrEmpty(regexp, "regexp");
    this.regexp = regexp;

    /** One week in milliseconds. */
    batchJobTimeout = 7 * Constants.ONE_DAY_IN_MILLIES;
  }

  /**
   * Does nothing, no initialisation is needed.
   *
   * @param os Not used.
   */
  @Override
  public void initialize(OutputStream os) {}

  @Override
  public ArchiveBatchFilter getFilter() {
    return new ArchiveBatchFilter("OnlyCrawlLog") {
      public boolean accept(ArchiveRecordBase record) {
        String URL = record.getHeader().getUrl();
        if (URL == null) {
          return false;
        } else {
          return URL.startsWith(SETUP_URL_FORMAT);
        }
      }
    };
  }

  /**
   * Process a record on crawl log concerning the given domain to result.
   *
   * @param record The record to process.
   * @param os The output stream for the result.
   * @throws ArgumentNotValid on null parameters
   * @throws IOFailure on trouble processing the record.
   */
  @Override
  public void processRecord(ArchiveRecordBase record, OutputStream os) {
    ArgumentNotValid.checkNotNull(record, "ArchiveRecordBase record");
    ArgumentNotValid.checkNotNull(os, "OutputStream os");
    BufferedReader arcreader = new BufferedReader(new InputStreamReader(record.getInputStream()));
    try {
      for (String line = arcreader.readLine(); line != null; line = arcreader.readLine()) {
        if (line.matches(regexp)) {
          os.write(line.getBytes("UTF-8"));
          os.write('\n');
        }
      }
    } catch (IOException e) {
      throw new IOFailure("Unable to process (w)arc record", e);
    } finally {
      try {
        arcreader.close();
      } catch (IOException e) {
        log.warn("unable to close arcreader probably", e);
      }
    }
  }

  /**
   * Does nothing, no finishing is needed.
   *
   * @param os Not used.
   */
  @Override
  public void finish(OutputStream os) {}

  @Override
  public String toString() {
    return getClass().getName()
        + ", with arguments: Regexp = "
        + regexp
        + ", Filter = "
        + getFilter();
  }
}
  @Test
  public void testProcessMissingRequest() throws Exception {

    Settings.set(
        ArchiveSettings.DIR_ARCREPOSITORY_BITPRESERVATION, TestInfo.WORKING_DIR.getAbsolutePath());
    Settings.set(ArchiveSettings.DIRS_ARCREPOSITORY_ADMIN, TestInfo.WORKING_DIR.getAbsolutePath());

    // Ensure that a admin data exists before we start.
    AdminData.getUpdateableInstance();

    MockFileBasedActiveBitPreservation mockabp = new MockFileBasedActiveBitPreservation();
    MockHttpServletRequest request = new MockHttpServletRequest();
    String replicaID1 = "ONE";
    String replicaID2 = "TWO";
    String filename1 = "foo";
    String filename2 = "bar";
    Locale defaultLocale = new Locale("da");

    // First test a working set of params
    Map<String, String[]> args = new HashMap<String, String[]>();
    args.put(
        ADD_COMMAND,
        new String[] {
          Replica.getReplicaFromId(replicaID1).getName() + STRING_FILENAME_SEPARATOR + filename1
        });
    request.setupAddParameter(
        ADD_COMMAND,
        new String[] {
          Replica.getReplicaFromId(replicaID1).getName() + STRING_FILENAME_SEPARATOR + filename1
        });
    args.put(GET_INFO_COMMAND, new String[] {filename1});
    request.setupAddParameter(GET_INFO_COMMAND, new String[] {filename1});
    args.put(BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID1).getName()});
    request.setupAddParameter(
        BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID1).getName()});
    request.setupGetParameterMap(args);
    request.setupGetParameterNames(new Vector<String>(args.keySet()).elements());
    Map<String, PreservationState> status =
        BitpreserveFileState.processMissingRequest(
            WebinterfaceTestCase.getDummyPageContext(defaultLocale, request), new StringBuilder());
    assertEquals("Should have one call to reestablish", 1, mockabp.getCallCount(ADD_METHOD));
    assertEquals(
        "Should have one call to getFilePreservationStatus",
        1,
        mockabp.getCallCount(GET_INFO_METHOD));
    assertEquals("Should have one info element (with mock results)", null, status.get(filename1));

    // Check that we can call without any params
    mockabp.calls.clear();
    request = new MockHttpServletRequest();
    args.clear();
    args.put(BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID1).getName()});
    request.setupAddParameter(
        BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID1).getName()});
    request.setupGetParameterMap(args);
    status =
        BitpreserveFileState.processMissingRequest(
            WebinterfaceTestCase.getDummyPageContext(defaultLocale, request), new StringBuilder());
    assertEquals("Should have no call to restablish", 0, mockabp.getCallCount(ADD_METHOD));
    assertEquals(
        "Should have no call to getFilePreservationStatus",
        0,
        mockabp.getCallCount(GET_INFO_METHOD));
    assertEquals("Should have no status", 0, status.size());

    // Check that we can handle more than one call to each and that the
    // args are correct.
    mockabp.calls.clear();
    request = new MockHttpServletRequest();
    args.clear();
    args.put(BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID2).getName()});
    request.setupAddParameter(
        BITARCHIVE_NAME_PARAM, new String[] {Replica.getReplicaFromId(replicaID2).getName()});
    request.setupAddParameter(
        ADD_COMMAND,
        new String[] {
          Replica.getReplicaFromId(replicaID2).getName() + STRING_FILENAME_SEPARATOR + filename1,
          Replica.getReplicaFromId(replicaID2).getName() + STRING_FILENAME_SEPARATOR + filename1
        });
    args.put(
        ADD_COMMAND,
        new String[] {
          Replica.getReplicaFromId(replicaID2).getName() + STRING_FILENAME_SEPARATOR + filename1,
          Replica.getReplicaFromId(replicaID2).getName() + STRING_FILENAME_SEPARATOR + filename1
        });
    request.setupAddParameter(GET_INFO_COMMAND, new String[] {filename1, filename2, filename1});
    args.put(GET_INFO_COMMAND, new String[] {filename1, filename2, filename1});
    request.setupGetParameterMap(args);
    status =
        BitpreserveFileState.processMissingRequest(
            WebinterfaceTestCase.getDummyPageContext(defaultLocale, request), new StringBuilder());
    assertEquals("Should have two calls to restablish", 2, mockabp.getCallCount(ADD_METHOD));
    assertEquals(
        "Should have three calls to getFilePreservationStatus",
        3,
        mockabp.getCallCount(GET_INFO_METHOD));
    assertEquals("Should have two info elements", 2, status.size());
    assertEquals("Should have info for filename1", null, status.get(filename1));
    assertEquals("Should have info for filename2", null, status.get(filename2));

    // Iterator<String> it = mockabp.calls.get(ADD_METHOD).iterator();
    // while (it.hasNext()) {
    // System.out.println(it.next());
    // }

    CollectionAsserts.assertIteratorEquals(
        "Should have the args given add",
        Arrays.asList(new String[] {filename1 + "," + replicaID2, filename1 + "," + replicaID2})
            .iterator(),
        mockabp.calls.get(ADD_METHOD).iterator());

    CollectionAsserts.assertIteratorEquals(
        "Should have the args given info",
        Arrays.asList(new String[] {filename1, filename2, filename1}).iterator(),
        mockabp.calls.get(GET_INFO_METHOD).iterator());
  }
/**
 * A base class for {@link JobGenerator} implementations. It is recommended to extend this class to
 * implement a new job generator.
 *
 * <p>The base algorithm iterates over domain configurations within the harvest definition, and
 * according to the configuration ({@link HarvesterSettings#JOBGEN_DOMAIN_CONFIG_SUBSET_SIZE},
 * constitutes a subset of domain configurations from which one or more jobs will be generated.
 */
abstract class AbstractJobGenerator implements JobGenerator {

  /** Logger for this class. */
  private static Log log = LogFactory.getLog(AbstractJobGenerator.class);

  /** How many domain configurations to process in one go. */
  private final long DOMAIN_CONFIG_SUBSET_SIZE =
      Settings.getLong(HarvesterSettings.JOBGEN_DOMAIN_CONFIG_SUBSET_SIZE);

  /** Is deduplication enabled or disabled. * */
  private final boolean DEDUPLICATION_ENABLED =
      Settings.getBoolean(HarvesterSettings.DEDUPLICATION_ENABLED);

  @Override
  public int generateJobs(HarvestDefinition harvest) {
    log.info("Generating jobs for harvestdefinition # " + harvest.getOid());
    int jobsMade = 0;
    final Iterator<DomainConfiguration> domainConfigurations = harvest.getDomainConfigurations();

    while (domainConfigurations.hasNext()) {
      List<DomainConfiguration> subset = new ArrayList<DomainConfiguration>();
      while (domainConfigurations.hasNext() && subset.size() < DOMAIN_CONFIG_SUBSET_SIZE) {
        subset.add(domainConfigurations.next());
      }

      Collections.sort(subset, getDomainConfigurationSubsetComparator(harvest));
      if (log.isTraceEnabled()) {
        log.trace(
            subset.size()
                + " domainconfigs now sorted and ready to processing "
                + "for harvest #"
                + harvest.getOid());
      }
      jobsMade += processDomainConfigurationSubset(harvest, subset.iterator());
    }
    harvest.setNumEvents(harvest.getNumEvents() + 1);

    if (!harvest.isSnapShot()) {
      PartialHarvest focused = (PartialHarvest) harvest;
      Schedule schedule = focused.getSchedule();
      int numEvents = harvest.getNumEvents();

      // Calculate next event
      Date now = new Date();
      Date nextEvent = schedule.getNextEvent(focused.getNextDate(), numEvents);

      // Refuse to schedule event in the past
      if (nextEvent != null && nextEvent.before(now)) {
        int eventsSkipped = 0;
        while (nextEvent != null && nextEvent.before(now)) {
          nextEvent = schedule.getNextEvent(nextEvent, numEvents);
          eventsSkipped++;
        }
        if (log.isWarnEnabled()) {
          log.warn(
              "Refusing to schedule harvest definition '"
                  + harvest.getName()
                  + "' in the past. Skipped "
                  + eventsSkipped
                  + " events. Old nextDate was "
                  + focused.getNextDate()
                  + " new nextDate is "
                  + nextEvent);
        }
      }

      // Set next event
      focused.setNextDate(nextEvent);
      if (log.isTraceEnabled()) {
        log.trace(
            "Next event for harvest definition "
                + harvest.getName()
                + " happens: "
                + (nextEvent == null ? "Never" : nextEvent.toString()));
      }
    }

    log.info(
        "Finished generating " + jobsMade + " jobs for harvestdefinition # " + harvest.getOid());
    return jobsMade;
  }

  /**
   * Instantiates a new job.
   *
   * @param cfg the {@link DomainConfiguration} being processed
   * @param harvest the {@link HarvestDefinition} being processed
   * @return an instance of {@link Job}
   */
  public static Job getNewJob(HarvestDefinition harvest, DomainConfiguration cfg) {
    HarvestChannelDAO harvestChannelDao = HarvestChannelDAO.getInstance();
    HarvestChannel channel = harvestChannelDao.getChannelForHarvestDefinition(harvest.getOid());
    if (channel == null) {
      log.info(
          "No channel mapping registered for harvest id "
              + harvest.getOid()
              + ", will use default.");
      channel = harvestChannelDao.getDefaultChannel(harvest.isSnapShot());
    }
    if (harvest.isSnapShot()) {
      return Job.createSnapShotJob(
          harvest.getOid(),
          channel,
          cfg,
          harvest.getMaxCountObjects(),
          harvest.getMaxBytes(),
          ((FullHarvest) harvest).getMaxJobRunningTime(),
          harvest.getNumEvents());
    }
    return Job.createJob(harvest.getOid(), channel, cfg, harvest.getNumEvents());
  }

  /**
   * Returns a comparator used to sort the subset of {@link #DOMAIN_CONFIG_SUBSET_SIZE}
   * configurations that are scanned at each iteration.
   *
   * @param harvest the {@link HarvestDefinition} being processed.
   * @return a comparator
   */
  protected abstract Comparator<DomainConfiguration> getDomainConfigurationSubsetComparator(
      HarvestDefinition harvest);

  /**
   * Create new jobs from a collection of configurations. All configurations must use the same
   * order.xml file.Jobs
   *
   * @param harvest the {@link HarvestDefinition} being processed.
   * @param domainConfSubset the configurations to use to create the jobs
   * @return The number of jobs created
   * @throws ArgumentNotValid if any of the parameters is null or if the cfglist does not contain
   *     any configurations
   */
  protected abstract int processDomainConfigurationSubset(
      HarvestDefinition harvest, Iterator<DomainConfiguration> domainConfSubset);

  @Override
  public boolean canAccept(Job job, DomainConfiguration cfg) {
    if (!checkAddDomainConfInvariant(job, cfg)) {
      return false;
    }
    return checkSpecificAcceptConditions(job, cfg);
  }

  /**
   * Called by {@link #canAccept(Job, DomainConfiguration)}. Tests the implementation-specific
   * conditions to accept the given {@link DomainConfiguration} in the given {@link Job}. It is
   * assumed that {@link #checkAddDomainConfInvariant(Job, DomainConfiguration)} has already passed.
   *
   * @param job the {@link Job} n=being built
   * @param cfg the {@link DomainConfiguration} to test
   * @return true if the configuration passes the conditions.
   */
  protected abstract boolean checkSpecificAcceptConditions(Job job, DomainConfiguration cfg);

  /**
   * Once the job has been filled with {@link DomainConfiguration}s, performs the following
   * operations:
   *
   * <ol>
   *   <li>Edit the harvest template to add/remove deduplicator configuration.
   *   <li>
   * </ol>
   *
   * @param job the job
   */
  protected void editJobOrderXml(Job job) {
    Document doc = job.getOrderXMLdoc();
    if (DEDUPLICATION_ENABLED) {
      // Check that the Deduplicator element is present in the
      // OrderXMl and enabled. If missing or disabled log a warning
      if (!HeritrixTemplate.isDeduplicationEnabledInTemplate(doc)) {
        if (log.isWarnEnabled()) {
          log.warn(
              "Unable to perform deduplication for this job"
                  + " as the required DeDuplicator element is "
                  + "disabled or missing from template");
        }
      }
    } else {
      // Remove deduplicator Element from OrderXML if present
      Node xpathNode = doc.selectSingleNode(HeritrixTemplate.DEDUPLICATOR_XPATH);
      if (xpathNode != null) {
        xpathNode.detach();
        job.setOrderXMLDoc(doc);
        if (log.isInfoEnabled()) {
          log.info("Removed DeDuplicator element because " + "Deduplication is disabled");
        }
      }
    }
  }

  /**
   * Tests that:
   *
   * <ol>
   *   <li>The given domain configuration and job are not null.
   *   <li>The job does not already contain the given domain configuration.
   *   <li>The domain configuration has the same order xml name as the first inserted domain config.
   * </ol>
   *
   * @param job a given Job
   * @param cfg a given DomainConfiguration
   * @return true, if the given DomainConfiguration can be inserted into the given job
   */
  private boolean checkAddDomainConfInvariant(Job job, DomainConfiguration cfg) {
    ArgumentNotValid.checkNotNull(job, "job");
    ArgumentNotValid.checkNotNull(cfg, "cfg");

    // check if domain in DomainConfiguration cfg is not already in this job
    // domainName is used as key in domainConfigurationMap
    if (job.getDomainConfigurationMap().containsKey(cfg.getDomainName())) {
      if (log.isDebugEnabled()) {
        log.debug("Job already has a configuration for Domain '" + cfg.getDomainName() + "'.");
      }
      return false;
    }

    // check if template is same as this job.
    String orderXMLname = job.getOrderXMLName();
    if (!orderXMLname.equals(cfg.getOrderXmlName())) {
      if (log.isDebugEnabled()) {
        log.debug(
            "This Job only accept configurations "
                + "using the harvest template '"
                + orderXMLname
                + "'. This configuration uses the harvest template '"
                + cfg.getOrderXmlName()
                + "'.");
      }
      return false;
    }

    return true;
  }
}
/**
 * This class implements a generator for an history chart of a running job. The chart traces the
 * progress percentage and the queued URI count over the crawl time. Charts are rendered in a PNG
 * image file, generated in the webapp directory.
 */
class StartedJobHistoryChartGen {

  /** Time units used to scale the crawl time values and generate the chart's time axis ticks. */
  protected static enum TimeAxisResolution {
    /** One second. Tick step is 10s. */
    second(1, 1, 10),
    /** One minute. Tick step is 5m. */
    minute(60, 60, 5),
    /** One hour. Tick step is 1h. */
    hour(60 * minute.seconds, 60 * minute.seconds, 1),
    /** Twelve hours. Tick step is 2h. */
    half_day(12 * 60 * minute.seconds, 60 * minute.seconds, 2),
    /** One day. Tick step is 0.5d. */
    day(24 * hour.seconds, 24 * hour.seconds, 0.5),
    /** One week. Tick step is 1w. */
    week(7 * day.seconds, 7 * day.seconds, 1);

    /** The time unit in seconds. */
    private final int seconds;

    /** The scale in seconds. */
    private final int scaleSeconds;

    /** The step between two tick units. */
    private final double tickStep;

    /**
     * Builds a time axis resolution.
     *
     * @param seconds the actual resolution in seconds
     * @param scaleSeconds the actual "scale" of ticks
     * @param tickStep the number of ticks in one step.
     */
    TimeAxisResolution(int seconds, int scaleSeconds, double tickStep) {
      this.seconds = seconds;
      this.scaleSeconds = scaleSeconds;
      this.tickStep = tickStep;
    }

    /**
     * Scale down an array of seconds.
     *
     * @param timeInSeconds An array of seconds
     * @return a scaled down version of the given array of seconds
     */
    double[] scale(double[] timeInSeconds) {
      double[] scaledTime = new double[timeInSeconds.length];
      for (int i = 0; i < timeInSeconds.length; i++) {
        scaledTime[i] = timeInSeconds[i] / this.scaleSeconds;
      }
      return scaledTime;
    }

    /**
     * @param seconds the seconds
     * @return the proper timeUnit for the given argument
     */
    static TimeAxisResolution findTimeUnit(double seconds) {

      TimeAxisResolution[] allTus = values();
      for (int i = 0; i < allTus.length - 1; i++) {
        TimeAxisResolution nextGreater = allTus[i + 1];
        if (seconds < nextGreater.seconds) {
          return allTus[i];
        }
      }
      return week; // largest unit
    }
  }

  /** A chart generation task. Generates a PNG image for a job progress history. */
  private static class ChartGen implements Runnable {
    /** The process that generates the Charts. */
    private final StartedJobHistoryChartGen gen;

    /**
     * Constructor of a ChartGen objector.
     *
     * @param gen the process that generates the charts.
     */
    ChartGen(StartedJobHistoryChartGen gen) {
      super();
      this.gen = gen;
    }

    @Override
    public void run() {

      synchronized (gen) {
        gen.chartFile = null;
      }

      long jobId = gen.jobId;

      StartedJobInfo[] fullHistory = RunningJobsInfoDAO.getInstance().getFullJobHistory(jobId);

      LinkedList<Double> timeValues = new LinkedList<Double>();
      LinkedList<Double> progressValues = new LinkedList<Double>();
      LinkedList<Double> urlValues = new LinkedList<Double>();

      for (StartedJobInfo sji : fullHistory) {
        timeValues.add((double) sji.getElapsedSeconds());
        progressValues.add(sji.getProgress());
        urlValues.add((double) sji.getQueuedFilesCount());
      }

      // Refresh the history png image for the job.
      File pngFile = new File(gen.outputFolder, jobId + "-history.png");

      File newPngFile;
      try {
        newPngFile =
            File.createTempFile(jobId + "-history", "." + System.currentTimeMillis() + ".png");
      } catch (IOException e) {
        LOG.warn("Failed to create temp PNG file for job " + jobId);
        return;
      }

      long startTime = System.currentTimeMillis();
      gen.generatePngChart(
          newPngFile,
          CHART_RESOLUTION[0],
          CHART_RESOLUTION[1],
          null, // no chart title
          I18N.getString(gen.locale, "running.job.details.chart.legend.crawlTime"),
          new String[] {
            I18N.getString(gen.locale, "running.job.details.chart.legend.progress"),
            I18N.getString(gen.locale, "running.job.details.chart.legend.queuedUris")
          },
          NumberUtils.toPrimitiveArray(timeValues),
          new double[][] {new double[] {0, 100}, null},
          new double[][] {
            NumberUtils.toPrimitiveArray(progressValues), NumberUtils.toPrimitiveArray(urlValues)
          },
          new Color[] {Color.blue, Color.green.darker()},
          new String[] {"%", ""},
          false,
          Color.lightGray.brighter().brighter());

      long genTime = System.currentTimeMillis() - startTime;
      LOG.info(
          "Generated history chart for job "
              + jobId
              + " in "
              + (genTime < TimeUtils.SECOND_IN_MILLIS
                  ? genTime + " ms"
                  : StringUtils.formatDuration(genTime / TimeUtils.SECOND_IN_MILLIS))
              + ".");

      synchronized (gen) {
        // Overwrite old file, then delete temp file
        try {
          FileUtils.copyFile(newPngFile, pngFile);
          FileUtils.remove(newPngFile);
        } catch (IOFailure iof) {
          LOG.error("IOFailure while copying PNG file", iof);
        }
        gen.chartFile = pngFile;
      }
    }
  }

  /** The class logger. */
  static final Log LOG = LogFactory.getLog(StartedJobHistoryChartGen.class);

  /** Internationalisation object. */
  private static final I18n I18N = new I18n(dk.netarkivet.harvester.Constants.TRANSLATIONS_BUNDLE);

  /** Rate in seconds at which history charts should be generated. */
  private static final long GEN_INTERVAL =
      Settings.getLong(HarvesterSettings.HARVEST_MONITOR_HISTORY_CHART_GEN_INTERVAL);

  /** The chart image resolution. */
  private static final int[] CHART_RESOLUTION = new int[] {600, 450};
  /** The dimension of the chart axis. */
  private static final double CHART_AXIS_DIMENSION = 10.0;
  /** The relative path of the output. */
  private static final String OUTPUT_REL_PATH = "History" + File.separator + "webapp";

  /** The job id. */
  private final long jobId;

  /** The folder where image files are output. */
  private final File outputFolder;

  /** The chart image file. */
  private File chartFile = null;

  /** The locale for internationalizing the chart. The locale is set to the system default. */
  private final Locale locale;
  /** The process controlling the cyclic regeneration of charts. */
  private PeriodicTaskExecutor genExec = null;

  /**
   * Constructor. Start generating charts for data belonging to the given job.
   *
   * @param jobId a job id.
   */
  StartedJobHistoryChartGen(long jobId) {
    super();

    this.outputFolder = new File(FileUtils.getTempDir() + File.separator + OUTPUT_REL_PATH);

    this.jobId = jobId;

    // Set the locale to the system default
    this.locale = Locale.getDefault();

    genExec = new PeriodicTaskExecutor("ChartGen", new ChartGen(this), 0, GEN_INTERVAL);
  }

  /**
   * Returns the image file.
   *
   * @return the image file. Might return null if no file is currently available.
   */
  public synchronized File getChartFile() {
    return chartFile;
  }

  /** Deletes the chart image if it exists and stops the generation schedule. */
  public void cleanup() {

    if (chartFile != null && chartFile.exists()) {
      if (!chartFile.delete()) {
        chartFile.deleteOnExit();
      }
    }

    if (genExec != null) {
      genExec.shutdown();
    }
  }

  /**
   * Generates a chart in PNG format.
   *
   * @param outputFile the output file, it should exist.
   * @param pxWidth the image width in pixels.
   * @param pxHeight the image height in pixels.
   * @param chartTitle the chart title, may be null.
   * @param xAxisTitle the x axis title
   * @param yDataSeriesRange the axis range (null for auto)
   * @param yDataSeriesTitles the Y axis titles.
   * @param timeValuesInSeconds the time values in seconds
   * @param yDataSeries the Y axis value series.
   * @param yDataSeriesColors the Y axis value series drawing colors.
   * @param yDataSeriesTickSuffix TODO explain argument yDataSeriesTickSuffix
   * @param drawBorder draw, or not, the border.
   * @param backgroundColor the chart background color.
   */
  final void generatePngChart(
      File outputFile,
      int pxWidth,
      int pxHeight,
      String chartTitle,
      String xAxisTitle,
      String[] yDataSeriesTitles,
      double[] timeValuesInSeconds,
      double[][] yDataSeriesRange,
      double[][] yDataSeries,
      Color[] yDataSeriesColors,
      String[] yDataSeriesTickSuffix,
      boolean drawBorder,
      Color backgroundColor) {

    // Domain axis
    NumberAxis xAxis = new NumberAxis(xAxisTitle);
    xAxis.setFixedDimension(CHART_AXIS_DIMENSION);
    xAxis.setLabelPaint(Color.black);
    xAxis.setTickLabelPaint(Color.black);

    double maxSeconds = getMaxValue(timeValuesInSeconds);
    TimeAxisResolution xAxisRes = TimeAxisResolution.findTimeUnit(maxSeconds);
    xAxis.setTickUnit(new NumberTickUnit(xAxisRes.tickStep));
    double[] scaledTimeValues = xAxisRes.scale(timeValuesInSeconds);

    String tickSymbol =
        I18N.getString(locale, "running.job.details.chart.timeunit.symbol." + xAxisRes.name());
    xAxis.setNumberFormatOverride(new DecimalFormat("###.##'" + tickSymbol + "'"));

    // First dataset
    String firstDataSetTitle = yDataSeriesTitles[0];
    XYDataset firstDataSet = createXYDataSet(firstDataSetTitle, scaledTimeValues, yDataSeries[0]);
    Color firstDataSetColor = yDataSeriesColors[0];

    // First range axis
    NumberAxis firstYAxis = new NumberAxis(firstDataSetTitle);

    firstYAxis.setFixedDimension(CHART_AXIS_DIMENSION);
    setAxisRange(firstYAxis, yDataSeriesRange[0]);
    firstYAxis.setLabelPaint(firstDataSetColor);
    firstYAxis.setTickLabelPaint(firstDataSetColor);
    String firstAxisTickSuffix = yDataSeriesTickSuffix[0];
    if (firstAxisTickSuffix != null && !firstAxisTickSuffix.isEmpty()) {
      firstYAxis.setNumberFormatOverride(new DecimalFormat("###.##'" + firstAxisTickSuffix + "'"));
    }

    // Create the plot with domain axis and first range axis
    XYPlot plot = new XYPlot(firstDataSet, xAxis, firstYAxis, null);

    XYLineAndShapeRenderer firstRenderer = new XYLineAndShapeRenderer(true, false);
    plot.setRenderer(firstRenderer);

    plot.setOrientation(PlotOrientation.VERTICAL);
    plot.setBackgroundPaint(Color.lightGray);
    plot.setDomainGridlinePaint(Color.white);
    plot.setRangeGridlinePaint(Color.white);

    plot.setAxisOffset(new RectangleInsets(5.0, 5.0, 5.0, 5.0));
    firstRenderer.setSeriesPaint(0, firstDataSetColor);

    // Now iterate on next axes
    for (int i = 1; i < yDataSeries.length; i++) {
      // Create axis
      String seriesTitle = yDataSeriesTitles[i];
      Color seriesColor = yDataSeriesColors[i];
      NumberAxis yAxis = new NumberAxis(seriesTitle);

      yAxis.setFixedDimension(CHART_AXIS_DIMENSION);
      setAxisRange(yAxis, yDataSeriesRange[i]);

      yAxis.setLabelPaint(seriesColor);
      yAxis.setTickLabelPaint(seriesColor);

      String yAxisTickSuffix = yDataSeriesTickSuffix[i];
      if (yAxisTickSuffix != null && !yAxisTickSuffix.isEmpty()) {
        yAxis.setNumberFormatOverride(new DecimalFormat("###.##'" + yAxisTickSuffix + "'"));
      }

      // Create dataset and add axis to plot
      plot.setRangeAxis(i, yAxis);
      plot.setRangeAxisLocation(i, AxisLocation.BOTTOM_OR_LEFT);
      plot.setDataset(i, createXYDataSet(seriesTitle, scaledTimeValues, yDataSeries[i]));
      plot.mapDatasetToRangeAxis(i, i);
      XYItemRenderer renderer = new StandardXYItemRenderer();
      renderer.setSeriesPaint(0, seriesColor);
      plot.setRenderer(i, renderer);
    }

    // Create the chart
    JFreeChart chart = new JFreeChart(chartTitle, JFreeChart.DEFAULT_TITLE_FONT, plot, false);

    // Customize rendering
    chart.setBackgroundPaint(Color.white);
    chart.setBorderVisible(true);
    chart.setBorderPaint(Color.BLACK);

    // Render image
    try {
      ChartUtilities.saveChartAsPNG(outputFile, chart, pxWidth, pxHeight);
    } catch (IOException e) {
      LOG.error("Chart export failed", e);
    }
  }

  /**
   * Create a XYDataset based on the given arguments.
   *
   * @param name The name
   * @param timeValues The timevalues
   * @param values the values
   * @return a DefaultXYDataset.
   */
  private XYDataset createXYDataSet(String name, double[] timeValues, double[] values) {

    DefaultXYDataset ds = new DefaultXYDataset();
    ds.addSeries(name, new double[][] {timeValues, values});

    return ds;
  }

  /**
   * Find the maximum of the values given. If this maximum is less than {@link Double#MIN_VALUE}
   * then {@link Double#MIN_VALUE} is returned.
   *
   * @param values an array of doubles
   * @return the maximum of the values given
   */
  private double getMaxValue(double[] values) {
    double max = Double.MIN_VALUE;
    for (double v : values) {
      max = Math.max(v, max);
    }
    return max;
  }

  /**
   * Set the axis range.
   *
   * @param axis a numberAxis
   * @param range a range
   */
  private void setAxisRange(NumberAxis axis, double[] range) {
    if (range == null || range.length != 2) {
      axis.setAutoRange(true);
    } else {
      double lower = range[0];
      double upper = range[1];
      ArgumentNotValid.checkTrue(lower < upper, "Incorrect range");
      axis.setAutoRange(false);
      axis.setRange(new Range(lower, upper));
    }
  }
}
Example #15
0
  /**
   * Package private constructor for common initialisation.
   *
   * @param harvestID the id of the harvestdefinition
   * @param cfg the configuration to base the Job on
   * @param orderXMLdoc
   * @param channel the channel on which the job will be submitted.
   * @param forceMaxObjectsPerDomain the maximum number of objects harvested from a domain,
   *     overrides individual configuration settings. -1 means no limit
   * @param forceMaxBytesPerDomain The maximum number of objects harvested from a domain, or -1 for
   *     no limit.
   * @param forceMaxJobRunningTime The max time in seconds given to the harvester for this job
   * @param harvestNum the run number of the harvest definition
   * @throws ArgumentNotValid if cfg or priority is null or harvestID is invalid, or if any limit <
   *     -1
   */
  public Job(
      Long harvestID,
      DomainConfiguration cfg,
      HeritrixTemplate orderXMLdoc,
      HarvestChannel channel,
      long forceMaxObjectsPerDomain,
      long forceMaxBytesPerDomain,
      long forceMaxJobRunningTime,
      int harvestNum)
      throws ArgumentNotValid {
    ArgumentNotValid.checkNotNull(cfg, "cfg");
    ArgumentNotValid.checkNotNull(harvestID, "harvestID");
    ArgumentNotValid.checkNotNegative(harvestID, "harvestID");
    ArgumentNotValid.checkNotNull(channel, "channel");

    if (forceMaxObjectsPerDomain < -1) {
      String msg = "forceMaxObjectsPerDomain must be either -1 or positive";
      log.debug(msg);
      throw new ArgumentNotValid(msg);
    }
    if (forceMaxBytesPerDomain < -1) {
      String msg = "forceMaxBytesPerDomain must be either -1 or positive";
      log.debug(msg);
      throw new ArgumentNotValid(msg);
    }

    if (forceMaxBytesPerDomain == 0L) {
      log.warn(
          "forceMaxBytesPerDomain should probably not be 0.Means 0 bytes downloaded per domain");
    }

    if (forceMaxObjectsPerDomain == 0L) {
      log.warn(
          "forceMaxObjectsPerDomain should probably not be 0.Means 0 objects downloaded per domain");
    }

    // setup initial members
    domainConfigurationMap = new HashMap<>();
    origHarvestDefinitionID = harvestID;
    orderXMLname = cfg.getOrderXmlName();
    this.orderXMLdoc = orderXMLdoc;

    setHarvestChannel(channel);

    long maxObjects = NumberUtils.minInf(forceMaxObjectsPerDomain, cfg.getMaxObjects());
    setMaxObjectsPerDomain(maxObjects);
    configurationSetsObjectLimit = (maxObjects != forceMaxObjectsPerDomain);

    long maxBytes = NumberUtils.minInf(forceMaxBytesPerDomain, cfg.getMaxBytes());
    setMaxBytesPerDomain(maxBytes);
    configurationSetsByteLimit = (maxBytes != forceMaxBytesPerDomain);

    long expectation =
        cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain);
    maxCountObjects = expectation;
    minCountObjects = expectation;
    this.harvestNum = harvestNum;

    addConfiguration(cfg);

    setMaxJobRunningTime(forceMaxJobRunningTime);

    setArchiveFormatInTemplate(Settings.get(HarvesterSettings.HERITRIX_ARCHIVE_FORMAT));

    setAttributes(cfg.getAttributesAndTypes());

    orderXMLdoc.enableOrDisableDeduplication(
        Settings.getBoolean(HarvesterSettings.DEDUPLICATION_ENABLED));

    status = JobStatus.NEW;
  }
Example #16
0
/**
 * This class represents one job to run by Heritrix. It's based on a number of configurations all
 * based on the same order.xml and at most one configuration for each domain. Each job consists of
 * configurations of the approximate same size; that is the difference in expectation from the
 * smallest configuration to the largest configuration is within a factor of each other defined as
 * limMaxRelSize (although differences smaller than limMinAbsSize are ignored) There is a limit
 * limMaxTotalSize on the total size of the job in objects.
 *
 * <p>A job may also be limited on bytes or objects, defined either by the configurations in the job
 * or the harvest definition the job is generated by.
 *
 * <p>The job contains the order file, the seedlist and the current status of the job, as well as
 * the ID of the harvest definition that defined it and names of all the configurations it is based
 * on.
 */
@SuppressWarnings({"serial"})
public class Job implements Serializable, JobInfo {
  private static final transient Logger log = LoggerFactory.getLogger(Job.class);

  // Persistent fields stored in and read from DAO
  /** The persistent ID of this job. */
  private Long jobID;
  /** The Id of the harvestdefinition, that generated this job. */
  protected Long origHarvestDefinitionID;
  /** The status of the job. See the JobStatus class for the possible states. */
  protected JobStatus status;
  /** The name of the {@link HarvestChannel} on which this job will be posted. */
  private String channel;

  /** Whether the job belongs to a snapshot or partial harvest. */
  private boolean isSnapshot;
  /**
   * Overrides the individual configurations maximum setting for objects retrieved from a domain
   * when set to a positive value.
   */
  private long forceMaxObjectsPerDomain = Constants.HERITRIX_MAXOBJECTS_INFINITY;
  /**
   * Overrides the individual configurations maximum setting for bytes retrieved from a domain when
   * set to other than -1.
   */
  private long forceMaxBytesPerDomain = Constants.HERITRIX_MAXBYTES_INFINITY;
  /** The name of the harvest template used by the job. */
  private String orderXMLname;
  /** The harvest template used by the job. */
  private HeritrixTemplate orderXMLdoc;
  /** The list of Heritrix settings files. */
  private File[] settingsXMLfiles;

  /** The corresponding Dom4j Documents for these files. */
  // private Document[] settingsXMLdocs;

  /**
   * A set of seeds involved in this job. Outside the SetSeedList() method, the set of seeds is
   * updated in the addConfiguration() method.
   */
  private Set<String> seedListSet = new HashSet<String>();
  /** Which run of the harvest definition this is. */
  private int harvestNum;
  /** Errors during harvesting. */
  private String harvestErrors;
  /** Details about errors during harvesting. */
  private String harvestErrorDetails;
  /** Errors during upload of the harvested data. */
  private String uploadErrors;
  /** Details about errors during upload of the harvested data. */
  private String uploadErrorDetails;
  /** The starting point of the job. */
  private Date actualStart;
  /** The ending point of the job. */
  private Date actualStop;
  /** The time when this job was submitted. */
  private Date submittedDate;
  /** The time when this job was created. */
  private Date creationDate;

  /** Edition is used by the DAO to keep track of changes. */
  private long edition = -1;

  /** Resubmitted as the Job with this ID. If null, this job has not been resubmitted. */
  private Long resubmittedAsJobWithID;

  /** Continuation of this job. */
  private Long continuationOF;

  /**
   * A map (domainName, domainConfigurationName), must be accessible in order to update job
   * information (see Ass. 2.4.3)
   */
  private Map<String, String> domainConfigurationMap;
  /**
   * A hint to the DAO that configurations have changed. Since configurations are large, the DAO can
   * use that this is false to avoid updating the config list. The DAO can set it to false after
   * saving configurations.
   */
  boolean configsChanged = false;

  // Intermediate fields, non-persistent and only used while building objects

  /**
   * Whether the maxObjects field was defined by the harvest definition or the configuration limit.
   * This is deciding for whether we accept smaller configurations or not when building jobs. True
   * means the limit is defined by the configuration, false means that it is defined by the harvest
   * definition.
   */
  private boolean configurationSetsObjectLimit;

  /**
   * Whether the maxBytes field was defined by the harvest definition or the configuration limit.
   * This is deciding for whether we accept smaller configurations or not when building jobs. True
   * means the limit is defined by the configuration, false means by the harvest definition.
   */
  private boolean configurationSetsByteLimit;

  /** The lowest number of objects expected by a configuration. */
  private long minCountObjects;

  /** The highest number of objects expected by a configuration. */
  private long maxCountObjects;

  /** The total number of objects expected by all added configurations. */
  private long totalCountObjects;

  /** The max time in seconds given to the harvester for this job. 0 is unlimited. */
  private long forceMaxRunningTime;

  /**
   * If true, this job object is still undergoing changes due to having more configurations added.
   * When set to false, the object is no longer considered immutable except for updating status.
   *
   * <p>Jobs loaded from the DAO are never under construction anymore.
   */
  private boolean underConstruction = true;

  // Constants

  // Note: The following constants are intentionally left non-static for easy
  // unit testing

  private boolean maxObjectsIsSetByQuotaEnforcer =
      Settings.getBoolean(HarvesterSettings.OBJECT_LIMIT_SET_BY_QUOTA_ENFORCER);

  /**
   * The harvestname prefix used in the files generated by Heritrix. Is set using an
   * ArchiveFileNaming class when the jobID is available.
   */
  private String harvestnamePrefix;

  /** This variable is right now the same as harvestdefinitions.audience field. */
  private String harvestAudience;

  protected Job() {
    this.status = JobStatus.NEW;
  }

  /**
   * Package private constructor for common initialisation.
   *
   * @param harvestID the id of the harvestdefinition
   * @param cfg the configuration to base the Job on
   * @param orderXMLdoc
   * @param channel the channel on which the job will be submitted.
   * @param forceMaxObjectsPerDomain the maximum number of objects harvested from a domain,
   *     overrides individual configuration settings. -1 means no limit
   * @param forceMaxBytesPerDomain The maximum number of objects harvested from a domain, or -1 for
   *     no limit.
   * @param forceMaxJobRunningTime The max time in seconds given to the harvester for this job
   * @param harvestNum the run number of the harvest definition
   * @throws ArgumentNotValid if cfg or priority is null or harvestID is invalid, or if any limit <
   *     -1
   */
  public Job(
      Long harvestID,
      DomainConfiguration cfg,
      HeritrixTemplate orderXMLdoc,
      HarvestChannel channel,
      long forceMaxObjectsPerDomain,
      long forceMaxBytesPerDomain,
      long forceMaxJobRunningTime,
      int harvestNum)
      throws ArgumentNotValid {
    ArgumentNotValid.checkNotNull(cfg, "cfg");
    ArgumentNotValid.checkNotNull(harvestID, "harvestID");
    ArgumentNotValid.checkNotNegative(harvestID, "harvestID");
    ArgumentNotValid.checkNotNull(channel, "channel");

    if (forceMaxObjectsPerDomain < -1) {
      String msg = "forceMaxObjectsPerDomain must be either -1 or positive";
      log.debug(msg);
      throw new ArgumentNotValid(msg);
    }
    if (forceMaxBytesPerDomain < -1) {
      String msg = "forceMaxBytesPerDomain must be either -1 or positive";
      log.debug(msg);
      throw new ArgumentNotValid(msg);
    }

    if (forceMaxBytesPerDomain == 0L) {
      log.warn(
          "forceMaxBytesPerDomain should probably not be 0.Means 0 bytes downloaded per domain");
    }

    if (forceMaxObjectsPerDomain == 0L) {
      log.warn(
          "forceMaxObjectsPerDomain should probably not be 0.Means 0 objects downloaded per domain");
    }

    // setup initial members
    domainConfigurationMap = new HashMap<>();
    origHarvestDefinitionID = harvestID;
    orderXMLname = cfg.getOrderXmlName();
    this.orderXMLdoc = orderXMLdoc;

    setHarvestChannel(channel);

    long maxObjects = NumberUtils.minInf(forceMaxObjectsPerDomain, cfg.getMaxObjects());
    setMaxObjectsPerDomain(maxObjects);
    configurationSetsObjectLimit = (maxObjects != forceMaxObjectsPerDomain);

    long maxBytes = NumberUtils.minInf(forceMaxBytesPerDomain, cfg.getMaxBytes());
    setMaxBytesPerDomain(maxBytes);
    configurationSetsByteLimit = (maxBytes != forceMaxBytesPerDomain);

    long expectation =
        cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain);
    maxCountObjects = expectation;
    minCountObjects = expectation;
    this.harvestNum = harvestNum;

    addConfiguration(cfg);

    setMaxJobRunningTime(forceMaxJobRunningTime);

    setArchiveFormatInTemplate(Settings.get(HarvesterSettings.HERITRIX_ARCHIVE_FORMAT));

    setAttributes(cfg.getAttributesAndTypes());

    orderXMLdoc.enableOrDisableDeduplication(
        Settings.getBoolean(HarvesterSettings.DEDUPLICATION_ENABLED));

    status = JobStatus.NEW;
  }

  public void setAttributes(List<AttributeAndType> attributesAndTypes) {
    orderXMLdoc.insertAttributes(attributesAndTypes);
  }

  /** Update the order template according to the chosen archive format (arc/warc). */
  private void setArchiveFormatInTemplate(String archiveFormat) {
    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }
    orderXMLdoc.setArchiveFormat(archiveFormat);
  }

  /**
   * Create a new Job object from basic information stored in the DAO.
   *
   * @param harvestID the id of the harvestdefinition
   * @param configurations the configurations to base the Job on
   * @param channel the name of the channel on which the job will be submitted.
   * @param snapshot whether the job belongs to a snapshot harvest
   * @param forceMaxObjectsPerDomain the maximum number of objects harvested from a domain,
   *     overrides individual configuration settings. 0 means no limit.
   * @param forceMaxBytesPerDomain The maximum number of objects harvested from a domain, or -1 for
   *     no limit.
   * @param forceMaxJobRunningTime The max time in seconds given to the harvester for this job
   * @param status the current status of the job.
   * @param orderXMLname the name of the order template used.
   * @param orderXMLdoc the (possibly modified) template
   * @param seedlist the combined seedlist from all configs.
   * @param harvestNum the run number of the harvest definition
   */
  Job(
      Long harvestID,
      Map<String, String> configurations,
      String channel,
      boolean snapshot,
      long forceMaxObjectsPerDomain,
      long forceMaxBytesPerDomain,
      long forceMaxJobRunningTime,
      JobStatus status,
      String orderXMLname,
      HeritrixTemplate orderXMLdoc,
      String seedlist,
      int harvestNum,
      Long continuationOf) {
    origHarvestDefinitionID = harvestID;
    domainConfigurationMap = configurations;
    this.channel = channel;
    this.isSnapshot = snapshot;
    this.forceMaxBytesPerDomain = forceMaxBytesPerDomain;
    this.forceMaxObjectsPerDomain = forceMaxObjectsPerDomain;
    this.forceMaxRunningTime = forceMaxJobRunningTime;
    this.status = status;
    this.orderXMLname = orderXMLname;
    this.orderXMLdoc = orderXMLdoc;
    this.setSeedList(seedlist);
    this.harvestNum = harvestNum;
    this.continuationOF = continuationOf;

    underConstruction = false;
  }

  /**
   * Adds a configuration to this Job. Seedlists and settings are updated accordingly.
   *
   * @param cfg the configuration to add
   * @throws ArgumentNotValid if cfg is null or cfg uses a different orderxml than this job or if
   *     this job already contains a configuration associated with domain of configuration cfg.
   */
  public void addConfiguration(DomainConfiguration cfg) {
    ArgumentNotValid.checkNotNull(cfg, "cfg");
    if (domainConfigurationMap.containsKey(cfg.getDomainName())) {
      throw new ArgumentNotValid(
          "Job already has a configuration for Domain " + cfg.getDomainName());
    }

    if (log.isTraceEnabled()) {
      log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName());
    }

    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }

    if (!cfg.getOrderXmlName().equals(getOrderXMLName())) {
      throw new ArgumentNotValid(
          "Job requires the orderxml file:'"
              + getOrderXMLName()
              + "' not:'"
              + cfg.getOrderXmlName()
              + "' used by the configuration:'"
              + cfg.getName());
    }

    domainConfigurationMap.put(cfg.getDomainName(), cfg.getName());

    // Add the seeds from the configuration to the Job seeds.
    // Take care of duplicates.
    for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext(); ) {
      SeedList seed = itt.next();
      List<String> seeds = seed.getSeeds();
      for (String seedUrl : seeds) {
        seedListSet.add(seedUrl); // duplicates is silently ignored

        // TODO remove when heritrix implements this functionality
        // try to convert a seed into a Internationalized Domain Name
        try {
          String seedASCII = seedUrl;
          // It is rare to see these seeds, but they need to be
          // correctly idnaized
          if (seedUrl.contains(":") || seedUrl.contains("/")) {
            String normalizedUrl = seedUrl;
            if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) {
              // If no protocol is given, assume http
              normalizedUrl = "http://" + normalizedUrl;
            }
            URL url = new URL(normalizedUrl);
            String domainName = url.getHost();
            String domainNameASCII = IDNA.toASCII(domainName);
            if (!domainName.equals(domainNameASCII)) {
              // If the domain name changed, replace that in the
              // seed.
              seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII);
            }
          } else {
            seedASCII = IDNA.toASCII(seedUrl);
          }
          if (!seedASCII.equals(seedUrl)) {
            log.trace("Converted {} to {}", seedUrl, seedASCII);
            // Note that duplicates is silently ignored
            seedListSet.add(seedASCII);
          }
        } catch (IDNAException e) {
          log.trace("Cannot convert seed {} to ASCII", seedUrl, e);
        } catch (MalformedURLException e) {
          log.trace("Cannot convert seed {} to ASCII", seedUrl, e);
        }
      }
    }

    orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg);

    // TODO update limits in settings files - see also bug 269

    // Update estimates of job size
    long expectation =
        cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain);
    maxCountObjects = Math.max(expectation, maxCountObjects);
    minCountObjects = Math.min(expectation, minCountObjects);
    totalCountObjects += expectation;

    configsChanged = true;

    assert (maxCountObjects >= minCountObjects) : "basic invariant";
  }

  /**
   * Get the name of the order XML file used by this Job.
   *
   * @return the name of the orderXML file
   */
  public String getOrderXMLName() {
    return orderXMLname;
  }

  /**
   * Get the actual time when this job was stopped/completed.
   *
   * @return the time as Date
   */
  public Date getActualStop() {
    return actualStop;
  }

  /**
   * Get the actual time when this job was started.
   *
   * @return the time as Date
   */
  public Date getActualStart() {
    return actualStart;
  }

  /**
   * Get the time when this job was submitted.
   *
   * @return the time as Date
   */
  public Date getSubmittedDate() {
    return submittedDate;
  }

  /**
   * Get the time when this job was created.
   *
   * @return the creation time as a <code>Date</code>
   */
  public Date getCreationDate() {
    return creationDate;
  }

  /**
   * Get a list of Heritrix settings.xml files. Note that these files have nothing to do with
   * NetarchiveSuite settings files. They are files that supplement the Heritrix order.xml files,
   * and contain overrides for specific domains.
   *
   * @return the list of Files as an array
   */
  public File[] getSettingsXMLfiles() {
    return settingsXMLfiles;
  }

  /**
   * Get the id of the HarvestDefinition from which this job originates.
   *
   * @return the id as a Long
   */
  public Long getOrigHarvestDefinitionID() {
    return origHarvestDefinitionID;
  }

  /**
   * Get the id of this Job.
   *
   * @return the id as a Long
   */
  public Long getJobID() {
    return jobID;
  }

  /**
   * Set the id of this Job.
   *
   * @param id The Id for this job.
   */
  public void setJobID(Long id) {
    jobID = id;
  }

  /**
   * Get's the total number of different domains harvested by this job.
   *
   * @return the number of configurations added to this domain
   */
  public int getCountDomains() {
    return domainConfigurationMap.size();
  }

  /**
   * Set the actual time when this job was started.
   *
   * <p>Sends a notification, if actualStart is set to a time after actualStop.
   *
   * @param actualStart A Date object representing the time when this job was started.
   */
  public void setActualStart(Date actualStart) {
    ArgumentNotValid.checkNotNull(actualStart, "actualStart");
    if (actualStop != null && actualStop.before(actualStart)) {
      log.warn(
          "Job("
              + getJobID()
              + "): Start time ("
              + actualStart
              + ") is after end time: "
              + actualStop);
    }
    this.actualStart = (Date) actualStart.clone();
  }

  /**
   * Set the actual time when this job was stopped/completed. Sends a notification, if actualStop is
   * set to a time before actualStart.
   *
   * @param actualStop A Date object representing the time when this job was stopped.
   * @throws ArgumentNotValid
   */
  public void setActualStop(Date actualStop) throws ArgumentNotValid {
    ArgumentNotValid.checkNotNull(actualStop, "actualStop");
    if (actualStart == null) {
      log.warn("Job(" + getJobID() + "): actualStart should be defined before setting actualStop");
    } else if (actualStop.before(actualStart)) {
      log.warn(
          "Job("
              + getJobID()
              + "): actualStop ("
              + actualStop
              + ") is before actualStart: "
              + actualStart);
    }
    this.actualStop = (Date) actualStop.clone();
  }

  /**
   * Set the orderxml for this job.
   *
   * @param doc A orderxml to be used by this job
   */
  public void setOrderXMLDoc(HeritrixTemplate doc) {
    ArgumentNotValid.checkNotNull(doc, "doc");
    this.orderXMLdoc = doc;
  }

  /**
   * Gets a document representation of the order.xml associated with this Job.
   *
   * @return the XML as a org.dom4j.Document
   */
  public HeritrixTemplate getOrderXMLdoc() {
    return orderXMLdoc;
  }

  //    /**
  //     * Gets a list of document representations of the settings.xml's associated with this Job.
  //     *
  //     * @return the XML as an array of org.dom4j.Document
  //     */
  //    public Document[] getSettingsXMLdocs() {
  //        return settingsXMLdocs;
  //    }

  /**
   * Set the seedlist of the job from the seedList argument. Individual seeds are separated by a
   * '\n' character. Duplicate seeds are removed.
   *
   * @param seedList List of seeds as one String
   */
  public void setSeedList(String seedList) {
    ArgumentNotValid.checkNotNullOrEmpty(seedList, "seedList");
    seedListSet = new HashSet<>();
    BufferedReader reader = new BufferedReader(new StringReader(seedList));
    String seed;
    try {
      while ((seed = reader.readLine()) != null) {
        seedListSet.add(seed); // add to seedlist if not already there
      }
    } catch (IOException e) {
      // This never happens, as we're reading from a string!
      throw new IOFailure("IOException reading from seed string", e);
    } finally {
      IOUtils.closeQuietly(reader);
    }
  }

  /**
   * Get the seedlist as a String. The individual seeds are separated by the character '\n'. The
   * order of the seeds are unknown.
   *
   * @return the seedlist as a String
   */
  public String getSeedListAsString() {
    return StringUtils.conjoin("\n", seedListSet);
  }

  /**
   * Get the current status of this Job.
   *
   * @return the status as an int in the range 0 to 4.
   */
  public JobStatus getStatus() {
    return status;
  }

  /**
   * Sets status of this job.
   *
   * @param newStatus Must be one of the values STATUS_NEW, ..., STATUS_FAILED
   * @throws ArgumentNotValid in case of invalid status argument or invalid status change
   */
  public void setStatus(JobStatus newStatus) {
    ArgumentNotValid.checkNotNull(newStatus, "newStatus");
    if (!status.legalChange(newStatus)) {
      final String message =
          "Status change from " + status + " to " + newStatus + " is not allowed";
      log.debug(message);
      throw new ArgumentNotValid(message);
    }

    if ((this.status == JobStatus.NEW || this.status == JobStatus.RESUBMITTED)
        && newStatus == JobStatus.SUBMITTED) {
      orderXMLdoc.configureQuotaEnforcer(
          maxObjectsIsSetByQuotaEnforcer, forceMaxBytesPerDomain, forceMaxObjectsPerDomain);
    }

    if (this.status == JobStatus.SUBMITTED && newStatus == JobStatus.STARTED) {
      setActualStart(new Date());
    }
    if (this.status == JobStatus.STARTED
        && (newStatus == JobStatus.DONE || newStatus == JobStatus.FAILED)) {
      setActualStop(new Date());
    }
    status = newStatus;
  }

  /**
   * Returns a map of domain names and name of their corresponding configuration.
   *
   * <p>The returned Map cannot be changed.
   *
   * @return a read-only Map (<String>, <String>)
   */
  public Map<String, String> getDomainConfigurationMap() {
    return Collections.unmodifiableMap(domainConfigurationMap);
  }

  /**
   * Gets the maximum number of objects harvested per domain.
   *
   * @return The maximum number of objects harvested per domain. 0 means no limit.
   */
  public long getMaxObjectsPerDomain() {
    return forceMaxObjectsPerDomain;
  }

  /**
   * Gets the maximum number of bytes harvested per domain.
   *
   * @return The maximum number of bytes harvested per domain. -1 means no limit.
   */
  public long getMaxBytesPerDomain() {
    return forceMaxBytesPerDomain;
  }

  /**
   * Get the edition number.
   *
   * @return The edition number
   */
  long getEdition() {
    return edition;
  }

  /**
   * Set the edition number.
   *
   * @param edition the new edition number
   */
  void setEdition(long edition) {
    this.edition = edition;
  }

  public void setHarvestChannel(HarvestChannel harvestChannel) {
    this.channel = harvestChannel.getName();
    this.isSnapshot = harvestChannel.isSnapshot();
  }

  /** @return the associated {@link HarvestChannel} name. */
  public String getChannel() {
    return channel;
  }

  /**
   * Sets the associated {@link HarvestChannel} name.
   *
   * @param channel the channel name
   */
  public void setChannel(String channel) {
    this.channel = channel;
  }

  /**
   * @return true if the job belongs to a snapshot harvest, false if it belongs to a focused
   *     harvest.
   */
  public boolean isSnapshot() {
    return isSnapshot;
  }

  /**
   * Sets whether job belongs to a snapshot or focused harvest.
   *
   * @param isSnapshot true if the job belongs to a snapshot harvest, false if it belongs to a
   *     focused harvest.
   */
  public void setSnapshot(boolean isSnapshot) {
    this.isSnapshot = isSnapshot;
  }

  @Override
  public String toString() {
    return "Job "
        + getJobID()
        + " (state = "
        + getStatus()
        + ", HD = "
        + getOrigHarvestDefinitionID()
        + ", channel = "
        + getChannel()
        + ", snapshot = "
        + isSnapshot()
        + ", forcemaxcount = "
        + getForceMaxObjectsPerDomain()
        + ", forcemaxbytes = "
        + getMaxBytesPerDomain()
        + ", forcemaxrunningtime = "
        + forceMaxRunningTime
        + ", orderxml = "
        + getOrderXMLName()
        + ", numconfigs = "
        + getDomainConfigurationMap().size()
        + ", created = "
        + getCreationDate()
        + (getSubmittedDate() != null ? ", submitted = " + getSubmittedDate() : "")
        + (getActualStart() != null ? ", started = " + getActualStart() : "")
        + (getActualStop() != null ? ", stopped = " + getActualStop() : "")
        + ")";
  }

  /** @return Returns the forceMaxObjectsPerDomain. 0 means no limit. */
  public long getForceMaxObjectsPerDomain() {
    return forceMaxObjectsPerDomain;
  }

  /**
   * Sets the maxObjectsPerDomain value.
   *
   * @param maxObjectsPerDomain The forceMaxObjectsPerDomain to set. 0 means no limit.
   * @throws IOFailure Thrown from auxiliary method editOrderXML_maxObjectsPerDomain.
   */
  protected void setMaxObjectsPerDomain(long maxObjectsPerDomain) {
    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }

    this.forceMaxObjectsPerDomain = maxObjectsPerDomain;
    orderXMLdoc.setMaxObjectsPerDomain(
        maxObjectsPerDomain); // FIXME? add argument to maxObjectsIsSetByQuotaEnforcer to method
    // setMaxObjectsPerDomain
    // orderXMLdoc.editOrderXML_maxObjectsPerDomain(orderXMLdoc, maxObjectsPerDomain,
    //        maxObjectsIsSetByQuotaEnforcer);

    if (0L == maxObjectsPerDomain && 0L != forceMaxBytesPerDomain) {
      setMaxBytesPerDomain(0L);
    }
  }

  /**
   * Set the maxbytes per domain value.
   *
   * @param maxBytesPerDomain The maxBytesPerDomain to set, or -1 for no limit.
   */
  protected void setMaxBytesPerDomain(long maxBytesPerDomain) {
    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }
    this.forceMaxBytesPerDomain = maxBytesPerDomain;
    orderXMLdoc.setMaxBytesPerDomain(maxBytesPerDomain);

    if (0L == maxBytesPerDomain && 0L != forceMaxObjectsPerDomain) {
      setMaxObjectsPerDomain(0L);
    }
  }

  /**
   * Set the maxJobRunningTime value.
   *
   * @param maxJobRunningTime The maxJobRunningTime in seconds to set, or 0 for no limit.
   */
  protected void setMaxJobRunningTime(long maxJobRunningTime) {
    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }
    this.forceMaxRunningTime = maxJobRunningTime;
    orderXMLdoc.setMaxJobRunningTime(maxJobRunningTime);
  }

  /** @return Returns the MaxJobRunningTime. 0 means no limit. */
  public long getMaxJobRunningTime() {
    return forceMaxRunningTime;
  }

  /**
   * Get the harvestNum for this job. The number reflects which run of the harvest definition this
   * is.
   *
   * @return the harvestNum for this job.
   */
  public int getHarvestNum() {
    return harvestNum;
  }

  /**
   * Set the harvestNum for this job. The number reflects which run of the harvest definition this
   * is. ONLY TO BE USED IN THE CONSTRUCTION PHASE.
   *
   * @param harvestNum a given harvestNum
   */
  public void setHarvestNum(int harvestNum) {
    if (!underConstruction) {
      final String msg = "Cannot modify job " + this + " as it is no longer under construction";
      log.debug(msg);
      throw new IllegalState(msg);
    }
    this.harvestNum = harvestNum;
  }

  /**
   * Get the list of harvest errors for this job. If no harvest errors, null is returned This value
   * is not meaningful until the job is finished (FAILED,DONE, RESUBMITTED)
   *
   * @return the harvest errors for this job or null if no harvest errors.
   */
  public String getHarvestErrors() {
    return harvestErrors;
  }

  /**
   * Append to the list of harvest errors for this job. Nothing happens, if argument harvestErrors
   * is null.
   *
   * @param harvestErrors a string containing harvest errors (may be null)
   */
  public void appendHarvestErrors(String harvestErrors) {
    if (harvestErrors != null) {
      if (this.harvestErrors == null) {
        this.harvestErrors = harvestErrors;
      } else {
        this.harvestErrors += "\n" + harvestErrors;
      }
    }
  }

  /**
   * Get the list of harvest error details for this job. If no harvest error details, null is
   * returned This value is not meaningful until the job is finished (FAILED,DONE, RESUBMITTED)
   *
   * @return the list of harvest error details for this job or null if no harvest error details.
   */
  public String getHarvestErrorDetails() {
    return harvestErrorDetails;
  }

  /**
   * Append to the list of harvest error details for this job. Nothing happens, if argument
   * harvestErrorDetails is null.
   *
   * @param harvestErrorDetails a string containing harvest error details.
   */
  public void appendHarvestErrorDetails(String harvestErrorDetails) {
    if (harvestErrorDetails != null) {
      if (this.harvestErrorDetails == null) {
        this.harvestErrorDetails = harvestErrorDetails;
      } else {
        this.harvestErrorDetails += "\n" + harvestErrorDetails;
      }
    }
  }

  /**
   * Get the list of upload errors. If no upload errors, null is returned. This value is not
   * meaningful until the job is finished (FAILED,DONE, RESUBMITTED)
   *
   * @return the list of upload errors as String, or null if no upload errors.
   */
  public String getUploadErrors() {
    return uploadErrors;
  }

  /**
   * Append to the list of upload errors. Nothing happens, if argument uploadErrors is null.
   *
   * @param uploadErrors a string containing upload errors.
   */
  public void appendUploadErrors(String uploadErrors) {
    if (uploadErrors != null) {
      if (this.uploadErrors == null) {
        this.uploadErrors = uploadErrors;
      } else {
        this.uploadErrors += "\n" + uploadErrors;
      }
    }
  }

  /**
   * Get the list of upload error details. If no upload error details, null is returned. This value
   * is not meaningful until the job is finished (FAILED,DONE, RESUBMITTED)
   *
   * @return the list of upload error details as String, or null if no upload error details
   */
  public String getUploadErrorDetails() {
    return uploadErrorDetails;
  }

  /**
   * Append to the list of upload error details. Nothing happens, if argument uploadErrorDetails is
   * null.
   *
   * @param uploadErrorDetails a string containing upload error details.
   */
  public void appendUploadErrorDetails(String uploadErrorDetails) {
    if (uploadErrorDetails != null) {
      if (this.uploadErrorDetails == null) {
        this.uploadErrorDetails = uploadErrorDetails;
      } else {
        this.uploadErrorDetails += "\n" + uploadErrorDetails;
      }
    }
  }

  /**
   * Get the ID for the job which this job was resubmitted as. If null, this job has not been
   * resubmitted.
   *
   * @return this ID.
   */
  public Long getResubmittedAsJob() {
    return resubmittedAsJobWithID;
  }

  /**
   * Set the Date for when this job was submitted. If null, this job has not been submitted.
   *
   * @param submittedDate The date when this was submitted
   */
  public void setSubmittedDate(Date submittedDate) {
    this.submittedDate = submittedDate;
  }

  /**
   * Set the Date for when this job was created. If null, this job has not been created.
   *
   * @param creationDate The date when this was created
   */
  public void setCreationDate(Date creationDate) {
    this.creationDate = creationDate;
  }

  /**
   * Set the ID for the job which this job was resubmitted as.
   *
   * @param resubmittedAsJob An Id for a new job.
   */
  public void setResubmittedAsJob(Long resubmittedAsJob) {
    this.resubmittedAsJobWithID = resubmittedAsJob;
  }

  /**
   * @return id of the job that this job is supposed to continue using Heritrix recover-log or null
   *     if it starts from scratch.
   */
  public Long getContinuationOf() {
    return this.continuationOF;
  }

  @Override
  public String getHarvestFilenamePrefix() {
    if (this.harvestnamePrefix == null) {
      log.warn(
          "HarvestnamePrefix not yet set for job {}. Set it by using the naming scheme. "
              + "This should only happen for old jobs being read",
          this.jobID);
      setDefaultHarvestNamePrefix();
    }
    return this.harvestnamePrefix;
  }

  /** @param prefix */
  public void setHarvestFilenamePrefix(String prefix) {
    this.harvestnamePrefix = prefix;
  }

  /** @return the forceMaxBytesPerDomain */
  public long getForceMaxBytesPerDomain() {
    return forceMaxBytesPerDomain;
  }

  /** @return the configurationSetsObjectLimit */
  public boolean isConfigurationSetsObjectLimit() {
    return configurationSetsObjectLimit;
  }

  /** @return the configurationSetsByteLimit */
  public boolean isConfigurationSetsByteLimit() {
    return configurationSetsByteLimit;
  }

  /** @return the minCountObjects */
  public long getMinCountObjects() {
    return minCountObjects;
  }

  /** @return the maxCountObjects */
  public long getMaxCountObjects() {
    return maxCountObjects;
  }

  /** @return the totalCountObjects */
  public long getTotalCountObjects() {
    return totalCountObjects;
  }

  void setDefaultHarvestNamePrefix() {
    if (getJobID() != null) {
      ArchiveFileNaming naming = ArchiveFileNamingFactory.getInstance();
      log.debug("Applying the default ArchiveFileNaming class '{}'.", naming.getClass().getName());
      final String prefix = naming.getPrefix(this);
      setHarvestFilenamePrefix(prefix);
      log.debug("The harvestPrefix of this job is: {}", prefix);
    } else {
      log.warn(
          "The harvestnamePrefix is not set now, as it depends on the JobID, which is not set yet");
    }
  }

  /** @return the harvest-audience. */
  public String getHarvestAudience() {
    return harvestAudience;
  }

  /**
   * Set the harvest audience for this job. Taken from the harvestdefinition that generated this
   * job.
   *
   * @param theAudience the harvest-audience.
   */
  public void setHarvestAudience(String theAudience) {
    this.harvestAudience = theAudience;
  }

  ///////////// The following two methods are needed by harvestStatus-jobdetails.jsp
  // ////////////////////////////////////
  /**
   * Returns a list of sorted seeds for this job. The sorting is by domain, and inside each domain,
   * the list is sorted by url
   *
   * @return a list of sorted seeds for this job.
   */
  public List<String> getSortedSeedList() {
    Map<String, Set<String>> urlMap = new HashMap<String, Set<String>>();
    for (String seed : seedListSet) {
      String url;
      // Assume the protocol is http://, if it is missing
      if (!seed.matches(Constants.PROTOCOL_REGEXP)) {
        url = "http://" + seed;
      } else {
        url = seed;
      }
      String domain = getDomain(url);
      if (domain == null) {
        // stop processing this url, and continue to the next seed
        continue;
      }
      Set<String> set;
      if (urlMap.containsKey(domain)) {
        set = urlMap.get(domain);
      } else {
        set = new TreeSet<String>();
        urlMap.put(domain, set);
      }
      set.add(seed);
    }
    List<String> result = new ArrayList<String>();
    for (Set<String> set : urlMap.values()) {
      result.addAll(set);
    }
    return result;
  }
  /**
   * Get the domain, that the given URL belongs to.
   *
   * @param url an URL
   * @return the domain, that the given URL belongs to, or null if unable to do so.
   */
  private String getDomain(String url) {
    try {
      URL uri = new URL(url);
      return DomainUtils.domainNameFromHostname(uri.getHost());
    } catch (MalformedURLException e) {
      log.warn("The string '{}' is not a valid URL", url);
      return null;
    }
  }
}
/** Unit test for the archive upload tool. */
public class UploadTester {
  private UseTestRemoteFile ulrf = new UseTestRemoteFile();
  private PreventSystemExit pse = new PreventSystemExit();
  private PreserveStdStreams pss = new PreserveStdStreams(true);
  private MoveTestFiles mtf = new MoveTestFiles(TestInfo.DATA_DIR, TestInfo.WORKING_DIR);
  private MockupJMS mjms = new MockupJMS();
  private MockupArcRepositoryClient marc = new MockupArcRepositoryClient();
  ReloadSettings rs = new ReloadSettings();

  /** Max number of store retries. */
  private final int storeRetries =
      Settings.getInt(JMSArcRepositoryClient.ARCREPOSITORY_STORE_RETRIES);

  @Before
  public void setUp() {
    rs.setUp();
    ulrf.setUp();
    mjms.setUp();
    mtf.setUp();
    pss.setUp();
    pse.setUp();
    marc.setUp();
    Settings.set(CommonSettings.NOTIFICATIONS_CLASS, RememberNotifications.class.getName());
  }

  @After
  public void tearDown() {
    marc.tearDown();
    pse.tearDown();
    pss.tearDown();
    mtf.tearDown();
    mjms.tearDown();
    ulrf.tearDown();
    RememberNotifications.resetSingleton();
    rs.tearDown();
  }

  @Test
  public void testConstructor() {
    ReflectUtils.testUtilityConstructor(Upload.class);
  }

  /** Verify that uploading a single ARC file works as expected and deletes the file locally. */
  @Test
  public void testMainOneFile() {
    Upload.main(new String[] {TestInfo.ARC1.getAbsolutePath()});
    assertMsgCount(1, 0);
    assertStoreStatus(0, TestInfo.ARC1, true);
  }

  /**
   * Verify that uploading more than one ARC file works as expected and deletes the files locally.
   */
  @Test
  public void testMainSeveralFiles() {
    Upload.main(new String[] {TestInfo.ARC1.getAbsolutePath(), TestInfo.ARC2.getAbsolutePath()});
    assertMsgCount(2, 0);
    assertStoreStatus(0, TestInfo.ARC1, true);
    assertStoreStatus(1, TestInfo.ARC2, true);
  }

  /**
   * Verify that non-ARC files are rejected and execution fails. Also verifies that nothing is
   * stored in that case.
   */
  @Test
  public void testMainNonArc() {
    try {
      Upload.main(
          new String[] {TestInfo.ARC1.getAbsolutePath(), TestInfo.INDEX_DIR.getAbsolutePath()});
      fail("Calling Upload with non-arc file should System.exit");
    } catch (SecurityException e) {
      // Expected
      assertMsgCount(0, 0);
    }
  }

  /** Verify that uploading a single WARC file works as expected and deletes the file locally. */
  @Test
  public void testMainOneWarcFile() {
    Upload.main(new String[] {TestInfo.WARC1.getAbsolutePath()});
    assertMsgCount(1, 0);
    assertStoreStatus(0, TestInfo.WARC1, true);
  }

  /**
   * Verify that the system fails as expected when the store operation fails on the server side.
   * (Local files must NOT be deleted).
   */
  @Test
  public void testMainStoreFails1() {
    marc.failOnFile(TestInfo.ARC1.getName());

    Upload.main(
        new String[] {
          TestInfo.ARC1.getAbsolutePath(),
          TestInfo.ARC2.getAbsolutePath(),
          TestInfo.ARC3.getAbsolutePath()
        });
    assertMsgCount(2, 1);
    int index = 0;
    for (int i = 0; i < storeRetries; i++) {
      assertStoreStatus(index, TestInfo.ARC1, false);
      index++;
    }
    assertStoreStatus(index, TestInfo.ARC2, true);
    index++;
    assertStoreStatus(index, TestInfo.ARC3, true);
  }

  /**
   * Verify that the system fails as expected when the store operation fails on the server side.
   * (Local files must NOT be deleted).
   */
  @Test
  public void testMainStoreFails2() {
    marc.failOnFile(TestInfo.ARC2.getName());

    Upload.main(
        new String[] {
          TestInfo.ARC1.getAbsolutePath(),
          TestInfo.ARC2.getAbsolutePath(),
          TestInfo.ARC3.getAbsolutePath()
        });
    assertMsgCount(2, 1);
    int index = 0;
    assertStoreStatus(index, TestInfo.ARC1, true);
    index++;
    for (int i = 0; i < storeRetries; i++) {
      assertStoreStatus(index, TestInfo.ARC2, false);
      index++;
    }
    assertStoreStatus(index, TestInfo.ARC3, true);
  }

  /**
   * Verify that the system fails as expected when the store operation fails on the server side.
   * (Local files must NOT be deleted).
   */
  @Test
  public void testMainStoreFails3() {
    marc.failOnFile(TestInfo.ARC3.getName());

    Upload.main(
        new String[] {
          TestInfo.ARC1.getAbsolutePath(),
          TestInfo.ARC2.getAbsolutePath(),
          TestInfo.ARC3.getAbsolutePath()
        });
    assertMsgCount(2, 1);
    int index = 0;
    assertStoreStatus(index, TestInfo.ARC1, true);
    index++;
    assertStoreStatus(index, TestInfo.ARC2, true);
    index++;
    for (int i = 0; i < storeRetries; i++) {
      assertStoreStatus(index, TestInfo.ARC3, false);
      index++;
    }
  }

  /**
   * Verifies that calling Upload without arguments fails. Also verifies that nothing is stored in
   * that case.
   */
  @Test
  public void testNoArguments() {
    try {
      Upload.main(new String[] {});
      fail("Calling Upload without arguments should System.exit");
    } catch (SecurityException e) {
      // Expected
      assertMsgCount(0, 0);
    }
  }

  /**
   * Asserts that we got the expected number of StoreMessages.
   *
   * @param succeeded Number of files successfully stored
   * @param failed Number of files that never got stored
   */
  private void assertMsgCount(int succeeded, int failed) {
    int expected = succeeded + failed * storeRetries;
    assertEquals(
        "Upload should generate exactly 1 StoreMessage "
            + "per succeeded arc file and "
            + storeRetries
            + " per failed store",
        expected,
        marc.getMsgCount());
  }

  /**
   * Asserts that the nth StoreMessage is regarding the given arc file and that the arc file is
   * delete if and only if store succeeded.
   *
   * @param n The relevant index to marc.getStoreMsgs()
   * @param arcFile The arc file that was stored
   * @param shouldSucceed Whether store was supposed to succeed
   */
  private void assertStoreStatus(int n, File arcFile, boolean shouldSucceed) {
    StoreMessage sm = marc.getStoreMsgs().get(n);
    assertEquals(
        "Upload should attempt to upload the specified files",
        arcFile.getName(),
        sm.getArcfileName());
    if (shouldSucceed) {
      assertFalse("Upload should delete a properly uploaded file", arcFile.exists());
    } else {
      assertTrue(
          "Upload should not delete a file that wasn't " + " properly uploaded", arcFile.exists());
    }
  }
}