/** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
  void configureKeyStores(Configuration config) {
    Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
    for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
      String id = (String) iter.next();
      Configuration oneKs = allKs.getConfigTree(id);
      try {
        LockssKeyStore lk = createLockssKeyStore(oneKs);
        String name = lk.getName();
        if (name == null) {
          log.error("KeyStore definition missing name: " + oneKs);
          continue;
        }
        LockssKeyStore old = keystoreMap.get(name);
        if (old != null && !lk.equals(old)) {
          log.warning(
              "Keystore "
                  + name
                  + " redefined.  "
                  + "New definition may not take effect until daemon restart");
        }

        log.debug("Adding keystore " + name);
        keystoreMap.put(name, lk);

      } catch (Exception e) {
        log.error("Couldn't create keystore: " + oneKs, e);
      }
    }
  }
Exemple #2
0
 public void export() {
   log.debug("export(" + au.getName() + ")");
   log.debug(
       "dir: "
           + dir
           + ", pref: "
           + prefix
           + ", size: "
           + maxSize
           + ", ver: "
           + maxVersions
           + (compress ? ", (C)" : ""));
   checkArgs();
   try {
     start();
   } catch (IOException e) {
     recordError("Error opening file", e);
     return;
   }
   writeFiles();
   try {
     finish();
   } catch (IOException e) {
     if (!isDiskFull) {
       // If we already knew (and reported) disk full, also reporting it
       // as a close error is misleading.
       recordError("Error closing file", e);
     }
   }
 }
 protected void initFeatureVersions() throws PluginException.InvalidDefinition {
   if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
     Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
     Map<String, String> spec =
         (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
     log.debug2("features: " + spec);
     for (Map.Entry<String, String> ent : spec.entrySet()) {
       try {
         // Prefix version string with feature name to create separate
         // namespace for each feature
         String key = ent.getKey();
         map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
       } catch (RuntimeException e) {
         log.warning(
             getPluginName()
                 + " set unknown feature: "
                 + ent.getKey()
                 + " to version "
                 + ent.getValue(),
             e);
         throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
       }
     }
     featureVersion = map;
   } else {
     featureVersion = null;
   }
 }
Exemple #4
0
 private void initTestPolls() throws Exception {
   testV1polls = new V1Poll[testV1msg.length];
   for (int i = 0; i < testV1polls.length; i++) {
     log.debug3("initTestPolls: V1 " + i);
     BasePoll p = pollmanager.makePoll(testV1msg[i]);
     assertNotNull(p);
     assertNotNull(p.getMessage());
     log.debug("initTestPolls: V1 " + i + " returns " + p);
     assertTrue(p instanceof V1Poll);
     switch (i) {
       case 0:
         assertTrue(p instanceof V1NamePoll);
         break;
       case 1:
         assertTrue(p instanceof V1ContentPoll);
         break;
       case 2:
         assertTrue(p instanceof V1VerifyPoll);
         break;
     }
     testV1polls[i] = (V1Poll) p;
     assertNotNull(testV1polls[i]);
     log.debug3("initTestPolls: " + i + " " + p.toString());
   }
 }
 public String getDefaultArticleMimeType() {
   String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
   log.debug3("DefaultArticleMimeType " + ret);
   if (ret == null) {
     ret = super.getDefaultArticleMimeType();
     log.debug3("DefaultArticleMimeType from super " + ret);
   }
   return ret;
 }
Exemple #6
0
 /** test for method scheduleVote(..) */
 public void testScheduleVote() {
   V1Poll p = testV1polls[1];
   assertTrue(p instanceof V1ContentPoll);
   log.debug3("testScheduleVote 1");
   p.scheduleVote();
   log.debug3("testScheduleVote 2");
   assertNotNull(p.m_voteTime);
   assertTrue(p.m_voteTime.getRemainingTime() < p.m_deadline.getRemainingTime());
   log.debug3("at end of testScheduleVote");
 }
 void stopThread() {
   if (sizeCalcThread != null) {
     log.debug2("Stopping thread");
     sizeCalcThread.stopSizeCalc();
     sizeCalcThread = null;
   }
 }
 protected void logParams() {
   Enumeration en = req.getParameterNames();
   while (en.hasMoreElements()) {
     String name = (String) en.nextElement();
     String vals[];
     String dispval;
     if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
       dispval = req.getParameter(name).length() == 0 ? "" : "********";
     } else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
       dispval = StringUtil.separatedString(vals, ", ");
     } else {
       dispval = req.getParameter(name);
     }
     log.debug(name + " = " + dispval);
   }
 }
  /** Create LockssKeystore from a config subtree */
  LockssKeyStore createLockssKeyStore(Configuration config) {
    log.debug2("Creating LockssKeyStore from config: " + config);
    String name = config.get(KEYSTORE_PARAM_NAME);
    LockssKeyStore lk = new LockssKeyStore(name);

    String file = config.get(KEYSTORE_PARAM_FILE);
    String resource = config.get(KEYSTORE_PARAM_RESOURCE);
    String url = config.get(KEYSTORE_PARAM_URL);

    if (!StringUtil.isNullString(file)) {
      lk.setLocation(file, LocationType.File);
    } else if (!StringUtil.isNullString(resource)) {
      lk.setLocation(resource, LocationType.Resource);
    } else if (!StringUtil.isNullString(url)) {
      lk.setLocation(url, LocationType.Url);
    }

    lk.setType(config.get(KEYSTORE_PARAM_TYPE, defaultKeyStoreType));
    lk.setProvider(config.get(KEYSTORE_PARAM_PROVIDER, defaultKeyStoreProvider));
    lk.setPassword(config.get(KEYSTORE_PARAM_PASSWORD));
    lk.setKeyPassword(config.get(KEYSTORE_PARAM_KEY_PASSWORD));
    lk.setKeyPasswordFile(config.get(KEYSTORE_PARAM_KEY_PASSWORD_FILE));
    lk.setMayCreate(config.getBoolean(KEYSTORE_PARAM_CREATE, DEFAULT_CREATE));
    return lk;
  }
Exemple #10
0
 private void checkSubstance(ArchivalUnit au) {
   SubstanceChecker subChecker = new SubstanceChecker(au);
   if (!subChecker.isEnabled()) {
     errMsg = "No substance patterns defined for plugin.";
     return;
   }
   AuState auState = AuUtil.getAuState(au);
   SubstanceChecker.State oldState = auState.getSubstanceState();
   SubstanceChecker.State newState = subChecker.findSubstance();
   String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
   switch (newState) {
     case Unknown:
       log.error("Shouldn't happen: SubstanceChecker returned Unknown");
       errMsg = "Error in SubstanceChecker; see log.";
       break;
     case Yes:
       statusMsg = "AU has substance " + chtxt + ": " + au.getName();
       auState.setSubstanceState(SubstanceChecker.State.Yes);
       break;
     case No:
       statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
       auState.setSubstanceState(SubstanceChecker.State.No);
       break;
   }
 }
/** Export to an ARC file */
public class ArcExporter extends Exporter {

  private static Logger log = Logger.getLogger("ArcExporter");

  protected CIProperties arcProps = null;
  String arcFilePrefix = "SimulatedCrawl";
  AtomicInteger serialNo = new AtomicInteger(0);
  ARCWriter aw;
  boolean isResponse;

  public ArcExporter(LockssDaemon daemon, ArchivalUnit au, boolean isResponse) {
    super(daemon, au);
    this.isResponse = isResponse;
  }

  protected void start() {
    aw = makeARCWriter();
  }

  protected void finish() throws IOException {
    aw.close();
  }

  private ARCWriter makeARCWriter() {
    return new ARCWriter(
        serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
  }

  protected void writeCu(CachedUrl cu) throws IOException {
    String url = cu.getUrl();
    long contentSize = cu.getContentSize();
    CIProperties props = cu.getProperties();
    long fetchTime = Long.parseLong(props.getProperty(CachedUrl.PROPERTY_FETCH_TIME));
    InputStream contentIn = cu.getUnfilteredInputStream();
    try {
      if (isResponse) {
        String hdrString = getHttpResponseString(cu);
        long size = contentSize + hdrString.length();
        InputStream headerIn = new ReaderInputStream(new StringReader(hdrString));
        InputStream concat = new SequenceInputStream(headerIn, contentIn);
        try {
          aw.write(xlateFilename(url), cu.getContentType(), getHostIp(), fetchTime, size, concat);
        } finally {
          IOUtil.safeClose(concat);
        }
      } else {
        aw.write(
            xlateFilename(url),
            cu.getContentType(),
            getHostIp(),
            fetchTime,
            cu.getContentSize(),
            contentIn);
      }
    } finally {
      AuUtil.safeRelease(cu);
    }
  }
}
 /** engqueue a size calculation for the node */
 public void queueSizeCalc(RepositoryNode node) {
   synchronized (sizeCalcQueue) {
     if (sizeCalcQueue.add(node)) {
       log.debug2("Queue size calc: " + node);
       startOrKickThread();
     }
   }
 }
 void startOrKickThread() {
   if (sizeCalcThread == null) {
     log.debug2("Starting thread");
     sizeCalcThread = new SizeCalcThread();
     sizeCalcThread.start();
     sizeCalcThread.waitRunning();
   }
   sizeCalcSem.give();
 }
Exemple #14
0
 protected String getHostIp() {
   try {
     IPAddr localHost = IPAddr.getLocalHost();
     return localHost.getHostAddress();
   } catch (UnknownHostException e) {
     log.error("getHostIp()", e);
     return "1.1.1.1";
   }
 }
 /**
  * Checks the consistency of the node, and continues with its children if it's consistent.
  *
  * @param node RepositoryNodeImpl the node to check
  */
 private void recurseConsistencyCheck(RepositoryNodeImpl node) {
   logger.debug2("Checking node '" + node.getNodeUrl() + "'...");
   // check consistency at each node
   // correct/deactivate as necessary
   // 'checkNodeConsistency()' will repair if possible
   if (node.checkNodeConsistency()) {
     logger.debug3("Node consistent; recursing on children...");
     List children = node.getNodeList(null, false);
     Iterator iter = children.iterator();
     while (iter.hasNext()) {
       RepositoryNodeImpl child = (RepositoryNodeImpl) iter.next();
       recurseConsistencyCheck(child);
     }
   } else {
     logger.debug3("Node inconsistent; deactivating...");
     deactivateInconsistentNode(node);
   }
 }
 public PlatformUtil.DF getRepositoryDF(String repoName) {
   String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
   log.debug("path: " + path);
   //    try {
   return platInfo.getJavaDF(path);
   //    } catch (PlatformUtil.UnsupportedException e) {
   //      return null;
   //    }
 }
Exemple #17
0
 private void forceV3Poll() {
   ArchivalUnit au = getAu();
   if (au == null) return;
   try {
     callV3ContentPoll(au);
   } catch (Exception e) {
     log.error("Can't start poll", e);
     errMsg = "Error: " + e.toString();
   }
 }
Exemple #18
0
 private void forceReindexMetadata() {
   ArchivalUnit au = getAu();
   if (au == null) return;
   try {
     startReindexingMetadata(au, true);
   } catch (RuntimeException e) {
     log.error("Can't reindex metadata", e);
     errMsg = "Error: " + e.toString();
   }
 }
Exemple #19
0
 private void doDisableMetadataIndexing() {
   ArchivalUnit au = getAu();
   if (au == null) return;
   try {
     disableMetadataIndexing(au, false);
   } catch (RuntimeException e) {
     log.error("Can't disable metadata indexing", e);
     errMsg = "Error: " + e.toString();
   }
 }
Exemple #20
0
 private void doCheckSubstance() {
   ArchivalUnit au = getAu();
   if (au == null) return;
   try {
     checkSubstance(au);
   } catch (RuntimeException e) {
     log.error("Error in SubstanceChecker", e);
     errMsg = "Error in SubstanceChecker; see log.";
   }
 }
Exemple #21
0
  private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
    if (metadataMgr == null) {
      errMsg = "Metadata processing is not enabled.";
      return false;
    }

    if (!force) {
      if (!AuUtil.hasCrawled(au)) {
        errMsg = "Au has never crawled. Click again to reindex metadata";
        showForceReindexMetadata = true;
        return false;
      }

      AuState auState = AuUtil.getAuState(au);
      switch (auState.getSubstanceState()) {
        case No:
          errMsg = "Au has no substance. Click again to reindex metadata";
          showForceReindexMetadata = true;
          return false;
        case Unknown:
          errMsg = "Unknown substance for Au. Click again to reindex metadata.";
          showForceReindexMetadata = true;
          return false;
        case Yes:
          // fall through
      }
    }

    // Fully reindex metadata with the highest priority.
    Connection conn = null;
    PreparedStatement insertPendingAuBatchStatement = null;

    try {
      conn = dbMgr.getConnection();
      insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);

      if (metadataMgr.enableAndAddAuToReindex(
          au, conn, insertPendingAuBatchStatement, false, true)) {
        statusMsg = "Reindexing metadata for " + au.getName();
        return true;
      }
    } catch (DbException dbe) {
      log.error("Cannot reindex metadata for " + au.getName(), dbe);
    } finally {
      DbManager.safeCloseStatement(insertPendingAuBatchStatement);
      DbManager.safeRollbackAndClose(conn);
    }

    if (force) {
      errMsg = "Still cannot reindex metadata for " + au.getName();
    } else {
      errMsg = "Cannot reindex metadata for " + au.getName();
    }
    return false;
  }
 void loadKeyStores() {
   List<LockssKeyStore> lst = new ArrayList<LockssKeyStore>(keystoreMap.values());
   for (LockssKeyStore lk : lst) {
     try {
       lk.load();
     } catch (Exception e) {
       log.error("Can't load keystore " + lk.getName(), e);
       keystoreMap.remove(lk.getName());
     }
   }
 }
 static void saveAuIdProperties(String location, Properties props) {
   // XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
   // single-point corruption
   File propDir = new File(location);
   if (!propDir.exists()) {
     logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
     propDir.mkdirs();
   }
   File propFile = new File(propDir, AU_ID_FILE);
   try {
     logger.debug3("Saving au id properties at '" + location + "'.");
     OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
     props.store(os, "ArchivalUnit id info");
     os.close();
     propFile.setReadOnly();
   } catch (IOException ioe) {
     logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
     throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
   }
 }
 static LocalRepository getLocalRepository(String repoRoot) {
   synchronized (localRepositories) {
     LocalRepository localRepo = (LocalRepository) localRepositories.get(repoRoot);
     if (localRepo == null) {
       logger.debug2("Creating LocalRepository(" + repoRoot + ")");
       localRepo = new LocalRepository(repoRoot);
       localRepositories.put(repoRoot, localRepo);
     }
     return localRepo;
   }
 }
 static Properties getAuIdProperties(String location) {
   File propFile = new File(location + File.separator + AU_ID_FILE);
   try {
     InputStream is = new BufferedInputStream(new FileInputStream(propFile));
     Properties idProps = new Properties();
     idProps.load(is);
     is.close();
     return idProps;
   } catch (Exception e) {
     logger.warning("Error loading au id from " + propFile.getPath() + ".");
     return null;
   }
 }
 String getLocalIPAddr() {
   if (localAddr == null) {
     try {
       IPAddr localHost = IPAddr.getLocalHost();
       localAddr = localHost.getHostAddress();
     } catch (UnknownHostException e) {
       // shouldn't happen
       log.error("LockssServlet: getLocalHost: " + e.toString());
       return "???";
     }
   }
   return localAddr;
 }
 protected PermissionCheckerFactory getPermissionCheckerFactory() {
   if (permissionCheckerFact == null) {
     String permissionCheckerFactoryClass =
         definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
     if (permissionCheckerFactoryClass != null) {
       permissionCheckerFact =
           (PermissionCheckerFactory)
               newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
       log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
     }
   }
   return permissionCheckerFact;
 }
Exemple #28
0
 protected String getHostName() {
   String res = ConfigManager.getPlatformHostname();
   if (res == null) {
     try {
       InetAddress inet = InetAddress.getLocalHost();
       return inet.getHostName();
     } catch (UnknownHostException e) {
       log.warning("Can't get hostname", e);
       return "unknown";
     }
   }
   return res;
 }
  /**
   * This function returns a RepositoryNode with a canonicalized path.
   *
   * @param url the url in String form
   * @param create true iff the node should be created if absent
   * @return RepositoryNode the node
   * @throws MalformedURLException
   */
  private synchronized RepositoryNode getNode(String url, boolean create)
      throws MalformedURLException {
    String canonUrl;
    boolean isAuUrl = false;
    if (AuUrl.isAuUrl(url)) {
      // path information is lost here, but is unimportant if it's an AuUrl
      canonUrl = AuUrl.PROTOCOL;
      isAuUrl = true;
    } else {
      // create a canonical path, handling all illegal path traversal
      canonUrl = canonicalizePath(url);
    }

    // check LRUMap cache for node
    RepositoryNode node = (RepositoryNode) nodeCache.get(nodeCacheKey(canonUrl));
    if (node != null) {
      return node;
    }

    String nodeLocation;
    if (isAuUrl) {
      // base directory of ArchivalUnit
      nodeLocation = rootLocation;
      node = new AuNodeImpl(canonUrl, nodeLocation, this);
    } else {
      // determine proper node location
      nodeLocation =
          LockssRepositoryImpl.mapUrlToFileLocation(rootLocation, canonUrl)
              .replace("?", "")
              .replace("COM8", "COMEIGHT")
              .replace("%5c", "/"); // //windows folder structure fix
      node = new RepositoryNodeImpl(canonUrl, nodeLocation, this);
    }

    if (!create) {
      // if not creating, check for existence
      File nodeDir = new File(nodeLocation);
      if (!nodeDir.exists()) {
        // return null if the node doesn't exist and shouldn't be created
        return null;
      }
      if (!nodeDir.isDirectory()) {
        logger.error("Cache file not a directory: " + nodeLocation);
        throw new LockssRepository.RepositoryStateException("Invalid cache file.");
      }
    }

    // add to node cache
    nodeCache.put(nodeCacheKey(canonUrl), node);
    return node;
  }
Exemple #30
0
  /** test for method checkVote(..) */
  public void testCheckVote() throws Exception {
    V1LcapMessage msg = null;
    log.debug3("starting testCheeckVote");
    msg =
        V1LcapMessage.makeReplyMsg(
            testV1polls[0].getMessage(),
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            null,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID);
    log.debug3("testCheeckVote 2");
    V1Poll p = null;
    p = createCompletedPoll(theDaemon, testau, msg, 8, 2, pollmanager);
    assertTrue(p instanceof V1NamePoll);
    log.debug3("testCheeckVote 3");
    assertNotNull(p);
    PeerIdentity id = msg.getOriginatorId();
    assertNotNull(id);
    assertNotNull(p.m_tally);
    int rep = p.m_tally.wtAgree + idmgr.getReputation(id);

    // good vote check

    p.checkVote(msg.getHashed(), new Vote(msg, false));
    assertEquals(9, p.m_tally.numAgree);
    assertEquals(2, p.m_tally.numDisagree);
    assertEquals(rep, p.m_tally.wtAgree);

    rep = p.m_tally.wtDisagree + idmgr.getReputation(id);

    // bad vote check
    p.checkVote(ByteArray.makeRandomBytes(20), new Vote(msg, false));
    assertEquals(9, p.m_tally.numAgree);
    assertEquals(3, p.m_tally.numDisagree);
    assertEquals(rep, p.m_tally.wtDisagree);
  }