Ejemplo n.º 1
1
  public void testHeadNotPopular() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 3 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(0);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(1);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(2);
    assertSameElements(SetUtil.set(participant1, participant2), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(3);
    assertEmpty(repairCandidates.keySet());
  }
Ejemplo n.º 2
0
 public Map<String, PlatformUtil.DF> getRepositoryMap() {
   Map<String, PlatformUtil.DF> repoMap = new LinkedMap();
   for (String repo : getRepositoryList()) {
     repoMap.put(repo, getRepositoryDF(repo));
   }
   return repoMap;
 }
Ejemplo n.º 3
0
  public void testMultipleIdenticalVersions() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 1 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(2);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    // With only three candidates, no version should reach a threshold
    // of 4, unless counting multiples is wrong.
    repairCandidates = versionCounts.getRepairCandidates(4);
    assertEmpty(repairCandidates.keySet());
  }
Ejemplo n.º 4
0
 protected void initFeatureVersions() throws PluginException.InvalidDefinition {
   if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
     Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
     Map<String, String> spec =
         (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
     log.debug2("features: " + spec);
     for (Map.Entry<String, String> ent : spec.entrySet()) {
       try {
         // Prefix version string with feature name to create separate
         // namespace for each feature
         String key = ent.getKey();
         map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
       } catch (RuntimeException e) {
         log.warning(
             getPluginName()
                 + " set unknown feature: "
                 + ent.getKey()
                 + " to version "
                 + ent.getValue(),
             e);
         throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
       }
     }
     featureVersion = map;
   } else {
     featureVersion = null;
   }
 }
  /** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
  void configureKeyStores(Configuration config) {
    Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
    for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
      String id = (String) iter.next();
      Configuration oneKs = allKs.getConfigTree(id);
      try {
        LockssKeyStore lk = createLockssKeyStore(oneKs);
        String name = lk.getName();
        if (name == null) {
          log.error("KeyStore definition missing name: " + oneKs);
          continue;
        }
        LockssKeyStore old = keystoreMap.get(name);
        if (old != null && !lk.equals(old)) {
          log.warning(
              "Keystore "
                  + name
                  + " redefined.  "
                  + "New definition may not take effect until daemon restart");
        }

        log.debug("Adding keystore " + name);
        keystoreMap.put(name, lk);

      } catch (Exception e) {
        log.error("Couldn't create keystore: " + oneKs, e);
      }
    }
  }
Ejemplo n.º 6
0
 /**
  * Return the request parameters as a Map<String,String>. Only the first value of multivalued
  * parameters is included.
  */
 Map<String, String> getParamsAsMap() {
   Map<String, String> map = new HashMap<String, String>();
   for (Enumeration en = req.getParameterNames(); en.hasMoreElements(); ) {
     String name = (String) en.nextElement();
     map.put(name, req.getParameter(name));
   }
   return map;
 }
 private Map makeRow(CrawlerStatus status, String mimeType, String key) {
   Map row = new HashMap();
   row.put(MIME_TYPE_NAME, mimeType);
   row.put(
       MIME_TYPE_NUM_URLS,
       makeRefIfColl(status.getMimeTypeCtr(mimeType), key, MIMETYPES_URLS_KEY + ":" + mimeType));
   return row;
 }
Ejemplo n.º 8
0
 // Break the line at commas, return a map of the resulting strings
 // broken at equals sign.  (<i>Ie</i>, name value pairs.)
 Map getRow(String line) {
   Map map = new HashMap();
   for (Iterator iter = StringUtil.breakAt(line, ',').iterator(); iter.hasNext(); ) {
     String item = (String) iter.next();
     List pair = StringUtil.breakAt(item, '=');
     map.put(pair.get(0), pair.get(1));
   }
   return map;
 }
Ejemplo n.º 9
0
 // hack only local
 public synchronized LockssRepositoryImpl getRepositoryFromPath(String path) {
   LockssRepositoryImpl repo = (LockssRepositoryImpl) localRepos.get(path);
   if (repo == null) {
     repo = new LockssRepositoryImpl(path);
     repo.initService(getDaemon());
     repo.startService();
     localRepos.put(path, repo);
   }
   return repo;
 }
Ejemplo n.º 10
0
 protected void initAuFeatureMap() {
   if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
     Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
     for (Map.Entry ent : featMap.entrySet()) {
       Object val = ent.getValue();
       if (val instanceof Map) {
         ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
       }
     }
   }
 }
 void loadKeyStores() {
   List<LockssKeyStore> lst = new ArrayList<LockssKeyStore>(keystoreMap.values());
   for (LockssKeyStore lk : lst) {
     try {
       lk.load();
     } catch (Exception e) {
       log.error("Can't load keystore " + lk.getName(), e);
       keystoreMap.remove(lk.getName());
     }
   }
 }
 static LocalRepository getLocalRepository(String repoRoot) {
   synchronized (localRepositories) {
     LocalRepository localRepo = (LocalRepository) localRepositories.get(repoRoot);
     if (localRepo == null) {
       logger.debug2("Creating LocalRepository(" + repoRoot + ")");
       localRepo = new LocalRepository(repoRoot);
       localRepositories.put(repoRoot, localRepo);
     }
     return localRepo;
   }
 }
Ejemplo n.º 13
0
 public String findLeastFullRepository(Map<String, PlatformUtil.DF> repoMap) {
   String mostFree = null;
   for (String repo : repoMap.keySet()) {
     PlatformUtil.DF df = repoMap.get(repo);
     if (df != null) {
       if (mostFree == null || (repoMap.get(mostFree)).getAvail() < df.getAvail()) {
         mostFree = repo;
       }
     }
   }
   return mostFree;
 }
  /*
   * When testing no-pdf-check basic XML parsing, you will get partial MD records
   * depending on whether the info comes from dataset.xml or from main.xml
   */
  private void validateDatasetMetadataRecord(ArticleMetadata am) {
    log.debug3("valideDatasetMetadatRecord");
    String doi_val = am.get(MetadataField.FIELD_DOI);
    assertEquals(common_issn, am.get(MetadataField.FIELD_ISSN));

    log.debug3("doi val is: " + doi_val);
    // The dataset doesn't set this value, it'll fail over the main.xml value
    if (doi_val.equals("10.1016/S0140-1111(14)61865-1")) {
      assertEquals(null, am.get(MetadataField.FIELD_DATE));
    } else {
      assertEquals(dateMap.get(doi_val), am.get(MetadataField.FIELD_DATE));
    }
    assertEquals(pubTitleMap.get(doi_val), am.get(MetadataField.FIELD_PUBLICATION_TITLE));
  }
 static String canonRoot(String root) {
   synchronized (canonicalRoots) {
     String canon = (String) canonicalRoots.get(root);
     if (canon == null) {
       try {
         canon = new File(root).getCanonicalPath();
         canonicalRoots.put(root, canon);
       } catch (IOException e) {
         logger.warning("Can't canonicalize: " + root, e);
         return root;
       }
     }
     return canon;
   }
 }
Ejemplo n.º 16
0
 /**
  * Return the number of TdbTitles in this Tdb.
  *
  * @return the total TdbTitle count
  */
 public int getTdbTitleCount() {
   int titleCount = 0;
   for (TdbPublisher publisher : tdbPublisherMap.values()) {
     titleCount += publisher.getTdbTitleCount();
   }
   return titleCount;
 }
  /*
   * You will have to tell it the DOI and the schema because those normally come from dataset
   */
  private void validateSingleMainMetadataRecord(ArticleMetadata am, String doi_val, String schema) {
    log.debug3("valideSingleMainMetadatRecord");
    if ("simple-article".equals(schema)) {
      assertEquals(common_simple_article_title, am.get(MetadataField.FIELD_ARTICLE_TITLE));
    } else {
      assertEquals(common_article_title, am.get(MetadataField.FIELD_ARTICLE_TITLE));
    }

    log.debug3("doi val is: " + doi_val);
    assertEquals(authorMap.get(doi_val), am.getList(MetadataField.FIELD_AUTHOR));
    assertEquals(volMap.get(doi_val), am.get(MetadataField.FIELD_VOLUME));
    assertEquals(issueMap.get(doi_val), am.get(MetadataField.FIELD_ISSUE));
    assertEquals("Comment", am.getRaw(ElsevierMainDTD5XmlSchemaHelper.common_dochead));
    assertEquals(doi_val, am.getRaw(ElsevierMainDTD5XmlSchemaHelper.common_doi));
    assertEquals("2014", am.getRaw(ElsevierMainDTD5XmlSchemaHelper.common_copyright));
  }
Ejemplo n.º 18
0
 /**
  * Add TdbAus for like (starts with) the specified TdbAu name.
  *
  * @param tdbAuName the name of the AU to select
  * @param tdbAus the collection to add to
  * @return <code>true</code> if TdbAus were added to the collection
  */
 public boolean getTdbAusLikeName(String tdbAuName, Collection<TdbAu> aus) {
   boolean added = false;
   for (TdbPublisher publisher : tdbPublisherMap.values()) {
     added |= publisher.getTdbAusLikeName(tdbAuName, aus);
   }
   return added;
 }
 /**
  * Finds the directory for this AU. If none found in the map, designates a new dir for it.
  *
  * @param auid AU id representing the au
  * @param repoRoot path to the root of the repository
  * @return the dir String
  */
 static String getAuDir(String auid, String repoRoot, boolean create) {
   String repoCachePath = extendCacheLocation(repoRoot);
   LocalRepository localRepo = getLocalRepository(repoRoot);
   synchronized (localRepo) {
     Map aumap = localRepo.getAuMap();
     String auPathSlash = (String) aumap.get(auid);
     if (auPathSlash != null) {
       return auPathSlash;
     }
     if (!create) {
       return null;
     }
     logger.debug3("Creating new au directory for '" + auid + "'.");
     String auDir = localRepo.getPrevAuDir();
     for (int cnt = RepositoryManager.getMaxUnusedDirSearch(); cnt > 0; cnt--) {
       // loop through looking for an available dir
       auDir = getNextDirName(auDir);
       File testDir = new File(repoCachePath, auDir);
       if (logger.isDebug3()) logger.debug3("Probe for unused: " + testDir);
       if (!testDir.exists()) {
         if (RepositoryManager.isStatefulUnusedDirSearch()) {
           localRepo.setPrevAuDir(auDir);
         }
         String auPath = testDir.toString();
         logger.debug3("New au directory: " + auPath);
         auPathSlash = auPath + File.separator;
         // write the new au property file to the new dir
         // XXX this data should be backed up elsewhere to avoid single-point
         // corruption
         Properties idProps = new Properties();
         idProps.setProperty(AU_ID_PROP, auid);
         saveAuIdProperties(auPath, idProps);
         aumap.put(auid, auPathSlash);
         return auPathSlash;
       } else {
         if (logger.isDebug3()) {
           logger.debug3("Existing directory found at '" + auDir + "'.  Checking next...");
         }
       }
     }
   }
   throw new RuntimeException(
       "Can't find unused repository dir after "
           + RepositoryManager.getMaxUnusedDirSearch()
           + " tries in "
           + repoCachePath);
 }
Ejemplo n.º 20
0
 public void setRepoMap(Map<String, PlatformUtil.DF> repoMap) {
   List repos = new ArrayList();
   this.repoMap = repoMap;
   for (String repo : repoMap.keySet()) {
     repos.add(repo);
   }
   setRepos(repos);
 }
Ejemplo n.º 21
0
 /**
  * Adds to a collection of TdbTitles like (starts with) the specified title name across all
  * publishers.
  *
  * @param titleName the title name
  * @param titles a collection of matching titles
  * @return a collection of TdbTitles that match the title name
  */
 public boolean getTdbTitlesLikeName(String titleName, Collection<TdbTitle> titles) {
   boolean added = false;
   if (titleName != null) {
     for (TdbPublisher publisher : tdbPublisherMap.values()) {
       added |= publisher.getTdbTitlesLikeName(titleName, titles);
     }
   }
   return added;
 }
Ejemplo n.º 22
0
 /** Add to a collection of TdbAus for this TDB that match the ISBN. */
 public boolean getTdbAusByIsbn(String isbn, Collection<TdbAu> matchingTdbAus) {
   boolean added = false;
   if (isbn != null) {
     for (TdbPublisher tdbPublisher : tdbPublisherMap.values()) {
       added |= tdbPublisher.getTdbAusByIsbn(matchingTdbAus, isbn);
     }
   }
   return added;
 }
Ejemplo n.º 23
0
 /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
 void processOverrides(TypedEntryMap map) {
   String testMode = getTestingMode();
   if (StringUtil.isNullString(testMode)) {
     return;
   }
   Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
   if (o == null) {
     return;
   }
   if (o instanceof Map) {
     Map overrideMap = (Map) o;
     for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
       String key = (String) entry.getKey();
       Object val = entry.getValue();
       log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
       map.setMapElement(key, val);
     }
   }
 }
Ejemplo n.º 24
0
 /**
  * Get the title for the specified titleId.
  *
  * @param titleId the titleID
  * @return the title for the titleId or <code>null</code. if not found
  */
 public TdbTitle getTdbTitleById(String titleId) {
   if (titleId != null) {
     for (TdbPublisher publisher : tdbPublisherMap.values()) {
       TdbTitle title = publisher.getTdbTitleById(titleId);
       if (title != null) {
         return title;
       }
     }
   }
   return null;
 }
Ejemplo n.º 25
0
  public void testText() throws Exception {
    MockStatusAccessor statusAccessor =
        MockStatusAccessor.generateStatusAccessor(colArray1, rowArray1);
    statusAccessor.setTitle("testtbl", null);
    statSvc.registerStatusAccessor("testtbl", statusAccessor);

    WebResponse resp = getTable("testtbl", true);
    assertResponseOk(resp);
    assertEquals("Content type", "text/plain", resp.getContentType());
    log.debug(resp.getText());
    List lines = getLines(resp);
    assertEquals(rowArray1.length + 3, lines.size());
    Map row0 = getRow((String) lines.get(0));
    assertEquals("2.4.6.8", row0.get("host"));

    Map row2 = getRow((String) lines.get(2));
    assertEquals("testtbl", row2.get("table"));

    assertEqualTables(table1, lines);
  }
Ejemplo n.º 26
0
 /**
  * Get a title for the specified issn.
  *
  * @param issn the issn
  * @return the title for the titleId or <code>null</code. if not found
  */
 public TdbTitle getTdbTitleByIssn(String issn) {
   if (issn != null) {
     for (TdbPublisher publisher : tdbPublisherMap.values()) {
       TdbTitle title = publisher.getTdbTitleByIssn(issn);
       if (title != null) {
         return title;
       }
     }
   }
   return null;
 }
Ejemplo n.º 27
0
  public void testHeadNotAllowed() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(0);
    assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());

    // Same, but with an excluded version that doesn't matter.
    repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash2)));
    assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());

    // Same, but with an excluded version that does matter
    repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash1)));
    assertEmpty(repairCandidates);
  }
Ejemplo n.º 28
0
  /**
   * Adds a collection of pluginIds for TdbAus that are different from those in this Tdb.
   *
   * @param pluginIds the set of pluginIds
   * @param otherTdb a Tdb
   */
  private void addPluginIdsForDifferences(Set<String> pluginIds, Tdb otherTdb) {
    Map<String, TdbPublisher> tdbPublishers = otherTdb.getAllTdbPublishers();

    for (TdbPublisher tdbPublisher : tdbPublishers.values()) {
      if (!this.tdbPublisherMap.containsKey(tdbPublisher.getName())) {
        // add pluginIds for publishers in tdb that are not in this Tdb
        tdbPublisher.addAllPluginIds(pluginIds);
      }
    }

    for (TdbPublisher thisPublisher : tdbPublisherMap.values()) {
      TdbPublisher tdbPublisher = tdbPublishers.get(thisPublisher.getName());
      if (tdbPublisher == null) {
        // add pluginIds for publisher in this Tdb that is not in tdb
        thisPublisher.addAllPluginIds(pluginIds);
      } else {
        // add pluginIds for publishers in both Tdbs that are different
        thisPublisher.addPluginIdsForDifferences(pluginIds, tdbPublisher);
      }
    }
  }
Ejemplo n.º 29
0
  /**
   * Add a new TdbAu to this title database. The TdbAu must have its pluginID, and title set. The
   * TdbAu''s title must also have its titleId and publisher set. The publisher name must be unique
   * to all publishers in this Tdb.
   *
   * @param au the TdbAu to add.
   * @throws TdbException if Tdb is sealed, this is a duplicate au, or the au's publisher is a
   *     duplicate
   */
  public void addTdbAu(TdbAu au) throws TdbException {
    if (au == null) {
      throw new IllegalArgumentException("TdbAu cannot be null");
    }

    // verify not sealed
    if (isSealed()) {
      throw new TdbException("Cannot add TdbAu to sealed Tdb");
    }

    // validate title
    TdbTitle title = au.getTdbTitle();
    if (title == null) {
      throw new IllegalArgumentException("TdbAu's title not set");
    }

    // validate publisher
    TdbPublisher publisher = title.getTdbPublisher();
    if (publisher == null) {
      throw new IllegalArgumentException("TdbAu's publisher not set");
    }

    // make sure publisher is not a duplicate
    String pubName = publisher.getName();
    TdbPublisher oldPublisher = tdbPublisherMap.put(pubName, publisher);
    if ((oldPublisher != null) && (oldPublisher != publisher)) {
      // restore old publisher and report error
      tdbPublisherMap.put(pubName, oldPublisher);
      throw new TdbException("New au publisher with duplicate name: " + pubName);
    }

    // register the au with this instance
    if (!addTdbAuForPlugin(au)) {
      // remove new publisher and report error
      if (oldPublisher == null) {
        tdbPublisherMap.remove(pubName);
      }
      throw new TdbException("Cannot register au " + au.getName());
    }
  }
    /**
     * Return the auid -> au-subdir-path mapping. Enumerating the directories if necessary to
     * initialize the map
     */
    Map getAuMap() {
      if (auMap == null) {
        logger.debug3("Loading name map for '" + repoCacheFile + "'.");
        auMap = new HashMap();
        if (!repoCacheFile.exists()) {
          logger.debug3("Creating cache dir:" + repoCacheFile + "'.");
          if (!repoCacheFile.mkdirs()) {
            logger.critical("Couldn't create directory, check owner/permissions: " + repoCacheFile);
            // return empty map
            return auMap;
          }
        } else {
          // read each dir's property file and store mapping auid -> dir
          File[] auDirs = repoCacheFile.listFiles();
          for (int ii = 0; ii < auDirs.length; ii++) {
            String dirName = auDirs[ii].getName();
            //       if (dirName.compareTo(lastPluginDir) == 1) {
            //         // adjust the 'lastPluginDir' upwards if necessary
            //         lastPluginDir = dirName;
            //       }

            String path = auDirs[ii].getAbsolutePath();
            Properties idProps = getAuIdProperties(path);
            if (idProps != null) {
              String auid = idProps.getProperty(AU_ID_PROP);
              StringBuilder sb = new StringBuilder(path.length() + File.separator.length());
              sb.append(path);
              sb.append(File.separator);
              auMap.put(auid, sb.toString());
              logger.debug3("Mapping to: " + auMap.get(auid) + ": " + auid);
            } else {
              logger.debug3("Not mapping " + path + ", no auid file.");
            }
          }
        }
      }
      return auMap;
    }