public void setUp() throws Exception {
   super.setUp();
   translator = new ItemToObjectTranslator(Model.getInstanceByName("testmodel"), null);
   translator.setObjectStore(ObjectStoreFactory.getObjectStore("os.unittest"));
   translator.idToNamespace.put(new Integer(0), "fish");
   translator.namespaceToId.put("fish", new Integer(0));
 }
 public static void oneTimeSetUp() throws Exception {
   os =
       new ObjectStoreTranslatingImpl(
           Model.getInstanceByName("testmodel"),
           ObjectStoreFactory.getObjectStore("os.unittest"),
           new DummyTranslator());
   ObjectStoreAbstractImplTestCase.oneTimeSetUp();
 }
 private ObjectStore getObjectStore() throws Exception {
   if (osAlias == null) {
     throw new BuildException("objectStoreWriter attribute is not set");
   }
   if (os == null) {
     System.out.println("Connecting to db: " + osAlias);
     os = ObjectStoreFactory.getObjectStore(osAlias);
   }
   return os;
 }
  /** {@inheritDoc} */
  @Override
  public void execute() {
    if (integrationWriter == null) {
      throw new BuildException("integrationWriter attribute is not set");
    }
    if (source == null) {
      throw new BuildException("source attribute is not set");
    }

    try {
      IntegrationWriter iw = IntegrationWriterFactory.getIntegrationWriter(integrationWriter);
      PriorityConfig.verify(iw.getModel(), allSources);
      iw.setIgnoreDuplicates(ignoreDuplicates);
      if (queryClass != null) {
        Class<?> tmpQueryClass = Class.forName(queryClass);
        if (!FastPathObject.class.isAssignableFrom(tmpQueryClass)) {
          throw new ClassCastException(
              "Class " + queryClass + " is not a subclass of " + "FastPathObject");
        }
        @SuppressWarnings("unchecked")
        Class<? extends FastPathObject> tmp2QueryClass = (Class) tmpQueryClass;
        new ObjectStoreDataLoader(iw)
            .process(
                ObjectStoreFactory.getObjectStore(source),
                iw.getMainSource(sourceName, sourceType),
                iw.getSkeletonSource(sourceName, sourceType),
                tmp2QueryClass);

      } else {
        new ObjectStoreDataLoader(iw)
            .process(
                ObjectStoreFactory.getObjectStore(source),
                iw.getMainSource(sourceName, sourceType),
                iw.getSkeletonSource(sourceName, sourceType));
      }
    } catch (Exception e) {
      throw new BuildException(e);
    }
  }
 public void testTranslation() throws Exception {
   ObjectStore os2 =
       new ObjectStoreTranslatingImpl(
           Model.getInstanceByName("testmodel"),
           ObjectStoreFactory.getObjectStore("os.unittest"),
           new CompanyTranslator());
   Query q = new Query();
   QueryClass qc = new QueryClass(InterMineObject.class);
   q.addToSelect(qc);
   q.addFrom(qc);
   Results res = os2.execute(q);
   assertEquals(2, res.size());
   assertEquals("CompanyA", ((Bank) ((ResultsRow) res.get(0)).get(0)).getName());
   assertEquals("CompanyB", ((Bank) ((ResultsRow) res.get(1)).get(0)).getName());
 }
Beispiel #6
0
  public void setUp() throws Exception {
    os = ObjectStoreFactory.getObjectStore("os.unittest");

    item = new Item();
    item.setClassName("Department");
    item.setImplementations("Broke");
    item.setIdentifier("1");
    Attribute attr1 = new Attribute();
    attr1.setName("name");
    attr1.setValue("Department1");
    item.addAttribute(attr1);
    Attribute attr2 = new Attribute();
    attr2.setName("debt");
    attr2.setValue("10");
    item.addAttribute(attr2);
    Reference ref1 = new Reference();
    ref1.setName("address");
    ref1.setRefId("2");
    item.addReference(ref1);
    ReferenceList col1 = new ReferenceList();
    col1.setName("employees");
    col1.addRefId("3");
    col1.addRefId("4");
    item.addCollection(col1);

    dbItem = new org.intermine.model.fulldata.Item();
    dbItem.setClassName("Department");
    dbItem.setImplementations("Broke");
    dbItem.setIdentifier("1");
    dbItem.setId(1001);
    org.intermine.model.fulldata.Attribute dbAttr1 = new org.intermine.model.fulldata.Attribute();
    dbAttr1.setName("name");
    dbAttr1.setValue("Department1");
    dbAttr1.setItem(dbItem);
    dbItem.addAttributes(dbAttr1);
    org.intermine.model.fulldata.Attribute dbAttr2 = new org.intermine.model.fulldata.Attribute();
    dbAttr2.setName("debt");
    dbAttr2.setValue("10");
    dbAttr2.setItem(dbItem);
    dbItem.addAttributes(dbAttr2);
    org.intermine.model.fulldata.Reference dbRef1 = new org.intermine.model.fulldata.Reference();
    dbRef1.setName("address");
    dbRef1.setRefId("2");
    dbRef1.setItem(dbItem);
    dbItem.addReferences(dbRef1);
    org.intermine.model.fulldata.ReferenceList dbCol1 =
        new org.intermine.model.fulldata.ReferenceList();
    dbCol1.setName("employees");
    dbCol1.setRefIds("3 4");
    dbCol1.setItem(dbItem);
    dbItem.addCollections(dbCol1);

    referenceList = new ReferenceList();
    referenceList.setName("employees");
    referenceList.setRefIds(Arrays.asList("3", "4"));

    dbReferenceList = new org.intermine.model.fulldata.ReferenceList();
    dbReferenceList.setName("employees");
    dbReferenceList.setRefIds("3 4");
    departmentProxy = new ProxyReference(os, 1, Department.class);
    dbReferenceList.proxyItem(departmentProxy);
    // dbReferenceList.setId(2002);
  }
  public void setUp() throws Exception {
    super.setUp();

    os = ObjectStoreFactory.getObjectStore("os.unittest");
  }
  /**
   * Synchronize publications with pubmed using pmid
   *
   * @throws Exception if an error occurs
   */
  public void execute() throws Exception {
    // Needed so that STAX can find it's implementation classes
    ClassLoader cl = Thread.currentThread().getContextClassLoader();

    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());

    Database db = null;
    Transaction txn = null;
    try {
      if (osAlias == null) {
        throw new BuildException("osAlias attribute is not set");
      }
      if (outputFile == null) {
        throw new BuildException("outputFile attribute is not set");
      }

      // environment is transactional
      EnvironmentConfig envConfig = new EnvironmentConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);

      Environment env = new Environment(new File(cacheDirName), envConfig);

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setAllowCreate(true);
      dbConfig.setSortedDuplicates(true);

      db = env.openDatabase(null, "publications_db", dbConfig);

      txn = env.beginTransaction(null, null);

      LOG.info("Starting EntrezPublicationsRetriever");

      Writer writer = new FileWriter(outputFile);
      ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);

      Set<Integer> idsToFetch = new HashSet<Integer>();
      itemFactory = new ItemFactory(os.getModel(), "-1_");
      writer.write(FullRenderer.getHeader() + ENDL);
      for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
        String pubMedId = iter.next().getPubMedId();
        Integer pubMedIdInteger;
        try {
          pubMedIdInteger = Integer.valueOf(pubMedId);
        } catch (NumberFormatException e) {
          // not a pubmed id
          continue;
        }

        if (seenPubMeds.contains(pubMedIdInteger)) {
          continue;
        }
        DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
        DatabaseEntry data = new DatabaseEntry();
        if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
          try {
            ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
            ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
            Map<String, Object> pubMap = (Map) deserializer.readObject();
            writeItems(writer, mapToItems(itemFactory, pubMap));
            seenPubMeds.add(pubMedIdInteger);
          } catch (EOFException e) {
            // ignore and fetch it again
            System.err.println(
                "found in cache, but igored due to cache problem: " + pubMedIdInteger);
          }
        } else {
          idsToFetch.add(pubMedIdInteger);
        }
      }

      Iterator<Integer> idIter = idsToFetch.iterator();
      Set<Integer> thisBatch = new HashSet<Integer>();
      while (idIter.hasNext()) {
        Integer pubMedIdInteger = idIter.next();
        thisBatch.add(pubMedIdInteger);
        if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
          try {
            // the server may return less publications than we ask for, so keep a Map
            Map<String, Map<String, Object>> fromServerMap = null;

            for (int i = 0; i < MAX_TRIES; i++) {
              BufferedReader br = new BufferedReader(getReader(thisBatch));
              StringBuffer buf = new StringBuffer();
              String line;
              while ((line = br.readLine()) != null) {
                buf.append(line + "\n");
              }
              fromServerMap = new HashMap<String, Map<String, Object>>();
              Throwable throwable = null;
              try {
                if (loadFullRecord) {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new FullRecordHandler(fromServerMap),
                      false);
                } else {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new SummaryRecordHandler(fromServerMap),
                      false);
                }
              } catch (Throwable e) {
                LOG.error("Couldn't parse PubMed XML", e);
                // try again or re-throw the Throwable
                throwable = e;
              }
              if (i == MAX_TRIES) {
                throw new RuntimeException(
                    "failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
                    throwable);
              } else {
                if (throwable != null) {
                  // try again
                  continue;
                }
              }

              for (String id : fromServerMap.keySet()) {
                writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
              }
              addToDb(txn, db, fromServerMap);
              break;
            }
            thisBatch.clear();
          } finally {
            txn.commit();
            // start a new transaction incase there is an exception while parsing
            txn = env.beginTransaction(null, null);
          }
        }
      }
      writeItems(writer, authorMap.values());
      writeItems(writer, meshTerms.values());
      writer.write(FullRenderer.getFooter() + ENDL);
      writer.flush();
      writer.close();
    } catch (Throwable e) {
      throw new RuntimeException("failed to get all publications", e);
    } finally {
      txn.commit();
      db.close();
      Thread.currentThread().setContextClassLoader(cl);
    }
  }