public void setUp() throws Exception {
super.setUp();
translator = new ItemToObjectTranslator(Model.getInstanceByName("testmodel"), null);
translator.setObjectStore(ObjectStoreFactory.getObjectStore("os.unittest"));
translator.idToNamespace.put(new Integer(0), "fish");
translator.namespaceToId.put("fish", new Integer(0));
}
public static void oneTimeSetUp() throws Exception {
os =
new ObjectStoreTranslatingImpl(
Model.getInstanceByName("testmodel"),
ObjectStoreFactory.getObjectStore("os.unittest"),
new DummyTranslator());
ObjectStoreAbstractImplTestCase.oneTimeSetUp();
}
private ObjectStore getObjectStore() throws Exception {
if (osAlias == null) {
throw new BuildException("objectStoreWriter attribute is not set");
}
if (os == null) {
System.out.println("Connecting to db: " + osAlias);
os = ObjectStoreFactory.getObjectStore(osAlias);
}
return os;
}
/** {@inheritDoc} */
@Override
public void execute() {
if (integrationWriter == null) {
throw new BuildException("integrationWriter attribute is not set");
}
if (source == null) {
throw new BuildException("source attribute is not set");
}
try {
IntegrationWriter iw = IntegrationWriterFactory.getIntegrationWriter(integrationWriter);
PriorityConfig.verify(iw.getModel(), allSources);
iw.setIgnoreDuplicates(ignoreDuplicates);
if (queryClass != null) {
Class<?> tmpQueryClass = Class.forName(queryClass);
if (!FastPathObject.class.isAssignableFrom(tmpQueryClass)) {
throw new ClassCastException(
"Class " + queryClass + " is not a subclass of " + "FastPathObject");
}
@SuppressWarnings("unchecked")
Class<? extends FastPathObject> tmp2QueryClass = (Class) tmpQueryClass;
new ObjectStoreDataLoader(iw)
.process(
ObjectStoreFactory.getObjectStore(source),
iw.getMainSource(sourceName, sourceType),
iw.getSkeletonSource(sourceName, sourceType),
tmp2QueryClass);
} else {
new ObjectStoreDataLoader(iw)
.process(
ObjectStoreFactory.getObjectStore(source),
iw.getMainSource(sourceName, sourceType),
iw.getSkeletonSource(sourceName, sourceType));
}
} catch (Exception e) {
throw new BuildException(e);
}
}
public void testTranslation() throws Exception {
ObjectStore os2 =
new ObjectStoreTranslatingImpl(
Model.getInstanceByName("testmodel"),
ObjectStoreFactory.getObjectStore("os.unittest"),
new CompanyTranslator());
Query q = new Query();
QueryClass qc = new QueryClass(InterMineObject.class);
q.addToSelect(qc);
q.addFrom(qc);
Results res = os2.execute(q);
assertEquals(2, res.size());
assertEquals("CompanyA", ((Bank) ((ResultsRow) res.get(0)).get(0)).getName());
assertEquals("CompanyB", ((Bank) ((ResultsRow) res.get(1)).get(0)).getName());
}
public void setUp() throws Exception {
os = ObjectStoreFactory.getObjectStore("os.unittest");
item = new Item();
item.setClassName("Department");
item.setImplementations("Broke");
item.setIdentifier("1");
Attribute attr1 = new Attribute();
attr1.setName("name");
attr1.setValue("Department1");
item.addAttribute(attr1);
Attribute attr2 = new Attribute();
attr2.setName("debt");
attr2.setValue("10");
item.addAttribute(attr2);
Reference ref1 = new Reference();
ref1.setName("address");
ref1.setRefId("2");
item.addReference(ref1);
ReferenceList col1 = new ReferenceList();
col1.setName("employees");
col1.addRefId("3");
col1.addRefId("4");
item.addCollection(col1);
dbItem = new org.intermine.model.fulldata.Item();
dbItem.setClassName("Department");
dbItem.setImplementations("Broke");
dbItem.setIdentifier("1");
dbItem.setId(1001);
org.intermine.model.fulldata.Attribute dbAttr1 = new org.intermine.model.fulldata.Attribute();
dbAttr1.setName("name");
dbAttr1.setValue("Department1");
dbAttr1.setItem(dbItem);
dbItem.addAttributes(dbAttr1);
org.intermine.model.fulldata.Attribute dbAttr2 = new org.intermine.model.fulldata.Attribute();
dbAttr2.setName("debt");
dbAttr2.setValue("10");
dbAttr2.setItem(dbItem);
dbItem.addAttributes(dbAttr2);
org.intermine.model.fulldata.Reference dbRef1 = new org.intermine.model.fulldata.Reference();
dbRef1.setName("address");
dbRef1.setRefId("2");
dbRef1.setItem(dbItem);
dbItem.addReferences(dbRef1);
org.intermine.model.fulldata.ReferenceList dbCol1 =
new org.intermine.model.fulldata.ReferenceList();
dbCol1.setName("employees");
dbCol1.setRefIds("3 4");
dbCol1.setItem(dbItem);
dbItem.addCollections(dbCol1);
referenceList = new ReferenceList();
referenceList.setName("employees");
referenceList.setRefIds(Arrays.asList("3", "4"));
dbReferenceList = new org.intermine.model.fulldata.ReferenceList();
dbReferenceList.setName("employees");
dbReferenceList.setRefIds("3 4");
departmentProxy = new ProxyReference(os, 1, Department.class);
dbReferenceList.proxyItem(departmentProxy);
// dbReferenceList.setId(2002);
}
public void setUp() throws Exception {
super.setUp();
os = ObjectStoreFactory.getObjectStore("os.unittest");
}
/**
* Synchronize publications with pubmed using pmid
*
* @throws Exception if an error occurs
*/
public void execute() throws Exception {
// Needed so that STAX can find it's implementation classes
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
Database db = null;
Transaction txn = null;
try {
if (osAlias == null) {
throw new BuildException("osAlias attribute is not set");
}
if (outputFile == null) {
throw new BuildException("outputFile attribute is not set");
}
// environment is transactional
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment env = new Environment(new File(cacheDirName), envConfig);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setSortedDuplicates(true);
db = env.openDatabase(null, "publications_db", dbConfig);
txn = env.beginTransaction(null, null);
LOG.info("Starting EntrezPublicationsRetriever");
Writer writer = new FileWriter(outputFile);
ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);
Set<Integer> idsToFetch = new HashSet<Integer>();
itemFactory = new ItemFactory(os.getModel(), "-1_");
writer.write(FullRenderer.getHeader() + ENDL);
for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
String pubMedId = iter.next().getPubMedId();
Integer pubMedIdInteger;
try {
pubMedIdInteger = Integer.valueOf(pubMedId);
} catch (NumberFormatException e) {
// not a pubmed id
continue;
}
if (seenPubMeds.contains(pubMedIdInteger)) {
continue;
}
DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
DatabaseEntry data = new DatabaseEntry();
if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
try {
ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
Map<String, Object> pubMap = (Map) deserializer.readObject();
writeItems(writer, mapToItems(itemFactory, pubMap));
seenPubMeds.add(pubMedIdInteger);
} catch (EOFException e) {
// ignore and fetch it again
System.err.println(
"found in cache, but igored due to cache problem: " + pubMedIdInteger);
}
} else {
idsToFetch.add(pubMedIdInteger);
}
}
Iterator<Integer> idIter = idsToFetch.iterator();
Set<Integer> thisBatch = new HashSet<Integer>();
while (idIter.hasNext()) {
Integer pubMedIdInteger = idIter.next();
thisBatch.add(pubMedIdInteger);
if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
try {
// the server may return less publications than we ask for, so keep a Map
Map<String, Map<String, Object>> fromServerMap = null;
for (int i = 0; i < MAX_TRIES; i++) {
BufferedReader br = new BufferedReader(getReader(thisBatch));
StringBuffer buf = new StringBuffer();
String line;
while ((line = br.readLine()) != null) {
buf.append(line + "\n");
}
fromServerMap = new HashMap<String, Map<String, Object>>();
Throwable throwable = null;
try {
if (loadFullRecord) {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new FullRecordHandler(fromServerMap),
false);
} else {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new SummaryRecordHandler(fromServerMap),
false);
}
} catch (Throwable e) {
LOG.error("Couldn't parse PubMed XML", e);
// try again or re-throw the Throwable
throwable = e;
}
if (i == MAX_TRIES) {
throw new RuntimeException(
"failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
throwable);
} else {
if (throwable != null) {
// try again
continue;
}
}
for (String id : fromServerMap.keySet()) {
writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
}
addToDb(txn, db, fromServerMap);
break;
}
thisBatch.clear();
} finally {
txn.commit();
// start a new transaction incase there is an exception while parsing
txn = env.beginTransaction(null, null);
}
}
}
writeItems(writer, authorMap.values());
writeItems(writer, meshTerms.values());
writer.write(FullRenderer.getFooter() + ENDL);
writer.flush();
writer.close();
} catch (Throwable e) {
throw new RuntimeException("failed to get all publications", e);
} finally {
txn.commit();
db.close();
Thread.currentThread().setContextClassLoader(cl);
}
}
