/**
* Generate a .hashCode() method for the given class.
*
* @param cld descriptor for the class in question
* @return generate java code as a string
*/
protected String generateHashCode(ClassDescriptor cld) {
if (cld.getFieldDescriptorByName("id") != null) {
StringBuffer sb = new StringBuffer();
sb.append(INDENT)
.append("@Override public int hashCode() { ")
.append("return (id != null) ? id.hashCode() : super.hashCode(); ")
.append("}" + ENDL);
return sb.toString();
} else {
return "";
}
}
private void setNames(
List<?> names,
String symbol,
List<String> synonyms,
Set<Item> synonymsToAdd,
String primaryIdentifier,
Item feature,
ClassDescriptor cd) {
if (cd.getFieldDescriptorByName("symbol") == null) { // if symbol is not in the model
String name = (String) names.get(0);
feature.setAttribute("name", name);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
} else {
if (symbol == null) {
symbol = (String) names.get(0);
}
feature.setAttribute("symbol", symbol);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
}
}
/**
* Generate a .equals() method for the given class.
*
* @param cld descriptor for class in question
* @return generated java code as string
*/
protected String generateEquals(ClassDescriptor cld) {
if (cld.getFieldDescriptorByName("id") != null) {
String unqualifiedName = TypeUtil.unqualifiedName(cld.getName());
StringBuffer sb = new StringBuffer();
sb.append(INDENT)
.append("@Override public boolean equals(Object o) { return (o instanceof ")
.append(unqualifiedName)
.append(" && id != null) ? id.equals(((")
.append(unqualifiedName)
.append(")o).getId()) : this == o; }" + ENDL);
return sb.toString();
} else {
return "";
}
}
/**
* Returns whether this primary key can be fetched now.
*
* @param pk the PrimaryKey
* @param cld the ClassDescriptor that the PrimaryKey is in
* @param pksNotDone a Map of pks not yet fetched
* @return a boolean
*/
protected boolean canDoPkNow(
PrimaryKey pk, ClassDescriptor cld, Map<PrimaryKey, ClassDescriptor> pksNotDone) {
boolean canDoPkNow = true;
Iterator<String> fieldNameIter = pk.getFieldNames().iterator();
while (fieldNameIter.hasNext() && canDoPkNow) {
String fieldName = fieldNameIter.next();
FieldDescriptor fd = cld.getFieldDescriptorByName(fieldName);
if (fd.isReference()) {
Iterator<ClassDescriptor> otherCldIter = pksNotDone.values().iterator();
while (otherCldIter.hasNext() && canDoPkNow) {
ClassDescriptor otherCld = otherCldIter.next();
Class<? extends FastPathObject> fieldClass =
((ReferenceDescriptor) fd).getReferencedClassDescriptor().getType();
if (otherCld.getType().isAssignableFrom(fieldClass)
|| fieldClass.isAssignableFrom(otherCld.getType())) {
canDoPkNow = false;
}
}
}
}
return canDoPkNow;
}
private void setNames(
List<?> names, Set<Item> synonymsToAdd, String recordId, Item feature, ClassDescriptor cd) {
if (cd.getFieldDescriptorByName("symbol") == null) {
String name = (String) names.get(0);
feature.setAttribute("name", name);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(recordId) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
} else {
String symbol = (String) names.get(0);
feature.setAttribute("symbol", (String) names.get(0));
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(recordId) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
}
/**
* Create a new DelimitedFileConfiguration from an InputStream.
*
* @param model The model to use when looking for ClassDescriptors and FieldDescriptors
* @param inputStream The InputStream to read the configuration from
* @throws IOException throws if the read fails
*/
public DelimitedFileConfiguration(Model model, InputStream inputStream) throws IOException {
Properties properties = new Properties();
properties.load(inputStream);
String className = properties.getProperty("className");
if (className == null) {
throw new IllegalArgumentException(
"className not set in property file for " + "DelimitedFileConfiguration");
}
configClassDescriptor = model.getClassDescriptorByName(className);
if (configClassDescriptor == null) {
throw new IllegalArgumentException("cannot find ClassDescriptor for: " + className);
}
Map columnFieldDescriptorMap = new TreeMap();
Enumeration enumeration = properties.propertyNames();
while (enumeration.hasMoreElements()) {
String key = (String) enumeration.nextElement();
if (key.startsWith("column.")) {
String columnNumberString = key.substring(7);
try {
int keyColumnNumber = Integer.valueOf(columnNumberString).intValue();
String fieldName = properties.getProperty(key);
FieldDescriptor columnFD = configClassDescriptor.getFieldDescriptorByName(fieldName);
if (columnFD == null) {
throw new IllegalArgumentException(
"cannot find FieldDescriptor for " + fieldName + " in " + className);
}
if (!columnFD.isAttribute()) {
String message =
"field: "
+ fieldName
+ " in "
+ className
+ " is not an attribute field so cannot be used as a "
+ "className in DelimitedFileConfiguration";
throw new IllegalArgumentException(message);
}
columnFieldDescriptorMap.put(new Integer(keyColumnNumber), columnFD);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"column number ("
+ key
+ ") not parsable "
+ "in property file for "
+ "DelimitedFileConfiguration");
}
}
}
int mapMax = findMapMaxKey(columnFieldDescriptorMap);
columnFieldDescriptors = new ArrayList(mapMax + 1);
for (int columnNumber = 0; columnNumber < mapMax + 1; columnNumber++) {
FieldDescriptor columnFD =
(FieldDescriptor) columnFieldDescriptorMap.get(new Integer(columnNumber));
columnFieldDescriptors.add(columnFD);
}
}
/**
* process GFF3 record and give a xml presentation
*
* @param record GFF3Record
* @throws ObjectStoreException if an error occurs storing items
*/
public void process(GFF3Record record) throws ObjectStoreException {
String identifier = record.getId();
String refId = identifierMap.get(identifier);
// get rid of previous record Items from handler
handler.clear();
List<?> names = record.getNames();
Item seq = getSeq(record.getSequenceID());
String term = record.getType();
String className = TypeUtil.javaiseClassName(term);
String fullClassName = tgtModel.getPackageName() + "." + className;
ClassDescriptor cd = tgtModel.getClassDescriptorByName(fullClassName);
if (cd == null) {
throw new IllegalArgumentException(
"no class found in model for: "
+ className
+ " (original GFF record type: "
+ term
+ ") for "
+ "record: "
+ record);
}
Set<Item> synonymsToAdd = new HashSet<Item>();
Item feature = null;
// new feature
if (refId == null) {
feature = createItem(className);
refId = feature.getIdentifier();
}
if (!"chromosome".equals(record.getType()) && seq != null) {
boolean makeLocation =
record.getStart() >= 1
&& record.getEnd() >= 1
&& !dontCreateLocations
&& handler.createLocations(record);
if (makeLocation) {
Item location = getLocation(record, refId, seq, cd);
if (feature == null) {
// this feature has already been created and stored
// we only wanted the location, we're done here.
store(location);
return;
}
int length = getLength(record);
feature.setAttribute("length", String.valueOf(length));
handler.setLocation(location);
if ("Chromosome".equals(seqClsName)
&& (cd.getFieldDescriptorByName("chromosome") != null)) {
feature.setReference("chromosome", seq.getIdentifier());
feature.setReference("chromosomeLocation", location);
}
}
}
if (feature == null) {
// this feature has already been created and stored
// feature with discontinous location, this location wasn't valid for some reason
return;
}
if (identifier != null) {
feature.setAttribute("primaryIdentifier", identifier);
}
handler.setFeature(feature);
identifierMap.put(identifier, feature.getIdentifier());
if (names != null) {
setNames(names, synonymsToAdd, record.getId(), feature, cd);
}
List<String> parents = record.getParents();
if (parents != null && !parents.isEmpty()) {
setRefsAndCollections(parents, feature);
}
feature.addReference(getOrgRef());
feature.addToCollection("dataSets", dataSet);
handler.addDataSet(dataSet);
Double score = record.getScore();
if (score != null && !"".equals(String.valueOf(score))) {
feature.setAttribute("score", String.valueOf(score));
feature.setAttribute("scoreType", record.getSource());
}
for (Item synonym : synonymsToAdd) {
handler.addItem(synonym);
}
handler.process(record);
if (handler.getDataSetReferenceList().getRefIds().size() > 0) {
feature.addCollection(handler.getDataSetReferenceList());
}
handler.clearDataSetReferenceList();
if (handler.getPublicationReferenceList().getRefIds().size() > 0) {
feature.addCollection(handler.getPublicationReferenceList());
}
handler.clearPublicationReferenceList();
try {
Iterator<Item> iter = handler.getItems().iterator();
while (iter.hasNext()) {
store(iter.next());
}
} catch (ObjectStoreException e) {
LOG.error("Problem writing item to the itemwriter");
throw e;
}
}
/**
* process GFF3 record and give a xml presentation
*
* @param record GFF3Record
* @throws ObjectStoreException if an error occurs storing items
* @throws IOException
*/
public void process(GFF3Record record) throws ObjectStoreException {
String term = record.getType();
if (config_exclude != null
&& !config_exclude.isEmpty()) { // don't process terms in the exclude list
if (config_exclude.containsKey(this.orgTaxonId)) {
if (config_exclude.get(this.orgTaxonId).contains(term)) {
return;
}
}
}
if (config_term != null && !config_term.isEmpty()) { // otherwise all terms are processed
if (config_term.containsKey(this.orgTaxonId)) {
if (!config_term.get(this.orgTaxonId).contains(term)) {
return;
}
}
}
// By default, use ID field in attributes
String primaryIdentifier = record.getId();
// If pid set in gff_config.propeties, look for the attribute field, e.g. locus_tag
if (config_attr.containsKey(this.orgTaxonId)) {
if (config_attr.get(this.orgTaxonId).containsKey("primaryIdentifier")) {
String cls = config_attr_class.get(this.orgTaxonId).get("primaryIdentifier");
if ("all".equals(cls) || term.equals(cls)) {
String pidAttr = config_attr.get(this.orgTaxonId).get("primaryIdentifier");
if (pidAttr.contains("Dbxref") && record.getDbxrefs() != null) {
String pidAttrPrefix = pidAttr.split("\\.")[1];
for (Iterator<?> i = record.getDbxrefs().iterator(); i.hasNext(); ) {
String xref = (String) i.next();
if (xref.contains(pidAttrPrefix)) {
primaryIdentifier = xref.split(":")[1];
break;
}
}
} else {
if (record.getAttributes().get(pidAttr) != null) {
primaryIdentifier = record.getAttributes().get(pidAttr).get(0);
}
}
}
}
}
String refId = identifierMap.get(primaryIdentifier);
// get rid of previous record Items from handler
handler.clear();
Item seq = getSeq(record.getSequenceID());
String className = TypeUtil.javaiseClassName(term);
String fullClassName = tgtModel.getPackageName() + "." + className;
ClassDescriptor cd = tgtModel.getClassDescriptorByName(fullClassName);
if (cd == null) {
throw new IllegalArgumentException(
"no class found in model for: "
+ className
+ " (original GFF record type: "
+ term
+ ") for "
+ "record: "
+ record);
}
Set<Item> synonymsToAdd = new HashSet<Item>();
Item feature = null;
// new feature
if (refId == null) {
feature = createItem(className);
refId = feature.getIdentifier();
}
if (!"chromosome".equals(term) && seq != null) {
boolean makeLocation =
record.getStart() >= 1
&& record.getEnd() >= 1
&& !dontCreateLocations
&& handler.createLocations(record);
if (makeLocation) {
Item location = getLocation(record, refId, seq, cd);
if (feature == null) {
// this feature has already been created and stored
// we only wanted the location, we're done here.
store(location);
return;
}
int length = getLength(record);
feature.setAttribute("length", String.valueOf(length));
handler.setLocation(location);
if ("Chromosome".equals(seqClsName)
&& (cd.getFieldDescriptorByName("chromosome") != null)) {
feature.setReference("chromosome", seq.getIdentifier());
feature.setReference("chromosomeLocation", location);
}
}
}
if (feature == null) {
// this feature has already been created and stored
// feature with discontinous location, this location wasn't valid for some reason
return;
}
if (primaryIdentifier != null) {
feature.setAttribute("primaryIdentifier", primaryIdentifier);
}
handler.setFeature(feature);
identifierMap.put(primaryIdentifier, feature.getIdentifier());
List<?> names = record.getNames();
String symbol = null;
List<String> synonyms = new ArrayList<String>();
// get the attribute set for symbol
if (config_attr.containsKey(this.orgTaxonId)) {
if (config_attr.get(this.orgTaxonId).containsKey("symbol")) {
String cls = config_attr_class.get(this.orgTaxonId).get("symbol");
if ("all".equals(cls) || term.equals(cls)) {
String symbolAttr = config_attr.get(this.orgTaxonId).get("symbol");
if (symbolAttr.contains("Dbxref") && record.getDbxrefs() != null) {
String symbolAttrPrefix = symbolAttr.split("\\.")[1];
for (Iterator<?> i = record.getDbxrefs().iterator(); i.hasNext(); ) {
String xref = (String) i.next();
if (xref.contains(symbolAttrPrefix)) {
symbol = xref.split(":")[1];
break;
}
}
} else {
if (record.getAttributes().get(symbolAttr) != null) {
symbol = record.getAttributes().get(symbolAttr).get(0);
}
}
}
}
}
// get the attribute set for synonym
if (config_attr.containsKey(this.orgTaxonId)) {
if (config_attr.get(this.orgTaxonId).containsKey("synonym")) {
String cls = config_attr_class.get(this.orgTaxonId).get("synonym");
if ("all".equals(cls) || term.equals(cls)) {
String synonymAttr = config_attr.get(this.orgTaxonId).get("synonym");
if (synonymAttr.contains("Dbxref") && record.getDbxrefs() != null) {
String synonymAttrPrefix = synonymAttr.split("\\.")[1];
Set<String> synSet = new HashSet<String>();
for (Iterator<?> i = record.getDbxrefs().iterator(); i.hasNext(); ) {
String xref = (String) i.next();
if (xref.contains(synonymAttrPrefix)) {
synSet.add(xref.split(":")[1]);
}
}
synonyms.addAll(synSet);
} else {
synonyms = record.getAttributes().get(synonymAttr);
}
// synonyms.removeAll(Collections.singleton(null));
}
}
}
if (names != null) {
setNames(names, symbol, synonyms, synonymsToAdd, primaryIdentifier, feature, cd);
}
// Other attributes
List<String> primeAttrList = Arrays.asList("primaryIdentifier", "symbol", "synonym");
if (config_attr.containsKey(this.orgTaxonId)) {
Map<String, String> attrMapOrg = config_attr.get(this.orgTaxonId);
Map<String, String> attrMapClone = new HashMap<String, String>();
// Deep copy of a map
for (Entry<String, String> e : attrMapOrg.entrySet()) {
attrMapClone.put(e.getKey(), e.getValue());
}
for (String pa : primeAttrList) {
attrMapClone.remove(pa);
}
for (Entry<String, String> e : attrMapClone.entrySet()) {
String cls = config_attr_class.get(this.orgTaxonId).get(e.getKey());
if ("all".equals(cls) || term.equals(cls)) {
String attr = e.getValue();
if (attr.contains("Dbxref") && record.getDbxrefs() != null) {
String attrPrefix = attr.split("\\.")[1];
for (Iterator<?> i = record.getDbxrefs().iterator(); i.hasNext(); ) {
String xref = (String) i.next();
if (xref.contains(attrPrefix)) {
if (feature.checkAttribute(e.getKey())) {
feature.setAttribute(e.getKey(), xref.split(":")[1]);
}
break;
}
}
} else {
if (record.getAttributes().get(attr) != null) {
String attrVal = record.getAttributes().get(attr).get(0);
if (attrVal != null) {
if (feature.checkAttribute(e.getKey())) {
feature.setAttribute(e.getKey(), attrVal);
}
}
}
}
}
}
}
List<String> parents = record.getParents();
if (parents != null && !parents.isEmpty()) {
setRefsAndCollections(parents, feature);
}
feature.addReference(getOrgRef());
feature.addToCollection("dataSets", dataSet);
handler.addDataSet(dataSet);
Double score = record.getScore();
if (score != null && !"".equals(String.valueOf(score))) {
feature.setAttribute("score", String.valueOf(score));
feature.setAttribute("scoreType", record.getSource());
}
for (Item synonym : synonymsToAdd) {
handler.addItem(synonym);
}
handler.process(record);
if (handler.getDataSetReferenceList().getRefIds().size() > 0) {
feature.addCollection(handler.getDataSetReferenceList());
}
handler.clearDataSetReferenceList();
if (handler.getPublicationReferenceList().getRefIds().size() > 0) {
feature.addCollection(handler.getPublicationReferenceList());
}
handler.clearPublicationReferenceList();
try {
Iterator<Item> iter = handler.getItems().iterator();
while (iter.hasNext()) {
store(iter.next());
}
} catch (ObjectStoreException e) {
LOG.error("Problem writing item to the itemwriter");
throw e;
}
}
/**
* Look a the values of the given primary key in the object and return true if and only if some
* part of the primary key is null. If the primary key contains a reference it is sufficient for
* any of the primary keys of the referenced object to be non-null (ie objectPrimaryKeyIsNull()
* returning true).
*
* @param model the Model in which to find ClassDescriptors
* @param obj the Object to check
* @param cld one of the classes that obj is. Only primary keys for this classes will be checked
* @param pk the primary key to check
* @param source the Source database
* @param idMap an IntToIntMap from source IDs to destination IDs
* @return true if the the given primary key is non-null for the given object
* @throws MetaDataException if anything goes wrong
*/
public static boolean objectPrimaryKeyNotNull(
Model model,
InterMineObject obj,
ClassDescriptor cld,
PrimaryKey pk,
Source source,
IntToIntMap idMap)
throws MetaDataException {
for (String fieldName : pk.getFieldNames()) {
FieldDescriptor fd = cld.getFieldDescriptorByName(fieldName);
if (fd instanceof AttributeDescriptor) {
Object value;
try {
value = obj.getFieldValue(fieldName);
} catch (IllegalAccessException e) {
throw new MetaDataException(
"Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
}
if (value == null) {
return false;
}
} else if (fd instanceof CollectionDescriptor) {
throw new MetaDataException(
"Primary key "
+ pk.getName()
+ " for class "
+ cld.getName()
+ " cannot contain collection "
+ fd.getName()
+ ": collections cannot be part of a primary key. Please edit"
+ model.getName()
+ "_keyDefs.properties");
} else if (fd instanceof ReferenceDescriptor) {
InterMineObject refObj;
try {
refObj = (InterMineObject) obj.getFieldProxy(fieldName);
} catch (IllegalAccessException e) {
throw new MetaDataException(
"Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
}
if (refObj == null) {
return false;
}
if ((refObj.getId() != null) && (idMap.get(refObj.getId()) != null)) {
// We have previously loaded the object in this reference.
continue;
}
if (refObj instanceof ProxyReference) {
refObj = ((ProxyReference) refObj).getObject();
}
boolean foundNonNullKey = false;
boolean foundKey = false;
Set<ClassDescriptor> classDescriptors =
model.getClassDescriptorsForClass(refObj.getClass());
CLDS:
for (ClassDescriptor refCld : classDescriptors) {
Set<PrimaryKey> primaryKeys;
if (source == null) {
primaryKeys =
new LinkedHashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(refCld).values());
} else {
primaryKeys = DataLoaderHelper.getPrimaryKeys(refCld, source, null);
}
for (PrimaryKey refPK : primaryKeys) {
foundKey = true;
if (objectPrimaryKeyNotNull(model, refObj, refCld, refPK, source, idMap)) {
foundNonNullKey = true;
break CLDS;
}
}
}
if (foundKey && (!foundNonNullKey)) {
return false;
}
}
}
return true;
}