@Override
public void close() throws TableException {
try {
csv.close();
countStream.close();
} catch (IOException e) {
throw new TableException(e);
}
}
@Override
public Iterator<Tuple> iterator() {
try {
this.reset();
} catch (Exception e) {
// should not happen as this is second load
e.printStackTrace();
}
if (getLimit() > 0 || getOffset() > 0 || getColOffset() > 0 || getColLimit() > 0) {
return new TupleIterator(csv, getLimit(), getOffset(), getColLimit(), getColOffset());
}
return csv.iterator();
}
private boolean rangeIsValid(DesignParameters p) {
try {
CsvReader fileReader = new CsvFileReader(new File(p.getGenotype()));
int rows = fileReader.rownames().size();
if (p.getRangeStart() != null && p.getRangeEnd() != null)
for (int i = 0; i < p.getRangeStart().size(); i++) {
int start = Integer.parseInt((String) p.getRangeStart().get(i));
int end = Integer.parseInt((String) p.getRangeEnd().get(i));
logger.debug("" + start + "-" + end);
if (start < 1 || end > rows) {
this.setArgMissing(
"Marker range should be in [1,maxindex makers="
+ rows
+ ")], found [start="
+ start
+ ",end="
+ end
+ "]");
return false;
} else if (end < start) {
this.setArgMissing(
"marker range 'start' should be lower than 'end', found [start="
+ start
+ ",end="
+ end
+ "]");
return false;
}
}
} catch (Exception e) {
e.printStackTrace();
this.setArgMissing(e.getMessage());
return false;
}
return true;
}
/**
* Imports UploadBatch from tab/comma delimited File
*
* @param db database to import into
* @param reader csv reader to load data from
* @param defaults to set default values for each row
* @param dbAction indicating wether to add,update,remove etc
* @param missingValues indicating what value in the csv is treated as 'null' (e.g. "" or "NA")
* @return number of elements imported
*/
public int importCsv(
final Database db,
CsvReader reader,
final Tuple defaults,
final DatabaseAction dbAction,
final String missingValues)
throws DatabaseException, IOException, Exception {
// cache for entities of which xrefs couldn't be resolved (e.g. if there
// is a self-refence)
// these entities can be updated with their xrefs in a second round when
// all entities are in the database
// final List<UploadBatch> uploadBatchsMissingRefs = new
// ArrayList<UploadBatch>();
final MutationService mutationService = new MutationService();
mutationService.setDatabase(db);
final UploadService uploadService = new UploadService();
uploadService.setDatabase(db);
final Submission submission = new Submission();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
submission.setDate(dateFormat.format(new Date()));
submission.setReleasedate(dateFormat.format(new Date()));
submission.setIdentifier("S" + new Date());
if (db instanceof JDBCDatabase) {
List<Integer> submitters = new ArrayList<Integer>();
submitters.add(db.getLogin().getUserId());
submission.setSubmitters_Id(submitters);
} else if (db instanceof JpaDatabase) {
List<MolgenisUser> submitters = new ArrayList<MolgenisUser>();
submitters.add(db.findById(MolgenisUser.class, db.getLogin().getUserId()));
// submission.setSubmitters(submitters);
}
db.add(submission);
// cache for objects to be imported from file (in batch)
// TODO: Danny: Use or loose
/* final List<PatientSummaryVO> patientList = */ new ArrayList<PatientSummaryVO>(BATCH_SIZE);
// wrapper to count
final IntegerWrapper total = new IntegerWrapper(0);
reader.setMissingValues(missingValues);
Integer mutationIdentifier = uploadService.getMaxMutationIdentifier();
Integer patientIdentifier = uploadService.getMaxPatientIdentifier();
for (Tuple tuple : reader) {
// parse object, setting defaults and values from file
// if (lineNo > 5) return;
PatientSummaryVO patientSummaryVO = new PatientSummaryVO();
patientSummaryVO.setSubmissionDate(submission.getDate());
patientSummaryVO.setPatientNumber(tuple.getString("Local patient number"));
patientSummaryVO.setPatientGender(
ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Gender")), "unknown"));
patientSummaryVO.setPatientEthnicity(tuple.getString("Ethnicity"));
patientSummaryVO.setPatientAge(ObjectUtils.toString(tuple.getString("Age"), "unknown"));
patientSummaryVO.setPatientDeceased(
ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Deceased")), "unknown"));
patientSummaryVO.setPatientConsent(
ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Signed consent")), "no"));
patientSummaryVO.setVariantSummaryVOList(new ArrayList<MutationSummaryVO>());
if (StringUtils.isNotEmpty(tuple.getString("cDNA change_1"))) {
MutationUploadVO mutationUploadVO = new MutationUploadVO();
mutationUploadVO.setGeneSymbol("COL7A1");
mutationUploadVO.setMutation(new Mutation());
mutationUploadVO.getMutation().setCdna_Notation("c." + tuple.getString("cDNA change_1"));
mutationService.assignValuesFromNotation(mutationUploadVO);
if (StringUtils.isNotEmpty(tuple.getString("Protein change_1")))
mutationUploadVO.getMutation().setAa_Notation("p." + tuple.getString("Protein change_1"));
if (StringUtils.isNotEmpty(tuple.getString("Consequence_1")))
mutationUploadVO
.getMutation()
.setConsequence(ObjectUtils.toString(tuple.getString("Consequence_1"), ""));
mutationUploadVO
.getMutation()
.setInheritance(
ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Inheritance_1")), ""));
mutationIdentifier = mutationIdentifier + 1;
mutationUploadVO.getMutation().setIdentifier("M" + mutationIdentifier);
mutationUploadVO.getMutation().setName("M" + mutationIdentifier);
// Insert mutation if it does not exist already
MutationSearchCriteriaVO criteria = new MutationSearchCriteriaVO();
criteria.setVariation(mutationUploadVO.getMutation().getCdna_Notation());
List<MutationSummaryVO> results = mutationService.findMutations(criteria);
if (results.size() != 1) {
uploadService.insert(mutationUploadVO);
System.out.println(">>>Inserted mutation: " + mutationUploadVO.getMutation().toString());
}
MutationSummaryVO mutationSummaryVO = new MutationSummaryVO();
mutationSummaryVO.setCdnaNotation(mutationUploadVO.getMutation().getCdna_Notation());
patientSummaryVO.getVariantSummaryVOList().add(mutationSummaryVO);
}
// Second mutation can be 'NA' or 'unknown':
// Leave xref == null and add remark in mutation2remark
if (StringUtils.isNotEmpty(tuple.getString("cDNA change_2"))) {
if (tuple.getString("cDNA change_2").equalsIgnoreCase("na"))
patientSummaryVO.setVariantComment(tuple.getString("cDNA change_2").toUpperCase());
else if (tuple.getString("cDNA change_2").equalsIgnoreCase("unknown"))
patientSummaryVO.setVariantComment(tuple.getString("cDNA change_2").toUpperCase());
else {
MutationUploadVO mutationUploadVO = new MutationUploadVO();
mutationUploadVO.setGeneSymbol("COL7A1");
mutationUploadVO.setMutation(new Mutation());
mutationUploadVO.getMutation().setCdna_Notation("c." + tuple.getString("cDNA change_2"));
mutationService.assignValuesFromNotation(mutationUploadVO);
if (StringUtils.isNotEmpty(tuple.getString("Protein change_2")))
mutationUploadVO
.getMutation()
.setAa_Notation("p." + tuple.getString("Protein change_2"));
if (StringUtils.isNotEmpty(tuple.getString("Consequence_2")))
mutationUploadVO
.getMutation()
.setConsequence(ObjectUtils.toString(tuple.getString("Consequence_2"), ""));
mutationUploadVO
.getMutation()
.setInheritance(
ObjectUtils.toString(
StringUtils.lowerCase(tuple.getString("Inheritance_2")), ""));
mutationIdentifier = mutationIdentifier + 1;
mutationUploadVO.getMutation().setIdentifier("M" + mutationIdentifier);
mutationUploadVO.getMutation().setName("M" + mutationIdentifier);
// Insert mutation if it does not exist already
MutationSearchCriteriaVO criteria = new MutationSearchCriteriaVO();
criteria.setVariation(mutationUploadVO.getMutation().getCdna_Notation());
List<MutationSummaryVO> results = mutationService.findMutations(criteria);
if (results.size() != 1) {
uploadService.insert(mutationUploadVO);
System.out.println(
">>>Inserted mutation: " + mutationUploadVO.getMutation().toString());
}
MutationSummaryVO mutationSummaryVO = new MutationSummaryVO();
mutationSummaryVO.setCdnaNotation(mutationUploadVO.getMutation().getCdna_Notation());
patientSummaryVO.getVariantSummaryVOList().add(mutationSummaryVO);
}
}
patientSummaryVO.setPhenotypeMajor(
StringUtils.upperCase(tuple.getString("Phenotype major type")));
patientSummaryVO.setPhenotypeSub(StringUtils.lowerCase(tuple.getString("Phenotype Subtype")));
// PhenotypeDetailsVO phenotypeDetailsVO = new PhenotypeDetailsVO();
// patientSummaryVO.getPhenotypeDetails().setLocation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Location")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setBlistering(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Blistering")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setHands(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Hands")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setFeet(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Feet")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setArms(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Arms")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setLegs(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Legs")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setProximal_Body_Flexures(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Proximal body flexures")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setTrunk(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Trunk")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setMucous_Membranes(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mucosa")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setSkin_Atrophy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Skin atrophy")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setMilia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Milia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setNail_Dystrophy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Nail dystrophy")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setAlbopapuloid_Papules(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Albopapuloid papules")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setPruritic_Papules(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Pruritic papules")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setAlopecia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Alopecia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setSquamous_Cell_Carcinomas(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Squamous cell carcinoma(s)")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setRevertant_Skin_Patch(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Revertant skin patch(es)")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setMechanism(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mechanism")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setFlexion_Contractures(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Flexion contractures")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setPseudosyndactyly_Hands(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Pseudosyndactyly (hands)")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setMicrostomia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Microstomia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setAnkyloglossia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Ankyloglossia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setDysphagia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Swallowing difficulties/ dysphagia/ oesophagus strictures")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setGrowth_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Growth retardation")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setAnemia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Anaemia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setRenal_Failure(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Renal failure")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setDilated_Cardiomyopathy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Dilated cardiomyopathy")),
// "unknown"));
//
// patientSummaryVO.getPhenotypeDetails().setColoboma(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Coloboma")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setCongenital_Heart_Defect(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Congenital heart defect")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setClp(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("C(L)P")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setChoanal_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Choanal anomaly")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setMental_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mental retardation")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setGrowth_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Growth retardation")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setGenital_Hypoplasia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Genital hypoplasia")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setExternal_Ear_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("External ear anomaly")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setSemicircular_Canal_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Semicircular canal anomaly")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setHearing_Loss(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Hearing loss")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setTe_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("TE anomaly")),
// "unknown"));
// patientSummaryVO.getPhenotypeDetails().setCn_Dysfunction(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("CN dysfunction")),
// "unknown"));
// for (String field :
// patientSummaryVO.getPhenotypeDetails().getFields(true))
// {
// if ("".equals(patientSummaryVO.getPhenotypeDetails().get(field)))
// patientSummaryVO.getPhenotypeDetails().set(field, "unknown");
// }
// patientSummaryVO.getPhenotypeDetails().set(tuple.getString(""));
if (tuple.getString("PubMed ID") != null && tuple.getString("Reference") != null) {
List<PublicationVO> publicationVOs = new ArrayList<PublicationVO>();
String[] pubmeds = tuple.getString("PubMed ID").split(";");
String[] titles = tuple.getString("Reference").split(";");
for (int i = 0; i < pubmeds.length; i++) {
PublicationVO publicationVO = new PublicationVO();
publicationVO.setPubmedId(pubmeds[i]);
publicationVO.setName(titles[i]);
publicationVO.setTitle(titles[i]);
publicationVOs.add(publicationVO);
}
patientSummaryVO.setPublicationVOList(publicationVOs);
patientSummaryVO.setPatientConsent("publication");
}
patientIdentifier = patientIdentifier + 1;
patientSummaryVO.setPatientIdentifier("P" + patientIdentifier);
patientSummaryVO.setPatientName("P" + patientIdentifier);
patientSummaryVO.setObservedValueVOList(new ArrayList<ObservedValueVO>());
ObservedValueVO observedValueVO;
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName("LH7:2 Amount of type VII collagen");
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(ObjectUtils.toString(tuple.getString("IF LH7:2"), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName("IF Retention of type VII Collagen in basal cells");
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(
ObjectUtils.toString(tuple.getString("IF Retention COLVII"), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName("Anchoring fibrils Number");
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(ObjectUtils.toString(tuple.getString("EM AF_no"), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName("Anchoring fibrils Ultrastructure");
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(ObjectUtils.toString(tuple.getString("EM AF_structure"), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName("EM Retention of type VII Collagen in basal cells");
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(
ObjectUtils.toString(tuple.getString("EM_Retention COLVII"), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
for (int i = 34; ; i++) {
System.out.println(">>>i==" + i);
String colName = tuple.getColName(i);
System.out.println(">>>colName==" + colName);
if (colName == null) break;
observedValueVO = new ObservedValueVO();
observedValueVO.setFeatureName(colName);
observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
observedValueVO.setValue(ObjectUtils.toString(tuple.getString(colName), "unknown"));
patientSummaryVO.getObservedValueVOList().add(observedValueVO);
}
uploadService.insert(patientSummaryVO);
total.set(total.get() + 1);
// patientList.add(patientSummaryVO);
//
// //add to db when batch size is reached
// if(patientList.size() == BATCH_SIZE)
// {
// //resolve foreign keys and copy those entities that could not be
// resolved to the missingRefs list
// uploadBatchsMissingRefs.addAll(resolveForeignKeys(db,
// uploadBatchList));
//
// //update objects in the database using xref_label defined
// secondary key(s) 'id' defined in xref_label
// db.update(uploadBatchList,dbAction, "id");
//
// //clear for next batch
// uploadBatchList.clear();
//
// //keep count
// total.set(total.get() + BATCH_SIZE);
// }
}
// add remaining elements to the database
// if(!uploadBatchList.isEmpty())
// {
// //resolve foreign keys, again keeping track of those entities that
// could not be solved
// uploadBatchsMissingRefs.addAll(resolveForeignKeys(db,
// uploadBatchList));
// //update objects in the database using xref_label defined secondary
// key(s) 'id' defined in xref_label
// db.update(uploadBatchList,dbAction, "id");
// }
//
// //second import round, try to resolve FK's for entities again as they
// might have referred to entities in the imported list
// List<UploadBatch> uploadBatchsStillMissingRefs =
// resolveForeignKeys(db, uploadBatchsMissingRefs);
//
// //if there are still missing references, throw error and rollback
// if(uploadBatchsStillMissingRefs.size() > 0){
// throw new
// Exception("Import of 'UploadBatch' objects failed: attempting to resolve in-list references,
// but there are still UploadBatchs referring to UploadBatchs that are neither in the database
// nor in the list of to-be imported UploadBatchs. (the first one being:
// "+uploadBatchsStillMissingRefs.get(0)+")");
// }
// //else update the entities in the database with the found references
// and return total
// else
// {
// db.update(uploadBatchsMissingRefs,DatabaseAction.UPDATE, "id");
//
// //output count
// total.set(total.get() + uploadBatchList.size());
// logger.info("imported "+total.get()+" uploadBatch from CSV");
//
// return total.get();
// }
return total.get();
}
/**
* Helper method to load the Field metadata
*
* @throws Exception
*/
private void loadColumns() throws Exception {
for (String name : csv.colnames()) {
Field f = new Field(name);
columns.add(f);
}
}