Пример #1
0
 @Override
 public void close() throws TableException {
   try {
     csv.close();
     countStream.close();
   } catch (IOException e) {
     throw new TableException(e);
   }
 }
Пример #2
0
  @Override
  public Iterator<Tuple> iterator() {
    try {
      this.reset();
    } catch (Exception e) {
      // should not happen as this is second load
      e.printStackTrace();
    }

    if (getLimit() > 0 || getOffset() > 0 || getColOffset() > 0 || getColLimit() > 0) {
      return new TupleIterator(csv, getLimit(), getOffset(), getColLimit(), getColOffset());
    }
    return csv.iterator();
  }
 private boolean rangeIsValid(DesignParameters p) {
   try {
     CsvReader fileReader = new CsvFileReader(new File(p.getGenotype()));
     int rows = fileReader.rownames().size();
     if (p.getRangeStart() != null && p.getRangeEnd() != null)
       for (int i = 0; i < p.getRangeStart().size(); i++) {
         int start = Integer.parseInt((String) p.getRangeStart().get(i));
         int end = Integer.parseInt((String) p.getRangeEnd().get(i));
         logger.debug("" + start + "-" + end);
         if (start < 1 || end > rows) {
           this.setArgMissing(
               "Marker range should be in [1,maxindex makers="
                   + rows
                   + ")], found [start="
                   + start
                   + ",end="
                   + end
                   + "]");
           return false;
         } else if (end < start) {
           this.setArgMissing(
               "marker range 'start' should be lower than 'end', found [start="
                   + start
                   + ",end="
                   + end
                   + "]");
           return false;
         }
       }
   } catch (Exception e) {
     e.printStackTrace();
     this.setArgMissing(e.getMessage());
     return false;
   }
   return true;
 }
  /**
   * Imports UploadBatch from tab/comma delimited File
   *
   * @param db database to import into
   * @param reader csv reader to load data from
   * @param defaults to set default values for each row
   * @param dbAction indicating wether to add,update,remove etc
   * @param missingValues indicating what value in the csv is treated as 'null' (e.g. "" or "NA")
   * @return number of elements imported
   */
  public int importCsv(
      final Database db,
      CsvReader reader,
      final Tuple defaults,
      final DatabaseAction dbAction,
      final String missingValues)
      throws DatabaseException, IOException, Exception {
    // cache for entities of which xrefs couldn't be resolved (e.g. if there
    // is a self-refence)
    // these entities can be updated with their xrefs in a second round when
    // all entities are in the database
    // final List<UploadBatch> uploadBatchsMissingRefs = new
    // ArrayList<UploadBatch>();

    final MutationService mutationService = new MutationService();
    mutationService.setDatabase(db);
    final UploadService uploadService = new UploadService();
    uploadService.setDatabase(db);

    final Submission submission = new Submission();
    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
    submission.setDate(dateFormat.format(new Date()));
    submission.setReleasedate(dateFormat.format(new Date()));
    submission.setIdentifier("S" + new Date());
    if (db instanceof JDBCDatabase) {
      List<Integer> submitters = new ArrayList<Integer>();
      submitters.add(db.getLogin().getUserId());
      submission.setSubmitters_Id(submitters);
    } else if (db instanceof JpaDatabase) {
      List<MolgenisUser> submitters = new ArrayList<MolgenisUser>();
      submitters.add(db.findById(MolgenisUser.class, db.getLogin().getUserId()));
      // submission.setSubmitters(submitters);
    }
    db.add(submission);

    // cache for objects to be imported from file (in batch)
    // TODO: Danny: Use or loose
    /* final List<PatientSummaryVO> patientList = */ new ArrayList<PatientSummaryVO>(BATCH_SIZE);
    // wrapper to count
    final IntegerWrapper total = new IntegerWrapper(0);
    reader.setMissingValues(missingValues);

    Integer mutationIdentifier = uploadService.getMaxMutationIdentifier();
    Integer patientIdentifier = uploadService.getMaxPatientIdentifier();

    for (Tuple tuple : reader) {

      // parse object, setting defaults and values from file
      // if (lineNo > 5) return;
      PatientSummaryVO patientSummaryVO = new PatientSummaryVO();

      patientSummaryVO.setSubmissionDate(submission.getDate());

      patientSummaryVO.setPatientNumber(tuple.getString("Local patient number"));
      patientSummaryVO.setPatientGender(
          ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Gender")), "unknown"));
      patientSummaryVO.setPatientEthnicity(tuple.getString("Ethnicity"));
      patientSummaryVO.setPatientAge(ObjectUtils.toString(tuple.getString("Age"), "unknown"));
      patientSummaryVO.setPatientDeceased(
          ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Deceased")), "unknown"));
      patientSummaryVO.setPatientConsent(
          ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Signed consent")), "no"));

      patientSummaryVO.setVariantSummaryVOList(new ArrayList<MutationSummaryVO>());

      if (StringUtils.isNotEmpty(tuple.getString("cDNA change_1"))) {
        MutationUploadVO mutationUploadVO = new MutationUploadVO();
        mutationUploadVO.setGeneSymbol("COL7A1");
        mutationUploadVO.setMutation(new Mutation());
        mutationUploadVO.getMutation().setCdna_Notation("c." + tuple.getString("cDNA change_1"));
        mutationService.assignValuesFromNotation(mutationUploadVO);
        if (StringUtils.isNotEmpty(tuple.getString("Protein change_1")))
          mutationUploadVO.getMutation().setAa_Notation("p." + tuple.getString("Protein change_1"));
        if (StringUtils.isNotEmpty(tuple.getString("Consequence_1")))
          mutationUploadVO
              .getMutation()
              .setConsequence(ObjectUtils.toString(tuple.getString("Consequence_1"), ""));
        mutationUploadVO
            .getMutation()
            .setInheritance(
                ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Inheritance_1")), ""));

        mutationIdentifier = mutationIdentifier + 1;
        mutationUploadVO.getMutation().setIdentifier("M" + mutationIdentifier);
        mutationUploadVO.getMutation().setName("M" + mutationIdentifier);

        // Insert mutation if it does not exist already

        MutationSearchCriteriaVO criteria = new MutationSearchCriteriaVO();
        criteria.setVariation(mutationUploadVO.getMutation().getCdna_Notation());
        List<MutationSummaryVO> results = mutationService.findMutations(criteria);

        if (results.size() != 1) {
          uploadService.insert(mutationUploadVO);
          System.out.println(">>>Inserted mutation: " + mutationUploadVO.getMutation().toString());
        }
        MutationSummaryVO mutationSummaryVO = new MutationSummaryVO();
        mutationSummaryVO.setCdnaNotation(mutationUploadVO.getMutation().getCdna_Notation());

        patientSummaryVO.getVariantSummaryVOList().add(mutationSummaryVO);
      }

      // Second mutation can be 'NA' or 'unknown':
      // Leave xref == null and add remark in mutation2remark
      if (StringUtils.isNotEmpty(tuple.getString("cDNA change_2"))) {
        if (tuple.getString("cDNA change_2").equalsIgnoreCase("na"))
          patientSummaryVO.setVariantComment(tuple.getString("cDNA change_2").toUpperCase());
        else if (tuple.getString("cDNA change_2").equalsIgnoreCase("unknown"))
          patientSummaryVO.setVariantComment(tuple.getString("cDNA change_2").toUpperCase());
        else {
          MutationUploadVO mutationUploadVO = new MutationUploadVO();
          mutationUploadVO.setGeneSymbol("COL7A1");
          mutationUploadVO.setMutation(new Mutation());
          mutationUploadVO.getMutation().setCdna_Notation("c." + tuple.getString("cDNA change_2"));
          mutationService.assignValuesFromNotation(mutationUploadVO);
          if (StringUtils.isNotEmpty(tuple.getString("Protein change_2")))
            mutationUploadVO
                .getMutation()
                .setAa_Notation("p." + tuple.getString("Protein change_2"));
          if (StringUtils.isNotEmpty(tuple.getString("Consequence_2")))
            mutationUploadVO
                .getMutation()
                .setConsequence(ObjectUtils.toString(tuple.getString("Consequence_2"), ""));
          mutationUploadVO
              .getMutation()
              .setInheritance(
                  ObjectUtils.toString(
                      StringUtils.lowerCase(tuple.getString("Inheritance_2")), ""));

          mutationIdentifier = mutationIdentifier + 1;
          mutationUploadVO.getMutation().setIdentifier("M" + mutationIdentifier);
          mutationUploadVO.getMutation().setName("M" + mutationIdentifier);

          // Insert mutation if it does not exist already

          MutationSearchCriteriaVO criteria = new MutationSearchCriteriaVO();
          criteria.setVariation(mutationUploadVO.getMutation().getCdna_Notation());
          List<MutationSummaryVO> results = mutationService.findMutations(criteria);

          if (results.size() != 1) {
            uploadService.insert(mutationUploadVO);
            System.out.println(
                ">>>Inserted mutation: " + mutationUploadVO.getMutation().toString());
          }
          MutationSummaryVO mutationSummaryVO = new MutationSummaryVO();
          mutationSummaryVO.setCdnaNotation(mutationUploadVO.getMutation().getCdna_Notation());

          patientSummaryVO.getVariantSummaryVOList().add(mutationSummaryVO);
        }
      }

      patientSummaryVO.setPhenotypeMajor(
          StringUtils.upperCase(tuple.getString("Phenotype major type")));
      patientSummaryVO.setPhenotypeSub(StringUtils.lowerCase(tuple.getString("Phenotype Subtype")));

      // PhenotypeDetailsVO phenotypeDetailsVO = new PhenotypeDetailsVO();

      // patientSummaryVO.getPhenotypeDetails().setLocation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Location")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setBlistering(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Blistering")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setHands(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Hands")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setFeet(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Feet")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setArms(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Arms")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setLegs(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Legs")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setProximal_Body_Flexures(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Proximal body flexures")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setTrunk(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Trunk")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setMucous_Membranes(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mucosa")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setSkin_Atrophy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Skin atrophy")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setMilia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Milia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setNail_Dystrophy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Nail dystrophy")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setAlbopapuloid_Papules(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Albopapuloid papules")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setPruritic_Papules(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Pruritic papules")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setAlopecia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Alopecia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setSquamous_Cell_Carcinomas(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Squamous cell carcinoma(s)")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setRevertant_Skin_Patch(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Revertant skin patch(es)")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setMechanism(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mechanism")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setFlexion_Contractures(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Flexion contractures")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setPseudosyndactyly_Hands(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Pseudosyndactyly (hands)")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setMicrostomia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Microstomia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setAnkyloglossia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Ankyloglossia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setDysphagia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Swallowing difficulties/ dysphagia/ oesophagus strictures")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setGrowth_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Growth retardation")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setAnemia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Anaemia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setRenal_Failure(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Renal failure")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setDilated_Cardiomyopathy(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Dilated cardiomyopathy")),
      // "unknown"));
      //
      // patientSummaryVO.getPhenotypeDetails().setColoboma(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Coloboma")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setCongenital_Heart_Defect(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Congenital heart defect")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setClp(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("C(L)P")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setChoanal_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Choanal anomaly")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setMental_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Mental retardation")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setGrowth_Retardation(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Growth retardation")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setGenital_Hypoplasia(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Genital hypoplasia")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setExternal_Ear_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("External ear anomaly")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setSemicircular_Canal_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Semicircular canal anomaly")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setHearing_Loss(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("Hearing loss")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setTe_Anomaly(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("TE anomaly")),
      // "unknown"));
      // patientSummaryVO.getPhenotypeDetails().setCn_Dysfunction(ObjectUtils.toString(StringUtils.lowerCase(tuple.getString("CN dysfunction")),
      // "unknown"));

      // for (String field :
      // patientSummaryVO.getPhenotypeDetails().getFields(true))
      // {
      // if ("".equals(patientSummaryVO.getPhenotypeDetails().get(field)))
      // patientSummaryVO.getPhenotypeDetails().set(field, "unknown");
      // }
      // patientSummaryVO.getPhenotypeDetails().set(tuple.getString(""));

      if (tuple.getString("PubMed ID") != null && tuple.getString("Reference") != null) {
        List<PublicationVO> publicationVOs = new ArrayList<PublicationVO>();
        String[] pubmeds = tuple.getString("PubMed ID").split(";");
        String[] titles = tuple.getString("Reference").split(";");
        for (int i = 0; i < pubmeds.length; i++) {
          PublicationVO publicationVO = new PublicationVO();
          publicationVO.setPubmedId(pubmeds[i]);
          publicationVO.setName(titles[i]);
          publicationVO.setTitle(titles[i]);
          publicationVOs.add(publicationVO);
        }
        patientSummaryVO.setPublicationVOList(publicationVOs);
        patientSummaryVO.setPatientConsent("publication");
      }

      patientIdentifier = patientIdentifier + 1;
      patientSummaryVO.setPatientIdentifier("P" + patientIdentifier);
      patientSummaryVO.setPatientName("P" + patientIdentifier);

      patientSummaryVO.setObservedValueVOList(new ArrayList<ObservedValueVO>());

      ObservedValueVO observedValueVO;

      observedValueVO = new ObservedValueVO();
      observedValueVO.setFeatureName("LH7:2 Amount of type VII collagen");
      observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
      observedValueVO.setValue(ObjectUtils.toString(tuple.getString("IF LH7:2"), "unknown"));
      patientSummaryVO.getObservedValueVOList().add(observedValueVO);

      observedValueVO = new ObservedValueVO();
      observedValueVO.setFeatureName("IF Retention of type VII Collagen in basal cells");
      observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
      observedValueVO.setValue(
          ObjectUtils.toString(tuple.getString("IF Retention COLVII"), "unknown"));
      patientSummaryVO.getObservedValueVOList().add(observedValueVO);

      observedValueVO = new ObservedValueVO();
      observedValueVO.setFeatureName("Anchoring fibrils Number");
      observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
      observedValueVO.setValue(ObjectUtils.toString(tuple.getString("EM AF_no"), "unknown"));
      patientSummaryVO.getObservedValueVOList().add(observedValueVO);

      observedValueVO = new ObservedValueVO();
      observedValueVO.setFeatureName("Anchoring fibrils Ultrastructure");
      observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
      observedValueVO.setValue(ObjectUtils.toString(tuple.getString("EM AF_structure"), "unknown"));
      patientSummaryVO.getObservedValueVOList().add(observedValueVO);

      observedValueVO = new ObservedValueVO();
      observedValueVO.setFeatureName("EM Retention of type VII Collagen in basal cells");
      observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
      observedValueVO.setValue(
          ObjectUtils.toString(tuple.getString("EM_Retention COLVII"), "unknown"));
      patientSummaryVO.getObservedValueVOList().add(observedValueVO);

      for (int i = 34; ; i++) {
        System.out.println(">>>i==" + i);
        String colName = tuple.getColName(i);
        System.out.println(">>>colName==" + colName);

        if (colName == null) break;

        observedValueVO = new ObservedValueVO();
        observedValueVO.setFeatureName(colName);
        observedValueVO.setTargetName(patientSummaryVO.getPatientIdentifier());
        observedValueVO.setValue(ObjectUtils.toString(tuple.getString(colName), "unknown"));
        patientSummaryVO.getObservedValueVOList().add(observedValueVO);
      }

      uploadService.insert(patientSummaryVO);

      total.set(total.get() + 1);
      // patientList.add(patientSummaryVO);
      //
      // //add to db when batch size is reached
      // if(patientList.size() == BATCH_SIZE)
      // {
      // //resolve foreign keys and copy those entities that could not be
      // resolved to the missingRefs list
      // uploadBatchsMissingRefs.addAll(resolveForeignKeys(db,
      // uploadBatchList));
      //
      // //update objects in the database using xref_label defined
      // secondary key(s) 'id' defined in xref_label
      // db.update(uploadBatchList,dbAction, "id");
      //
      // //clear for next batch
      // uploadBatchList.clear();
      //
      // //keep count
      // total.set(total.get() + BATCH_SIZE);
      // }
    }

    // add remaining elements to the database
    // if(!uploadBatchList.isEmpty())
    // {
    // //resolve foreign keys, again keeping track of those entities that
    // could not be solved
    // uploadBatchsMissingRefs.addAll(resolveForeignKeys(db,
    // uploadBatchList));
    // //update objects in the database using xref_label defined secondary
    // key(s) 'id' defined in xref_label
    // db.update(uploadBatchList,dbAction, "id");
    // }
    //
    // //second import round, try to resolve FK's for entities again as they
    // might have referred to entities in the imported list
    // List<UploadBatch> uploadBatchsStillMissingRefs =
    // resolveForeignKeys(db, uploadBatchsMissingRefs);
    //
    // //if there are still missing references, throw error and rollback
    // if(uploadBatchsStillMissingRefs.size() > 0){
    // throw new
    // Exception("Import of 'UploadBatch' objects failed: attempting to resolve in-list references,
    // but there are still UploadBatchs referring to UploadBatchs that are neither in the database
    // nor in the list of to-be imported UploadBatchs. (the first one being:
    // "+uploadBatchsStillMissingRefs.get(0)+")");
    // }
    // //else update the entities in the database with the found references
    // and return total
    // else
    // {
    // db.update(uploadBatchsMissingRefs,DatabaseAction.UPDATE, "id");
    //
    // //output count
    // total.set(total.get() + uploadBatchList.size());
    // logger.info("imported "+total.get()+" uploadBatch from CSV");
    //
    // return total.get();
    // }

    return total.get();
  }
Пример #5
0
 /**
  * Helper method to load the Field metadata
  *
  * @throws Exception
  */
 private void loadColumns() throws Exception {
   for (String name : csv.colnames()) {
     Field f = new Field(name);
     columns.add(f);
   }
 }