@Before
  public void setUp() throws DaoException {
    DaoCancerStudy.reCacheAll();
    DaoGeneOptimized.getInstance().reCache();
    ProgressMonitor.resetWarnings();

    studyId = DaoCancerStudy.getCancerStudyByStableId("study_tcga_pub").getInternalId();

    GeneticProfile newGeneticProfile = new GeneticProfile();
    newGeneticProfile.setCancerStudyId(studyId);
    newGeneticProfile.setGeneticAlterationType(GeneticAlterationType.COPY_NUMBER_ALTERATION);
    newGeneticProfile.setStableId("study_tcga_pub_test");
    newGeneticProfile.setProfileName("Barry CNA Results");
    newGeneticProfile.setDatatype("test");
    DaoGeneticProfile.addGeneticProfile(newGeneticProfile);

    geneticProfileId =
        DaoGeneticProfile.getGeneticProfileByStableId("study_tcga_pub_test").getGeneticProfileId();

    sample1 =
        DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-A1-A0SB-01").getInternalId();
    sample2 =
        DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-A1-A0SD-01").getInternalId();
    sample3 =
        DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-A1-A0SE-01").getInternalId();
    sample4 =
        DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-A1-A0SF-01").getInternalId();
    sample5 =
        DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-A1-A0SG-01").getInternalId();
  }
  private void runImportRnaData1() throws DaoException, IOException {

    DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance();
    DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance();

    daoGene.addGene(new CanonicalGene(999999780, "A"));
    daoGene.addGene(new CanonicalGene(999995982, "B"));
    daoGene.addGene(new CanonicalGene(999993310, "C"));
    daoGene.addGene(new CanonicalGene(999997849, "D"));
    daoGene.addGene(new CanonicalGene(999992978, "E"));
    daoGene.addGene(new CanonicalGene(999997067, "F"));
    daoGene.addGene(new CanonicalGene(999911099, "G"));
    daoGene.addGene(new CanonicalGene(999999675, "6352"));

    GeneticProfile geneticProfile = new GeneticProfile();

    geneticProfile.setCancerStudyId(studyId);
    geneticProfile.setStableId("gbm_mrna");
    geneticProfile.setGeneticAlterationType(GeneticAlterationType.MRNA_EXPRESSION);
    geneticProfile.setDatatype("CONTINUOUS");
    geneticProfile.setProfileName("MRNA Data");
    geneticProfile.setProfileDescription("mRNA Data");
    DaoGeneticProfile.addGeneticProfile(geneticProfile);

    int newGeneticProfileId =
        DaoGeneticProfile.getGeneticProfileByStableId("gbm_mrna").getGeneticProfileId();

    ProgressMonitor.setConsoleMode(true);
    // TBD: change this to use getResourceAsStream()
    File file = new File("src/test/resources/mrna_test.txt");
    ImportTabDelimData parser = new ImportTabDelimData(file, newGeneticProfileId, null);
    int numLines = FileUtil.getNumLines(file);
    parser.importData(numLines);
    ConsoleUtil.showMessages();

    int sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "DD639").getInternalId();
    String value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999992978);
    assertEquals("2.01", value);

    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "DD638").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997849);
    assertEquals("0.55", value);
  }
  private void processRequestStatistics(
      HttpServletRequest request, HttpServletResponse response, String type)
      throws ServletException, IOException {
    String cancerStudyIds = request.getParameter(QueryBuilder.CANCER_STUDY_ID);

    try {
      boolean includeMut = "mut".equalsIgnoreCase(type);
      boolean includeCna = "cna".equalsIgnoreCase(type);

      Map<String, Map<String, Object>> data = new HashMap<String, Map<String, Object>>();
      // get list of cancer studies
      AccessControl accessControl = getaccessControl();
      List<CancerStudy> cancerStudies;
      if (cancerStudyIds == null || cancerStudyIds.isEmpty()) {
        cancerStudies = accessControl.getCancerStudies();
      } else {
        cancerStudies = new ArrayList<CancerStudy>();
        for (String studyId : cancerStudyIds.split("[ ,]+")) {
          CancerStudy study = DaoCancerStudy.getCancerStudyByStableId(studyId);
          if (study != null && !accessControl.isAccessibleCancerStudy(studyId).isEmpty()) {
            cancerStudies.add(study);
          }
        }
      }
      for (CancerStudy cancerStudy : cancerStudies) {
        if (cancerStudy.getCancerStudyStableId().equalsIgnoreCase("all")) {
          continue;
        }
        Map<String, Object> row = new HashMap<String, Object>();
        data.put(cancerStudy.getCancerStudyStableId(), row);

        if (!includeMut && !includeMut) {
          row.put("name", cancerStudy.getName());
          String pmid = cancerStudy.getPmid();
          if (pmid != null) {
            row.put("pmid", pmid);
          }
          String citation = cancerStudy.getCitation();
          if (citation != null) {
            row.put("citation", citation);
          }
          row.put(
              "cases", DaoPatient.getPatientsByCancerStudyId(cancerStudy.getInternalId()).size());
        }

        if (includeMut) {
          GeneticProfile mutProfile = cancerStudy.getMutationProfile();
          if (mutProfile == null) {
            row.put("mut", 0);
          } else {
            int mutEvents = DaoMutation.countMutationEvents(mutProfile.getGeneticProfileId());
            int samplesWithMut =
                DaoSampleProfile.countSamplesInProfile(mutProfile.getGeneticProfileId());
            row.put("mut", 1.0 * mutEvents / samplesWithMut);
          }
        }

        if (includeCna) {
          GeneticProfile cnaProfile = cancerStudy.getCopyNumberAlterationProfile(false);
          if (cnaProfile == null) {
            row.put("cna", 0);
          } else {
            List<Integer> samples =
                DaoSampleProfile.getAllSampleIdsInProfile(cnaProfile.getGeneticProfileId());
            Map<Integer, Double> fracs =
                DaoCopyNumberSegment.getCopyNumberActeredFraction(
                    samples,
                    cnaProfile.getCancerStudyId(),
                    GlobalProperties.getPatientViewGenomicOverviewCnaCutoff()[0]);
            double aveFrac = 0;
            for (double frac : fracs.values()) {
              aveFrac += frac;
            }
            aveFrac /= samples.size();
            row.put("cna", aveFrac);
          }
        }
      }

      response.setContentType("application/json");

      PrintWriter out = response.getWriter();
      try {
        JSONValue.writeJSONString(data, out);
      } finally {
        out.close();
      }
    } catch (DaoException ex) {
      throw new ServletException(ex);
    } catch (ProtocolException ex) {
      throw new ServletException(ex);
    }
  }
  /**
   * Test importing of data_rppa file.
   *
   * @throws Exception All Errors.
   */
  @Test
  public void testImportRppaData() throws Exception {
    MySQLbulkLoader.bulkLoadOn();

    DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance();
    DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance();

    // Genes with alias:
    daoGene.addGene(makeGeneWithAlias(999999931, "TESTACACA", "TESTACC1"));
    daoGene.addGene(makeGeneWithAlias(999999207, "TESTAKT1", "TESTAKT"));
    daoGene.addGene(makeGeneWithAlias(999999597, "TESTSANDER", "TESTACC1"));
    daoGene.addGene(makeGeneWithAlias(999997158, "TESTTP53BP1", "TEST53BP1"));
    // test for NA being a special case in RPPA, and not the usual alias
    daoGene.addGene(makeGeneWithAlias(999997504, "XK", "NA"));
    // Other genes:
    daoGene.addGene(new CanonicalGene(999999932, "TESTACACB"));
    daoGene.addGene(new CanonicalGene(999999208, "TESTAKT2"));
    daoGene.addGene(new CanonicalGene(999999369, "TESTARAF"));
    daoGene.addGene(new CanonicalGene(999991978, "TESTEIF4EBP1"));
    daoGene.addGene(new CanonicalGene(999995562, "TESTPRKAA1"));
    daoGene.addGene(new CanonicalGene(999997531, "TESTYWHAE"));
    daoGene.addGene(new CanonicalGene(999910000, "TESTAKT3"));
    daoGene.addGene(new CanonicalGene(999995578, "TESTPRKCA"));

    GeneticProfile geneticProfile = new GeneticProfile();

    geneticProfile.setCancerStudyId(studyId);
    geneticProfile.setStableId("gbm_rppa");
    geneticProfile.setGeneticAlterationType(GeneticAlterationType.PROTEIN_LEVEL);
    geneticProfile.setDatatype("LOG2-VALUE");
    geneticProfile.setProfileName("RPPA Data");
    geneticProfile.setProfileDescription("RPPA Data");
    DaoGeneticProfile.addGeneticProfile(geneticProfile);

    int newGeneticProfileId =
        DaoGeneticProfile.getGeneticProfileByStableId("gbm_rppa").getGeneticProfileId();

    ProgressMonitor.setConsoleMode(true);
    // TBD: change this to use getResourceAsStream()
    File file = new File("src/test/resources/tabDelimitedData/data_rppa.txt");
    ImportTabDelimData parser = new ImportTabDelimData(file, newGeneticProfileId, null);
    int numLines = FileUtil.getNumLines(file);
    parser.importData(numLines);
    ConsoleUtil.showMessages();

    int sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE1").getInternalId();
    String value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997531);
    assertEquals("1.5", value);

    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE4").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997531);
    assertEquals("2", value);

    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE4").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997504);
    assertEquals(
        "NaN",
        value); // "NA" is not expected to be stored because of workaround for bug in firehose. See
                // also https://github.com/cBioPortal/cbioportal/issues/839#issuecomment-203523078

    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE1").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999995578);
    assertEquals("1.5", value);
  }
  /**
   * Test importing of data_expression file.
   *
   * @throws Exception All Errors.
   */
  @Test
  public void testImportmRnaData2() throws Exception {
    MySQLbulkLoader.bulkLoadOn();

    DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance();
    DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance();

    // Gene with alias:
    daoGene.addGene(makeGeneWithAlias(999997504, "TESTXK", "NA"));
    // Other genes:
    daoGene.addGene(new CanonicalGene(999999999, "TESTNAT1"));

    daoGene.addGene(new CanonicalGene(999997124, "TESTTNF"));
    daoGene.addGene(new CanonicalGene(999991111, "TESTCHEK1"));
    daoGene.addGene(new CanonicalGene(999999919, "TESTABCA1"));
    // will get generated negative id:
    daoGene.addGene(new CanonicalGene(-1, "TESTphosphoprotein"));

    GeneticProfile geneticProfile = new GeneticProfile();

    geneticProfile.setCancerStudyId(studyId);
    geneticProfile.setStableId("gbm_mrna");
    geneticProfile.setGeneticAlterationType(GeneticAlterationType.MRNA_EXPRESSION);
    geneticProfile.setDatatype("CONTINUOUS");
    geneticProfile.setProfileName("MRNA Data");
    geneticProfile.setProfileDescription("mRNA Data");
    DaoGeneticProfile.addGeneticProfile(geneticProfile);

    int newGeneticProfileId =
        DaoGeneticProfile.getGeneticProfileByStableId("gbm_mrna").getGeneticProfileId();

    ProgressMonitor.setConsoleMode(true);
    // TBD: change this to use getResourceAsStream()
    File file = new File("src/test/resources/tabDelimitedData/data_expression2.txt");
    ImportTabDelimData parser = new ImportTabDelimData(file, newGeneticProfileId, null);
    int numLines = FileUtil.getNumLines(file);
    parser.importData(numLines);

    // check if expected warnings are given:
    ArrayList<String> warnings = ProgressMonitor.getWarnings();
    int countDuplicatedRowWarnings = 0;
    int countInvalidEntrez = 0;
    int countSkippedWarnings = 0;
    for (String warning : warnings) {
      if (warning.contains("Duplicated row")) {
        countDuplicatedRowWarnings++;
      }
      if (warning.contains("invalid Entrez_Id")) {
        // invalid Entrez
        countInvalidEntrez++;
      }
      if (warning.contains("Record will be skipped")) {
        // Entrez is a valid number, but not found
        countSkippedWarnings++;
      }
    }
    // check that we have 11 warning messages:
    assertEquals(2, countDuplicatedRowWarnings);
    assertEquals(3, countInvalidEntrez);
    assertEquals(6, countSkippedWarnings);

    Set<Long> entrezGeneIds = DaoGeneticAlteration.getGenesIdInProfile(newGeneticProfileId);
    // data will be loaded for 5 of the genes
    assertEquals(5, entrezGeneIds.size());
    HashMap<Long, HashMap<Integer, String>> dataMap =
        dao.getGeneticAlterationMap(newGeneticProfileId, entrezGeneIds);
    assertEquals(5, dataMap.entrySet().size());

    int sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE1").getInternalId();
    String value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997124);
    assertEquals("770", value);

    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE3").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997124);
    assertEquals("220", value);

    // gene should also be loaded via its alias "NA" as defined above:
    sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "SAMPLE3").getInternalId();
    value = dao.getGeneticAlteration(newGeneticProfileId, sampleId, 999997504);
    assertEquals("9940", value);
  }
  /**
   * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
   *
   * @param request servlet request
   * @param response servlet response
   * @throws javax.servlet.ServletException if a servlet-specific error occurs
   * @throws java.io.IOException if an I/O error occurs
   */
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws IOException, ServletException {
    XDebug xdebug = new XDebug();
    xdebug.startTimer();

    response.setContentType("application/json");
    PrintWriter writer = response.getWriter();

    try {
      List resultsList = new LinkedList();

      // Get the gene list
      String geneList = request.getParameter(QueryBuilder.GENE_LIST);
      if (request instanceof XssRequestWrapper) {
        geneList = ((XssRequestWrapper) request).getRawParameter(QueryBuilder.GENE_LIST);
      }
      String cancerStudyIdListString = request.getParameter(QueryBuilder.CANCER_STUDY_LIST);
      String[] cancerStudyIdList = cancerStudyIdListString.split(",");

      // Get the priority
      Integer dataTypePriority;
      try {
        dataTypePriority =
            Integer.parseInt(request.getParameter(QueryBuilder.DATA_PRIORITY).trim());
      } catch (NumberFormatException e) {
        dataTypePriority = 0;
      }

      //  Cancer All Cancer Studies
      List<CancerStudy> cancerStudiesList = accessControl.getCancerStudies();
      HashMap<String, Boolean> studyMap = new HashMap<>();
      for (String studyId : cancerStudyIdList) {
        studyMap.put(studyId, Boolean.TRUE);
      }
      for (CancerStudy cancerStudy : cancerStudiesList) {
        String cancerStudyId = cancerStudy.getCancerStudyStableId();
        if (!studyMap.containsKey(cancerStudyId)) {
          continue;
        }
        if (cancerStudyId.equalsIgnoreCase("all")) continue;

        Map cancerMap = new LinkedHashMap();
        cancerMap.put("studyId", cancerStudyId);
        cancerMap.put("typeOfCancer", cancerStudy.getTypeOfCancerId());

        //  Get all Genetic Profiles Associated with this Cancer Study ID.
        ArrayList<GeneticProfile> geneticProfileList =
            GetGeneticProfiles.getGeneticProfiles(cancerStudyId);

        //  Get all Patient Lists Associated with this Cancer Study ID.
        ArrayList<SampleList> sampleSetList = GetSampleLists.getSampleLists(cancerStudyId);

        //  Get the default patient set
        AnnotatedSampleSets annotatedSampleSets =
            new AnnotatedSampleSets(sampleSetList, dataTypePriority);
        SampleList defaultSampleSet = annotatedSampleSets.getDefaultSampleList();
        if (defaultSampleSet == null) {
          continue;
        }

        List<String> sampleIds = defaultSampleSet.getSampleList();

        //  Get the default genomic profiles
        CategorizedGeneticProfileSet categorizedGeneticProfileSet =
            new CategorizedGeneticProfileSet(geneticProfileList);
        HashMap<String, GeneticProfile> defaultGeneticProfileSet = null;
        switch (dataTypePriority) {
          case 2:
            defaultGeneticProfileSet = categorizedGeneticProfileSet.getDefaultCopyNumberMap();
            break;
          case 1:
            defaultGeneticProfileSet = categorizedGeneticProfileSet.getDefaultMutationMap();
            break;
          case 0:
          default:
            defaultGeneticProfileSet =
                categorizedGeneticProfileSet.getDefaultMutationAndCopyNumberMap();
        }

        String mutationProfile = "", cnaProfile = "";
        for (GeneticProfile geneticProfile : defaultGeneticProfileSet.values()) {
          GeneticAlterationType geneticAlterationType = geneticProfile.getGeneticAlterationType();
          if (geneticAlterationType.equals(GeneticAlterationType.COPY_NUMBER_ALTERATION)) {
            cnaProfile = geneticProfile.getStableId();
          } else if (geneticAlterationType.equals(GeneticAlterationType.MUTATION_EXTENDED)) {
            mutationProfile = geneticProfile.getStableId();
          }
        }

        cancerMap.put("mutationProfile", mutationProfile);
        cancerMap.put("cnaProfile", cnaProfile);

        cancerMap.put("caseSetId", defaultSampleSet.getStableId());
        cancerMap.put("caseSetLength", sampleIds.size());

        ProfileDataSummary genomicData =
            getGenomicData(
                cancerStudyId,
                defaultGeneticProfileSet,
                defaultSampleSet,
                geneList,
                sampleSetList,
                request,
                response);

        ArrayList<GeneWithScore> geneFrequencyList = genomicData.getGeneFrequencyList();
        ArrayList<String> genes = new ArrayList<String>();
        for (GeneWithScore geneWithScore : geneFrequencyList) genes.add(geneWithScore.getGene());
        int noOfMutated = 0,
            noOfCnaUp = 0,
            noOfCnaDown = 0,
            noOfCnaLoss = 0,
            noOfCnaGain = 0,
            noOfOther = 0,
            noOfAll = 0;

        boolean skipStudy = defaultGeneticProfileSet.isEmpty();
        if (!skipStudy) {

          for (String sampleId : sampleIds) {
            if (sampleId == null) {
              continue;
            }
            if (!genomicData.isCaseAltered(sampleId)) continue;

            boolean isAnyMutated = false,
                isAnyCnaUp = false,
                isAnyCnaDown = false,
                isAnyCnaLoss = false,
                isAnyCnaGain = false;

            for (String gene : genes) {
              isAnyMutated |= genomicData.isGeneMutated(gene, sampleId);
              GeneticTypeLevel cnaLevel = genomicData.getCNALevel(gene, sampleId);
              boolean isCnaUp = cnaLevel != null && cnaLevel.equals(GeneticTypeLevel.Amplified);
              isAnyCnaUp |= isCnaUp;
              boolean isCnaDown =
                  cnaLevel != null && cnaLevel.equals(GeneticTypeLevel.HomozygouslyDeleted);
              isAnyCnaDown |= isCnaDown;
              boolean isCnaLoss =
                  cnaLevel != null && cnaLevel.equals(GeneticTypeLevel.HemizygouslyDeleted);
              isAnyCnaLoss |= isCnaLoss;
              boolean isCnaGain = cnaLevel != null && cnaLevel.equals(GeneticTypeLevel.Gained);
              isAnyCnaGain |= isCnaGain;
            }

            boolean isAnyCnaChanged = isAnyCnaUp || isAnyCnaDown;
            if (isAnyMutated && !isAnyCnaChanged) noOfMutated++;
            else if (isAnyMutated && isAnyCnaChanged) noOfOther++;
            else if (isAnyCnaUp) noOfCnaUp++;
            else if (isAnyCnaDown) noOfCnaDown++;
            else if (isAnyCnaGain) noOfCnaGain++;
            else if (isAnyCnaLoss) noOfCnaLoss++;

            noOfAll++;
          }
        }

        Map alterations = new LinkedHashMap();
        cancerMap.put("alterations", alterations);
        alterations.put("all", noOfAll);
        alterations.put("mutation", noOfMutated);
        alterations.put("cnaUp", noOfCnaUp);
        alterations.put("cnaDown", noOfCnaDown);
        alterations.put("cnaLoss", noOfCnaLoss);
        alterations.put("cnaGain", noOfCnaGain);
        alterations.put("other", noOfOther);
        cancerMap.put("genes", genes);
        cancerMap.put("skipped", skipStudy);

        resultsList.add(cancerMap);
      }

      JSONValue.writeJSONString(resultsList, writer);
    } catch (DaoException e) {
      throw new ServletException(e);
    } catch (ProtocolException e) {
      throw new ServletException(e);
    } finally {
      writer.close();
    }
  }