コード例 #1
0
ファイル: SGExporterTask.java プロジェクト: EBIBioSamples/X-S
  void countSample(BioSample s) {
    stat.incUniqSampleCounter();

    if (AbstractXMLFormatter.isSamplePublic(s, stat.getNowDate()))
      stat.incSamplePublicUniqCounter();

    if (hasSourcesByName || hasSourcesByAcc) {

      for (MSI gmsi : s.getMSIs()) {
        for (DatabaseRecordRef db : gmsi.getDatabaseRecordRefs()) {
          if (hasSourcesByAcc) {
            String scrNm = db.getAcc();

            if (scrNm != null) {
              scrNm = scrNm.trim();

              if (scrNm.length() != 0) stat.addToSourceByAcc(scrNm, 1);
            }
          }

          if (hasSourcesByName) {
            String scrNm = db.getDbName();

            if (scrNm != null) {
              scrNm = scrNm.trim();

              if (scrNm.length() != 0) stat.addToSourceByAcc(scrNm, 1);
            }
          }
        }
      }
    }
  }
コード例 #2
0
ファイル: SGExporterTask.java プロジェクト: EBIBioSamples/X-S
  @Override
  public void run() {
    procThread = Thread.currentThread();

    procThread.setName(procThread.getName() + "-ExporterTask-gen" + (++genNo) + "-lane" + laneNo);

    AuxInfo auxInf = null;

    if (myEqFactory != null) auxInf = new AuxInfoImpl(myEqFactory);

    int objCount = 0;

    try {
      StringBuilder sb = new StringBuilder();

      while (true) {

        if (needGroupLoop) {
          Collection<BioSampleGroup> grps = null;

          int restart = 0;

          while (true) {
            try {
              grps = sgQM.getGroups();
              break;
            } catch (PersistenceException e) {
              sgQM.close();

              restart++;

              stat.incRecoverAttempt();

              if (restart > MaxErrorRecoverAttempts) throw e;
            }
          }

          if (grps.size() == 0) {
            log.debug("({}) No more groups to process", Thread.currentThread().getName());
            needGroupLoop = false;
          }

          for (BioSampleGroup grp : grps) {
            if (stopFlag.get()) {
              log.debug(
                  "({}) Stop flag set. Sending FINISH message", Thread.currentThread().getName());
              putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_FINISH, this));
              return;
            }

            if (!stat.addGroup(grp.getId())) continue;

            objCount++;

            //      System.out.printf("=MSI (L%d-G%d-N%d) %s  Processing group %s %n"
            //        ,laneNo,genNo,msiCount,msi.getAcc(), g.getAcc());

            stat.incGroupCounter();

            boolean grpPub = AbstractXMLFormatter.isGroupPublic(grp, stat.getNowDate());

            if (grpPub) stat.incGroupPublicCounter();

            for (FormattingTask ft : tasks) {
              if (ft.getGroupQueue() == null) continue;

              if (!ft.confirmOutput()) continue;

              restart = 0;

              while (true) {
                try {
                  sb.setLength(0);
                  ft.getFormatter().exportGroup(grp, auxInf, sb, false);
                  break;
                } catch (PersistenceException e) {
                  restart++;

                  stat.incRecoverAttempt();

                  if (restart > MaxErrorRecoverAttempts) throw e;
                }
              }

              putIntoQueue(ft.getGroupQueue(), sb.toString());
            }

            if (hasGroupedSmp
                || maxObjsPerThr <= 0
                || maxObjsPerThr - objCount > sgQM.getChunkSize()) {
              for (BioSample s : grp.getSamples()) {
                if (!hasGroupedSmp
                    && maxObjsPerThr > 0
                    && maxObjsPerThr - objCount <= sgQM.getChunkSize()) break;

                if (!stat.addSample(s.getId())) continue;

                objCount++;

                countSample(s);

                for (FormattingTask ft : tasks) {
                  if (ft.getSampleQueue() == null) continue;

                  if (!ft.confirmOutput()) continue;

                  restart = 0;

                  while (true) {
                    try {
                      sb.setLength(0);
                      ft.getFormatter().exportSample(s, auxInf, sb, false);
                      break;
                    } catch (PersistenceException e) {
                      restart++;

                      stat.incRecoverAttempt();

                      if (restart > MaxErrorRecoverAttempts) throw e;
                    }
                  }

                  putIntoQueue(ft.getSampleQueue(), sb.toString());
                }
              }
            }

            if (stopFlag.get()) {
              log.debug(
                  "({}) Stop flag set. Sending FINISH message", Thread.currentThread().getName());
              putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_FINISH, this));
              return;
            }
          }
        }

        if (hasUngroupedSmp) {
          Collection<BioSample> smpls = null;

          int restart = 0;

          while (true) {
            try {
              smpls = sgQM.getSamples();
              break;
            } catch (PersistenceException e) {
              sgQM.close();

              restart++;

              stat.incRecoverAttempt();

              if (restart > MaxErrorRecoverAttempts) throw e;
            }
          }

          if (smpls.size() == 0 && !needGroupLoop) {
            log.debug("({}) No more data to process", Thread.currentThread().getName());
            break;
          }

          for (BioSample s : smpls) {

            if (!stat.addSample(s.getId())) continue;

            objCount++;

            if (stopFlag.get()) {
              log.debug(
                  "({}) Stop flag set. Sending FINISH message", Thread.currentThread().getName());
              putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_FINISH, this));
              return;
            }

            countSample(s);

            for (FormattingTask ft : tasks) {
              if (ft.getSampleQueue() == null || ft.isGroupedSamplesOnly()) continue;

              if (!ft.confirmOutput()) continue;

              restart = 0;

              while (true) {
                try {
                  sb.setLength(0);
                  ft.getFormatter().exportSample(s, auxInf, sb, false);
                  break;
                } catch (PersistenceException e) {
                  restart++;

                  stat.incRecoverAttempt();

                  if (restart > MaxErrorRecoverAttempts) throw e;
                }
              }

              putIntoQueue(ft.getSampleQueue(), sb.toString());
            }
          }

        } else if (!needGroupLoop) {
          log.debug("({}) No more data to process", Thread.currentThread().getName());
          break;
        }

        boolean needMoreData = false;

        for (FormattingTask ft : tasks) {
          if (ft.confirmOutput()) {
            needMoreData = true;
            break;
          }
        }

        if (!needMoreData) {
          log.debug(
              "({}) Output tasks don't need more data. Breaking loop",
              Thread.currentThread().getName());
          break;
        }

        if (maxObjsPerThr > 0 && objCount >= maxObjsPerThr) {
          log.debug(
              "({}) Thread TTL expared. Processed {} objects. Sending TTL message",
              Thread.currentThread().getName(),
              objCount);

          if (auxInf != null) auxInf.destroy();

          auxInf = null;

          sgQM.close();

          putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_TTL, this));
          return;
        }

        if (auxInf != null) auxInf.clear();
      }

      log.debug(
          "({}) Thread terminating. Sending FINISH message", Thread.currentThread().getName());
      putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_FINISH, this));

    } catch (Throwable e) {
      e.printStackTrace();

      putIntoQueue(controlQueue, new ControlMessage(Type.PROCESS_ERROR, this, e));
    } finally {
      if (auxInf != null) auxInf.destroy();

      sgQM.close();
    }
  }