Beispiel #1
0
  public static void main(final String[] arg)
      throws IOException, JSAPException, NoSuchMethodException {

    final SimpleJSAP jsap =
        new SimpleJSAP(
            BloomFilter.class.getName(),
            "Creates a Bloom filter reading from standard input a newline-separated list of terms.",
            new Parameter[] {
              new FlaggedOption(
                  "bufferSize",
                  IntSizeStringParser.getParser(),
                  "64Ki",
                  JSAP.NOT_REQUIRED,
                  'b',
                  "buffer-size",
                  "The size of the I/O buffer used to read terms."),
              new FlaggedOption(
                  "encoding",
                  ForNameStringParser.getParser(Charset.class),
                  "UTF-8",
                  JSAP.NOT_REQUIRED,
                  'e',
                  "encoding",
                  "The term file encoding."),
              new UnflaggedOption(
                  "bloomFilter",
                  JSAP.STRING_PARSER,
                  JSAP.NO_DEFAULT,
                  JSAP.REQUIRED,
                  JSAP.NOT_GREEDY,
                  "The filename for the serialised front-coded list."),
              new UnflaggedOption(
                  "size",
                  JSAP.INTSIZE_PARSER,
                  JSAP.NO_DEFAULT,
                  JSAP.REQUIRED,
                  JSAP.NOT_GREEDY,
                  "The size of the filter (i.e., the expected number of elements in the filter; usually, the number of terms)."),
              new UnflaggedOption(
                  "precision",
                  JSAP.INTEGER_PARSER,
                  JSAP.NO_DEFAULT,
                  JSAP.REQUIRED,
                  JSAP.NOT_GREEDY,
                  "The precision of the filter.")
            });

    JSAPResult jsapResult = jsap.parse(arg);
    if (jsap.messagePrinted()) return;

    final int bufferSize = jsapResult.getInt("bufferSize");
    final String filterName = jsapResult.getString("bloomFilter");
    final Charset encoding = (Charset) jsapResult.getObject("encoding");

    BloomFilter filter = new BloomFilter(jsapResult.getInt("size"), jsapResult.getInt("precision"));
    final ProgressLogger pl = new ProgressLogger();
    pl.itemsName = "terms";
    pl.start("Reading terms...");
    MutableString s = new MutableString();
    FastBufferedReader reader =
        new FastBufferedReader(new InputStreamReader(System.in, encoding), bufferSize);
    while (reader.readLine(s) != null) {
      filter.add(s);
      pl.lightUpdate();
    }
    pl.done();

    BinIO.storeObject(filter, filterName);
  }
  /** Computes the next step of the Power Method. */
  public void step() throws IOException {
    double[] oldRank = rank, newRank = previousRank;
    DoubleArrays.fill(newRank, 0.0);

    // for each node, calculate its outdegree and redistribute its rank among pointed nodes
    double accum = 0.0;

    progressLogger.expectedUpdates = numNodes;
    progressLogger.start("Iteration " + (++iterationNumber) + "...");

    final ArcLabelledNodeIterator nodeIterator = g.nodeIterator();
    int i, outdegree, j, n = numNodes;
    int[] succ;
    Label[] lab;

    while (n-- != 0) {
      i = nodeIterator.nextInt();
      outdegree = nodeIterator.outdegree();

      if (outdegree == 0 || buckets != null && buckets.get(i)) accum += oldRank[i];
      else {
        j = outdegree;
        succ = nodeIterator.successorArray();
        lab = nodeIterator.labelArray();
        while (j-- != 0) {
          newRank[succ[j]] += (oldRank[i] * lab[j].getFloat()) / sumoutweight[i];
        }
      }
      progressLogger.update();
    }
    progressLogger.done();

    final double accumOverNumNodes = accum / numNodes;

    final double oneOverNumNodes = 1.0 / numNodes;
    if (preference != null)
      if (preferentialAdjustment == null)
        for (i = numNodes; i-- != 0; )
          newRank[i] =
              alpha * newRank[i]
                  + (1 - alpha) * preference.getDouble(i)
                  + alpha * accumOverNumNodes;
      else
        for (i = numNodes; i-- != 0; )
          newRank[i] =
              alpha * newRank[i]
                  + (1 - alpha) * preference.getDouble(i)
                  + alpha * accum * preferentialAdjustment.getDouble(i);
    else if (preferentialAdjustment == null)
      for (i = numNodes; i-- != 0; )
        newRank[i] = alpha * newRank[i] + (1 - alpha) * oneOverNumNodes + alpha * accumOverNumNodes;
    else
      for (i = numNodes; i-- != 0; )
        newRank[i] =
            alpha * newRank[i]
                + (1 - alpha) * oneOverNumNodes
                + alpha * accum * preferentialAdjustment.getDouble(i);

    // make the rank just computed the new rank
    rank = newRank;
    previousRank = oldRank;

    // Compute derivatives.
    n = iterationNumber;

    if (subset == null) {
      for (i = 0; i < order.length; i++) {
        final int k = order[i];
        final double alphak = Math.pow(alpha, k);
        final double nFallingK = Util.falling(n, k);
        for (j = 0; j < numNodes; j++)
          derivative[i][j] += nFallingK * (rank[j] - previousRank[j]) / alphak;
      }
    } else {
      for (i = 0; i < order.length; i++) {
        final int k = order[i];
        final double alphak = Math.pow(alpha, k);
        final double nFallingK = Util.falling(n, k);

        for (int t : subset) derivative[i][t] += nFallingK * (rank[t] - previousRank[t]) / alphak;
      }
    }

    // Compute coefficients, if required.

    if (coeffBasename != null) {
      final DataOutputStream coefficients =
          new DataOutputStream(
              new FastBufferedOutputStream(
                  new FileOutputStream(coeffBasename + "-" + (iterationNumber))));
      final double alphaN = Math.pow(alpha, n);
      for (i = 0; i < numNodes; i++) coefficients.writeDouble((rank[i] - previousRank[i]) / alphaN);
      coefficients.close();
    }
  }
  /**
   * Perform the split transcripts mode.
   *
   * @throws IOException error reading / writing
   */
  @Override
  public void execute() throws IOException {
    // Load the gene to transcripts file
    if (!config.validate()) {
      throw new IOException("Invalid SplitTranscripts configuration");
    }
    final GeneTranscriptRelationships gtr = new GeneTranscriptRelationships();
    final IndexedIdentifier transcriptIdents = new IndexedIdentifier();
    final Int2ObjectMap<MutableString> transcriptIndexToIdMap =
        new Int2ObjectOpenHashMap<MutableString>();
    final List<FastXEntry> fastxEntries = new LinkedList<FastXEntry>();
    //
    // Pass through the file once to collect the transcript - gene relationships
    //
    int entryCount = 0;
    try {
      for (final FastXEntry entry : new FastXReader(config.getInputFile())) {
        entryCount++;
        parseHeader(entry.getEntryHeader());
        final MutableString transcriptId = transcriptHeader.get("transcriptId");
        final MutableString geneId = transcriptHeader.get("geneId");

        final int transcriptIndex = transcriptIdents.registerIdentifier(transcriptId);
        gtr.addRelationship(geneId, transcriptIndex);

        transcriptIndexToIdMap.put(transcriptIndex, transcriptId);

        fastxEntries.add(entry.clone());
      }
    } catch (CloneNotSupportedException e) {
      LOG.error("Couldn't clone for some reason", e);
      throw new GobyRuntimeException("Couldn't clone for some reason", e);
    }

    LOG.info("Loading map of genes-transcripts complete.");

    //
    // Scan through the transcript-gene relationships to determine which
    // transcript id goes into which file
    //
    final Int2IntMap transcriptIndex2FileIndex = new Int2IntOpenHashMap();
    final String configOutputFilename = config.getOutputBase() + ".config";
    final String configOutputPath = FilenameUtils.getFullPath(configOutputFilename);
    if (StringUtils.isNotBlank(configOutputPath)) {
      LOG.info("Creating output directory: " + configOutputPath);
      FileUtils.forceMkdir(new File(configOutputPath));
    }

    PrintWriter configOutput = null;
    try {
      configOutput = new PrintWriter(configOutputFilename);
      configOutput.println("Ensembl Gene ID\tEnsembl Transcript ID");

      final Int2IntMap fileIndex2NumberOfEntries = new Int2IntOpenHashMap();
      fileIndex2NumberOfEntries.defaultReturnValue(0);
      transcriptIndex2FileIndex.defaultReturnValue(-1);

      final int initialNumberOfFiles = getNumberOfFiles(gtr, transcriptIndex2FileIndex);

      for (int geneIndex = 0; geneIndex < gtr.getNumberOfGenes(); geneIndex++) {
        final MutableString geneId = gtr.getGeneId(geneIndex);
        final IntSet transcriptIndices = gtr.getTranscriptSet(geneIndex);
        int fileNum = 0;

        for (final int transcriptIndex : transcriptIndices) {
          if (transcriptIndex2FileIndex.get(transcriptIndex) != -1) {
            LOG.warn("Skipping repeated transcriptIndex: " + transcriptIndex);
            continue;
          }
          final int maxEntriesPerFile = config.getMaxEntriesPerFile();
          final int numberOfEntriesInOriginalBucket = fileIndex2NumberOfEntries.get(fileNum);
          final int adjustedFileIndex =
              fileNum
                  + initialNumberOfFiles * (numberOfEntriesInOriginalBucket / maxEntriesPerFile);

          transcriptIndex2FileIndex.put(transcriptIndex, adjustedFileIndex);
          fileIndex2NumberOfEntries.put(fileNum, fileIndex2NumberOfEntries.get(fileNum) + 1);
          final MutableString transcriptId = transcriptIndexToIdMap.get(transcriptIndex);
          configOutput.printf("%s\t%s%n", geneId, transcriptId);

          fileNum++;
        }
      }
    } finally {
      IOUtils.closeQuietly(configOutput);
    }

    final int numFiles = getFileIndices(transcriptIndex2FileIndex).size();
    if (LOG.isInfoEnabled()) {
      LOG.info(
          NumberFormat.getInstance().format(entryCount)
              + " entries will be written to "
              + numFiles
              + " files");
      final int maxEntriesPerFile = config.getMaxEntriesPerFile();
      if (maxEntriesPerFile < Integer.MAX_VALUE) {
        LOG.info("Each file will contain at most " + maxEntriesPerFile + " entries");
      }
    }

    // formatter for uniquely numbering files each with the same number of digits
    final NumberFormat fileNumberFormatter = getNumberFormatter(numFiles - 1);

    final ProgressLogger progressLogger = new ProgressLogger();
    progressLogger.expectedUpdates = entryCount;
    progressLogger.itemsName = "entries";
    progressLogger.start();

    // Write each file one at a time rather than in the order they appear in the input file
    // to avoid the issue of having too many streams open at the same or continually opening
    // and closing streams which is quite costly.  We could store the gene/transcripts in
    // memory and then just write the files at the end but that could be worse.
    for (final int fileIndex : getFileIndices(transcriptIndex2FileIndex)) {
      final String filename =
          config.getOutputBase() + "." + fileNumberFormatter.format(fileIndex) + ".fa.gz";
      PrintStream printStream = null;
      try {
        // each file is compressed
        printStream = new PrintStream(new GZIPOutputStream(new FileOutputStream(filename)));

        //
        // Read through the input file get the actual sequence information
        //
        final Iterator<FastXEntry> entries = fastxEntries.iterator();
        while (entries.hasNext()) {
          final FastXEntry entry = entries.next();
          parseHeader(entry.getEntryHeader());
          final MutableString transcriptId = transcriptHeader.get("transcriptId");
          final MutableString geneId = transcriptHeader.get("geneId");
          final int transcriptIndex = transcriptIdents.getInt(transcriptId);
          final int transcriptFileIndex = transcriptIndex2FileIndex.get(transcriptIndex);
          if (transcriptFileIndex == fileIndex) {
            printStream.print(entry.getHeaderSymbol());
            printStream.print(transcriptId);
            printStream.print(" gene:");
            printStream.println(geneId);
            printStream.println(entry.getEntrySansHeader());
            entries.remove();
            progressLogger.lightUpdate();
          }
        }
      } finally {
        IOUtils.closeQuietly(printStream);
      }
    }

    assert progressLogger.count == entryCount : "Some entries were not processed!";
    progressLogger.done();
  }