/**
  * Creates an option container
  *
  * @param index The index of the option container (0..n).
  */
 public OptionContainer(int index) throws OptionException {
   this.index = index;
   savedOptionMap = Collections.synchronizedSortedMap(new TreeMap<Option, Object>());
   dependenciesResolvedOptionMap =
       Collections.synchronizedSortedMap(new TreeMap<Option, Object>());
   commandLineOptionMap = Collections.synchronizedSortedMap(new TreeMap<Option, Object>());
   optionFileOptionMap = Collections.synchronizedSortedMap(new TreeMap<Option, Object>());
 }
Example #2
0
 /** @param iCapacity Maximum number of entries that cache can hold */
 public DBSubsetCache(int iCapacity) {
   iUsed = 0;
   iTopIndex = 0;
   iCacheCapacity = iCapacity;
   LRUList = new String[iCacheCapacity];
   for (int s = 0; s < iCacheCapacity; s++) LRUList[s] = null;
   oCache = Collections.synchronizedSortedMap(new TreeMap<String, DBCacheEntry>());
 }
  /** Creates a new instance of CMapFormat0 */
  protected CMapFormat4(short language) {
    super((short) 4, language);

    segments = Collections.synchronizedSortedMap(new TreeMap<Segment, Object>());

    char[] map = new char[1];
    map[0] = (char) 0;
    addSegment((short) 0xffff, (short) 0xffff, map);
  }
Example #4
0
  /**
   * 計測対象オブジェクトを与えて初期化
   *
   * @param measuredObject 計測対象オブジェクト
   */
  public MetricsInfo(final T measuredObject) {

    if (null == measuredObject) {
      throw new NullPointerException();
    }

    this.measuredObject = measuredObject;
    this.metrics =
        Collections.synchronizedSortedMap(
            new TreeMap<AbstractPlugin, Number>(new MetricTypeAndNamePluginComparator()));
  }
  public BoteMailbox(EmailFolder folder, long uidValidity, long uid) {
    super(new MailboxPath("I2P-Bote", "bote", folder.getName()), uidValidity);
    this.folder = folder;
    this.messageMap =
        Collections.synchronizedSortedMap(
            new TreeMap<Email, BoteMessage>(
                new Comparator<Email>() {
                  @Override
                  public int compare(Email email1, Email email2) {
                    // Try received dates first, this is set for all received.
                    // emails. If not set, this is a sent email, use sent date.
                    Date msg1date = email1.getReceivedDate();
                    if (msg1date == null)
                      try {
                        msg1date = email1.getSentDate();
                      } catch (MessagingException e) {
                      }

                    Date msg2date = email2.getReceivedDate();
                    if (msg2date == null)
                      try {
                        msg2date = email2.getSentDate();
                      } catch (MessagingException e) {
                      }

                    if (msg1date != null && msg2date != null) return msg1date.compareTo(msg2date);

                    // Catch-all
                    return email1.getMessageID().compareTo(email2.getMessageID());
                  }
                }));
    this.uid = uid;

    modSeq = System.currentTimeMillis();
    modSeq <<= 32;

    startListening();
    try {
      List<Email> emails = folder.getElements();
      for (Email email : emails) messageMap.put(email, new BoteMessage(email, getFolderName()));
      updateMessages();
    } catch (PasswordException e) {
      throw new RuntimeException(_t("Password required or invalid password provided"), e);
    }
  }
/**
 * Regionserver which provides transactional support for atomic transactions. This is achieved with
 * optimistic concurrency control (see http://www.seas.upenn.edu/~zives/cis650/papers/opt-cc.pdf).
 * We keep track read and write sets for each transaction, and hold off on processing the writes. To
 * decide to commit a transaction we check its read sets with all transactions that have committed
 * while it was running for overlaps.
 *
 * <p>Because transactions can span multiple regions, all regions must agree to commit a
 * transactions. The client side of this commit protocol is encoded in
 * org.apache.hadoop.hbase.client.transactional.TransactionManger
 *
 * <p>In the event of an failure of the client mid-commit, (after we voted yes), we will have to
 * consult the transaction log to determine the final decision of the transaction. This is not yet
 * implemented.
 */
public class TransactionalRegion extends HRegion {

  private static final String OLD_TRANSACTION_FLUSH = "hbase.transaction.flush";
  private static final int DEFAULT_OLD_TRANSACTION_FLUSH = 100; // Do a flush if
  // we have this
  // many old
  // transactions..

  static final Log LOG = LogFactory.getLog(TransactionalRegion.class);

  // Collection of active transactions (PENDING) keyed by id.
  protected Map<String, TransactionState> transactionsById =
      new HashMap<String, TransactionState>();

  // Map of recent transactions that are COMMIT_PENDING or COMMITED keyed by
  // their sequence number
  private SortedMap<Integer, TransactionState> commitedTransactionsBySequenceNumber =
      Collections.synchronizedSortedMap(new TreeMap<Integer, TransactionState>());

  // Collection of transactions that are COMMIT_PENDING
  private Set<TransactionState> commitPendingTransactions =
      Collections.synchronizedSet(new HashSet<TransactionState>());

  private AtomicInteger nextSequenceId = new AtomicInteger(0);
  private Object commitCheckLock = new Object();
  private THLog transactionLog;
  private final int oldTransactionFlushTrigger;
  private Leases transactionLeases;

  /**
   * @param basedir
   * @param log
   * @param fs
   * @param conf
   * @param regionInfo
   * @param flushListener
   */
  public TransactionalRegion(
      final Path basedir,
      final HLog log,
      final FileSystem fs,
      final Configuration conf,
      final HRegionInfo regionInfo,
      final FlushRequester flushListener) {
    oldTransactionFlushTrigger = conf.getInt(OLD_TRANSACTION_FLUSH, DEFAULT_OLD_TRANSACTION_FLUSH);
  }

  /**
   * Open HRegion. Calls initialize and sets sequenceid to both regular WAL and trx WAL.
   *
   * @param reporter
   * @return Returns <code>this</code>
   * @throws IOException
   */
  @Override
  protected HRegion openHRegion(final CancelableProgressable reporter) throws IOException {
    super.openHRegion(reporter);
    if (this.transactionLog != null) {
      this.transactionLog.setSequenceNumber(super.getLog().getSequenceNumber());
    }
    return this;
  }

  /* protected long replayRecoveredEditsIfAny(final Path regiondir, final long minSeqId, final Progressable reporter)
          throws UnsupportedEncodingException, IOException {
      long maxSeqId = super.replayRecoveredEdits(regiondir, minSeqId, reporter);

      Path recoveredEdits = new Path(regiondir, HLogSplitter.RECOVERED_EDITS);

      doReconstructionLog(recoveredEdits, minSeqId, maxSeqId, reporter);

      return maxSeqId;
  }*/

  protected void doReconstructionLog(
      final Path oldCoreLogFile,
      final long minSeqId,
      final long maxSeqId,
      final Progressable reporter)
      throws UnsupportedEncodingException, IOException {

    Path trxPath = new Path(oldCoreLogFile.getParent(), THLog.HREGION_OLD_THLOGFILE_NAME);

    // We can ignore doing anything with the Trx Log table, it is
    // not-transactional.
    if (super.getTableDesc().getNameAsString().equals(HBaseBackedTransactionLogger.TABLE_NAME)) {
      return;
    }

    THLogRecoveryManager recoveryManager = new THLogRecoveryManager(this);
    Map<Long, WALEdit> commitedTransactionsById =
        recoveryManager.getCommitsFromLog(trxPath, minSeqId, reporter);

    if (commitedTransactionsById != null && commitedTransactionsById.size() > 0) {
      LOG.debug("found " + commitedTransactionsById.size() + " COMMITED transactions to recover.");

      for (Entry<Long, WALEdit> entry : commitedTransactionsById.entrySet()) {
        LOG.debug(
            "Writing " + entry.getValue().size() + " updates for transaction " + entry.getKey());
        WALEdit b = entry.getValue();

        for (KeyValue kv : b.getKeyValues()) {
          // FIXME need to convert these into puts and deletes. Not sure this is
          // the write way.
          // Could probably combine multiple KV's into single put/delete.
          // Also timestamps?
          if (kv.getType() == KeyValue.Type.Put.getCode()) {
            Put put = new Put();
            put.add(kv);
            super.put(put);
          } else if (kv.isDelete()) {
            Delete del = new Delete(kv.getRow());
            if (kv.isDeleteFamily()) {
              del.deleteFamily(kv.getFamily());
            } else if (kv.isDeleteType()) {
              del.deleteColumn(kv.getFamily(), kv.getQualifier());
            }
          }
        }
      }

      LOG.debug("Flushing cache"); // We must trigger a cache flush,
      // otherwise we will would ignore the log on subsequent failure
      if (!super.flushcache()) {
        LOG.warn("Did not flush cache");
      }
    }
  }

  /**
   * We need to make sure that we don't complete a cache flush between running transactions. If we
   * did, then we would not find all log messages needed to restore the transaction, as some of them
   * would be before the last "complete" flush id.
   */
  @Override
  protected long getCompleteCacheFlushSequenceId(final long currentSequenceId) {
    LinkedList<TransactionState> transactionStates;
    synchronized (transactionsById) {
      transactionStates = new LinkedList<TransactionState>(transactionsById.values());
    }

    long minPendingStartSequenceId = currentSequenceId;
    for (TransactionState transactionState : transactionStates) {
      minPendingStartSequenceId =
          Math.min(minPendingStartSequenceId, transactionState.getHLogStartSequenceId());
    }
    return minPendingStartSequenceId;
  }

  /**
   * @param transactionId
   * @throws IOException
   */
  public void beginTransaction(final long transactionId) throws IOException {
    checkClosing();
    String key = String.valueOf(transactionId);
    if (transactionsById.get(key) != null) {
      TransactionState alias = getTransactionState(transactionId);
      if (alias != null) {
        alias.setStatus(Status.ABORTED);
        retireTransaction(alias);
      }
      LOG.error(
          "Existing trasaction with id ["
              + key
              + "] in region ["
              + super.getRegionInfo().getRegionNameAsString()
              + "]");
      throw new IOException("Already exiting transaction id: " + key);
    }

    TransactionState state =
        new TransactionState(
            transactionId, super.getLog().getSequenceNumber(), super.getRegionInfo());

    state.setStartSequenceNumber(nextSequenceId.get());
    List<TransactionState> commitPendingCopy =
        new ArrayList<TransactionState>(commitPendingTransactions);
    for (TransactionState commitPending : commitPendingCopy) {
      state.addTransactionToCheck(commitPending);
    }

    synchronized (transactionsById) {
      transactionsById.put(key, state);
    }
    try {
      transactionLeases.createLease(getLeaseId(transactionId), new TransactionLeaseListener(key));
    } catch (LeaseStillHeldException e) {
      LOG.error(
          "Lease still held for ["
              + key
              + "] in region ["
              + super.getRegionInfo().getRegionNameAsString()
              + "]");
      throw new RuntimeException(e);
    }
    LOG.debug(
        "Begining transaction "
            + key
            + " in region "
            + super.getRegionInfo().getRegionNameAsString());

    maybeTriggerOldTransactionFlush();
  }

  private String getLeaseId(final long transactionId) {
    return super.getRegionInfo().getRegionNameAsString() + transactionId;
  }

  public Result get(final long transactionId, final Get get) throws IOException {
    Scan scan = new Scan(get);
    List<KeyValue> results = new ArrayList<KeyValue>();

    InternalScanner scanner = null;
    try {
      scanner = getScanner(transactionId, scan);
      scanner.next(results);
    } finally {
      if (scanner != null) {
        scanner.close();
      }
    }
    return new Result(results);
  }

  /** Get a transactional scanner. */
  public RegionScanner getScanner(final long transactionId, final Scan scan) throws IOException {
    checkClosing();

    TransactionState state = getTransactionState(transactionId);
    state.addScan(scan);
    List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(1);
    scanners.add(state.getScanner(scan));
    return super.getScanner(wrapWithDeleteFilter(scan, state), scanners);
  }

  private Scan wrapWithDeleteFilter(final Scan scan, final TransactionState state) {
    FilterBase deleteFilter =
        new FilterBase() {

          private boolean rowFiltered = false;

          @Override
          public void reset() {
            rowFiltered = false;
          }

          @Override
          public boolean hasFilterRow() {
            return true;
          }

          @Override
          public void filterRow(final List<KeyValue> kvs) {
            state.applyDeletes(kvs, scan.getTimeRange().getMin(), scan.getTimeRange().getMax());
            rowFiltered = kvs.isEmpty();
          }

          @Override
          public boolean filterRow() {
            return rowFiltered;
          }

          @Override
          public void write(final DataOutput out) throws IOException {
            // does nothing
          }

          @Override
          public void readFields(final DataInput in) throws IOException {
            // does nothing
          }
        };

    if (scan.getFilter() == null) {
      scan.setFilter(deleteFilter);
      return scan;
    }

    FilterList wrappedFilter = new FilterList(Arrays.asList(deleteFilter, scan.getFilter()));
    scan.setFilter(wrappedFilter);
    return scan;
  }

  /**
   * Add a write to the transaction. Does not get applied until commit process.
   *
   * @param transactionId
   * @param put
   * @throws IOException
   */
  public void put(final long transactionId, final Put put) throws IOException {
    checkClosing();

    TransactionState state = getTransactionState(transactionId);
    state.addWrite(put);
  }

  /**
   * Add multiple writes to the transaction. Does not get applied until commit process.
   *
   * @param transactionId
   * @param puts
   * @throws IOException
   */
  public void put(final long transactionId, final Put[] puts) throws IOException {
    checkClosing();

    TransactionState state = getTransactionState(transactionId);
    for (Put put : puts) {
      state.addWrite(put);
    }
  }

  /**
   * Add a delete to the transaction. Does not get applied until commit process.
   *
   * @param transactionId
   * @param delete
   * @throws IOException
   */
  public void delete(final long transactionId, final Delete delete) throws IOException {
    checkClosing();
    TransactionState state = getTransactionState(transactionId);
    state.addDelete(delete);
  }

  /**
   * @param transactionId
   * @return TransactionRegionInterface commit code
   * @throws IOException
   */
  public int commitRequest(final long transactionId) throws IOException {
    checkClosing();

    synchronized (commitCheckLock) {
      TransactionState state = getTransactionState(transactionId);
      if (state == null) {
        return TransactionalRegionInterface.COMMIT_UNSUCESSFUL;
      }

      if (hasConflict(state)) {
        state.setStatus(Status.ABORTED);
        retireTransaction(state);
        return TransactionalRegionInterface.COMMIT_UNSUCESSFUL;
      }

      // No conflicts, we can commit.
      LOG.trace(
          "No conflicts for transaction "
              + transactionId
              + " found in region "
              + super.getRegionInfo().getRegionNameAsString()
              + ". Voting for commit");

      // If there are writes we must keep record off the transaction
      if (state.hasWrite()) {
        // Order is important
        state.setStatus(Status.COMMIT_PENDING);
        commitPendingTransactions.add(state);
        state.setSequenceNumber(nextSequenceId.getAndIncrement());
        commitedTransactionsBySequenceNumber.put(state.getSequenceNumber(), state);
        transactionLog.writeCommitResuestToLog(getRegionInfo(), state);

        return TransactionalRegionInterface.COMMIT_OK;
      }
      // Otherwise we were read-only and commitable, so we can forget it.
      state.setStatus(Status.COMMITED);
      retireTransaction(state);
      return TransactionalRegionInterface.COMMIT_OK_READ_ONLY;
    }
  }

  /**
   * @param transactionId
   * @return true if commit is successful
   * @throws IOException
   */
  public boolean commitIfPossible(final long transactionId) throws IOException {
    int status = commitRequest(transactionId);

    if (status == TransactionalRegionInterface.COMMIT_OK) {
      commit(transactionId);
      return true;
    } else if (status == TransactionalRegionInterface.COMMIT_OK_READ_ONLY) {
      return true;
    }
    return false;
  }

  private boolean hasConflict(final TransactionState state) {
    // Check transactions that were committed while we were running
    for (int i = state.getStartSequenceNumber(); i < nextSequenceId.get(); i++) {
      TransactionState other = commitedTransactionsBySequenceNumber.get(i);
      if (other == null) {
        continue;
      }
      state.addTransactionToCheck(other);
    }

    return state.hasConflict();
  }

  /**
   * Commit the transaction.
   *
   * @param transactionId
   * @throws IOException
   */
  public void commit(final long transactionId) throws IOException {
    TransactionState state;
    try {
      state = getTransactionState(transactionId);
    } catch (UnknownTransactionException e) {
      LOG.fatal(
          "Asked to commit unknown transaction: "
              + transactionId
              + " in region "
              + super.getRegionInfo().getRegionNameAsString());
      // TODO. Anything to handle here?
      throw e;
    }

    if (!state.getStatus().equals(Status.COMMIT_PENDING)) {
      LOG.fatal("Asked to commit a non pending transaction");
      // TODO. Anything to handle here?
      throw new IOException("commit failure");
    }

    commit(state);
  }

  /**
   * Abort the transaction.
   *
   * @param transactionId
   * @throws IOException
   */
  public void abort(final long transactionId) throws IOException {
    // Not checking closing...
    TransactionState state;
    try {
      state = getTransactionState(transactionId);
    } catch (UnknownTransactionException e) {
      LOG.info(
          "Asked to abort unknown transaction ["
              + transactionId
              + "] in region ["
              + getRegionInfo().getRegionNameAsString()
              + "], ignoring");
      return;
    }

    state.setStatus(Status.ABORTED);

    if (state.hasWrite()) {
      this.transactionLog.writeAbortToLog(super.getRegionInfo(), state.getTransactionId());
    }

    // Following removes needed if we have voted
    if (state.getSequenceNumber() != null) {
      commitedTransactionsBySequenceNumber.remove(state.getSequenceNumber());
    }
    commitPendingTransactions.remove(state);

    retireTransaction(state);
  }

  private void commit(final TransactionState state) throws IOException {

    LOG.debug(
        "Commiting transaction: "
            + state.toString()
            + " to "
            + super.getRegionInfo().getRegionNameAsString());

    // Perform write operations timestamped to right now
    List<WriteAction> writeOrdering = state.getWriteOrdering();
    for (WriteAction action : writeOrdering) {
      Put put = action.getPut();
      if (null != put) {
        this.put(put, true);
      }

      Delete delete = action.getDelete();
      if (null != delete) {
        delete(delete, null, true);
      }
    }

    // Now the transactional writes live in the core WAL, we can write a commit
    // to the log
    // so we don't have to recover it from the transactional WAL.
    if (state.hasWrite()) {
      this.transactionLog.writeCommitToLog(super.getRegionInfo(), state.getTransactionId());
    }

    state.setStatus(Status.COMMITED);
    if (state.hasWrite() && !commitPendingTransactions.remove(state)) {
      LOG.fatal("Commiting a non-query transaction that is not in commitPendingTransactions");
      // Something has gone really wrong.
      throw new IOException("commit failure");
    }
    retireTransaction(state);
  }

  @Override
  public List<StoreFile> close(final boolean abort) throws IOException {
    prepareToClose();
    if (!commitPendingTransactions.isEmpty()) {
      LOG.warn(
          "Closing transactional region ["
              + getRegionInfo().getRegionNameAsString()
              + "], but still have ["
              + commitPendingTransactions.size()
              + "] transactions  that are pending commit.");
      // TODO resolve from the Global Trx Log.
    }
    return super.close(abort);
  }

  @Override
  protected void prepareToSplit() {
    prepareToClose();
  }

  boolean closing = false;
  private static final int CLOSE_WAIT_ON_COMMIT_PENDING = 1000;

  /** Get ready to close. */
  void prepareToClose() {
    if (closing) {
      return;
    }

    LOG.info("Preparing to close region " + getRegionInfo().getRegionNameAsString());
    closing = true;

    while (!commitPendingTransactions.isEmpty()) {
      LOG.info(
          "Preparing to closing transactional region ["
              + getRegionInfo().getRegionNameAsString()
              + "], but still have ["
              + commitPendingTransactions.size()
              + "] transactions that are pending commit. Sleeping");
      for (TransactionState s : commitPendingTransactions) {
        LOG.info("commit pending: " + s.toString());
      }
      try {
        Thread.sleep(CLOSE_WAIT_ON_COMMIT_PENDING);
      } catch (InterruptedException e) {
        throw new RuntimeException(e);
      }
    }
  }

  private void checkClosing() throws IOException {
    if (closing) {
      throw new IOException("closing region, no more transaction allowed");
    }
  }

  // Cancel leases, and removed from lease lookup. This transaction may still
  // live in commitedTransactionsBySequenceNumber and commitPendingTransactions
  private void retireTransaction(final TransactionState state) {
    String key = String.valueOf(state.getTransactionId());
    try {
      transactionLeases.cancelLease(getLeaseId(state.getTransactionId()));
    } catch (LeaseException e) {
      // Ignore
    }

    synchronized (transactionsById) {
      transactionsById.remove(key);
    }
  }

  protected TransactionState getTransactionState(final long transactionId)
      throws UnknownTransactionException {
    String key = String.valueOf(transactionId);
    TransactionState state = null;

    state = transactionsById.get(key);

    if (state == null) {
      LOG.debug(
          "Unknown transaction: ["
              + key
              + "], region: ["
              + getRegionInfo().getRegionNameAsString()
              + "]");
      throw new UnknownTransactionException(
          "transaction: [" + key + "], region: [" + getRegionInfo().getRegionNameAsString() + "]");
    }

    try {
      transactionLeases.renewLease(getLeaseId(transactionId));
    } catch (LeaseException e) {
      throw new RuntimeException(e);
    }

    return state;
  }

  private void maybeTriggerOldTransactionFlush() {
    if (commitedTransactionsBySequenceNumber.size() > oldTransactionFlushTrigger) {
      removeUnNeededCommitedTransactions();
    }
  }

  /** Cleanup references to committed transactions that are no longer needed. */
  synchronized void removeUnNeededCommitedTransactions() {
    Integer minStartSeqNumber = getMinStartSequenceNumber();
    if (minStartSeqNumber == null) {
      minStartSeqNumber = Integer.MAX_VALUE; // Remove all
    }

    int numRemoved = 0;
    // Copy list to avoid conc update exception
    for (Entry<Integer, TransactionState> entry :
        new LinkedList<Entry<Integer, TransactionState>>(
            commitedTransactionsBySequenceNumber.entrySet())) {
      if (entry.getKey() >= minStartSeqNumber) {
        break;
      }
      numRemoved =
          numRemoved
              + (commitedTransactionsBySequenceNumber.remove(entry.getKey()) == null ? 0 : 1);
      numRemoved++;
    }

    if (LOG.isDebugEnabled()) {
      StringBuilder debugMessage = new StringBuilder();
      if (numRemoved > 0) {
        debugMessage.append("Removed [").append(numRemoved).append("] commited transactions");

        if (minStartSeqNumber == Integer.MAX_VALUE) {
          debugMessage.append(" with any sequence number.");
        } else {
          debugMessage.append(" with sequence lower than [").append(minStartSeqNumber).append("].");
        }
        if (!commitedTransactionsBySequenceNumber.isEmpty()) {
          debugMessage
              .append(" Still have [")
              .append(commitedTransactionsBySequenceNumber.size())
              .append("] left.");
        } else {
          debugMessage.append(" None left.");
        }
        LOG.debug(debugMessage.toString());
      } else if (commitedTransactionsBySequenceNumber.size() > 0) {
        debugMessage
            .append("Could not remove any transactions, and still have ")
            .append(commitedTransactionsBySequenceNumber.size())
            .append(" left");
        LOG.debug(debugMessage.toString());
      }
    }
  }

  private Integer getMinStartSequenceNumber() {
    List<TransactionState> transactionStates;
    synchronized (transactionsById) {
      transactionStates = new ArrayList<TransactionState>(transactionsById.values());
    }
    Integer min = null;
    for (TransactionState transactionState : transactionStates) {
      if (min == null || transactionState.getStartSequenceNumber() < min) {
        min = transactionState.getStartSequenceNumber();
      }
    }
    return min;
  }

  private void resolveTransactionFromLog(final TransactionState transactionState)
      throws IOException {
    LOG.error(
        "Global transaction log is not Implemented. (Optimisticly) assuming transaction commit!");
    commit(transactionState);
    // throw new RuntimeException("Global transaction log is not Implemented");
  }

  private static final int MAX_COMMIT_PENDING_WAITS = 10;

  private class TransactionLeaseListener implements LeaseListener {

    private final String transactionName;

    TransactionLeaseListener(final String n) {
      this.transactionName = n;
    }

    @Override
    public void leaseExpired() {
      LOG.info(
          "Transaction ["
              + this.transactionName
              + "] expired in region ["
              + getRegionInfo().getRegionNameAsString()
              + "]");
      TransactionState s = null;
      synchronized (transactionsById) {
        s = transactionsById.remove(transactionName);
      }
      if (s == null) {
        LOG.warn("Unknown transaction expired " + this.transactionName);
        return;
      }

      switch (s.getStatus()) {
        case PENDING:
          s.setStatus(Status.ABORTED); // Other transactions may have a ref
          break;
        case COMMIT_PENDING:
          LOG.info("Transaction " + s.getTransactionId() + " expired in COMMIT_PENDING state");

          try {
            if (s.getCommitPendingWaits() > MAX_COMMIT_PENDING_WAITS) {
              LOG.info("Checking transaction status in transaction log");
              resolveTransactionFromLog(s);
              break;
            }
            LOG.info("renewing lease and hoping for commit");
            s.incrementCommitPendingWaits();
            String key = Long.toString(s.getTransactionId());
            transactionsById.put(key, s);
            try {
              transactionLeases.createLease(getLeaseId(s.getTransactionId()), this);
            } catch (LeaseStillHeldException e) {
              transactionLeases.renewLease(getLeaseId(s.getTransactionId()));
            }
          } catch (IOException e) {
            throw new RuntimeException(e);
          }

          break;
        default:
          LOG.warn("Unexpected status on expired lease");
      }
    }
  }

  public void setTransactionLog(final THLog trxHLog) {
    this.transactionLog = trxHLog;
  }

  public void setTransactionalLeases(final Leases transactionalLeases) {
    this.transactionLeases = transactionalLeases;
  }
}
Example #7
0
  public static void main(String[] args) {

    try {

      final Options options = new Options();

      {
        final Option d = new Option("d", "directory", true, "target directory");
        d.setArgName("directory");
        d.setArgs(1);
        d.setRequired(true);
        options.addOption(d);
      }

      {
        final Option ad = new Option("ad", "another-directory", true, "another target directory");
        ad.setArgName("another-directory");
        ad.setArgs(1);
        ad.setRequired(false);
        options.addOption(ad);
      }

      {
        final Option o = new Option("o", "output", true, "output file");
        o.setArgName("file");
        o.setArgs(1);
        o.setRequired(true);
        options.addOption(o);
      }

      {
        final Option s = new Option("s", "size", true, "size");
        s.setArgName("size");
        s.setArgs(1);
        s.setRequired(true);
        options.addOption(s);
      }

      {
        final Option t = new Option("t", "thread", true, "number of threads");
        t.setArgName("thread");
        t.setArgs(1);
        t.setRequired(false);
        options.addOption(t);
      }

      {
        final Option cross =
            new Option(
                "cross", "cross-project-only", true, "whether to detect cross project clones only");
        cross.setArgName("on or off");
        cross.setRequired(false);
        options.addOption(cross);
      }

      {
        final Option lowmem =
            new Option("lowmem", "low-memory-mode", true, "whether to run on the low memory mode");
        lowmem.setArgName("on or off");
        lowmem.setRequired(false);
        options.addOption(lowmem);
      }

      {
        final Option v = new Option("v", "verbose", true, "verbose output");
        v.setArgName("on or off");
        v.setRequired(false);
        options.addOption(v);
      }

      {
        final Option C = new Option("C", "control", true, "use of control dependency");
        C.setArgName("on or off");
        C.setArgs(1);
        C.setRequired(false);
        options.addOption(C);
      }

      {
        final Option D = new Option("D", "data", true, "use of data dependency");
        D.setArgName("on or off");
        D.setArgs(1);
        D.setRequired(false);
        options.addOption(D);
      }

      {
        final Option E = new Option("E", "execution", true, "use of execution dependency");
        E.setArgName("on or off");
        E.setArgs(1);
        E.setRequired(false);
        options.addOption(E);
      }

      {
        final Option M = new Option("M", "merging", true, "merging consecutive similar nodes");
        M.setArgName("on or off");
        M.setArgs(1);
        M.setRequired(false);
        options.addOption(M);
      }

      final CommandLineParser parser = new PosixParser();
      final CommandLine cmd = parser.parse(options, args);

      final File target = new File(cmd.getOptionValue("d"));
      if (!target.exists()) {
        System.err.println("specified directory or file does not exist.");
        System.exit(0);
      }

      final String output = cmd.getOptionValue("o");
      final int SIZE_THRESHOLD = Integer.parseInt(cmd.getOptionValue("s"));
      final int NUMBER_OF_THREADS =
          cmd.hasOption("t") ? Integer.parseInt(cmd.getOptionValue("t")) : 1;

      boolean useOfControl = !cmd.hasOption("C");
      if (!useOfControl) {
        if (cmd.getOptionValue("C").equals("on")) {
          useOfControl = true;
        } else if (cmd.getOptionValue("C").equals("off")) {
          useOfControl = false;
        } else {
          System.err.println("option of \"-C\" must be \"on\" or \"off\".");
        }
      }

      boolean useOfData = !cmd.hasOption("D");
      if (!useOfData) {
        if (cmd.getOptionValue("D").equals("on")) {
          useOfData = true;
        } else if (cmd.getOptionValue("D").equals("off")) {
          useOfData = false;
        } else {
          System.err.println("option of \"-D\" must be \"on\" or \"off\".");
        }
      }

      boolean useOfExecution = !cmd.hasOption("E");
      if (!useOfExecution) {
        if (cmd.getOptionValue("E").equals("on")) {
          useOfExecution = true;
        } else if (cmd.getOptionValue("E").equals("off")) {
          useOfExecution = false;
        } else {
          System.err.println("option of \"-E\" must be \"on\" or \"off\".");
        }
      }

      boolean useOfMerging = !cmd.hasOption("M");
      if (!useOfMerging) {
        if (cmd.getOptionValue("M").equals("on")) {
          useOfMerging = true;
        } else if (cmd.getOptionValue("M").equals("off")) {
          useOfMerging = false;
        } else {
          System.err.println("option of \"-M\" must be \"on\" or \"off\".");
        }
      }

      if (!useOfExecution && useOfMerging) {
        useOfMerging = false;
      }

      boolean crossProjectOnly = false;
      if (cmd.hasOption("cross")) {
        if (cmd.getOptionValue("cross").equals("on")) {
          crossProjectOnly = true;
        } else if (cmd.getOptionValue("cross").equals("off")) {
          crossProjectOnly = false;
        } else {
          System.err.println("option of \"-cross\" must be \"on\" or \"off\".");
        }
      }

      boolean lowMemoryMode = false;
      if (cmd.hasOption("lowmem")) {
        if (cmd.getOptionValue("lowmem").equals("on")) {
          lowMemoryMode = true;
        } else if (cmd.getOptionValue("lowmem").equals("off")) {
          lowMemoryMode = false;
        } else {
          System.err.println("option of \"-lowmem\" must be \"on\" or \"off\".");
        }
      }

      // default verbose level is "off"
      if (cmd.hasOption("v")) {
        if (cmd.getOptionValue("v").equals("on")) {
          Message.setVerbose(true);
        } else if (cmd.getOptionValue("v").equals("off")) {
          Message.setVerbose(false);
        } else {
          System.err.println("option of \"-v\" must be \"on\" or \"off\".");
        }
      }

      File anotherTarget = null;
      if (cmd.hasOption("ad")) {
        anotherTarget = new File(cmd.getOptionValue("ad"));
        if (!anotherTarget.exists()) {
          System.err.println("specified directory or file does not exist.");
          System.exit(0);
        }
      }

      if (crossProjectOnly && anotherTarget == null) {
        System.err.println(
            "detecting cross project only is ON, but no second directory or file has been specified");
        System.exit(0);
      }

      final long time1 = System.nanoTime();
      System.out.print("generating PDGs ... ");
      Message.log("");
      final PDG[] pdgArray;
      {
        final List<File> files = getFiles(target);

        if (anotherTarget != null) {
          files.addAll(getFiles(anotherTarget));
        }

        int count = 0;
        final int numOfFiles = files.size();

        final List<MethodInfo> methods = new ArrayList<MethodInfo>();
        for (final File file : files) {
          Message.log(
              "\t["
                  + (++count)
                  + "/"
                  + numOfFiles
                  + "] building an AST for "
                  + file.getAbsolutePath());

          final CompilationUnit unit = TinyPDGASTVisitor.createAST(file);
          final ASTVisitor visitor;

          if (lowMemoryMode) {
            visitor = new OffsetBasedTinyPDGASTVisitor(file.getAbsolutePath(), unit, methods);
          } else {
            visitor = new TinyPDGASTVisitor(file.getAbsolutePath(), unit, methods);
          }

          unit.accept(visitor);
        }

        long memoryElapsed = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
        System.out.println("MEMORY: " + (memoryElapsed / 1024));

        final SortedSet<PDG> pdgs = Collections.synchronizedSortedSet(new TreeSet<PDG>());
        final CFGNodeFactory cfgNodeFactory = new CFGNodeFactory();
        final PDGNodeFactory pdgNodeFactory = new PDGNodeFactory();
        final Thread[] pdgGenerationThreads = new Thread[NUMBER_OF_THREADS];
        for (int i = 0; i < pdgGenerationThreads.length; i++) {
          pdgGenerationThreads[i] =
              new Thread(
                  new PDGGenerationThread(
                      methods,
                      pdgs,
                      cfgNodeFactory,
                      pdgNodeFactory,
                      useOfControl,
                      useOfData,
                      useOfExecution,
                      useOfMerging,
                      SIZE_THRESHOLD));
          pdgGenerationThreads[i].start();
        }
        for (final Thread thread : pdgGenerationThreads) {
          try {
            thread.join();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
        pdgArray = pdgs.toArray(new PDG[0]);
      }
      System.out.print("done: ");
      final long time2 = System.nanoTime();
      printTime(time2 - time1);

      System.out.print("calculating hash values ... ");
      Message.log("");
      final SortedMap<PDG, SortedMap<PDGNode<?>, Integer>> mappingPDGToPDGNodes =
          Collections.synchronizedSortedMap(new TreeMap<PDG, SortedMap<PDGNode<?>, Integer>>());
      final SortedMap<PDG, SortedMap<PDGEdge, Integer>> mappingPDGToPDGEdges =
          Collections.synchronizedSortedMap(new TreeMap<PDG, SortedMap<PDGEdge, Integer>>());
      {
        final Thread[] hashCalculationThreads = new Thread[NUMBER_OF_THREADS];
        for (int i = 0; i < hashCalculationThreads.length; i++) {
          hashCalculationThreads[i] =
              new Thread(
                  new HashCalculationThread(pdgArray, mappingPDGToPDGNodes, mappingPDGToPDGEdges));
          hashCalculationThreads[i].start();
        }
        for (final Thread thread : hashCalculationThreads) {
          try {
            thread.join();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      }
      System.out.print("done: ");
      final long time3 = System.nanoTime();
      printTime(time3 - time2);

      System.out.print("detecting clone pairs ... ");
      Message.log("");
      final SortedSet<ClonePairInfo> clonepairs =
          Collections.synchronizedSortedSet(new TreeSet<ClonePairInfo>());
      {
        int numIgnored = 0;
        final List<PDGPairInfo> pdgpairs = new ArrayList<PDGPairInfo>();
        Message.log("\tmaking PDG pairs ... ");
        for (int i = 0; i < pdgArray.length; i++) {
          for (int j = i + 1; j < pdgArray.length; j++) {
            final PDG pdg1 = pdgArray[i];
            final PDG pdg2 = pdgArray[j];

            if (!crossProjectOnly || isCrossProject(pdg1, pdg2, target, anotherTarget)) {
              pdgpairs.add(new PDGPairInfo(pdgArray[i], pdgArray[j]));
            } else {
              // Message.log("\t\tignore the PDG pair \"" +
              // pdg1.unit.name + " in "
              // + pdg1.unit.path + "\" and \""
              // + pdg2.unit.name + " in " + pdg2.unit.path
              // + "\"");
              numIgnored++;
            }
          }
        }
        Message.log("\tdone: the number of ignored PDG pairs is " + numIgnored);

        final PDGPairInfo[] pdgpairArray = pdgpairs.toArray(new PDGPairInfo[0]);
        final Thread[] slicingThreads = new Thread[NUMBER_OF_THREADS];
        for (int i = 0; i < slicingThreads.length; i++) {
          slicingThreads[i] =
              new Thread(
                  new SlicingThread(
                      pdgpairArray,
                      pdgArray,
                      mappingPDGToPDGNodes,
                      mappingPDGToPDGEdges,
                      clonepairs,
                      SIZE_THRESHOLD,
                      crossProjectOnly));
          slicingThreads[i].start();
        }
        for (final Thread thread : slicingThreads) {
          try {
            thread.join();
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
        }
      }
      System.out.print("done: ");
      final long time4 = System.nanoTime();
      printTime(time4 - time3);

      System.out.print("writing to a file ... ");
      final Writer writer = new BellonWriter(output, clonepairs);
      writer.write();
      System.out.print("done: ");
      final long time5 = System.nanoTime();
      printTime(time5 - time4);

      System.out.print("total elapsed time: ");
      printTime(time5 - time1);

      System.out.print("number of comparisons: ");
      printNumberOfComparison(Slicing.getNumberOfComparison());

    } catch (Exception e) {
      System.err.println(e.getMessage());
      System.exit(0);
    }
  }
/** Created with IntelliJ IDEA. User: ex3ndr Date: 03.11.13 Time: 8:51 */
public class Scheduller {

  private final String TAG; //  = "MTProtoScheduller";

  // Share identity values across all connections to avoid collisions
  private static AtomicInteger messagesIds = new AtomicInteger(1);
  private static HashMap<Long, Long> idGenerationTime = new HashMap<Long, Long>();

  private static final int SCHEDULLER_TIMEOUT = 15 * 1000; // 15 sec

  private static final long CONFIRM_TIMEOUT = 60 * 1000; // 60 sec

  private static final int MAX_WORKLOAD_SIZE = 1024;
  private static final int BIG_MESSAGE_SIZE = 1024;
  private static final long RETRY_TIMEOUT = 15 * 1000;

  private static final int MAX_ACK_COUNT = 16;

  private SortedMap<Integer, SchedullerPackage> messages =
      Collections.synchronizedSortedMap(new TreeMap<Integer, SchedullerPackage>());
  private HashSet<Long> currentMessageGeneration = new HashSet<Long>();
  private HashSet<Long> confirmedMessages = new HashSet<Long>();

  private long firstConfirmTime = 0;

  private long lastMessageId = 0;
  private long lastDependId = 0;
  private int seqNo = 0;

  private CallWrapper wrapper;

  public Scheduller(MTProto mtProto, CallWrapper wrapper) {
    TAG = "MTProto#" + mtProto.getInstanceIndex() + "#Scheduller";
    this.wrapper = wrapper;
  }

  private synchronized long generateMessageId() {
    long messageId = TimeOverlord.getInstance().createWeakMessageId();
    if (messageId <= lastMessageId) {
      messageId = lastMessageId = lastMessageId + 4;
    }
    while (idGenerationTime.containsKey(messageId)) {
      messageId += 4;
    }
    lastMessageId = messageId;
    idGenerationTime.put(messageId, getCurrentTime());
    currentMessageGeneration.add(messageId);
    return messageId;
  }

  private synchronized int generateSeqNoWeak() {
    return seqNo * 2;
  }

  private synchronized int generateSeqNo() {
    int res = seqNo * 2 + 1;
    seqNo++;
    return res;
  }

  private synchronized void generateParams(SchedullerPackage schedullerPackage) {
    schedullerPackage.messageId = generateMessageId();
    schedullerPackage.seqNo = generateSeqNo();
    schedullerPackage.idGenerationTime = getCurrentTime();
    schedullerPackage.relatedMessageIds.add(schedullerPackage.messageId);
    schedullerPackage.generatedMessageIds.add(schedullerPackage.messageId);
  }

  private long getCurrentTime() {
    return System.nanoTime() / 1000000;
  }

  public long getMessageIdGenerationTime(long msgId) {
    if (idGenerationTime.containsKey(msgId)) {
      return idGenerationTime.get(msgId);
    }
    return 0;
  }

  public int postMessageDelayed(
      TLObject object,
      boolean isRpc,
      long timeout,
      int delay,
      int contextId,
      boolean highPrioroty) {
    int id = messagesIds.incrementAndGet();
    SchedullerPackage schedullerPackage = new SchedullerPackage(id);
    schedullerPackage.object = object;
    schedullerPackage.addTime = getCurrentTime();
    schedullerPackage.scheduleTime = schedullerPackage.addTime + delay;
    schedullerPackage.expiresTime = schedullerPackage.scheduleTime + timeout;
    schedullerPackage.ttlTime = schedullerPackage.scheduleTime + timeout * 2;
    schedullerPackage.isRpc = isRpc;
    schedullerPackage.queuedToChannel = contextId;
    schedullerPackage.priority = highPrioroty ? PRIORITY_HIGH : PRIORITY_NORMAL;
    schedullerPackage.isDepend = highPrioroty;
    schedullerPackage.supportTag = object.toString();
    messages.put(id, schedullerPackage);
    return id;
  }

  public int postMessage(TLObject object, boolean isApi, long timeout) {
    return postMessageDelayed(object, isApi, timeout, 0, -1, false);
  }

  public int postMessage(TLObject object, boolean isApi, long timeout, boolean highPrioroty) {
    return postMessageDelayed(object, isApi, timeout, 0, -1, highPrioroty);
  }

  public synchronized void prepareScheduller(PrepareSchedule prepareSchedule, int[] connectionIds) {
    long time = getCurrentTime();

    // Clear packages for unknown channels
    outer:
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.queuedToChannel != -1) {
        for (int id : connectionIds) {
          if (schedullerPackage.queuedToChannel == id) {
            continue outer;
          }
        }
        forgetMessage(schedullerPackage.id);
      }
    }

    if (connectionIds.length == 0) {
      prepareSchedule.setDelay(SCHEDULLER_TIMEOUT);
      prepareSchedule.setAllowedContexts(connectionIds);
      prepareSchedule.setDoWait(true);
      return;
    }

    long minDelay = SCHEDULLER_TIMEOUT;
    boolean allConnections = false;
    boolean doWait = true;
    HashSet<Integer> supportedConnections = new HashSet<Integer>();
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      boolean isPendingPackage = false;
      long packageTime = 0;

      if (schedullerPackage.state == STATE_QUEUED) {
        isPendingPackage = true;
        if (schedullerPackage.scheduleTime <= time) {
          packageTime = 0;
        } else {
          packageTime = Math.max(schedullerPackage.scheduleTime - time, 0);
        }
      } else if (schedullerPackage.state == STATE_SENT) {
        if (getCurrentTime() <= schedullerPackage.expiresTime) {
          if (schedullerPackage.serialized == null
              || schedullerPackage.serialized.length < BIG_MESSAGE_SIZE) {
            if (time - schedullerPackage.lastAttemptTime >= RETRY_TIMEOUT) {
              isPendingPackage = true;
              packageTime = 0;
            }
          }
        }
      }

      if (isPendingPackage) {
        if (schedullerPackage.queuedToChannel == -1) {
          allConnections = true;
        } else {
          supportedConnections.add(schedullerPackage.queuedToChannel);
        }

        if (packageTime == 0) {
          minDelay = 0;
          doWait = false;
        } else {
          minDelay = Math.min(packageTime, minDelay);
        }
      }
    }
    prepareSchedule.setDoWait(doWait);
    prepareSchedule.setDelay(minDelay);

    if (allConnections) {
      prepareSchedule.setAllowedContexts(connectionIds);
    } else {
      Integer[] allowedBoxed = supportedConnections.toArray(new Integer[0]);
      int[] allowed = new int[allowedBoxed.length];
      for (int i = 0; i < allowed.length; i++) {
        allowed[i] = allowedBoxed[i];
      }
      prepareSchedule.setAllowedContexts(allowed);
    }
  }

  public void registerFastConfirm(long msgId, int fastConfirm) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      boolean contains = false;
      for (Long relatedMsgId : schedullerPackage.relatedMessageIds) {
        if (relatedMsgId == msgId) {
          contains = true;
          break;
        }
      }
      if (contains) {
        schedullerPackage.relatedFastConfirm.add(fastConfirm);
      }
    }
  }

  public int mapSchedullerId(long msgId) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.generatedMessageIds.contains(msgId)) {
        return schedullerPackage.id;
      }
    }
    return 0;
  }

  public void resetMessageId() {
    lastMessageId = 0;
    lastDependId = 0;
  }

  public void resetSession() {
    lastMessageId = 0;
    lastDependId = 0;
    seqNo = 0;
    currentMessageGeneration.clear();
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      schedullerPackage.idGenerationTime = 0;
      schedullerPackage.dependMessageId = 0;
      schedullerPackage.messageId = 0;
      schedullerPackage.seqNo = 0;
    }
  }

  public boolean isMessageFromCurrentGeneration(long msgId) {
    return currentMessageGeneration.contains(msgId);
  }

  public void resendAsNewMessage(long msgId) {
    resendAsNewMessageDelayed(msgId, 0);
  }

  public void resendAsNewMessageDelayed(long msgId, int delay) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.relatedMessageIds.contains(msgId)) {
        schedullerPackage.idGenerationTime = 0;
        schedullerPackage.dependMessageId = 0;
        schedullerPackage.messageId = 0;
        schedullerPackage.seqNo = 0;
        schedullerPackage.state = STATE_QUEUED;
        schedullerPackage.scheduleTime = getCurrentTime() + delay;
        Logger.d(TAG, "Resending as new #" + schedullerPackage.id);
      }
    }
  }

  public void resendMessage(long msgId) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.relatedMessageIds.contains(msgId)) {
        // schedullerPackage.relatedMessageIds.clear();
        schedullerPackage.state = STATE_QUEUED;
        schedullerPackage.lastAttemptTime = 0;
      }
    }
  }

  public int[] mapFastConfirm(int fastConfirm) {
    ArrayList<Integer> res = new ArrayList<Integer>();
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.state == STATE_SENT) {
        if (schedullerPackage.relatedFastConfirm.contains(fastConfirm)) {
          res.add(schedullerPackage.id);
        }
      }
    }
    int[] res2 = new int[res.size()];
    for (int i = 0; i < res2.length; i++) {
      res2[i] = res.get(i);
    }
    return res2;
  }

  public void onMessageFastConfirmed(int fastConfirm) {
    //        for (SchedullerPackage schedullerPackage : messages.values().toArray(new
    // SchedullerPackage[0])) {
    //            if (schedullerPackage.state == STATE_SENT) {
    //                if (schedullerPackage.relatedFastConfirm.contains(fastConfirm)) {
    //                    schedullerPackage.state = STATE_CONFIRMED;
    //                }
    //            }
    //        }
  }

  public void onMessageConfirmed(long msgId) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.state == STATE_SENT) {
        if (schedullerPackage.relatedMessageIds.contains(msgId)) {
          schedullerPackage.state = STATE_CONFIRMED;
        }
      }
    }
  }

  public void confirmMessage(long msgId) {
    synchronized (confirmedMessages) {
      confirmedMessages.add(msgId);
      if (firstConfirmTime == 0) {
        firstConfirmTime = getCurrentTime();
      }
    }
  }

  public void unableToSendMessage(long messageId) {
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.state == STATE_SENT) {
        boolean contains = false;
        for (Long relatedMsgId : schedullerPackage.relatedMessageIds) {
          if (relatedMsgId == messageId) {
            contains = true;
            break;
          }
        }
        if (contains) {
          schedullerPackage.state = STATE_QUEUED;
        }
      }
    }
  }

  public synchronized void forgetMessageByMsgId(long msgId) {
    int scId = mapSchedullerId(msgId);
    if (scId > 0) {
      forgetMessage(scId);
    }
  }

  public synchronized void forgetMessage(int id) {
    Logger.d(TAG, "Forgetting message: #" + id);
    messages.remove(id);
  }

  private synchronized ArrayList<SchedullerPackage> actualPackages(int contextId) {
    ArrayList<SchedullerPackage> foundedPackages = new ArrayList<SchedullerPackage>();
    long time = getCurrentTime();
    for (SchedullerPackage schedullerPackage :
        messages.values().toArray(new SchedullerPackage[0])) {
      if (schedullerPackage.queuedToChannel != -1
          && contextId != schedullerPackage.queuedToChannel) {
        continue;
      }
      boolean isPendingPackage = false;

      if (schedullerPackage.ttlTime <= getCurrentTime()) {
        forgetMessage(schedullerPackage.id);
        continue;
      }

      if (schedullerPackage.state == STATE_QUEUED) {
        if (schedullerPackage.scheduleTime <= time) {
          isPendingPackage = true;
        }
      } else if (schedullerPackage.state == STATE_SENT) {
        if (getCurrentTime() <= schedullerPackage.expiresTime) {
          if (schedullerPackage.serialized == null
              || schedullerPackage.serialized.length < BIG_MESSAGE_SIZE) {
            if (getCurrentTime() - schedullerPackage.lastAttemptTime >= RETRY_TIMEOUT) {
              isPendingPackage = true;
            }
          }
        }
      }

      if (isPendingPackage) {
        if (schedullerPackage.serialized == null) {
          try {
            if (schedullerPackage.isRpc) {
              schedullerPackage.serialized =
                  wrapper.wrapObject((TLMethod) schedullerPackage.object).serialize();
            } else {
              schedullerPackage.serialized = schedullerPackage.object.serialize();
            }
          } catch (IOException e) {
            Logger.e(TAG, e);
            forgetMessage(schedullerPackage.id);
            continue;
          }
        }

        foundedPackages.add(schedullerPackage);
      }
    }
    return foundedPackages;
  }

  public synchronized PreparedPackage doSchedule(int contextId, boolean isInited) {
    ArrayList<SchedullerPackage> foundedPackages = actualPackages(contextId);

    synchronized (confirmedMessages) {
      if (foundedPackages.size() == 0
          && (confirmedMessages.size() <= MAX_ACK_COUNT
              || (System.nanoTime() - firstConfirmTime) < CONFIRM_TIMEOUT)) {
        return null;
      }
    }

    boolean useHighPriority = false;

    for (SchedullerPackage p : foundedPackages) {
      if (p.priority == PRIORITY_HIGH) {
        useHighPriority = true;
        break;
      }
    }

    ArrayList<SchedullerPackage> packages = new ArrayList<SchedullerPackage>();

    if (useHighPriority) {
      Logger.d("Scheduller", "Using high priority scheduling");
      int totalSize = 0;
      for (SchedullerPackage p : foundedPackages) {
        if (p.priority == PRIORITY_HIGH) {
          packages.add(p);
          totalSize += p.serialized.length;
          if (totalSize > MAX_WORKLOAD_SIZE) {
            break;
          }
        }
      }
    } else {
      int totalSize = 0;
      for (SchedullerPackage p : foundedPackages) {
        packages.add(p);
        Logger.d(
            "Scheduller", "Prepare package: " + p.supportTag + " of size " + p.serialized.length);
        totalSize += p.serialized.length;
        Logger.d("Scheduller", "Total size: " + totalSize);
        if (totalSize > MAX_WORKLOAD_SIZE) {
          break;
        }
      }
    }

    Logger.d(TAG, "Iteration: count: " + packages.size() + ", confirm:" + confirmedMessages.size());
    Logger.d(TAG, "Building package");
    if (foundedPackages.size() == 0 && confirmedMessages.size() != 0) {
      Long[] msgIds;
      synchronized (confirmedMessages) {
        msgIds = confirmedMessages.toArray(new Long[confirmedMessages.size()]);
        confirmedMessages.clear();
      }
      MTMsgsAck ack = new MTMsgsAck(msgIds);
      Logger.d(TAG, "Single msg_ack");
      try {
        return new PreparedPackage(
            generateSeqNoWeak(), generateMessageId(), ack.serialize(), useHighPriority);
      } catch (IOException e) {
        Logger.e(TAG, e);
        return null;
      }
    } else if (foundedPackages.size() == 1 && confirmedMessages.size() == 0) {
      SchedullerPackage schedullerPackage = foundedPackages.get(0);
      schedullerPackage.state = STATE_SENT;
      if (schedullerPackage.idGenerationTime == 0) {
        generateParams(schedullerPackage);
      }
      Logger.d(
          TAG,
          "Single package: #"
              + schedullerPackage.id
              + " "
              + schedullerPackage.supportTag
              + " ("
              + schedullerPackage.messageId
              + ", "
              + schedullerPackage.seqNo
              + ")");
      schedullerPackage.writtenToChannel = contextId;
      schedullerPackage.lastAttemptTime = getCurrentTime();
      return new PreparedPackage(
          schedullerPackage.seqNo,
          schedullerPackage.messageId,
          schedullerPackage.serialized,
          useHighPriority);
    } else {
      MTMessagesContainer container = new MTMessagesContainer();
      if ((confirmedMessages.size() > 0 && !useHighPriority) || (!isInited)) {
        try {
          Long[] msgIds;
          synchronized (confirmedMessages) {
            msgIds = confirmedMessages.toArray(new Long[0]);
            confirmedMessages.clear();
          }
          MTMsgsAck ack = new MTMsgsAck(msgIds);
          Logger.d(TAG, "Adding msg_ack: " + msgIds.length);
          container
              .getMessages()
              .add(new MTMessage(generateMessageId(), generateSeqNoWeak(), ack.serialize()));
        } catch (IOException e) {
          Logger.e(TAG, e);
        }
      }
      for (SchedullerPackage schedullerPackage : packages) {
        schedullerPackage.state = STATE_SENT;
        if (schedullerPackage.idGenerationTime == 0) {
          generateParams(schedullerPackage);
        }

        if (schedullerPackage.isDepend) {
          if (schedullerPackage.dependMessageId == 0) {
            if (lastDependId > 0) {
              schedullerPackage.dependMessageId = lastDependId;
            } else {
              schedullerPackage.dependMessageId = -1;
            }
          }

          lastDependId = schedullerPackage.messageId;
        }
        schedullerPackage.writtenToChannel = contextId;
        schedullerPackage.lastAttemptTime = getCurrentTime();
        if (schedullerPackage.isDepend && schedullerPackage.dependMessageId > 0) {

          Logger.d(
              TAG,
              "Adding package: #"
                  + schedullerPackage.id
                  + " "
                  + schedullerPackage.supportTag
                  + " ("
                  + schedullerPackage.messageId
                  + " on "
                  + schedullerPackage.dependMessageId
                  + ", "
                  + schedullerPackage.seqNo
                  + ")");

          MTInvokeAfter invokeAfter =
              new MTInvokeAfter(schedullerPackage.dependMessageId, schedullerPackage.serialized);
          try {
            container
                .getMessages()
                .add(
                    new MTMessage(
                        schedullerPackage.messageId,
                        schedullerPackage.seqNo,
                        invokeAfter.serialize()));
          } catch (IOException e) {
            Logger.e(TAG, e);
            // Never happens
          }
        } else {
          Logger.d(
              TAG,
              "Adding package: #"
                  + schedullerPackage.id
                  + " "
                  + schedullerPackage.supportTag
                  + " ("
                  + schedullerPackage.messageId
                  + ", "
                  + schedullerPackage.seqNo
                  + ")");
          container
              .getMessages()
              .add(
                  new MTMessage(
                      schedullerPackage.messageId,
                      schedullerPackage.seqNo,
                      schedullerPackage.serialized));
        }
      }

      long containerMessageId = generateMessageId();
      int containerSeq = generateSeqNoWeak();

      for (SchedullerPackage schedullerPackage : packages) {
        schedullerPackage.relatedMessageIds.add(containerMessageId);
      }

      Logger.d(TAG, "Sending Package (" + containerMessageId + ", " + containerSeq + ")");

      try {
        return new PreparedPackage(
            containerSeq, containerMessageId, container.serialize(), useHighPriority);
      } catch (IOException e) {
        // Might not happens
        Logger.e(TAG, e);
        return null;
      }
    }
  }

  public synchronized void onConnectionDies(int connectionId) {
    //        Logger.d(TAG, "Connection dies " + connectionId);
    //        for (SchedullerPackage schedullerPackage : messages.values().toArray(new
    // SchedullerPackage[0])) {
    //            if (schedullerPackage.writtenToChannel != connectionId) {
    //                continue;
    //            }
    //
    //            if (schedullerPackage.queuedToChannel != -1) {
    //                Logger.d(TAG, "Removing: #" + schedullerPackage.id + " " +
    // schedullerPackage.supportTag);
    //                forgetMessage(schedullerPackage.id);
    //            } else {
    //                if (schedullerPackage.isRpc) {
    //                    if (schedullerPackage.state == STATE_CONFIRMED || schedullerPackage.state
    // == STATE_QUEUED) {
    //                        if (schedullerPackage.serialized == null ||
    // schedullerPackage.serialized.length < BIG_MESSAGE_SIZE) {
    //                            Logger.d(TAG, "Re-schedule: #" + schedullerPackage.id + " " +
    // schedullerPackage.supportTag);
    //                            schedullerPackage.state = STATE_QUEUED;
    //                            schedullerPackage.lastAttemptTime = 0;
    //                        }
    //                    }
    //                } else {
    //                    if (schedullerPackage.state == STATE_SENT) {
    //                        Logger.d(TAG, "Re-schedule: #" + schedullerPackage.id + " " +
    // schedullerPackage.supportTag);
    //                        schedullerPackage.state = STATE_QUEUED;
    //                        schedullerPackage.lastAttemptTime = 0;
    //                    }
    //                }
    //
    //            }
    //        }
  }

  private static final int PRIORITY_HIGH = 1;
  private static final int PRIORITY_NORMAL = 0;

  private static final int STATE_QUEUED = 0;
  private static final int STATE_SENT = 1;
  private static final int STATE_CONFIRMED = 2;

  private class SchedullerPackage {

    public SchedullerPackage(int id) {
      this.id = id;
    }

    public String supportTag;

    public int id;

    public TLObject object;
    public byte[] serialized;

    public long addTime;
    public long scheduleTime;
    public long expiresTime;
    public long ttlTime;
    public long lastAttemptTime;

    public int writtenToChannel = -1;

    public int queuedToChannel = -1;

    public int state = STATE_QUEUED;

    public int priority = PRIORITY_NORMAL;

    public boolean isDepend;

    public long idGenerationTime;
    public long dependMessageId;
    public long messageId;
    public int seqNo;
    public HashSet<Integer> relatedFastConfirm = new HashSet<Integer>();
    public HashSet<Long> relatedMessageIds = new HashSet<Long>();
    public HashSet<Long> generatedMessageIds = new HashSet<Long>();

    public boolean isRpc;
  }
}
Example #9
0
  /** Key Reporter is used to help see any un-used variables quickly. */
  private static final class KeyReporter {

    /**
     * Get key report.
     *
     * @param source Source to get report on.
     * @return Key report.
     */
    KeyReport getKeyReport(final String source) {
      KeyReport rtn = keyReports.get(source);
      if (rtn == null) {
        rtn = new KeyReport();
        keyReports.put(source, rtn);
      }
      return rtn;
    }

    /**
     * Report used key.
     *
     * @param source Source where key is used.
     * @param key Key.
     */
    void reportUsedKey(final String source, final String key) {
      KeyReport keyReport = getKeyReport(source);
      if (!keyReport.getUsedKeys().contains(key)) {
        keyReport.getUsedKeys().add(key);
      }
    }

    /**
     * Report defined key.
     *
     * @param source Source where key is defined.
     * @param key Key.
     */
    void reportDefinedKey(final String source, final String key) {
      KeyReport keyReport = getKeyReport(source);
      if (!keyReport.getDefinedKeys().contains(key)) {
        keyReport.getDefinedKeys().add(key);
      }
    }

    /** Maximum numerics. */
    static final int MAXIMUM_NUMERICS = 100;

    /**
     * Report unused keys.
     *
     * @param file File.
     */
    void reportUnusedKeys(final String file) {
      StringWriter sw = new StringWriter();
      for (String source : keyReports.keySet()) {
        StringWriter hdr = new StringWriter();
        hdr.append("*********" + source + ":");
        hdr.append(Stringop.getEol());
        KeyReport keyReport = getKeyReport(source);
        for (String s : keyReport.getUsedKeys()) {
          keyReport.getDefinedKeys().remove(s);
        }
        for (int i = 0; i < MAXIMUM_NUMERICS; i++) {
          keyReport.getDefinedKeys().remove("" + i);
        }
        keyReport.getDefinedKeys().remove(ReservedWords.FILE);
        keyReport.getDefinedKeys().remove(ReservedWords.DEPTH_CHARGE);
        keyReport.getDefinedKeys().remove(ReservedWords.SCRIPT);
        keyReport.getDefinedKeys().remove(ReservedWords.SPLITS);
        keyReport.getDefinedKeys().remove(ReservedWords.TARGET);
        keyReport.getDefinedKeys().remove(ReservedWords.TEMPLATE);
        keyReport.getDefinedKeys().remove(ReservedWords.TOKENS);
        boolean firstTime = true;
        for (String s : keyReport.getDefinedKeys()) {
          if (firstTime) {
            sw.append(hdr.toString());
            firstTime = false;
          }
          sw.append(s);
          sw.append(Stringop.getEol());
        }
      }
      Fileop.saveStringToFile(sw.toString(), new File(file));
    }

    /** Key reports map. */
    private final Map<String, KeyReport> keyReports =
        Collections.synchronizedSortedMap(new TreeMap<String, KeyReport>());

    /** Key report. */
    public static class KeyReport {

      /** Used keys. */
      private final List<String> usedKeys = new ArrayList<String>();

      /** Defined keys. */
      private final Set<String> dfndKeys = Collections.synchronizedSortedSet(new TreeSet<String>());

      /** @return Defined keys. */
      public Set<String> getDefinedKeys() {
        return dfndKeys;
      }

      /**
       * Get used keys.
       *
       * @return Used keys.
       */
      public List<String> getUsedKeys() {
        return usedKeys;
      }
    }
  }
Example #10
0
/**
 * @author czarek @TODO ta klasa powinna byc singletonem @TODO Wybrac waska grupe akceptowanych
 *     szyfrowan w polaczeniach
 */
public class Server {

  SortedSet<PeerLoginInfo> loginInfo =
      Collections.synchronizedSortedSet(new TreeSet<PeerLoginInfo>());
  SortedMap<String, PeerInfo> peersInfo =
      Collections.synchronizedSortedMap(new TreeMap<String, PeerInfo>());

  private FileInputStream is;
  private KeyStore keystore;
  private KeyManagerFactory kmf;
  private SSLContext sc;
  private SSLServerSocketFactory ssf;
  private SSLServerSocket ss;
  PrivateKey caPrivKey;
  int listeningPort;

  public enum STATE {
    CONNECTING,
    IDLE,
    CONNECTED,
    LOGGEDIN,
    DONE,
    LOGGING
  }

  public Server(String keystoreLoc, char[] kestorePass) {
    try {
      is = new FileInputStream(keystoreLoc);
      this.keystore = KeyStore.getInstance(KeyStore.getDefaultType());
      keystore.load(is, "123456".toCharArray());
      kmf = KeyManagerFactory.getInstance("SunX509", "SunJSSE");
      kmf.init(keystore, ("123456").toCharArray());
      sc = SSLContext.getInstance("SSL");
      sc.init(kmf.getKeyManagers(), null, null);
      ssf = sc.getServerSocketFactory();
      readServerInfo("./res/server/ServerInfo.dat");
      ss = (SSLServerSocket) ssf.createServerSocket(listeningPort);
      caPrivKey = (PrivateKey) keystore.getKey("serverTrustedCert", "123456".toCharArray());

    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
      System.exit(1);
    }
  }

  public static void main(String[] args) throws Exception {

    Server server = new Server("./res/server/key/serverKeys", "123456".toCharArray());
    //	System.out.println(server.sc.getProvider());
    //	System.out.println(server.ss.getEnabledCipherSuites());
    for (int i = 0; i < server.ss.getEnabledCipherSuites().length; i++) {
      System.out.println(server.ss.getEnabledCipherSuites()[i]);
    }

    /** Zrob z tego w¹tek */
    while (true) {
      // new Server(ss.accept()).start();
      try {
        System.out.println("serwer czeka");
        new S2PConnection(server.ss.accept(), server);
      } catch (Exception e1) {
        e1.printStackTrace();
      }
    }
  }

  /** Metoda zaczytujaca z pliku loginy i skroty hasel @TODO dodac sol do hasel */
  private void readServerInfo(String serverInfo) {
    try {
      System.out.println("readLoginInfo");
      FileReader fr = new FileReader(serverInfo);

      BufferedReader br = new BufferedReader(fr);
      StringTokenizer st; // = new StringTokenizer();
      String line;
      boolean timeTobreak = false;
      while (!timeTobreak) {
        line = br.readLine();
        if (!(line.startsWith("#"))) {
          this.listeningPort = Integer.valueOf(line);
          timeTobreak = true;
        }
      }

      while ((line = br.readLine()) != null) {

        if (!(line.startsWith("#"))) {
          st = new StringTokenizer(line);
          if ((st.countTokens()) != 2) throw new Exception("Ivalid peerLoginInfo.dat file");

          loginInfo.add(new PeerLoginInfo(st.nextToken(), st.nextToken(), true));
        }
      }
      System.out.println("[Server.readLoginInfo()] " + loginInfo);

    } catch (Exception e) {
      e.printStackTrace();
      System.exit(1);
    }
  }

  /**
   * Sprawdza, czy dane od peera zgadzaja sie z tymi z pliku
   *
   * @param PeerLoginInfo pli
   * @return wynik weryfikacji
   */
  boolean verifyPeer(PeerLoginInfo pli) {
    PeerLoginInfo pliAtServer = loginInfo.tailSet(pli).first();
    if (pliAtServer != null && (pliAtServer.getPasswdHash()).equals(pli.getPasswdHash()))
      return true;
    else return false;
  }

  /**
   * Generowanie certyfikatu x509
   *
   * @param certInfo informacje ktore maja znalezc sie w certyfikacie
   * @return
   * @throws InvalidKeyException
   * @throws NoSuchProviderException
   * @throws SignatureException
   * @throws CertificateEncodingException
   * @throws IllegalStateException
   * @throws NoSuchAlgorithmException
   */
  public common.Pair<X509Certificate, KeyPair> generateV3Certificate(X500Principal certInfo)
      throws InvalidKeyException, NoSuchProviderException, SignatureException,
          CertificateEncodingException, IllegalStateException, NoSuchAlgorithmException,
          KeyStoreException {
    KeyPairGenerator keyGen;
    keyGen = KeyPairGenerator.getInstance("RSA");
    keyGen.initialize(1024);
    KeyPair keyPair = keyGen.generateKeyPair();
    Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());
    X509V3CertificateGenerator certGen = new X509V3CertificateGenerator();
    certGen.setSerialNumber(BigInteger.valueOf(System.currentTimeMillis()));
    certGen.setIssuerDN(
        ((X509Certificate) this.keystore.getCertificate("servertrustedcert"))
            .getSubjectX500Principal());
    certGen.setNotBefore(new Date(System.currentTimeMillis() - 7 * 24 * 3600 * 1000));
    certGen.setNotAfter(new Date(System.currentTimeMillis() + 7 * 24 * 3600 * 1000));
    certGen.setSubjectDN(certInfo);
    certGen.setPublicKey(keyPair.getPublic());
    certGen.setSignatureAlgorithm("SHA1withDSA");

    // return new Pair(certGen.generate(this.caPrivKey), keyPair);
    return new common.Pair<X509Certificate, KeyPair>(certGen.generate(this.caPrivKey), keyPair);
  }
}
 public LocationSensitiveUpdater(final A3DContainer container) {
   sensitives = Collections.synchronizedSortedMap(new TreeMap<Double, Object[]>());
   lastestPosition = new Vector3();
   currentPosition = new Vector3();
   this.container = container;
 }