public static void readTest(String tableStr, String row) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] rowkey = Bytes.toBytes(row);
      Get get = new Get(rowkey);
      get.addFamily(f0);

      Result result = table.get(get);
      NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0);

      if (m == null || m.isEmpty()) {
        System.err.println("Empty." + m);
        return;
      }

      for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
        String qualifier = Bytes.toString(entry.getKey());
        String value = Bytes.toString(entry.getValue());
        System.out.println(qualifier + ":" + value);
      }
      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
 public String toString() {
   if (variant == null
       || variant.isEmpty()
       || destinationName == null
       || destinationName.isEmpty()) {
     return "jms://";
   }
   final StringBuilder buf = new StringBuilder("jms:");
   buf.append(variant).append(':').append(encode(destinationName));
   if (!parameters.isEmpty()) {
     buf.append('?');
     boolean addDelimiter = false;
     for (Entry<String, String> param : parameters.entrySet()) {
       if (addDelimiter) {
         buf.append('&');
       } else {
         addDelimiter = true;
       }
       buf.append(encode(param.getKey()));
       final String value = param.getValue();
       if (value != null && value.length() > 0) {
         buf.append('=').append(encode(value));
       }
     }
   }
   return buf.toString();
 }
  @Override
  public boolean rollbackTx() throws Exception {
    if (consumingEntries.isEmpty()) {
      return true;
    }

    // Put the consuming entries back to cache
    entryCache.putAll(consumingEntries);

    // If not committed, no need to update HBase.
    if (!committed) {
      return true;
    }
    commitCount -= consumingEntries.size();

    // Revert changes in HBase rows
    // If it is FIFO, restore to the CLAIMED state. This instance will retry it on the next dequeue.
    if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
        && consumerConfig.getGroupSize() > 1) {
      byte[] stateContent = encodeStateColumn(ConsumerEntryState.CLAIMED);
      updateState(consumingEntries.keySet(), stateColumnName, stateContent);
    } else {
      undoState(consumingEntries.keySet(), stateColumnName);
    }
    return true;
  }
 public Word getRandomWord() {
   double value = random.nextDouble() * total;
   if (weightsMap.isEmpty()) {
     return new Word(null, 1.0);
   }
   return weightsMap.ceilingEntry(value).getValue();
 }
Exemple #5
0
    private int flushRing(int writeGroupsToFlush, int flushedGroups, boolean forceFlush)
        throws IOException {
      NavigableMap<GroupKey, WriteGroup> subMap = writeGroups.tailMap(lastGroupKey, false);

      if (!subMap.isEmpty()) {
        flushedGroups = iterateBySubRing(subMap, writeGroupsToFlush, 0, forceFlush);
        if (flushedGroups < writeGroupsToFlush) {
          if (!subMap.isEmpty()) {
            subMap = writeGroups.headMap(subMap.firstKey(), false);
            flushedGroups = iterateBySubRing(subMap, writeGroupsToFlush, flushedGroups, forceFlush);
          }
        }
      } else
        flushedGroups =
            iterateBySubRing(writeGroups, writeGroupsToFlush, flushedGroups, forceFlush);

      return flushedGroups;
    }
 public Word getWord(int position) throws Exception {
   if (!weightsMap.isEmpty() && position >= 0 && position < weightsMap.size()) {
     int count = 0;
     for (Entry<Double, Word> entry : weightsMap.entrySet()) {
       if (count == position) return entry.getValue();
       count++;
     }
   }
   throw new Exception("Wrong position \"" + position + "\" for dictionary \"" + name + "\".");
 }
  @Override
  public boolean commitTx() throws Exception {
    if (consumingEntries.isEmpty()) {
      return true;
    }

    byte[] stateContent = encodeStateColumn(ConsumerEntryState.PROCESSED);
    updateState(consumingEntries.keySet(), stateColumnName, stateContent);
    commitCount += consumingEntries.size();
    committed = true;
    return true;
  }
Exemple #8
0
 private NavigableMap<Number640, Data> getLatestInternalOrig(NavigableMap<Number640, Data> input) {
   // delete all predecessors
   NavigableMap<Number640, Data> result = new TreeMap<Number640, Data>();
   while (!input.isEmpty()) {
     // first entry is a latest version
     Entry<Number640, Data> latest = input.lastEntry();
     // store in results list
     result.put(latest.getKey(), latest.getValue());
     // delete all predecessors of latest entry
     deletePredecessors(latest.getKey(), input);
   }
   return result;
 }
  @Override
  protected ApacheLogEntry prefetch() throws Exception {
    while (sortedBuffer.size() < maxSortedBufferSize) {
      if (itEntry == null) {
        if (itFile.hasNext()) {
          itEntry = new ApacheLogEntryIterator(LogDirectory.open(itFile.next().getValue()), true);
        } else {
          break;
        }
      }

      if (!itEntry.hasNext()) {
        itEntry = null;
        continue;
      }

      ApacheLogEntry entry = itEntry.next();

      if (low != null) {
        int d = entry.getDate().compareTo(low);
        if (d < 0 || (d == 0 && !lowInclusive)) {
          continue;
        }
      }

      sortedBuffer.put(entry.getDate(), entry);
    }

    if (!sortedBuffer.isEmpty()) {
      ApacheLogEntry entry = sortedBuffer.pollFirstEntry().getValue();

      if (sanityCheckMonotonictyDate != null) {
        if (sanityCheckMonotonictyDate.compareTo(entry.getDate()) > 0) {
          throw new RuntimeException("Dates are not monoton");
        }
      }
      sanityCheckMonotonictyDate = entry.getDate();

      if (high != null) {
        int d = high.compareTo(entry.getDate());
        if (d < 0 || (d == 0 && !highInclusive)) {
          return finish();
        }
      }

      return entry;
    }

    return finish();
  }
  @Override
  public DequeueResult<byte[]> dequeue(int maxBatchSize) throws IOException {
    Preconditions.checkArgument(maxBatchSize > 0, "Batch size must be > 0.");

    // pre-compute the "claimed" state content in case of FIFO.
    byte[] claimedStateValue = null;
    if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
        && consumerConfig.getGroupSize() > 1) {
      claimedStateValue = encodeStateColumn(ConsumerEntryState.CLAIMED);
    }
    while (consumingEntries.size() < maxBatchSize && getEntries(consumingEntries, maxBatchSize)) {

      // ANDREAS: this while loop should stop once getEntries/populateCache reaches the end of the
      // queue. Currently, it
      // will retry as long as it gets at least one entry in every round, even if that is an entry
      // that must be ignored
      // because it cannot be claimed.
      // ANDREAS: It could be a problem that we always read to the end of the queue. This way one
      // flowlet instance may
      // always all entries, while others are idle.

      // For FIFO, need to try claiming the entry if group size > 1
      if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
          && consumerConfig.getGroupSize() > 1) {
        Iterator<Map.Entry<byte[], SimpleQueueEntry>> iterator =
            consumingEntries.entrySet().iterator();
        while (iterator.hasNext()) {
          SimpleQueueEntry entry = iterator.next().getValue();

          if (entry.getState() == null
              || QueueEntryRow.getStateInstanceId(entry.getState())
                  >= consumerConfig.getGroupSize()) {
            // If not able to claim it, remove it, and move to next one.
            if (!claimEntry(entry.getRowKey(), claimedStateValue)) {
              iterator.remove();
            }
          }
        }
      }
    }

    // If nothing get dequeued, return the empty result.
    if (consumingEntries.isEmpty()) {
      return EMPTY_RESULT;
    }

    return new SimpleDequeueResult(consumingEntries.values());
  }
 @Override
 public void postTxCommit() {
   if (scanStartRow != null) {
     if (!consumingEntries.isEmpty()) {
       // Start row can be updated to the largest rowKey in the consumingEntries (now is consumed)
       // that is smaller than or equal to scanStartRow
       byte[] floorKey = consumingEntries.floorKey(scanStartRow);
       if (floorKey != null) {
         startRow = floorKey;
       }
     } else {
       // If the dequeue has empty result, startRow can advance to scanStartRow
       startRow = Arrays.copyOf(scanStartRow, scanStartRow.length);
     }
   }
 }
  public static void scanTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] startRow = Bytes.toBytes("rowKey_0");
      byte[] stopRow = Bytes.toBytes("rowKey_6");
      byte[] family = f0;

      Scan scan = new Scan();
      scan.addFamily(family);
      scan.setMaxVersions(1);

      //            scan.setStartRow(startRow);
      //            scan.setStopRow(stopRow);
      int count = 0;
      ResultScanner scanner = table.getScanner(scan);
      Result result = scanner.next();
      while (result != null) {
        String rowKey = Bytes.toString(result.getRow());
        NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);

        if (m == null || m.isEmpty()) {
          System.err.println("Empty." + m);
          return;
        }
        for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
          String qualifier = Bytes.toString(entry.getKey());
          String value = Bytes.toString(entry.getValue());
          System.out.println(rowKey + ":" + qualifier + ":" + value);
        }
        result = scanner.next();
        count++;
        System.out.println("-----------------------------");
      }
      table.close(); // very important
      System.out.println("count:" + count);
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Exemple #13
0
 private static List<ExecutionScript> loadScripts(
     String flowId, ExecutionPhase phase, NavigableMap<String, String> contents) {
   assert flowId != null;
   assert phase != null;
   assert contents != null;
   if (contents.isEmpty()) {
     return Collections.emptyList();
   }
   List<ExecutionScript> results = new ArrayList<ExecutionScript>();
   Map<String, NavigableMap<String, String>> scripts = partitioning(contents);
   for (Map.Entry<String, NavigableMap<String, String>> entry : scripts.entrySet()) {
     String scriptId = entry.getKey();
     NavigableMap<String, String> scriptContents = entry.getValue();
     ExecutionScript script = loadScript(flowId, phase, scriptId, scriptContents);
     results.add(script);
   }
   checkBlockers(flowId, phase, results);
   return results;
 }
  @Override
  protected ApacheLogEntry prefetch() throws Exception {
    String line;
    while (sortedBuffer.size() < maxSortedBufferSize && ((line = reader.readLine()) != null)) {
      try {
        ApacheLogEntry entry = ApacheLogEntry.parse(line);
        sortedBuffer.put(entry.getDate(), entry);
      } catch (ParseException e) {
        // TODO Shouldn't happen, but the parser is sucky, so it happens
        e.printStackTrace();
      }
    }

    if (!sortedBuffer.isEmpty()) {
      return sortedBuffer.pollFirstEntry().getValue();
    }

    return finish();
  }
  public Word getNextWord(String previousWord, boolean limit) throws Exception {
    if (weightsMap.isEmpty()) {
      return new Word(null, 1.0);
    }
    if (null == previousWord || previousWord.equals(weightsMap.lastEntry().getValue().getValue())) {
      if (!limit) {
        return weightsMap.firstEntry().getValue();
      }
      throw new LimitReachedException(
          previousWord, "Last word in dictionary \"" + name + "\" already reached.");
    }

    Iterator<Word> iterator = weightsMap.values().iterator();
    while (iterator.hasNext()) {
      if (previousWord.equals(iterator.next().getValue())) {
        return iterator.next();
      }
    }
    throw new Exception(
        "No previous word \"" + previousWord + "\" was found in dictionary \"" + name + "\".");
  }
Exemple #16
0
 private static ExecutionScript loadScript(
     String flowId, ExecutionPhase phase, String nodeId, Map<String, String> contents) {
   assert flowId != null;
   assert phase != null;
   assert nodeId != null;
   assert contents != null;
   String prefix = getPrefix(flowId, phase, nodeId);
   String scriptId = extract(contents, prefix, KEY_ID);
   String kindSymbol = extract(contents, prefix, KEY_KIND);
   ExecutionScript.Kind kind = ExecutionScript.Kind.findFromSymbol(kindSymbol);
   String blockersString = extract(contents, prefix, KEY_BLOCKERS);
   Set<String> blockers = parseTokens(blockersString);
   Map<String, String> environmentVariables =
       PropertiesUtil.createPrefixMap(contents, KEY_ENV_PREFIX);
   ExecutionScript script;
   if (kind == ExecutionScript.Kind.COMMAND) {
     String profileName = extract(contents, prefix, KEY_PROFILE);
     String moduleName = extract(contents, prefix, KEY_MODULE);
     NavigableMap<String, String> commandMap =
         PropertiesUtil.createPrefixMap(contents, KEY_COMMAND_PREFIX);
     if (commandMap.isEmpty()) {
       throw new IllegalArgumentException(
           MessageFormat.format("\"{0}*\" is not defined", prefix + KEY_COMMAND_PREFIX));
     }
     List<String> command = new ArrayList<String>(commandMap.values());
     script =
         new CommandScript(
             scriptId, blockers, profileName, moduleName, command, environmentVariables);
   } else if (kind == ExecutionScript.Kind.HADOOP) {
     String className = extract(contents, prefix, KEY_CLASS_NAME);
     Map<String, String> properties = PropertiesUtil.createPrefixMap(contents, KEY_PROP_PREFIX);
     script = new HadoopScript(scriptId, blockers, className, properties, environmentVariables);
   } else {
     throw new IllegalArgumentException(
         MessageFormat.format("Unsupported kind in \"{0}\": {1}", prefix + KEY_KIND, kindSymbol));
   }
   LOG.trace("Loaded script {}* -> {}", script);
   return script;
 }
Exemple #17
0
 private static Map<String, NavigableMap<String, String>> partitioning(
     NavigableMap<String, String> map) {
   assert map != null;
   Map<String, NavigableMap<String, String>> results =
       new TreeMap<String, NavigableMap<String, String>>();
   while (map.isEmpty() == false) {
     String name = map.firstKey();
     int index = name.indexOf('.');
     if (index >= 0) {
       name = name.substring(0, index);
     }
     String first = name + '.';
     String last = name + (char) ('.' + 1);
     NavigableMap<String, String> partition = new TreeMap<String, String>();
     for (Map.Entry<String, String> entry : map.subMap(first, last).entrySet()) {
       String key = entry.getKey();
       partition.put(key.substring(name.length() + 1), entry.getValue());
     }
     results.put(name, partition);
     map.remove(name);
     map.subMap(first, last).clear();
   }
   return results;
 }
 @Override
 public boolean isEmpty() {
   return configMap.isEmpty();
 }
Exemple #19
0
  /**
   * HBaseの一行をItemDataに変換
   *
   * @param r row in HBase
   * @return 商品情報
   */
  public ItemData toDataStore(Result r) {
    if (r == null || r.isEmpty()) {
      return null;
    }

    ItemData item = null;

    // meta family
    NavigableMap<byte[], byte[]> map = r.getFamilyMap(META_FAMILY);
    if (map != null && !map.isEmpty()) {
      item = new ItemData();

      // ctime
      byte[] bCtime = map.get(Bytes.toBytes("ctime"));
      if (bCtime != null) {
        item.ctime = Bytes.toLong(bCtime);
      }
    }

    // data family
    map = r.getFamilyMap(DATA_FAMILY);
    if (map != null && !map.isEmpty()) {
      if (item == null) {
        item = new ItemData();
      }

      // shop ID
      byte[] bShopID = map.get(Bytes.toBytes("shop_id"));
      if (bShopID != null) {
        item.shopID = Bytes.toInt(bShopID);
      }

      // item ID
      byte[] bItemID = map.get(Bytes.toBytes("item_id"));
      if (bItemID != null) {
        item.itemID = Bytes.toInt(bItemID);
      }

      // genre ID
      byte[] bGenreID = map.get(Bytes.toBytes("genre_id"));
      if (bGenreID != null) {
        item.genreID = Bytes.toInt(bGenreID);
      }

      // item name
      byte[] bItemName = map.get(Bytes.toBytes("item_name"));
      if (bItemName != null) {
        item.itemName = Bytes.toString(bItemName);
      }

      // full item url
      byte[] bFullItemUrl = map.get(Bytes.toBytes("full_item_url"));
      if (bFullItemUrl != null) {
        item.fullItemUrl = Bytes.toString(bFullItemUrl);
      }

      // price
      byte[] bPrice = map.get(Bytes.toBytes("price"));
      if (bPrice != null) {
        item.price = Bytes.toString(bPrice);
      }
    }

    if (item == null) {
      item = new ItemData();
    }
    item.rowkey = r.getRow();

    return item;
  }
Exemple #20
0
  /**
   * Sends all messages currently in forward_table to the new coordinator (changing the dest field).
   * This needs to be done, so the underlying reliable unicast protocol (e.g. UNICAST) adds these
   * messages to its retransmission mechanism<br>
   * Note that we need to resend the messages in order of their seqnos ! We also need to prevent
   * other message from being inserted until we're done, that's why there's synchronization.<br>
   * Access to the forward_table doesn't need to be synchronized as there won't be any insertions
   * during flushing (all down-threads are blocked)
   */
  protected void flushMessagesInForwardTable() {
    if (is_coord) {
      for (Map.Entry<Long, Message> entry : forward_table.entrySet()) {
        Long key = entry.getKey();
        Message msg = entry.getValue();
        Buffer buf;
        try {
          buf = Util.streamableToBuffer(msg);
        } catch (Exception e) {
          log.error(Util.getMessage("FlushingBroadcastingFailed"), e);
          continue;
        }

        SequencerHeader hdr = new SequencerHeader(SequencerHeader.WRAPPED_BCAST, key);
        Message forward_msg = new Message(null, buf).putHeader(this.id, hdr);
        if (log.isTraceEnabled())
          log.trace(local_addr + ": flushing (broadcasting) " + local_addr + "::" + key);
        down_prot.down(new Event(Event.MSG, forward_msg));
      }
      return;
    }

    // for forwarded messages, we need to receive the forwarded message from the coordinator, to
    // prevent this case:
    // - V1={A,B,C}
    // - A crashes
    // - C installs V2={B,C}
    // - C forwards messages 3 and 4 to B (the new coord)
    // - B drops 3 because its view is still V1
    // - B installs V2
    // - B receives message 4 and broadcasts it
    // ==> C's message 4 is delivered *before* message 3 !
    // ==> By resending 3 until it is received, then resending 4 until it is received, we make sure
    // this won't happen
    // (see https://issues.jboss.org/browse/JGRP-1449)
    while (flushing && running && !forward_table.isEmpty()) {
      Map.Entry<Long, Message> entry = forward_table.firstEntry();
      final Long key = entry.getKey();
      Message msg = entry.getValue();
      Buffer buf;

      try {
        buf = Util.streamableToBuffer(msg);
      } catch (Exception e) {
        log.error(Util.getMessage("FlushingBroadcastingFailed"), e);
        continue;
      }

      while (flushing && running && !forward_table.isEmpty()) {
        SequencerHeader hdr = new SequencerHeader(SequencerHeader.FLUSH, key);
        Message forward_msg =
            new Message(coord, buf).putHeader(this.id, hdr).setFlag(Message.Flag.DONT_BUNDLE);
        if (log.isTraceEnabled())
          log.trace(
              local_addr
                  + ": flushing (forwarding) "
                  + local_addr
                  + "::"
                  + key
                  + " to coord "
                  + coord);
        ack_promise.reset();
        down_prot.down(new Event(Event.MSG, forward_msg));
        Long ack = ack_promise.getResult(500);
        if ((Objects.equals(ack, key)) || !forward_table.containsKey(key)) break;
      }
    }
  }
 boolean isEmpty() {
   return rangeMap.isEmpty();
 }
Exemple #22
0
  public Map<Number640, Enum<?>> putAll(
      final NavigableMap<Number640, Data> dataMap,
      PublicKey publicKey,
      boolean putIfAbsent,
      boolean domainProtection,
      boolean sendSelf) {
    if (dataMap.isEmpty()) {
      return Collections.emptyMap();
    }
    final Number640 min = dataMap.firstKey();
    final Number640 max = dataMap.lastKey();
    final Map<Number640, Enum<?>> retVal = new HashMap<Number640, Enum<?>>();
    final HashSet<Number480> keysToCheck = new HashSet<Number480>();
    final RangeLock<Number640>.Range lock = lock(min, max);
    try {
      for (Map.Entry<Number640, Data> entry : dataMap.entrySet()) {
        Number640 key = entry.getKey();
        keysToCheck.add(key.locationAndDomainAndContentKey());
        Data newData = entry.getValue();
        if (!securityDomainCheck(
            key.locationAndDomainKey(), publicKey, publicKey, domainProtection)) {
          retVal.put(key, PutStatus.FAILED_SECURITY);
          newData.release();
          continue;
        }

        // We need this check in case we did not use the encoder/decoder,
        // which is the case if we send the message to ourself. In that
        // case, the public key of the data is never set to the message
        // publick key, if the publick key of the data was null.
        final PublicKey dataKey;
        if (sendSelf && newData.publicKey() == null) {
          dataKey = publicKey;
        } else {
          dataKey = newData.publicKey();
        }

        if (!securityEntryCheck(
            key.locationAndDomainAndContentKey(), publicKey, dataKey, newData.isProtectedEntry())) {
          retVal.put(key, PutStatus.FAILED_SECURITY);
          newData.release();
          continue;
        }

        final Data oldDataGet = backend.get(key);
        if (oldDataGet != null) {
          if (putIfAbsent) {
            retVal.put(key, PutStatus.FAILED_NOT_ABSENT);
            newData.release();
            continue;
          }

          if (oldDataGet.isDeleted()) {
            retVal.put(key, PutStatus.DELETED);
            newData.release();
            continue;
          }
          if (!oldDataGet.basedOnSet().equals(newData.basedOnSet())) {
            retVal.put(key, PutStatus.VERSION_FORK);
            newData.release();
            continue;
          }
        }

        final Data oldDataPut = backend.put(key, newData);

        long expiration = newData.expirationMillis();
        // handle timeout
        backend.addTimeout(key, expiration);

        if (newData.hasPrepareFlag()) {
          retVal.put(key, PutStatus.OK_PREPARED);
        } else {
          retVal.put(key, PutStatus.OK);
        }
        if (oldDataPut != null && oldDataPut != newData) {
          oldDataPut.release();
        }
      }

      for (Number480 key : keysToCheck) {

        // now check for forks
        Number640 minVersion = new Number640(key, Number160.ZERO);
        Number640 maxVersion = new Number640(key, Number160.MAX_VALUE);
        NavigableMap<Number640, Data> tmp = backend.subMap(minVersion, maxVersion);
        tmp = filterCopyOrig(tmp, -1, true);
        NavigableMap<Number640, Data> heads = getLatestInternalOrig(tmp);

        final boolean forked = heads.size() > 1;
        for (final Map.Entry<Number640, Data> entry : heads.entrySet()) {
          if (forked) {
            if (retVal.containsKey(entry.getKey())) {
              retVal.put(entry.getKey(), PutStatus.VERSION_FORK);
            }
          }
        }

        // now remove old versions
        if (maxVersions > 0) {
          NavigableMap<Number640, Data> versions = backend.subMap(minVersion, maxVersion);

          while (!versions.isEmpty()
              && versions.firstKey().versionKey().timestamp() + maxVersions
                  <= versions.lastKey().versionKey().timestamp()) {
            Map.Entry<Number640, Data> entry = versions.pollFirstEntry();
            Data removed = backend.remove(entry.getKey(), true);
            if (removed != null) {
              removed.release();
            }
            backend.removeTimeout(entry.getKey());
          }
        }
      }
      return retVal;

    } finally {
      lock.unlock();
    }
  }
Exemple #23
0
 @MapFeature.Require(SUPPORTS_REMOVE)
 @CollectionSize.Require(ONE)
 public void testSingletonMapPollLast() {
   assertEquals(a, navigableMap.pollLastEntry());
   assertTrue(navigableMap.isEmpty());
 }
 public boolean isEmpty() {
   return data.isEmpty();
 }