public void load(Iterable<T> input) {

    log.debug("Starting import into table {}", tableName_);
    final ExecutorService workers = workers();

    // So modulo does not match right away, we set i != 0
    int i = 1;
    List<T> batch = new ArrayList<T>(BATCH_SIZE);
    for (T item : input) {
      batch.add(item);
      if (i % BATCH_SIZE == 0) {
        workers.submit(new Insert(batch));
        batch = new ArrayList<T>(BATCH_SIZE);
      }
      if (i % 50000 == 0) {
        log.info("Queued {} messages for table {}", i, tableName_);
      }
      ++i;
    }
    workers.submit(new Insert(batch));

    // Submit will block until it is safe to shut down:
    shutdownGracefully(workers);
    try {
      for (HTableInterface table : tables_) {
        table.flushCommits();
        factory_.release(table);
      }
    } catch (IOException flushFail) {
      log.error("Could not import buffer!", flushFail);
    }
  }
Esempio n. 2
0
  @Override
  public void mutateMany(Map<String, Map<ByteBuffer, KCVMutation>> mutations, StoreTransaction txh)
      throws StorageException {
    final long delTS = System.currentTimeMillis();
    final long putTS = delTS + 1;

    Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey = convertToCommands(mutations, putTS, delTS);
    List<Row> batch = new ArrayList<Row>(commandsPerKey.size()); // actual batch operation

    // convert sorted commands into representation required for 'batch' operation
    for (Pair<Put, Delete> commands : commandsPerKey.values()) {
      if (commands.getFirst() != null) batch.add(commands.getFirst());

      if (commands.getSecond() != null) batch.add(commands.getSecond());
    }

    try {
      HTableInterface table = null;

      try {
        table = connectionPool.getTable(tableName);
        table.batch(batch);
        table.flushCommits();
      } finally {
        IOUtils.closeQuietly(table);
      }
    } catch (IOException e) {
      throw new TemporaryStorageException(e);
    } catch (InterruptedException e) {
      throw new TemporaryStorageException(e);
    }

    waitUntil(putTS);
  }
Esempio n. 3
0
 @Override
 protected void cleanup(Context context) throws IOException, InterruptedException {
   super.cleanup(context);
   table.flushCommits();
   table.close();
   connection.close();
 }
Esempio n. 4
0
  public void removeQuery(final String creator, final String id) throws IOException {
    List<Query> queries = getQueries(creator);
    Iterator<Query> queryIter = queries.iterator();

    boolean changed = false;
    while (queryIter.hasNext()) {
      Query temp = queryIter.next();
      if (temp.getId().equals(id)) {
        queryIter.remove();
        changed = true;
        break;
      }
    }

    if (!changed) {
      return;
    }

    Query[] queryArray = new Query[queries.size()];
    byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
    HTableInterface htable = null;
    try {
      htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
      Put put = new Put(Bytes.toBytes(creator));
      put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);

      htable.put(put);
      htable.flushCommits();
    } finally {
      IOUtils.closeQuietly(htable);
    }
  }
Esempio n. 5
0
 @Override
 protected void deleteResourceImpl(String resPath) throws IOException {
   HTableInterface table = getConnection().getTable(getAllInOneTableName());
   try {
     Delete del = new Delete(Bytes.toBytes(resPath));
     table.delete(del);
     table.flushCommits();
   } finally {
     IOUtils.closeQuietly(table);
   }
 }
Esempio n. 6
0
  @Override
  protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    IOUtils.copy(content, bout);
    bout.close();

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    try {
      byte[] row = Bytes.toBytes(resPath);
      Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);

      table.put(put);
      table.flushCommits();
    } finally {
      IOUtils.closeQuietly(table);
    }
  }
Esempio n. 7
0
  public void saveQuery(final String creator, final Query query) throws IOException {
    List<Query> queries = getQueries(creator);
    queries.add(query);
    Query[] queryArray = new Query[queries.size()];

    byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
    HTableInterface htable = null;
    try {
      htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
      Put put = new Put(Bytes.toBytes(creator));
      put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);

      htable.put(put);
      htable.flushCommits();
    } finally {
      IOUtils.closeQuietly(htable);
    }
  }
Esempio n. 8
0
  @Override
  protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS)
      throws IOException, IllegalStateException {
    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    try {
      byte[] row = Bytes.toBytes(resPath);
      byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
      Put put = buildPut(resPath, newTS, row, content, table);

      boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
      if (!ok) {
        long real = getResourceTimestamp(resPath);
        throw new IllegalStateException(
            "Overwriting conflict " + resPath + ", expect old TS " + real + ", but it is " + oldTS);
      }

      table.flushCommits();

      return newTS;
    } finally {
      IOUtils.closeQuietly(table);
    }
  }