public void load(Iterable<T> input) {
log.debug("Starting import into table {}", tableName_);
final ExecutorService workers = workers();
// So modulo does not match right away, we set i != 0
int i = 1;
List<T> batch = new ArrayList<T>(BATCH_SIZE);
for (T item : input) {
batch.add(item);
if (i % BATCH_SIZE == 0) {
workers.submit(new Insert(batch));
batch = new ArrayList<T>(BATCH_SIZE);
}
if (i % 50000 == 0) {
log.info("Queued {} messages for table {}", i, tableName_);
}
++i;
}
workers.submit(new Insert(batch));
// Submit will block until it is safe to shut down:
shutdownGracefully(workers);
try {
for (HTableInterface table : tables_) {
table.flushCommits();
factory_.release(table);
}
} catch (IOException flushFail) {
log.error("Could not import buffer!", flushFail);
}
}
@Override
public void mutateMany(Map<String, Map<ByteBuffer, KCVMutation>> mutations, StoreTransaction txh)
throws StorageException {
final long delTS = System.currentTimeMillis();
final long putTS = delTS + 1;
Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey = convertToCommands(mutations, putTS, delTS);
List<Row> batch = new ArrayList<Row>(commandsPerKey.size()); // actual batch operation
// convert sorted commands into representation required for 'batch' operation
for (Pair<Put, Delete> commands : commandsPerKey.values()) {
if (commands.getFirst() != null) batch.add(commands.getFirst());
if (commands.getSecond() != null) batch.add(commands.getSecond());
}
try {
HTableInterface table = null;
try {
table = connectionPool.getTable(tableName);
table.batch(batch);
table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
} catch (IOException e) {
throw new TemporaryStorageException(e);
} catch (InterruptedException e) {
throw new TemporaryStorageException(e);
}
waitUntil(putTS);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
super.cleanup(context);
table.flushCommits();
table.close();
connection.close();
}
public void removeQuery(final String creator, final String id) throws IOException {
List<Query> queries = getQueries(creator);
Iterator<Query> queryIter = queries.iterator();
boolean changed = false;
while (queryIter.hasNext()) {
Query temp = queryIter.next();
if (temp.getId().equals(id)) {
queryIter.remove();
changed = true;
break;
}
}
if (!changed) {
return;
}
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
HTableInterface htable = null;
try {
htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
}
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
}
@Override
protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
IOUtils.copy(content, bout);
bout.close();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
}
public void saveQuery(final String creator, final Query query) throws IOException {
List<Query> queries = getQueries(creator);
queries.add(query);
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
HTableInterface htable = null;
try {
htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
}
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS)
throws IOException, IllegalStateException {
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
Put put = buildPut(resPath, newTS, row, content, table);
boolean ok = table.checkAndPut(row, B_FAMILY, B_COLUMN_TS, bOldTS, put);
if (!ok) {
long real = getResourceTimestamp(resPath);
throw new IllegalStateException(
"Overwriting conflict " + resPath + ", expect old TS " + real + ", but it is " + oldTS);
}
table.flushCommits();
return newTS;
} finally {
IOUtils.closeQuietly(table);
}
}
