public void saveRegisterModelList(List<RegisterModel> list) {
try {
List<Put> putList = new ArrayList<Put>();
for (RegisterModel o : list) {
String rowKey =
System.currentTimeMillis()
+ ":"
+ o.getProvinceId()
+ ":"
+ o.getCityId()
+ ":"
+ UUID.randomUUID().toString();
Put put = new Put(Bytes.toBytes(rowKey));
put.add(
Bytes.toBytes("info"), Bytes.toBytes(o.getTid()), Bytes.toBytes(JSON.toJSONString(o)));
putList.add(put);
}
table = HBaseDataSource.getTable("TH_TERMINAL_REGISTER");
if (table != null) {
table.put(putList);
} else {
table.put(putList);
}
} catch (IOException e) {
log.error("存储异常:" + e.getMessage(), e);
}
}
public void saveAuthModelList(List<AuthModel> list) {
try {
List<Put> putList = new ArrayList<Put>();
for (AuthModel o : list) {
String rowKey =
System.currentTimeMillis()
+ ":"
+ o.getResult()
+ ":"
+ o.getCommaddr()
+ ":"
+ UUID.randomUUID().toString();
Put put = new Put(Bytes.toBytes(rowKey));
put.add(
Bytes.toBytes("info"), Bytes.toBytes(o.getAkey()), Bytes.toBytes(JSON.toJSONString(o)));
putList.add(put);
}
table = HBaseDataSource.getTable("TH_VEHICLE_CHECKED");
if (table != null) {
table.put(putList);
} else {
table.put(putList);
}
} catch (IOException e) {
log.error("存储异常:" + e.getMessage(), e);
}
}
@Test
public void testSetLayout() throws Exception {
final TableLayoutDesc layoutDesc = KijiTableLayouts.getLayout(KijiTableLayouts.SIMPLE);
final KijiTableLayout layout = KijiTableLayout.newLayout(layoutDesc);
final Get expectedGet =
new Get(Bytes.toBytes(layout.getDesc().getName()))
.addColumn(
Bytes.toBytes(mFamily), Bytes.toBytes(HBaseTableLayoutDatabase.QUALIFIER_LAYOUT))
.setMaxVersions(1);
final Result expectedGetResult = new Result(Collections.<KeyValue>emptyList());
expect(mHTable.get(eqGet(expectedGet))).andReturn(expectedGetResult);
final Put expectedPut =
new Put(Bytes.toBytes(layout.getDesc().getName()))
.add(
Bytes.toBytes(mFamily),
Bytes.toBytes(HBaseTableLayoutDatabase.QUALIFIER_UPDATE),
encode(layoutDesc))
.add(
Bytes.toBytes(mFamily),
Bytes.toBytes(HBaseTableLayoutDatabase.QUALIFIER_LAYOUT),
encode(layout.getDesc()))
.add(
Bytes.toBytes(mFamily),
Bytes.toBytes(HBaseTableLayoutDatabase.QUALIFIER_LAYOUT_ID),
Bytes.toBytes("1"));
mHTable.put(eqPut(expectedPut));
replay(mHTable);
mDb.updateTableLayout(layout.getDesc().getName(), layoutDesc);
verify(mHTable);
}
/*
* 为表添加数据（适合知道有多少列族的固定表）
*
* @rowKey rowKey
*
* @tableName 表名
*
* @column1 第一个列族列表
*
* @value1 第一个列的值的列表
*
* @column2 第二个列族列表
*
* @value2 第二个列的值的列表
*/
public static void addData(
String rowKey,
String tableName,
String[] column1,
String[] value1,
String[] column2,
String[] value2)
throws IOException {
Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
HTableInterface htable = conn.getTable(tableName);
HColumnDescriptor[] columnFamilies =
htable
.getTableDescriptor() // 获取所有的列族
.getColumnFamilies();
for (int i = 0; i < columnFamilies.length; i++) {
String familyName = columnFamilies[i].getNameAsString(); // 获取列族名
if (familyName.equals("article")) { // article列族put数据
for (int j = 0; j < column1.length; j++) {
put.add(Bytes.toBytes(familyName), Bytes.toBytes(column1[j]), Bytes.toBytes(value1[j]));
}
}
if (familyName.equals("author")) { // author列族put数据
for (int j = 0; j < column2.length; j++) {
put.add(Bytes.toBytes(familyName), Bytes.toBytes(column2[j]), Bytes.toBytes(value2[j]));
}
}
}
htable.put(put);
// System.out.println("add data Success!");
}
public void removeQuery(final String creator, final String id) throws IOException {
List<Query> queries = getQueries(creator);
Iterator<Query> queryIter = queries.iterator();
boolean changed = false;
while (queryIter.hasNext()) {
Query temp = queryIter.next();
if (temp.getId().equals(id)) {
queryIter.remove();
changed = true;
break;
}
}
if (!changed) {
return;
}
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
HTableInterface htable = null;
try {
htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
}
public static void writeTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] family = f0;
List<Put> puts = new ArrayList<Put>();
for (int k = 0; k < 10; k++) // 写10行数据
{
byte[] rowkey = Bytes.toBytes("rowKey_" + k);
Put p = new Put(rowkey);
byte[] value_id = Bytes.toBytes("123456");
byte[] value_user = Bytes.toBytes("mengqinghao" + k);
p.add(family, qualifier_id, value_id);
p.add(family, qualifier_user, value_user);
puts.add(p);
}
table.put(puts);
System.out.println("Puts done: " + puts.size());
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
/*
* (non-Javadoc)
*
* @see com.hazelcast.core.MapStore#store(java.lang.Object,
* java.lang.Object)
*/
@Override
public void store(String key, String value) {
HTableInterface table = null;
try {
table = pool.getTable(tableName);
try {
byte[] rowId = prefixDate ? IdUtil.bucketizeId(key) : Bytes.toBytes(key);
Put p = new Put(rowId);
if (outputFormatType == StoreFormatType.SMILE) {
p.add(family, qualifier, jsonSmileConverter.convertToSmile(value));
} else {
p.add(family, qualifier, Bytes.toBytes(value));
}
table.put(p);
} catch (NumberFormatException nfe) {
LOG.error("Encountered bad key: " + key, nfe);
}
} catch (IOException e) {
LOG.error("Error during put", e);
} finally {
if (table != null) {
pool.putTable(table);
}
}
}
public <V> void addObject(String tableName, V v, HbaseMapper<V> mapper) throws Exception {
HTableInterface htable = dataSource.getConnection(tableName);
Put put = mapper.mapPut(v);
log.info("put is -[" + put + "]");
htable.put(put);
log.info("put one object to " + tableName + ", put info -[" + put + "]");
htable.close();
}
public static void createTable(String tableName) {
long start = System.currentTimeMillis();
try {
HTableInterface htable = conn.getTable(tableName);
List<Put> putList = new ArrayList<Put>();
HColumnDescriptor[] columnFamilies = htable.getTableDescriptor().getColumnFamilies();
String[] family = {"article", "author"};
creatTable(tableName, family);
String[] column1 = {"title", "content", "tag"};
String[] column2 = {"name", "nickname"};
for (int i = 1; i <= 1000000; i++) {
DecimalFormat format = new DecimalFormat("00000000");
String rowKey = format.format(i);
// System.out.println("==| insert"+rowKey);
String[] valueOfArticle = {"title" + i, "content" + i, "tag" + i};
String[] valueOfAuthor = {"name" + i, "nickname" + i};
// addData(rowKey, tableName, column1, valueOfArticle, column2,
// valueOfAuthor);
Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
for (int colIndex = 0; colIndex < columnFamilies.length; colIndex++) {
String familyName = columnFamilies[colIndex].getNameAsString(); // 获取列族名
if (familyName.equals("article")) { // article列族put数据
for (int k = 0; k < column1.length; k++) {
put.add(
Bytes.toBytes(familyName),
Bytes.toBytes(column1[k]),
Bytes.toBytes(valueOfArticle[k]));
}
}
if (familyName.equals("author")) { // author列族put数据
for (int k = 0; k < column2.length; k++) {
put.add(
Bytes.toBytes(familyName),
Bytes.toBytes(column2[k]),
Bytes.toBytes(valueOfAuthor[k]));
}
}
}
putList.add(put);
if (i % 10000 == 0) {
htable.put(putList);
putList = new ArrayList<Put>();
}
}
long end = System.currentTimeMillis();
long used = end - start;
System.out.println("data insert finished used " + used + " ms");
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void close() throws IOException {
super.close();
byte[] bytes = toByteArray();
Put put = new Put(blobKey);
put.add(BLOBS_COLUMN_FAMILY_BYTES, BLOB_COLUMN, bytes);
blobTable.put(put);
blob.setValue(blobKey);
}
/**
* 向表中插入一条新数据
*
* @param tableName 表名
* @param row 行键key
* @param columnFamily 列族
* @param column 列名
* @param data 要插入的数据
* @throws IOException
*/
public static void putData(
String tableName, String row, String columnFamily, String column, byte[] data)
throws IOException {
HTableInterface table = conn.getTable(tableName);
Put put = new Put(Bytes.toBytes(row));
put.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), data);
table.put(put);
System.out.println(
"PUT:" + row + "','" + columnFamily + ":" + column + "','" + Bytes.toString(data) + "'");
}
public void markProblematic(RowLogMessage message, String subscription) throws RowLogException {
byte[] rowKey = createRowKey(message, subscription, true);
Put put = new Put(rowKey);
put.add(MESSAGES_CF, MESSAGE_COLUMN, encodeMessage(message));
try {
table.put(put);
} catch (IOException e) {
throw new RowLogException("Failed to mark message as problematic", e);
}
removeMessage(message, subscription, false);
}
public <V> void addObjects(String tableName, List<V> objects, HbaseMapper<V> mapper)
throws Exception {
if (CollectionUtils.isEmpty(objects)) {
return;
}
HTableInterface htable = dataSource.getConnection(tableName);
List<Put> puts = new ArrayList<Put>();
for (V v : objects) {
puts.add(mapper.mapPut(v));
}
htable.put(puts);
log.info("put " + puts.size() + " objects to " + tableName);
htable.close();
}
@Override
protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
IOUtils.copy(content, bout);
bout.close();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
}
public void saveQuery(final String creator, final Query query) throws IOException {
List<Query> queries = getQueries(creator);
queries.add(query);
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
HTableInterface htable = null;
try {
htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
}
/**
*
*
* <pre>
* rowkey f/q1 f/q2 f/q3
* 1 1 2
* 2 4 8
* 3 16
* 4 32
* 5 64 NULL
* 6 NULL 128
* 7 NULL NULL
* 8 "test"
* </pre>
*/
private void fillData() throws Throwable {
Put put = new Put(rowKey1);
put.add(ColumnFamilyNameBytes, QName1, Bytes.toBytes(1L));
put.add(ColumnFamilyNameBytes, QName2, Bytes.toBytes(2L));
table.put(put);
put = new Put(rowKey2);
put.add(ColumnFamilyNameBytes, QName1, Bytes.toBytes(4L));
put.add(ColumnFamilyNameBytes, QName2, Bytes.toBytes(8L));
table.put(put);
put = new Put(rowKey3);
put.add(ColumnFamilyNameBytes, QName1, Bytes.toBytes(16L));
table.put(put);
put = new Put(rowKey4);
put.add(ColumnFamilyNameBytes, QName2, Bytes.toBytes(32L));
table.put(put);
// set null case.
put = new Put(rowKey5);
put.add(ColumnFamilyNameBytes, QName1, Bytes.toBytes(64L));
put.add(ColumnFamilyNameBytes, QName2, null);
table.put(put);
put = new Put(rowKey6);
put.add(ColumnFamilyNameBytes, QName1, null);
put.add(ColumnFamilyNameBytes, QName2, Bytes.toBytes(128L));
table.put(put);
put = new Put(rowKey7);
put.add(ColumnFamilyNameBytes, QName1, null);
put.add(ColumnFamilyNameBytes, QName2, null);
table.put(put);
// empty case.
put = new Put(rowKey8);
put.add(ColumnFamilyNameBytes, QName3, Bytes.toBytes("test"));
table.put(put);
}
@Test
public void testEmptyRowId() throws IOException {
deleteTable(TABLE);
HTablePool pool = new HTablePool();
HTableInterface table = pool.getTable(TABLE);
byte[] EMPTY = new byte[0];
Put put = new Put(EMPTY);
put.add(COLUMN_FAMILY, $("test").getBytes(), $("test").getBytes());
table.put(put);
Scan scan = new Scan();
boolean found = false;
for (Result r : table.getScanner(scan)) {
assertFalse(found);
found = true;
assertEquals(0, EMPTY.length);
}
assertTrue(found);
deleteTable(TABLE);
}
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String all[] = value.toString().split("/t");
if (all.length == 2) {
put = new Put(Bytes.toBytes(all[0]));
put.add(Bytes.toBytes("xxx"), Bytes.toBytes("20110313"), Bytes.toBytes(all[1]));
}
if (!wal) {
put.setDurability(Durability.SKIP_WAL);
}
table.put(put);
if ((++count % 100) == 0) {
context.setStatus(count + " DOCUMENTS done!");
context.progress();
System.out.println(count + " DOCUMENTS done!");
}
}
@Ignore
@Test
public void testNonTxToTxTableFailure() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
// Put table in SYSTEM schema to prevent attempts to update the cache after we disable
// SYSTEM.CATALOG
conn.createStatement()
.execute("CREATE TABLE SYSTEM.NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
conn.createStatement().execute("UPSERT INTO SYSTEM.NON_TX_TABLE VALUES (1)");
conn.commit();
// Reset empty column value to an empty value like it is pre-transactions
HTableInterface htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
Put put = new Put(PInteger.INSTANCE.toBytes(1));
put.add(
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
QueryConstants.EMPTY_COLUMN_BYTES,
ByteUtil.EMPTY_BYTE_ARRAY);
htable.put(put);
HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
try {
// This will succeed initially in updating the HBase metadata, but then will fail when
// the SYSTEM.CATALOG table is attempted to be updated, exercising the code to restore
// the coprocessors back to the non transactional ones.
conn.createStatement().execute("ALTER TABLE SYSTEM.NON_TX_TABLE SET TRANSACTIONAL=true");
fail();
} catch (SQLException e) {
assertTrue(
e.getMessage().contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " is disabled"));
} finally {
admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
admin.close();
}
ResultSet rs =
conn.createStatement().executeQuery("SELECT k FROM SYSTEM.NON_TX_TABLE WHERE v IS NULL");
assertTrue(rs.next());
assertEquals(1, rs.getInt(1));
assertFalse(rs.next());
htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
assertFalse(
htable
.getTableDescriptor()
.getCoprocessors()
.contains(TransactionProcessor.class.getName()));
assertEquals(
1,
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("SYSTEM.NON_TX_TABLE"))
.getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)
.getMaxVersions());
}
@Test
public void testNonTxToTxTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (1)");
conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (2, 'a')");
conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (3, 'b')");
conn.commit();
conn.createStatement().execute("CREATE INDEX IDX ON NON_TX_TABLE(v)");
// Reset empty column value to an empty value like it is pre-transactions
HTableInterface htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("NON_TX_TABLE"));
List<Put> puts =
Lists.newArrayList(
new Put(PInteger.INSTANCE.toBytes(1)),
new Put(PInteger.INSTANCE.toBytes(2)),
new Put(PInteger.INSTANCE.toBytes(3)));
for (Put put : puts) {
put.add(
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
QueryConstants.EMPTY_COLUMN_BYTES,
ByteUtil.EMPTY_BYTE_ARRAY);
}
htable.put(puts);
conn.createStatement().execute("ALTER TABLE NON_TX_TABLE SET TRANSACTIONAL=true");
htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("NON_TX_TABLE"));
assertTrue(
htable
.getTableDescriptor()
.getCoprocessors()
.contains(TransactionProcessor.class.getName()));
htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("IDX"));
assertTrue(
htable
.getTableDescriptor()
.getCoprocessors()
.contains(TransactionProcessor.class.getName()));
conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (4, 'c')");
ResultSet rs =
conn.createStatement()
.executeQuery("SELECT /*+ NO_INDEX */ k FROM NON_TX_TABLE WHERE v IS NULL");
assertTrue(
conn.unwrap(PhoenixConnection.class)
.getTable(new PTableKey(null, "NON_TX_TABLE"))
.isTransactional());
assertTrue(rs.next());
assertEquals(1, rs.getInt(1));
assertFalse(rs.next());
conn.commit();
conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (5, 'd')");
rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
assertTrue(
conn.unwrap(PhoenixConnection.class)
.getTable(new PTableKey(null, "IDX"))
.isTransactional());
assertTrue(rs.next());
assertEquals(1, rs.getInt(1));
assertTrue(rs.next());
assertEquals(2, rs.getInt(1));
assertTrue(rs.next());
assertEquals(3, rs.getInt(1));
assertTrue(rs.next());
assertEquals(4, rs.getInt(1));
assertTrue(rs.next());
assertEquals(5, rs.getInt(1));
assertFalse(rs.next());
conn.rollback();
rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
assertTrue(rs.next());
assertEquals(1, rs.getInt(1));
assertTrue(rs.next());
assertEquals(2, rs.getInt(1));
assertTrue(rs.next());
assertEquals(3, rs.getInt(1));
assertTrue(rs.next());
assertEquals(4, rs.getInt(1));
assertFalse(rs.next());
}