@TimeDepend
@Test
public void testGetWith_Ts() throws Exception {
recreateTable();
fillData();
Get get = new Get(rowKey_ForTest);
get.addColumn(ColumnFamilyNameBytes, QName1);
get.setMaxVersions(3);
get.setTimeStamp(3L);
Result result = table.get(get);
Assert.assertEquals(1, result.raw().length);
get.setTimeStamp(2L);
result = table.get(get);
Assert.assertEquals(1, result.raw().length);
get.setTimeStamp(1L);
result = table.get(get);
Assert.assertEquals(1, result.raw().length);
get.setTimeStamp(0L);
result = table.get(get);
Assert.assertEquals(0, result.raw().length);
get.setTimeRange(1, 4);
result = table.get(get);
Assert.assertEquals(3, result.raw().length);
recreateTable();
}
public static String get(String keyspace, String rowKey, String column, long timestamp)
throws Exception {
String columnValue = null;
HTable htable = new HTable(keyspace);
Get get = new Get(rowKey.getBytes());
get = get.setTimeStamp(timestamp);
get = get.setMaxVersions();
Result res = htable.get(get);
KeyValue[] data = res.raw();
for (int i = 0; i < data.length; i++) {
KeyValue d = data[i];
String family = new String(data[i].getFamily());
String qualifier = new String(data[i].getQualifier());
if (qualifier.toLowerCase().equals(column.toLowerCase())) {
columnValue = new String(d.getValue());
System.out.println(
data[i].toString()
+ " Family:"
+ family
+ " Qualifier:"
+ qualifier
+ " Value:"
+ columnValue);
break;
}
}
return columnValue;
}
public static List<String> getFreeBooks() throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.EQUAL,
Bytes.toBytes("0.00"));
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
/*
* for (KeyValue kv : r.raw()) { System.out.println(String.format(
* "row:%s, family:%s, qualifier:%s, qualifiervalue:%s, timestamp:%s."
* , Bytes.toString(kv.getRow()), Bytes.toString(kv.getFamily()),
* Bytes.toString(kv.getQualifier()), Bytes.toString(kv.getValue()),
* kv.getTimestamp())); }
*/
System.out.println(str);
list.add(str);
}
return list;
}
/**
* make a general method that takes a pair of lat/lon and a radius and give a boolean whether it
* was in or out.
*
* @throws IOException
*/
@Override
public Map<String, Integer> getAvailableBikesFromAPoint(
double lat, double lon, double radius, Get get) throws IOException {
Result r = ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().get(get, null);
log.debug("r is " + r);
log.debug(r.getMap().toString());
Map<String, Integer> result = new HashMap<String, Integer>();
try {
String s = null, latStr = null, lonStr = null;
for (KeyValue kv : r.raw()) {
s = Bytes.toString(kv.getValue());
log.debug("cell value is: " + s);
String[] sArr = s.split(BIXI_DELIMITER); // array of key=value pairs
latStr = sArr[3];
lonStr = sArr[4];
latStr = latStr.substring(latStr.indexOf("=") + 1);
lonStr = lonStr.substring(lonStr.indexOf("=") + 1);
log.debug("lon/lat values are: " + lonStr + "; " + latStr);
double distance =
giveDistance(Double.parseDouble(latStr), Double.parseDouble(lonStr), lat, lon) - radius;
log.debug("distance is : " + distance);
if (distance < 0) { // add it
result.put(sArr[0], getFreeBikes(kv));
}
}
} finally {
}
return result;
}
public static void main(String[] args) {
Integer id = Integer.parseInt(args[0]);
System.out.println(id);
byte[] k = new byte[5];
k[0] = (byte) 1;
byte[] col = getColIdbyte(id);
for (int i = 0; i < 4; i++) {
k[i + 1] = col[i];
}
System.out.println(Bytes.toStringBinary(k));
Get get = new Get(k);
// get.addColumn(Bytes.toBytes("A:i"));
HTable table;
try {
table = new HTable(hconf, "new");
Result result = table.get(get);
KeyValue[] g = result.raw();
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
for (int i = 0; i < g.length; i++) {
System.out.println(Bytes.toStringBinary(g[i].getValue()) + " kkkkkkkkkkkkkkkkkkkkkkk");
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static List<String> getBooksbyPrice(String min, String max) throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
FilterList filterList = new FilterList();
Filter maxFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.GREATER_OR_EQUAL,
Bytes.toBytes(min));
Filter minFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.LESS_OR_EQUAL,
Bytes.toBytes(max));
filterList.addFilter(maxFilter);
filterList.addFilter(minFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
System.out.println(str);
list.add(str);
}
return list;
}
public static HBaseTuple getHBaseTuple(Result result) {
HBaseTuple tuple = null;
KeyValue[] kvs = result.raw();
for (KeyValue kv : kvs) {
tuple = getHBaseTuple(kv);
break;
}
return tuple;
}
private static void assertGet(byte[] row, byte[] familiy, byte[] qualifier, byte[] value)
throws IOException {
// run a get and see if the value matches
Get get = new Get(row);
get.addColumn(familiy, qualifier);
Result result = region.get(get, null);
assertEquals(1, result.size());
KeyValue kv = result.raw()[0];
byte[] r = kv.getValue();
assertTrue(Bytes.compareTo(r, value) == 0);
}
/**
* Read a record from the database. Each field/value pair from the result will be stored in a
* HashMap.
*
* @param table The name of the table
* @param key The record key of the record to read.
* @param fields The list of fields to read, or null for all of them
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
public int read(
String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
try {
getHTable(table);
_table = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return ServerError;
}
}
Result r = null;
try {
if (_debug) {
System.out.println("Doing read from HBase columnfamily " + _columnFamily);
System.out.println("Doing read for key: " + key);
}
Get g = new Get(Bytes.toBytes(key));
if (fields == null) {
g.addFamily(_columnFamilyBytes);
} else {
for (String field : fields) {
g.addColumn(_columnFamilyBytes, Bytes.toBytes(field));
}
}
r = _hTable.get(g);
} catch (IOException e) {
System.err.println("Error doing get: " + e);
return ServerError;
} catch (ConcurrentModificationException e) {
// do nothing for now...need to understand HBase concurrency model better
return ServerError;
}
for (KeyValue kv : r.raw()) {
result.put(Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue()));
if (_debug) {
System.out.println(
"Result for field: "
+ Bytes.toString(kv.getQualifier())
+ " is: "
+ Bytes.toString(kv.getValue()));
}
}
return Ok;
}
public static void main(String[] args) throws IOException, SolrServerException {
final Configuration conf;
HttpSolrServer solrServer = new HttpSolrServer("http://c1master:8983/solr");
conf = HBaseConfiguration.create();
// Define Hbase Table Name
HTable table = new HTable(conf, "test_global_shop");
Scan scan = new Scan();
// Define Hbase Column Family
scan.addFamily(Bytes.toBytes("shop"));
scan.setCaching(1000);
scan.setCacheBlocks(false);
ResultScanner ss = table.getScanner(scan);
System.out.println("start Storing...");
int i = 0;
try {
for (Result r : ss) {
SolrInputDocument solrDoc = new SolrInputDocument();
solrDoc.addField("key", new String(r.getRow()));
for (KeyValue kv : r.raw()) {
String fieldName = new String(kv.getQualifier());
String fieldValue = new String(kv.getValue());
if (fieldName.equalsIgnoreCase("address")
|| fieldName.equalsIgnoreCase("category")
|| fieldName.equalsIgnoreCase("name")
|| fieldName.equalsIgnoreCase("province")
|| fieldName.equalsIgnoreCase("tel")) {
solrDoc.addField(fieldName, fieldValue);
}
}
solrServer.add(solrDoc);
solrServer.commit(true, true, true);
i = i + 1;
System.out.println("Already Succcess " + i + " number data");
}
ss.close();
table.close();
System.out.println("done !");
} catch (IOException e) {
} finally {
ss.close();
table.close();
System.out.println("error !");
}
}
public static String selectRowKey(String rowKey, int tableNum) throws IOException {
Get g = new Get(rowKey.getBytes());
Result rs = htable[tableNum].get(g);
String result = "";
for (KeyValue kv : rs.raw()) {
if (new String(kv.getQualifier()).equals("text")) {
result +=
new String(kv.getValue())
.replace("\\n", "\n")
.replace("\\t", "\t")
.replace("\\r", "\r")
.replace("\\\\", "\\")
+ "\n";
}
}
return result;
}
@Override
public void map(ImmutableBytesWritable row, Result value, Context context)
throws InterruptedException, IOException {
KeyValue[] kvs = value.raw();
if (lastRowKey == null) {
lastRowKey = row.get();
} else if (Bytes.compareTo(lastRowKey, row.get()) != 0) {
writeLine(context, Bytes.toString(lastRowKey));
columnValueMap.clear();
}
for (KeyValue kv : kvs) {
String qualifier = Bytes.toString(kv.getQualifier());
byte[] val = kv.getValue();
columnValueMap.put(qualifier, val);
}
}
@TimeDepend
@Test
public void testPutMultiVersionSameTime() throws Exception {
recreateTable();
Put put = new Put(rowKey_ForTest);
put.add(ColumnFamilyNameBytes, QName1, 1000, Bytes.toBytes("a"));
put.add(ColumnFamilyNameBytes, QName1, 2000, Bytes.toBytes("b"));
table.put(put);
Get get = new Get(rowKey_ForTest);
get.setMaxVersions(10);
Result result = table.get(get);
KeyValue[] keyValues = result.raw();
Assert.assertEquals(2, keyValues.length);
// have a and b both.
Assert.assertEquals('a' + 'b', keyValues[0].getValue()[0] + keyValues[1].getValue()[0]);
recreateTable();
}
public static PlaceModel Map(Result r) {
PlaceModel place = new PlaceModel();
place.rowkey = UUID.fromString(new String(r.getRow()));
for (KeyValue keyValue : r.raw()) {
if (new String(keyValue.getFamily()).compareTo(Constants.PLACE_INFO_FAMILYCOLUMNNAME) == 0)
place.Info.put(new String(keyValue.getQualifier()), new String(keyValue.getValue()));
else if (new String(keyValue.getFamily()).compareTo(Constants.PLACE_LOCATION_FAMILYCOLUMNNAME)
== 0)
place.Location.put(new String(keyValue.getQualifier()), new String(keyValue.getValue()));
else if (new String(keyValue.getFamily())
.compareTo(Constants.PLACE_OPENINGHOURS_FAMILYCOLUMNNAME)
== 0)
place.OpeningHours.put(
new String(keyValue.getQualifier()), new String(keyValue.getValue()));
else if (new String(keyValue.getFamily()).compareTo(Constants.PLACE_OTHERS_FAMILYCOLUMNNAME)
== 0)
place.OpeningHours.put(
new String(keyValue.getQualifier()), new String(keyValue.getValue()));
}
return place;
}
public void dump(String table, String[] rows, String[] fams, String[] quals) throws IOException {
HTable tbl = new HTable(conf, table);
List<Get> gets = new ArrayList<Get>();
for (String row : rows) {
Get get = new Get(Bytes.toBytes(row));
get.setMaxVersions();
if (fams != null) {
for (String fam : fams) {
for (String qual : quals) {
get.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual));
}
}
}
gets.add(get);
}
Result[] results = tbl.get(gets);
for (Result result : results) {
for (KeyValue kv : result.raw()) {
System.out.println("KV: " + kv + ", Value: " + Bytes.toString(kv.getValue()));
}
}
}
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the
* result will be stored in a HashMap.
*
* @param table The name of the table
* @param startkey The record key of the first record to read.
* @param recordcount The number of records to read
* @param fields The list of fields to read, or null for all of them
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one
* record
* @return Zero on success, a non-zero error code on error
*/
public int scan(
String table,
String startkey,
int recordcount,
Set<String> fields,
Vector<HashMap<String, ByteIterator>> result) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!_table.equals(table)) {
_hTable = null;
try {
getHTable(table);
_table = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return ServerError;
}
}
Scan s = new Scan(Bytes.toBytes(startkey));
// HBase has no record limit. Here, assume recordcount is small enough to bring back in one
// call.
// We get back recordcount records
s.setCaching(recordcount);
// add specified fields or else all fields
if (fields == null) {
s.addFamily(_columnFamilyBytes);
} else {
for (String field : fields) {
s.addColumn(_columnFamilyBytes, Bytes.toBytes(field));
}
}
// get results
ResultScanner scanner = null;
try {
scanner = _hTable.getScanner(s);
int numResults = 0;
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
// get row key
String key = Bytes.toString(rr.getRow());
if (_debug) {
System.out.println("Got scan result for key: " + key);
}
HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>();
for (KeyValue kv : rr.raw()) {
rowResult.put(
Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue()));
}
// add rowResult to result vector
result.add(rowResult);
numResults++;
if (numResults >= recordcount) // if hit recordcount, bail out
{
break;
}
} // done with row
} catch (IOException e) {
if (_debug) {
System.out.println("Error in getting/parsing scan result: " + e);
}
return ServerError;
} finally {
scanner.close();
}
return Ok;
}
/** @param args */
public static void main(String[] args) throws Exception {
// Configuration conf = HBaseConfiguration.create();
// change here if you want to change the HBase installation.
// conf.set("hbase.master", "localhost:60000");
Configuration config = HBaseConfiguration.create();
// config.set("hbase.master", "localhost:60020");
HTable table = new HTable(config, TABLE_NAME);
// Change here if you want to change the input file.
BufferedReader reader =
new BufferedReader(new FileReader("/home/cloudera/ebooks/Hbase/hdi-data.csv"));
try {
String line = null;
// skip first line
reader.readLine();
while ((line = reader.readLine()) != null) {
try {
String[] tokens = CSVLineParser.tokenizeCSV(line).toArray(new String[0]);
String country = tokens[1];
double lifeExpectacny = Double.parseDouble(tokens[3].replaceAll(",", ""));
double meanYearsOfSchooling = Double.parseDouble(tokens[4].replaceAll(",", ""));
double gnip = Double.parseDouble(tokens[6].replaceAll(",", ""));
Put put = new Put(Bytes.toBytes(country));
put.add(
"ByCountry".getBytes(),
Bytes.toBytes("lifeExpectacny"),
Bytes.toBytes(lifeExpectacny));
put.add(
"ByCountry".getBytes(),
Bytes.toBytes("meanYearsOfSchooling"),
Bytes.toBytes(meanYearsOfSchooling));
put.add("ByCountry".getBytes(), Bytes.toBytes("gnip"), Bytes.toBytes(gnip));
table.put(put);
} catch (Exception e) {
e.printStackTrace();
System.out.println("Error processing " + line + " caused by " + e.getMessage());
}
}
} catch (IOException e) {
try {
reader.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
// Following print back the results
Scan s = new Scan();
s.addFamily(Bytes.toBytes("ByCountry"));
ResultScanner results = table.getScanner(s);
try {
for (Result result : results) {
KeyValue[] keyValuePairs = result.raw();
System.out.println(new String(result.getRow()));
for (KeyValue keyValue : keyValuePairs) {
System.out.println(
new String(keyValue.getFamily())
+ " "
+ new String(keyValue.getQualifier())
+ "="
+ Bytes.toDouble(keyValue.getValue()));
}
}
} finally {
results.close();
}
}
/**
* Override the preAppend for checkAndPut and checkAndDelete, as we need the ability to a) set the
* TimeRange for the Get being done and b) return something back to the client to indicate
* success/failure
*/
@SuppressWarnings("deprecation")
@Override
public Result preAppend(
final ObserverContext<RegionCoprocessorEnvironment> e, final Append append)
throws IOException {
byte[] opBuf = append.getAttribute(OPERATION_ATTRIB);
if (opBuf == null) {
return null;
}
Op op = Op.values()[opBuf[0]];
long clientTimestamp = HConstants.LATEST_TIMESTAMP;
byte[] clientTimestampBuf = append.getAttribute(MAX_TIMERANGE_ATTRIB);
if (clientTimestampBuf != null) {
clientTimestamp = Bytes.toLong(clientTimestampBuf);
}
boolean hadClientTimestamp = (clientTimestamp != HConstants.LATEST_TIMESTAMP);
if (hadClientTimestamp) {
// Prevent race condition of creating two sequences at the same timestamp
// by looking for a sequence at or after the timestamp at which it'll be
// created.
if (op == Op.CREATE_SEQUENCE) {
clientTimestamp++;
}
} else {
clientTimestamp = EnvironmentEdgeManager.currentTimeMillis();
clientTimestampBuf = Bytes.toBytes(clientTimestamp);
}
RegionCoprocessorEnvironment env = e.getEnvironment();
// We need to set this to prevent region.append from being called
e.bypass();
e.complete();
HRegion region = env.getRegion();
byte[] row = append.getRow();
region.startRegionOperation();
try {
Integer lid = region.getLock(null, row, true);
try {
KeyValue keyValue = append.getFamilyMap().values().iterator().next().iterator().next();
byte[] family = keyValue.getFamily();
byte[] qualifier = keyValue.getQualifier();
Get get = new Get(row);
get.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimestamp);
get.addColumn(family, qualifier);
Result result = region.get(get);
if (result.isEmpty()) {
if (op == Op.DROP_SEQUENCE || op == Op.RESET_SEQUENCE) {
return getErrorResult(
row, clientTimestamp, SQLExceptionCode.SEQUENCE_UNDEFINED.getErrorCode());
}
} else {
if (op == Op.CREATE_SEQUENCE) {
return getErrorResult(
row, clientTimestamp, SQLExceptionCode.SEQUENCE_ALREADY_EXIST.getErrorCode());
}
}
Mutation m = null;
switch (op) {
case RESET_SEQUENCE:
KeyValue currentValueKV = result.raw()[0];
long expectedValue =
PDataType.LONG
.getCodec()
.decodeLong(append.getAttribute(CURRENT_VALUE_ATTRIB), 0, null);
long value =
PDataType.LONG
.getCodec()
.decodeLong(currentValueKV.getBuffer(), currentValueKV.getValueOffset(), null);
// Timestamp should match exactly, or we may have the wrong sequence
if (expectedValue != value || currentValueKV.getTimestamp() != clientTimestamp) {
return new Result(
Collections.singletonList(
KeyValueUtil.newKeyValue(
row,
PhoenixDatabaseMetaData.SEQUENCE_FAMILY_BYTES,
QueryConstants.EMPTY_COLUMN_BYTES,
currentValueKV.getTimestamp(),
ByteUtil.EMPTY_BYTE_ARRAY)));
}
m = new Put(row, currentValueKV.getTimestamp());
m.getFamilyMap().putAll(append.getFamilyMap());
break;
case DROP_SEQUENCE:
m = new Delete(row, clientTimestamp, null);
break;
case CREATE_SEQUENCE:
m = new Put(row, clientTimestamp);
m.getFamilyMap().putAll(append.getFamilyMap());
break;
}
if (!hadClientTimestamp) {
for (List<KeyValue> kvs : m.getFamilyMap().values()) {
for (KeyValue kv : kvs) {
kv.updateLatestStamp(clientTimestampBuf);
}
}
}
@SuppressWarnings("unchecked")
Pair<Mutation, Integer>[] mutations = new Pair[1];
mutations[0] = new Pair<Mutation, Integer>(m, lid);
region.batchMutate(mutations);
long serverTimestamp = MetaDataUtil.getClientTimeStamp(m);
// Return result with single KeyValue. The only piece of information
// the client cares about is the timestamp, which is the timestamp of
// when the mutation was actually performed (useful in the case of .
return new Result(
Collections.singletonList(
KeyValueUtil.newKeyValue(
row,
PhoenixDatabaseMetaData.SEQUENCE_FAMILY_BYTES,
QueryConstants.EMPTY_COLUMN_BYTES,
serverTimestamp,
SUCCESS_VALUE)));
} finally {
region.releaseRowLock(lid);
}
} catch (Throwable t) {
ServerUtil.throwIOException("Increment of sequence " + Bytes.toStringBinary(row), t);
return null; // Impossible
} finally {
region.closeRegionOperation();
}
}
