private static void waitUntil(long until) {
long now = System.currentTimeMillis();
while (now <= until) {
try {
Thread.sleep(1L);
now = System.currentTimeMillis();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
private void ensureColumnFamilyExists(String tableName, String columnFamily)
throws StorageException {
HBaseAdmin adm = getAdminInterface();
HTableDescriptor desc = ensureTableExists(tableName);
Preconditions.checkNotNull(desc);
HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
// Create our column family, if necessary
if (cf == null) {
try {
adm.disableTable(tableName);
desc.addFamily(
new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ));
adm.modifyTable(tableName.getBytes(), desc);
try {
logger.debug(
"Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
Thread.sleep(1000L);
} catch (InterruptedException ie) {
throw new TemporaryStorageException(ie);
}
adm.enableTable(tableName);
} catch (TableNotFoundException ee) {
logger.error("TableNotFoundException", ee);
throw new PermanentStorageException(ee);
} catch (org.apache.hadoop.hbase.TableExistsException ee) {
logger.debug("Swallowing exception {}", ee);
} catch (IOException ee) {
throw new TemporaryStorageException(ee);
}
} else { // check if compression was enabled, if not - enable it
if (cf.getCompressionType() == null
|| cf.getCompressionType() == Compression.Algorithm.NONE) {
try {
adm.disableTable(tableName);
adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ));
adm.enableTable(tableName);
} catch (IOException e) {
throw new TemporaryStorageException(e);
}
}
}
}
private static String lock(String lock) {
String realPath = "";
String parent = "/lock";
String lockName = parent + "/" + lock;
logger.debug("Getting lock " + lockName);
try {
if (zkInstance.exists(parent, false) == null)
zkInstance.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.fromFlag(0));
} catch (Exception E) {
logger.error("Error creating lock node: " + E.toString());
return null;
}
List<String> children = new LinkedList<String>();
try {
// List <ACL> ACLList = zkInstance.getACL(lockName, zkInstance.exists(lock, false));
realPath =
zkInstance.create(
lockName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
// children = zkInstance.getChildren(realPath, false);
checkLock:
while (true) {
children = zkInstance.getChildren(parent, false);
for (String curChild : children) {
String child = parent + "/" + curChild;
// System.out.println(child + " " + realPath + " " +
// Integer.toString(child.compareTo(realPath)));
if (child.compareTo(realPath) < 0
&& child.length() == realPath.length()
&& curChild.startsWith(lock)) {
// System.out.println(child + " cmp to " + realPath);
Thread.sleep(300);
continue checkLock;
}
}
logger.info("Got lock " + lockName);
return realPath;
}
} catch (Exception E) {
logger.error("Exception while trying to get lock " + lockName + " :" + E.toString());
E.printStackTrace();
return null;
}
}
@Test
public void insert_multithread() throws InterruptedException {
for (int k = 1; k <= 10; k++) {
final int j = k;
new Thread(
new Runnable() {
public void run() {
try {
// List<Put> puts = new ArrayList<Put>();
Configuration conf = new Configuration();
conf.set("hbase.zookeeper.quorum", "192.168.1.160");
HTable table = new HTable(conf, "wf:error");
table.setAutoFlushTo(false);
long t1 = System.currentTimeMillis();
for (int i = 0; i < 1000000; i++) {
String uuid =
UUID.randomUUID().toString().replaceAll("-", "").substring(0, 8);
Put put = new Put(Bytes.toBytes(uuid + "_" + "2015070" + j));
put.add(
fBytes,
Bytes.toBytes("stacktrace"),
Bytes.toBytes(
"java.io.IOException:file not found" + UUID.randomUUID().toString()));
// puts.add(put);
table.put(put);
if (i % 10000 == 0) {
table.flushCommits();
}
}
table.close();
long t2 = System.currentTimeMillis();
System.out.println(Thread.currentThread() + ",t2-t1=" + (t2 - t1));
} catch (IOException e) {
}
}
})
.start();
}
System.out.println("waiting.....");
Thread.sleep(1000 * 60 * 10);
System.out.println("completing.......");
}
public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
List<KeyValue> result = new ArrayList<>();
String strJson = null;
BufferedWriter writer = null;
Table table = null;
try (Connection connect = ConnectionFactory.createConnection(config);
Admin admin = connect.getAdmin()) {
TableName tablename = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tablename)) {
System.out.println("Table does not exist.");
return null;
}
table = connect.getTable(tablename);
Put put = null;
String start = null;
String end = null;
writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
for (KeyValue item : lstAllProcessStation) {
start = (String) item.getKey();
end = (String) item.getValue();
try {
try {
Thread.sleep(200);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
System.out.println("process : " + start + ":" + end);
strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
writer.write(start + ":" + end);
writer.newLine();
} catch (Exception e) {
System.out.println(start + ":" + end + "error");
e.printStackTrace();
break;
}
JSONObject jo = new JSONObject(strJson);
if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
JSONObject joData = jo.getJSONObject("data");
if (joData.has("flag") && joData.getBoolean("flag")) {
result.add(new DefaultKeyValue(start, end));
// 插入到hbase
String rowkey = start + ":" + end;
put = new Put(rowkey.getBytes());
put.addColumn(
CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
table.put(put);
System.out.println("start " + start + "\t end " + end + "\t has ticket");
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}