private static Long latestVersion(Hashtable<String, String> config, dbutil db_util)
throws Exception {
if (!config.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
return new Long(config.get("timestamp_stop"));
}
String rowName = config.get("file_id") + config.get("run_id") + "_";
if (config.get("task_id") != "") {
try {
rowName = rowName + String.format("%04d", new Integer(config.get("task_id")));
} catch (NumberFormatException E) {
rowName = rowName + config.get("task_id");
}
}
Get timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
Result timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
KeyValue tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
if (tsKv == null) {
rowName = config.get("file_id") + "_";
timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
}
if (tsKv == null) {
return new Long(Integer.MAX_VALUE);
}
Long latestVersion = new Long(tsKv.getTimestamp());
return latestVersion;
}
/** Assert CellUtil makes Cell toStrings same way we do KeyValue toStrings. */
@Test
public void testToString() {
byte[] row = Bytes.toBytes("row");
long ts = 123l;
// Make a KeyValue and a Cell and see if same toString result.
KeyValue kv =
new KeyValue(
row,
HConstants.EMPTY_BYTE_ARRAY,
HConstants.EMPTY_BYTE_ARRAY,
ts,
KeyValue.Type.Minimum,
HConstants.EMPTY_BYTE_ARRAY);
Cell cell =
CellUtil.createCell(
row,
HConstants.EMPTY_BYTE_ARRAY,
HConstants.EMPTY_BYTE_ARRAY,
ts,
KeyValue.Type.Minimum.getCode(),
HConstants.EMPTY_BYTE_ARRAY);
String cellToString = CellUtil.getCellKeyAsString(cell);
assertEquals(kv.toString(), cellToString);
// Do another w/ non-null family.
byte[] f = new byte[] {'f'};
byte[] q = new byte[] {'q'};
kv = new KeyValue(row, f, q, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY);
cell =
CellUtil.createCell(
row, f, q, ts, KeyValue.Type.Minimum.getCode(), HConstants.EMPTY_BYTE_ARRAY);
cellToString = CellUtil.getCellKeyAsString(cell);
assertEquals(kv.toString(), cellToString);
}
public void processEvent(KeyValue event) {
if (zk == null) {
try {
zk = new ZooKeeper("localhost:21810", 4000, this);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (!S4TestCase.registeredPEs.containsKey(getSafeKeeperId())) {
S4TestCase.registeredPEs.put(getSafeKeeperId(), this);
}
try {
if ("value1".equals(event.getKey())) {
setValue1(event.getValue());
zk.create("/value1Set", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} else if ("value2".equals(event.getKey())) {
setValue2(event.getValue());
zk.create("/value2Set", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} else if ("initiateCheckpoint".equals(event.getKey())) {
initiateCheckpoint();
} else {
throw new RuntimeException("unidentified event: " + event);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static boolean hasFile(
dbutil db_util, FileSystem fs, String db_name, String file_id, String file_path)
throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names == null) {
return false;
}
String all_files = new String(file_names.getValue());
String[] files = all_files.split("\n");
for (String line : files) {
if (line.equals(file_path)) {
if (fs.globStatus(new Path(line + "*")).length == 0) {
Put new_put = new Put(file_id.getBytes());
new_put.add(
"d".getBytes(),
"filenames".getBytes(),
all_files.replace(file_path + "\n", "").getBytes());
db_util.doPut(db_name, new_put);
return false;
}
return true;
}
}
return false;
}
private static boolean putFileEntry(
dbutil db_util,
FileSystem fs,
String db_name,
String file_id,
String file_path,
String source)
throws Exception {
String all_paths = file_path;
if (hasFile(db_util, fs, db_name, file_id, file_path)) {
logger.debug("File already found, putFileEntry aborting");
return false;
} else {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names != null) {
String paths = new String(file_names.getValue());
all_paths = paths + "\n" + file_path;
}
}
Put file_id_put = new Put(file_id.getBytes());
file_id_put.add("d".getBytes(), "source".getBytes(), source.getBytes());
if (!source.equals("fullfile")) {
file_id_put.add("d".getBytes(), "filenames".getBytes(), all_paths.getBytes());
}
db_util.doPut(db_name, file_id_put);
return true;
}
@Test
public void testGetKeyMethods() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getKeyLength());
ByteBufferUtils.copyFromArrayToBuffer(
buf, kvCell.getBuffer(), kvCell.getKeyOffset(), kvCell.getKeyLength());
ByteBufferedCell offheapKeyOnlyKV = new ByteBufferedKeyOnlyKeyValue(buf, 0, buf.capacity());
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getRowByteBuffer(),
offheapKeyOnlyKV.getRowPositionInByteBuffer(),
offheapKeyOnlyKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getFamilyByteBuffer(),
offheapKeyOnlyKV.getFamilyPositionInByteBuffer(),
offheapKeyOnlyKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKeyOnlyKV.getQualifierByteBuffer(),
offheapKeyOnlyKV.getQualifierPositionInByteBuffer(),
offheapKeyOnlyKV.getQualifierLength()));
assertEquals(0L, offheapKeyOnlyKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKeyOnlyKV.getTypeByte());
}
public String getContextByPostMethod(String url, List<KeyValue> params) {
HttpClient client = getHttpClient();
client.getParams().setParameter("http.protocol.content-charset", this.codeing);
PostMethod post = null;
String result = "";
try {
URL u = new URL(url);
client
.getHostConfiguration()
.setHost(
u.getHost(), u.getPort() == -1 ? u.getDefaultPort() : u.getPort(), u.getProtocol());
post = new PostMethod(u.getPath());
NameValuePair[] nvps = new NameValuePair[params.size()];
int i = 0;
for (KeyValue kv : params) {
nvps[i] = new NameValuePair(kv.getKey(), kv.getValue());
i++;
}
post.setRequestBody(nvps);
client.executeMethod(post);
result = post.getResponseBodyAsString();
} catch (Exception e) {
throw new NetException("HttpClient catch!", e);
} finally {
if (post != null) post.releaseConnection();
}
return result;
}
private Object[] getFromCache(RowMetaInterface keyMeta, Object[] keyData)
throws KettleValueException {
if (meta.isMemoryPreservationActive()) {
if (meta.isUsingSortedList()) {
KeyValue keyValue = new KeyValue(keyData, null);
int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
if (idx < 0) return null; // nothing found
keyValue = (KeyValue) data.list.get(idx);
return keyValue.getValue();
} else {
if (meta.isUsingIntegerPair()) {
Long value = data.longIndex.get(keyMeta.getInteger(keyData, 0));
if (value == null) return null;
return new Object[] {
value,
};
} else {
try {
byte[] value = data.hashIndex.get(RowMeta.extractData(keyMeta, keyData));
if (value == null) return null;
return RowMeta.getRow(data.valueMeta, value);
} catch (Exception e) {
logError("Oops", e);
throw new RuntimeException(e);
}
}
}
} else {
return (Object[]) data.look.get(new RowMetaAndData(keyMeta, keyData));
}
}
@Override
public void write(KeyValue keyValue) throws IOException {
try {
put(keyValue.getKey() + "_2", keyValue.getValue());
} catch (Exception e) {
throw new IOException(e);
}
}
private static String getSource(dbutil db_util, String db_name, String file_id) throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_source = file_result.getColumnLatest("d".getBytes(), "source".getBytes());
if (file_source == null) {
return "local";
}
return new String(file_source.getValue());
}
int removeFromList(String key) {
int count = keyValueList.size();
for (int i = 0; i < count; i++) {
KeyValue keyValue = (KeyValue) keyValueList.get(i);
if (keyValue.getKey().equals(key)) {
keyValueList.remove(i);
return i;
}
}
return -1;
}
private static void assertGet(byte[] row, byte[] familiy, byte[] qualifier, byte[] value)
throws IOException {
// run a get and see if the value matches
Get get = new Get(row);
get.addColumn(familiy, qualifier);
Result result = region.get(get, null);
assertEquals(1, result.size());
KeyValue kv = result.raw()[0];
byte[] r = kv.getValue();
assertTrue(Bytes.compareTo(r, value) == 0);
}
public static Cell createCell(
final byte[] row,
final byte[] family,
final byte[] qualifier,
final long timestamp,
final byte type,
final byte[] value,
final long memstoreTS) {
KeyValue keyValue =
new KeyValue(row, family, qualifier, timestamp, KeyValue.Type.codeToType(type), value);
keyValue.setSequenceId(memstoreTS);
return keyValue;
}
private static String isDatabase(dbutil db_util, String db_name, String file_id)
throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_db = file_result.getColumnLatest("d".getBytes(), "database".getBytes());
if (file_db == null) {
return "n";
}
String db = new String(file_db.getValue());
if (db.equals("y")) {
return "y";
} else {
return "n";
}
}
private boolean isVariablesListValid() {
List<KeyValue> variables = view.getEnvironmentVariables();
if (variables.isEmpty()) {
view.hideVariablesError();
return true;
}
for (KeyValue keyValue : variables) {
if (!OpenshiftValidator.isEnvironmentVariableNameValid(keyValue.getKey())) {
view.showVariablesError(locale.invalidVariablesError());
return false;
}
}
view.hideVariablesError();
return true;
}
private boolean isLabelListValid() {
List<KeyValue> labels = view.getLabels();
if (labels.isEmpty()) {
view.hideLabelsError();
return true;
}
for (KeyValue keyValue : labels) {
if (!OpenshiftValidator.isLabelNameValid(keyValue.getKey())
|| !OpenshiftValidator.isLabelValueValid(keyValue.getValue())) {
view.showLabelsError(locale.invalidLabelsError(), locale.invalidLabelsDetailError());
return false;
}
}
view.hideLabelsError();
return true;
}
static int insertData(Configuration conf, TableName tableName, String column, double prob)
throws IOException {
Random rng = new Random();
int count = 0;
HTable table = new HTable(conf, tableName);
byte[] k = new byte[3];
byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
for (byte b1 = 'a'; b1 < 'z'; b1++) {
for (byte b2 = 'a'; b2 < 'z'; b2++) {
for (byte b3 = 'a'; b3 < 'z'; b3++) {
if (rng.nextDouble() < prob) {
k[0] = b1;
k[1] = b2;
k[2] = b3;
Put put = new Put(k);
put.setDurability(Durability.SKIP_WAL);
put.add(famAndQf[0], famAndQf[1], k);
table.put(put);
count++;
}
}
}
}
table.flushCommits();
table.close();
return count;
}
@Test
public void testByteBufferBackedKeyValueWithTags() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), true, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
// change tags to handle both onheap and offheap stuff
List<Tag> resTags =
Tag.asList(offheapKV.getTagsArray(), offheapKV.getTagsOffset(), offheapKV.getTagsLength());
Tag tag1 = resTags.get(0);
assertEquals(t1.getType(), tag1.getType());
assertEquals(Bytes.toString(t1.getValue()), Bytes.toString(getTagValue(tag1)));
Tag tag2 = resTags.get(1);
assertEquals(tag2.getType(), tag2.getType());
assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
Tag res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 2);
assertEquals(Bytes.toString(t2.getValue()), Bytes.toString(getTagValue(tag2)));
res = Tag.getTag(offheapKV.getTagsArray(), 0, offheapKV.getTagsLength(), (byte) 3);
assertNull(res);
}
private static String getFileNames(
dbutil db_util, String db_name, String file_id, String file_path) throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names == null) {
return "";
}
String all_files = new String(file_names.getValue());
String[] files = all_files.split("\n");
for (String line : files) {
if (line.equals(file_path)) {
return all_files;
}
}
return all_files + "\n" + file_path;
}
public final void serialize(JsonContext context, T keyValuePairs) {
List<KeyValue<String, Object>> items = listItem(keyValuePairs);
if (items == null) {
context.context("{}");
return;
}
context.context('{');
int i = 0;
for (KeyValue<String, Object> item : items) {
String key = item.getKey();
Object value = item.getValue();
context.context('"');
context.context(JsonUtils.render(key.toString()));
context.context('"');
context.context(':');
JsonSerializeConfigHolder.addPro(key);
ObjectSerializer.getInstance().serialize(context, value);
if (i++ != items.size() - 1) {
context.context(",");
}
JsonSerializeConfigHolder.backPro();
}
context.context('}');
}
public java.util.List getConstraintColumns() {
ArrayList list = new ArrayList();
Iterator iter = identifier.getColumnIterator();
while (iter.hasNext()) list.add(iter.next());
return list;
}
@Override
public ReturnCode filterKeyValue(KeyValue kv) {
return navigate(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), Terminate.AFTER);
}
@Test
public void testByteBufferBackedKeyValue() throws Exception {
KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1);
ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
ByteBufferedCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
// Use the array() APIs
assertEquals(
ROW1,
Bytes.toStringBinary(
offheapKV.getRowArray(), offheapKV.getRowOffset(), offheapKV.getRowLength()));
assertEquals(
FAM1,
Bytes.toStringBinary(
offheapKV.getFamilyArray(), offheapKV.getFamilyOffset(), offheapKV.getFamilyLength()));
assertEquals(
QUAL1,
Bytes.toStringBinary(
offheapKV.getQualifierArray(),
offheapKV.getQualifierOffset(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
Bytes.toStringBinary(
offheapKV.getValueArray(), offheapKV.getValueOffset(), offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
kvCell = new KeyValue(row1, fam2, qual2, 0l, Type.Put, row1);
buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
FAM2,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
QUAL2,
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
byte[] nullQualifier = new byte[0];
kvCell = new KeyValue(row1, fam1, nullQualifier, 0L, Type.Put, row1);
buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l);
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getRowByteBuffer(),
offheapKV.getRowPositionInByteBuffer(),
offheapKV.getRowLength()));
assertEquals(
FAM1,
ByteBufferUtils.toStringBinary(
offheapKV.getFamilyByteBuffer(),
offheapKV.getFamilyPositionInByteBuffer(),
offheapKV.getFamilyLength()));
assertEquals(
"",
ByteBufferUtils.toStringBinary(
offheapKV.getQualifierByteBuffer(),
offheapKV.getQualifierPositionInByteBuffer(),
offheapKV.getQualifierLength()));
assertEquals(
ROW1,
ByteBufferUtils.toStringBinary(
offheapKV.getValueByteBuffer(),
offheapKV.getValuePositionInByteBuffer(),
offheapKV.getValueLength()));
assertEquals(0L, offheapKV.getTimestamp());
assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
/**
* Returns the currently selected direction.
*
* @return the selected direction
*/
public int getDirection() {
KeyValue item = (KeyValue) getSelectedItem();
return Integer.parseInt(item.getKey());
}
public String sqlCreateString(
Dialect dialect, Mapping p, String defaultCatalog, String defaultSchema)
throws HibernateException {
StringBuffer buf =
new StringBuffer("create table ")
.append(getQualifiedName(dialect, defaultCatalog, defaultSchema))
.append(" (");
boolean identityColumn = idValue != null && idValue.isIdentityColumn(dialect);
// Try to find out the name of the primary key to create it as identity if the IdentityGenerator
// is used
String pkname = null;
if (hasPrimaryKey() && identityColumn) {
pkname = ((Column) getPrimaryKey().getColumnIterator().next()).getQuotedName(dialect);
}
Iterator iter = getColumnIterator();
while (iter.hasNext()) {
Column col = (Column) iter.next();
buf.append(col.getQuotedName(dialect)).append(' ');
if (identityColumn && col.getQuotedName(dialect).equals(pkname)) {
// to support dialects that have their own identity data type
if (dialect.hasDataTypeInIdentityColumn()) {
buf.append(col.getSqlType(dialect, p));
}
buf.append(' ').append(dialect.getIdentityColumnString(col.getSqlTypeCode(p)));
} else {
buf.append(col.getSqlType(dialect, p));
String defaultValue = col.getDefaultValue();
if (defaultValue != null) {
buf.append(" default ").append(defaultValue);
}
if (col.isNullable()) {
buf.append(dialect.getNullColumnString());
} else {
buf.append(" not null");
}
}
boolean useUniqueConstraint =
col.isUnique() && (!col.isNullable() || dialect.supportsNotNullUnique());
if (useUniqueConstraint) {
if (dialect.supportsUnique()) {
buf.append(" unique");
} else {
UniqueKey uk = getOrCreateUniqueKey(col.getQuotedName(dialect) + '_');
uk.addColumn(col);
}
}
if (col.hasCheckConstraint() && dialect.supportsColumnCheck()) {
buf.append(" check (").append(col.getCheckConstraint()).append(")");
}
String columnComment = col.getComment();
if (columnComment != null) {
buf.append(dialect.getColumnComment(columnComment));
}
if (iter.hasNext()) buf.append(", ");
}
if (hasPrimaryKey()) {
buf.append(", ").append(getPrimaryKey().sqlConstraintString(dialect));
}
if (dialect.supportsUniqueConstraintInCreateAlterTable()) {
Iterator ukiter = getUniqueKeyIterator();
while (ukiter.hasNext()) {
UniqueKey uk = (UniqueKey) ukiter.next();
buf.append(", ").append(uk.sqlConstraintString(dialect));
}
}
/*Iterator idxiter = getIndexIterator();
while ( idxiter.hasNext() ) {
Index idx = (Index) idxiter.next();
buf.append(',').append( idx.sqlConstraintString(dialect) );
}*/
if (dialect.supportsTableCheck()) {
Iterator chiter = checkConstraints.iterator();
while (chiter.hasNext()) {
buf.append(", check (").append(chiter.next()).append(')');
}
}
buf.append(')');
if (comment != null) buf.append(dialect.getTableComment(comment));
return buf.append(dialect.getTableTypeString()).toString();
}
private int getQualifierLength(int rlength, int flength) {
return this.length - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0);
}
public String sqlCreateString(
Dialect dialect, Mapping p, String defaultCatalog, String defaultSchema) {
StringBuilder buf =
new StringBuilder(
hasPrimaryKey()
? dialect.getCreateTableString()
: dialect.getCreateMultisetTableString())
.append(' ')
.append(getQualifiedName(dialect, defaultCatalog, defaultSchema))
.append(" (");
boolean identityColumn =
idValue != null && idValue.isIdentityColumn(p.getIdentifierGeneratorFactory(), dialect);
// Try to find out the name of the primary key to create it as identity if the IdentityGenerator
// is used
String pkname = null;
if (hasPrimaryKey() && identityColumn) {
pkname = ((Column) getPrimaryKey().getColumnIterator().next()).getQuotedName(dialect);
}
Iterator iter = getColumnIterator();
while (iter.hasNext()) {
Column col = (Column) iter.next();
buf.append(col.getQuotedName(dialect)).append(' ');
if (identityColumn && col.getQuotedName(dialect).equals(pkname)) {
// to support dialects that have their own identity data type
if (dialect.getIdentityColumnSupport().hasDataTypeInIdentityColumn()) {
buf.append(col.getSqlType(dialect, p));
}
buf.append(' ')
.append(
dialect.getIdentityColumnSupport().getIdentityColumnString(col.getSqlTypeCode(p)));
} else {
buf.append(col.getSqlType(dialect, p));
String defaultValue = col.getDefaultValue();
if (defaultValue != null) {
buf.append(" default ").append(defaultValue);
}
if (col.isNullable()) {
buf.append(dialect.getNullColumnString());
} else {
buf.append(" not null");
}
}
if (col.isUnique()) {
String keyName = Constraint.generateName("UK_", this, col);
UniqueKey uk = getOrCreateUniqueKey(keyName);
uk.addColumn(col);
buf.append(dialect.getUniqueDelegate().getColumnDefinitionUniquenessFragment(col));
}
if (col.hasCheckConstraint() && dialect.supportsColumnCheck()) {
buf.append(" check (").append(col.getCheckConstraint()).append(")");
}
String columnComment = col.getComment();
if (columnComment != null) {
buf.append(dialect.getColumnComment(columnComment));
}
if (iter.hasNext()) {
buf.append(", ");
}
}
if (hasPrimaryKey()) {
buf.append(", ").append(getPrimaryKey().sqlConstraintString(dialect));
}
buf.append(dialect.getUniqueDelegate().getTableCreationUniqueConstraintsFragment(this));
if (dialect.supportsTableCheck()) {
for (String checkConstraint : checkConstraints) {
buf.append(", check (").append(checkConstraint).append(')');
}
}
buf.append(')');
if (comment != null) {
buf.append(dialect.getTableComment(comment));
}
return buf.append(dialect.getTableTypeString()).toString();
}
@Test
public void testFindCommonPrefixInFlatKey() {
// The whole key matching case
KeyValue kv1 = new KeyValue("r1".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
Assert.assertEquals(
kv1.getKeyLength(), CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, true));
Assert.assertEquals(
kv1.getKeyLength(), CellUtil.findCommonPrefixInFlatKey(kv1, kv1, false, true));
Assert.assertEquals(
kv1.getKeyLength() - KeyValue.TIMESTAMP_TYPE_SIZE,
CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, false));
// The rk length itself mismatch
KeyValue kv2 = new KeyValue("r12".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
Assert.assertEquals(1, CellUtil.findCommonPrefixInFlatKey(kv1, kv2, true, true));
// part of rk is same
KeyValue kv3 = new KeyValue("r14".getBytes(), "f1".getBytes(), "q1".getBytes(), null);
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE + "r1".getBytes().length,
CellUtil.findCommonPrefixInFlatKey(kv2, kv3, true, true));
// entire rk is same but different cf name
KeyValue kv4 = new KeyValue("r14".getBytes(), "f2".getBytes(), "q1".getBytes(), null);
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE
+ kv3.getRowLength()
+ KeyValue.FAMILY_LENGTH_SIZE
+ "f".getBytes().length,
CellUtil.findCommonPrefixInFlatKey(kv3, kv4, false, true));
// rk and family are same and part of qualifier
KeyValue kv5 = new KeyValue("r14".getBytes(), "f2".getBytes(), "q123".getBytes(), null);
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE
+ kv3.getRowLength()
+ KeyValue.FAMILY_LENGTH_SIZE
+ kv4.getFamilyLength()
+ kv4.getQualifierLength(),
CellUtil.findCommonPrefixInFlatKey(kv4, kv5, true, true));
// rk, cf and q are same. ts differs
KeyValue kv6 = new KeyValue("rk".getBytes(), 1234L);
KeyValue kv7 = new KeyValue("rk".getBytes(), 1235L);
// only last byte out of 8 ts bytes in ts part differs
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE
+ kv6.getRowLength()
+ KeyValue.FAMILY_LENGTH_SIZE
+ kv6.getFamilyLength()
+ kv6.getQualifierLength()
+ 7,
CellUtil.findCommonPrefixInFlatKey(kv6, kv7, true, true));
// rk, cf, q and ts are same. Only type differs
KeyValue kv8 = new KeyValue("rk".getBytes(), 1234L, Type.Delete);
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE
+ kv6.getRowLength()
+ KeyValue.FAMILY_LENGTH_SIZE
+ kv6.getFamilyLength()
+ kv6.getQualifierLength()
+ KeyValue.TIMESTAMP_SIZE,
CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, true));
// With out TS_TYPE check
Assert.assertEquals(
KeyValue.ROW_LENGTH_SIZE
+ kv6.getRowLength()
+ KeyValue.FAMILY_LENGTH_SIZE
+ kv6.getFamilyLength()
+ kv6.getQualifierLength(),
CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, false));
}
private DeploymentConfig generateDeploymentConfig(String namespace, Map<String, String> labels) {
Object exposedPorts =
osActiveStreamTag
.getImage()
.getDockerImageMetadata()
.getContainerConfig()
.getExposedPorts();
List<ContainerPort> ports = parsePorts(exposedPorts);
Map<String, String> templateLabels = new HashMap<>(labels);
templateLabels.put("deploymentconfig", osAppName);
List<EnvVar> env = newArrayList();
for (KeyValue variable : view.getEnvironmentVariables()) {
env.add(newDto(EnvVar.class).withName(variable.getKey()).withValue(variable.getValue()));
}
final String steamTagName = osAppName + ":latest";
PodSpec podSpec =
newDto(PodSpec.class)
.withContainers(
newArrayList(
newDto(Container.class)
.withImage(steamTagName)
.withName(osAppName)
.withPorts(ports)
.withEnv(env)));
PodTemplateSpec template =
newDto(PodTemplateSpec.class)
.withMetadata(newDto(ObjectMeta.class).withLabels(templateLabels))
.withSpec(podSpec);
DeploymentTriggerPolicy imageChange =
newDto(DeploymentTriggerPolicy.class)
.withType("ImageChange")
.withImageChangeParams(
newDto(DeploymentTriggerImageChangeParams.class)
.withAutomatic(true)
.withContainerNames(newArrayList(osAppName))
.withFrom(
newDto(ObjectReference.class)
.withKind("ImageStreamTag")
.withName(steamTagName)));
DeploymentTriggerPolicy configChange =
newDto(DeploymentTriggerPolicy.class).withType("ConfigChange");
return newDto(DeploymentConfig.class)
.withApiVersion(API_VERSION)
.withKind("DeploymentConfig")
.withMetadata(
newDto(ObjectMeta.class)
.withName(osAppName)
.withLabels(labels)
.withNamespace(namespace))
.withSpec(
newDto(DeploymentConfigSpec.class)
.withReplicas(1)
.withSelector(Collections.singletonMap("deploymentconfig", osAppName))
.withTemplate(template)
.withTriggers(newArrayList(imageChange, configChange)));
}
