@Test
public void testFindNextToCompact() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
long now = System.currentTimeMillis();
CompactionInfo ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
assertEquals("foo", ci.dbname);
assertEquals("bar", ci.tableName);
assertEquals("ds=today", ci.partName);
assertEquals(CompactionType.MINOR, ci.type);
assertNull(ci.runAs);
assertNull(txnHandler.findNextToCompact("fred"));
txnHandler.setRunAs(ci.id, "bob");
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
assertEquals(1, compacts.size());
ShowCompactResponseElement c = compacts.get(0);
assertEquals("foo", c.getDbname());
assertEquals("bar", c.getTablename());
assertEquals("ds=today", c.getPartitionname());
assertEquals(CompactionType.MINOR, c.getType());
assertEquals("working", c.getState());
assertTrue(c.getStart() - 5000 < now && c.getStart() + 5000 > now);
assertEquals("fred", c.getWorkerid());
assertEquals("bob", c.getRunAs());
}
@Test
public void testFindNextToClean() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
assertEquals(0, txnHandler.findReadyToClean().size());
CompactionInfo ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
assertEquals(0, txnHandler.findReadyToClean().size());
txnHandler.markCompacted(ci);
assertNull(txnHandler.findNextToCompact("fred"));
List<CompactionInfo> toClean = txnHandler.findReadyToClean();
assertEquals(1, toClean.size());
assertNull(txnHandler.findNextToCompact("fred"));
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
assertEquals(1, compacts.size());
ShowCompactResponseElement c = compacts.get(0);
assertEquals("foo", c.getDbname());
assertEquals("bar", c.getTablename());
assertEquals("ds=today", c.getPartitionname());
assertEquals(CompactionType.MINOR, c.getType());
assertEquals("ready for cleaning", c.getState());
assertNull(c.getWorkerid());
}
/** Performs a deep copy on <i>other</i>. */
public CompactionRequest(CompactionRequest other) {
if (other.isSetDbname()) {
this.dbname = other.dbname;
}
if (other.isSetTablename()) {
this.tablename = other.tablename;
}
if (other.isSetPartitionname()) {
this.partitionname = other.partitionname;
}
if (other.isSetType()) {
this.type = other.type;
}
if (other.isSetRunas()) {
this.runas = other.runas;
}
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, CompactionRequest struct)
throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
struct.dbname = iprot.readString();
struct.setDbnameIsSet(true);
struct.tablename = iprot.readString();
struct.setTablenameIsSet(true);
struct.type =
org.apache.hadoop.hive.metastore.api.CompactionType.findByValue(iprot.readI32());
struct.setTypeIsSet(true);
BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
struct.partitionname = iprot.readString();
struct.setPartitionnameIsSet(true);
}
if (incoming.get(1)) {
struct.runas = iprot.readString();
struct.setRunasIsSet(true);
}
}
public boolean equals(CompactionRequest that) {
if (that == null) return false;
boolean this_present_dbname = true && this.isSetDbname();
boolean that_present_dbname = true && that.isSetDbname();
if (this_present_dbname || that_present_dbname) {
if (!(this_present_dbname && that_present_dbname)) return false;
if (!this.dbname.equals(that.dbname)) return false;
}
boolean this_present_tablename = true && this.isSetTablename();
boolean that_present_tablename = true && that.isSetTablename();
if (this_present_tablename || that_present_tablename) {
if (!(this_present_tablename && that_present_tablename)) return false;
if (!this.tablename.equals(that.tablename)) return false;
}
boolean this_present_partitionname = true && this.isSetPartitionname();
boolean that_present_partitionname = true && that.isSetPartitionname();
if (this_present_partitionname || that_present_partitionname) {
if (!(this_present_partitionname && that_present_partitionname)) return false;
if (!this.partitionname.equals(that.partitionname)) return false;
}
boolean this_present_type = true && this.isSetType();
boolean that_present_type = true && that.isSetType();
if (this_present_type || that_present_type) {
if (!(this_present_type && that_present_type)) return false;
if (!this.type.equals(that.type)) return false;
}
boolean this_present_runas = true && this.isSetRunas();
boolean that_present_runas = true && that.isSetRunas();
if (this_present_runas || that_present_runas) {
if (!(this_present_runas && that_present_runas)) return false;
if (!this.runas.equals(that.runas)) return false;
}
return true;
}
@Test
public void testFindNextToCompact2() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=yesterday");
txnHandler.compact(rqst);
long now = System.currentTimeMillis();
boolean expectToday = false;
CompactionInfo ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
assertEquals("foo", ci.dbname);
assertEquals("bar", ci.tableName);
if ("ds=today".equals(ci.partName)) expectToday = false;
else if ("ds=yesterday".equals(ci.partName)) expectToday = true;
else fail("partition name should have been today or yesterday but was " + ci.partName);
assertEquals(CompactionType.MINOR, ci.type);
ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
assertEquals("foo", ci.dbname);
assertEquals("bar", ci.tableName);
if (expectToday) assertEquals("ds=today", ci.partName);
else assertEquals("ds=yesterday", ci.partName);
assertEquals(CompactionType.MINOR, ci.type);
assertNull(txnHandler.findNextToCompact("fred"));
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
List<ShowCompactResponseElement> compacts = rsp.getCompacts();
assertEquals(2, compacts.size());
for (ShowCompactResponseElement e : compacts) {
assertEquals("working", e.getState());
assertTrue(e.getStart() - 5000 < now && e.getStart() + 5000 > now);
assertEquals("fred", e.getWorkerid());
}
}
@Override
public void write(org.apache.thrift.protocol.TProtocol prot, CompactionRequest struct)
throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
oprot.writeString(struct.dbname);
oprot.writeString(struct.tablename);
oprot.writeI32(struct.type.getValue());
BitSet optionals = new BitSet();
if (struct.isSetPartitionname()) {
optionals.set(0);
}
if (struct.isSetRunas()) {
optionals.set(1);
}
oprot.writeBitSet(optionals, 2);
if (struct.isSetPartitionname()) {
oprot.writeString(struct.partitionname);
}
if (struct.isSetRunas()) {
oprot.writeString(struct.runas);
}
}
public void write(org.apache.thrift.protocol.TProtocol oprot, CompactionRequest struct)
throws org.apache.thrift.TException {
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
if (struct.dbname != null) {
oprot.writeFieldBegin(DBNAME_FIELD_DESC);
oprot.writeString(struct.dbname);
oprot.writeFieldEnd();
}
if (struct.tablename != null) {
oprot.writeFieldBegin(TABLENAME_FIELD_DESC);
oprot.writeString(struct.tablename);
oprot.writeFieldEnd();
}
if (struct.partitionname != null) {
if (struct.isSetPartitionname()) {
oprot.writeFieldBegin(PARTITIONNAME_FIELD_DESC);
oprot.writeString(struct.partitionname);
oprot.writeFieldEnd();
}
}
if (struct.type != null) {
oprot.writeFieldBegin(TYPE_FIELD_DESC);
oprot.writeI32(struct.type.getValue());
oprot.writeFieldEnd();
}
if (struct.runas != null) {
if (struct.isSetRunas()) {
oprot.writeFieldBegin(RUNAS_FIELD_DESC);
oprot.writeString(struct.runas);
oprot.writeFieldEnd();
}
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@Test
public void testMarkCleaned() throws Exception {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
rqst.setPartitionname("ds=today");
txnHandler.compact(rqst);
assertEquals(0, txnHandler.findReadyToClean().size());
CompactionInfo ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
assertEquals(0, txnHandler.findReadyToClean().size());
txnHandler.markCompacted(ci);
assertNull(txnHandler.findNextToCompact("fred"));
List<CompactionInfo> toClean = txnHandler.findReadyToClean();
assertEquals(1, toClean.size());
assertNull(txnHandler.findNextToCompact("fred"));
txnHandler.markCleaned(ci);
assertNull(txnHandler.findNextToCompact("fred"));
assertEquals(0, txnHandler.findReadyToClean().size());
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
assertEquals(1, rsp.getCompactsSize());
assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
}
public void read(org.apache.thrift.protocol.TProtocol iprot, CompactionRequest struct)
throws org.apache.thrift.TException {
org.apache.thrift.protocol.TField schemeField;
iprot.readStructBegin();
while (true) {
schemeField = iprot.readFieldBegin();
if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
case 1: // DBNAME
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.dbname = iprot.readString();
struct.setDbnameIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 2: // TABLENAME
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.tablename = iprot.readString();
struct.setTablenameIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 3: // PARTITIONNAME
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.partitionname = iprot.readString();
struct.setPartitionnameIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 4: // TYPE
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
struct.type =
org.apache.hadoop.hive.metastore.api.CompactionType.findByValue(iprot.readI32());
struct.setTypeIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 5: // RUNAS
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.runas = iprot.readString();
struct.setRunasIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
iprot.readFieldEnd();
}
iprot.readStructEnd();
struct.validate();
}
@Override
public int compareTo(CompactionRequest other) {
if (!getClass().equals(other.getClass())) {
return getClass().getName().compareTo(other.getClass().getName());
}
int lastComparison = 0;
lastComparison = Boolean.valueOf(isSetDbname()).compareTo(other.isSetDbname());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetDbname()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dbname, other.dbname);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetTablename()).compareTo(other.isSetTablename());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetTablename()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tablename, other.tablename);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetPartitionname()).compareTo(other.isSetPartitionname());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetPartitionname()) {
lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.partitionname, other.partitionname);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetType()).compareTo(other.isSetType());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetType()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, other.type);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetRunas()).compareTo(other.isSetRunas());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetRunas()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.runas, other.runas);
if (lastComparison != 0) {
return lastComparison;
}
}
return 0;
}
@Test
public void testMarkCleanedCleansTxnsAndTxnComponents() throws Exception {
long txnid = openTxn();
LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
comp.setTablename("mytable");
comp.setOperationType(DataOperationType.INSERT);
List<LockComponent> components = new ArrayList<LockComponent>(1);
components.add(comp);
LockRequest req = new LockRequest(components, "me", "localhost");
req.setTxnid(txnid);
LockResponse res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
txnid = openTxn();
comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
comp.setTablename("yourtable");
comp.setOperationType(DataOperationType.DELETE);
components = new ArrayList<LockComponent>(1);
components.add(comp);
req = new LockRequest(components, "me", "localhost");
req.setTxnid(txnid);
res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
txnid = openTxn();
comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
comp.setTablename("foo");
comp.setPartitionname("bar");
comp.setOperationType(DataOperationType.UPDATE);
components = new ArrayList<LockComponent>(1);
components.add(comp);
req = new LockRequest(components, "me", "localhost");
req.setTxnid(txnid);
res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
comp.setTablename("foo");
comp.setPartitionname("baz");
comp.setOperationType(DataOperationType.UPDATE);
components = new ArrayList<LockComponent>(1);
components.add(comp);
req = new LockRequest(components, "me", "localhost");
req.setTxnid(txnid);
res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
txnHandler.abortTxn(new AbortTxnRequest(txnid));
CompactionInfo ci = new CompactionInfo();
// Now clean them and check that they are removed from the count.
CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MAJOR);
txnHandler.compact(rqst);
assertEquals(0, txnHandler.findReadyToClean().size());
ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
txnHandler.markCompacted(ci);
List<CompactionInfo> toClean = txnHandler.findReadyToClean();
assertEquals(1, toClean.size());
txnHandler.markCleaned(ci);
// Check that we are cleaning up the empty aborted transactions
GetOpenTxnsResponse txnList = txnHandler.getOpenTxns();
assertEquals(3, txnList.getOpen_txnsSize());
txnHandler.cleanEmptyAbortedTxns();
txnList = txnHandler.getOpenTxns();
assertEquals(2, txnList.getOpen_txnsSize());
rqst = new CompactionRequest("mydb", "foo", CompactionType.MAJOR);
rqst.setPartitionname("bar");
txnHandler.compact(rqst);
assertEquals(0, txnHandler.findReadyToClean().size());
ci = txnHandler.findNextToCompact("fred");
assertNotNull(ci);
txnHandler.markCompacted(ci);
toClean = txnHandler.findReadyToClean();
assertEquals(1, toClean.size());
txnHandler.markCleaned(ci);
txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
txnHandler.cleanEmptyAbortedTxns();
txnList = txnHandler.getOpenTxns();
assertEquals(3, txnList.getOpen_txnsSize());
}
