@Test
  public void testFindNextToCompact() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    long now = System.currentTimeMillis();
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    assertEquals("ds=today", ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);
    assertNull(ci.runAs);
    assertNull(txnHandler.findNextToCompact("fred"));

    txnHandler.setRunAs(ci.id, "bob");

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MINOR, c.getType());
    assertEquals("working", c.getState());
    assertTrue(c.getStart() - 5000 < now && c.getStart() + 5000 > now);
    assertEquals("fred", c.getWorkerid());
    assertEquals("bob", c.getRunAs());
  }
  @Test
  public void testFindNextToClean() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean().size());
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);

    assertEquals(0, txnHandler.findReadyToClean().size());
    txnHandler.markCompacted(ci);
    assertNull(txnHandler.findNextToCompact("fred"));

    List<CompactionInfo> toClean = txnHandler.findReadyToClean();
    assertEquals(1, toClean.size());
    assertNull(txnHandler.findNextToCompact("fred"));

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(1, compacts.size());
    ShowCompactResponseElement c = compacts.get(0);
    assertEquals("foo", c.getDbname());
    assertEquals("bar", c.getTablename());
    assertEquals("ds=today", c.getPartitionname());
    assertEquals(CompactionType.MINOR, c.getType());
    assertEquals("ready for cleaning", c.getState());
    assertNull(c.getWorkerid());
  }
Beispiel #3
0
 /** Performs a deep copy on <i>other</i>. */
 public CompactionRequest(CompactionRequest other) {
   if (other.isSetDbname()) {
     this.dbname = other.dbname;
   }
   if (other.isSetTablename()) {
     this.tablename = other.tablename;
   }
   if (other.isSetPartitionname()) {
     this.partitionname = other.partitionname;
   }
   if (other.isSetType()) {
     this.type = other.type;
   }
   if (other.isSetRunas()) {
     this.runas = other.runas;
   }
 }
Beispiel #4
0
 @Override
 public void read(org.apache.thrift.protocol.TProtocol prot, CompactionRequest struct)
     throws org.apache.thrift.TException {
   TTupleProtocol iprot = (TTupleProtocol) prot;
   struct.dbname = iprot.readString();
   struct.setDbnameIsSet(true);
   struct.tablename = iprot.readString();
   struct.setTablenameIsSet(true);
   struct.type =
       org.apache.hadoop.hive.metastore.api.CompactionType.findByValue(iprot.readI32());
   struct.setTypeIsSet(true);
   BitSet incoming = iprot.readBitSet(2);
   if (incoming.get(0)) {
     struct.partitionname = iprot.readString();
     struct.setPartitionnameIsSet(true);
   }
   if (incoming.get(1)) {
     struct.runas = iprot.readString();
     struct.setRunasIsSet(true);
   }
 }
Beispiel #5
0
  public boolean equals(CompactionRequest that) {
    if (that == null) return false;

    boolean this_present_dbname = true && this.isSetDbname();
    boolean that_present_dbname = true && that.isSetDbname();
    if (this_present_dbname || that_present_dbname) {
      if (!(this_present_dbname && that_present_dbname)) return false;
      if (!this.dbname.equals(that.dbname)) return false;
    }

    boolean this_present_tablename = true && this.isSetTablename();
    boolean that_present_tablename = true && that.isSetTablename();
    if (this_present_tablename || that_present_tablename) {
      if (!(this_present_tablename && that_present_tablename)) return false;
      if (!this.tablename.equals(that.tablename)) return false;
    }

    boolean this_present_partitionname = true && this.isSetPartitionname();
    boolean that_present_partitionname = true && that.isSetPartitionname();
    if (this_present_partitionname || that_present_partitionname) {
      if (!(this_present_partitionname && that_present_partitionname)) return false;
      if (!this.partitionname.equals(that.partitionname)) return false;
    }

    boolean this_present_type = true && this.isSetType();
    boolean that_present_type = true && that.isSetType();
    if (this_present_type || that_present_type) {
      if (!(this_present_type && that_present_type)) return false;
      if (!this.type.equals(that.type)) return false;
    }

    boolean this_present_runas = true && this.isSetRunas();
    boolean that_present_runas = true && that.isSetRunas();
    if (this_present_runas || that_present_runas) {
      if (!(this_present_runas && that_present_runas)) return false;
      if (!this.runas.equals(that.runas)) return false;
    }

    return true;
  }
  @Test
  public void testFindNextToCompact2() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);

    rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=yesterday");
    txnHandler.compact(rqst);

    long now = System.currentTimeMillis();
    boolean expectToday = false;
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    if ("ds=today".equals(ci.partName)) expectToday = false;
    else if ("ds=yesterday".equals(ci.partName)) expectToday = true;
    else fail("partition name should have been today or yesterday but was " + ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);

    ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);
    assertEquals("foo", ci.dbname);
    assertEquals("bar", ci.tableName);
    if (expectToday) assertEquals("ds=today", ci.partName);
    else assertEquals("ds=yesterday", ci.partName);
    assertEquals(CompactionType.MINOR, ci.type);

    assertNull(txnHandler.findNextToCompact("fred"));

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
    assertEquals(2, compacts.size());
    for (ShowCompactResponseElement e : compacts) {
      assertEquals("working", e.getState());
      assertTrue(e.getStart() - 5000 < now && e.getStart() + 5000 > now);
      assertEquals("fred", e.getWorkerid());
    }
  }
Beispiel #7
0
 @Override
 public void write(org.apache.thrift.protocol.TProtocol prot, CompactionRequest struct)
     throws org.apache.thrift.TException {
   TTupleProtocol oprot = (TTupleProtocol) prot;
   oprot.writeString(struct.dbname);
   oprot.writeString(struct.tablename);
   oprot.writeI32(struct.type.getValue());
   BitSet optionals = new BitSet();
   if (struct.isSetPartitionname()) {
     optionals.set(0);
   }
   if (struct.isSetRunas()) {
     optionals.set(1);
   }
   oprot.writeBitSet(optionals, 2);
   if (struct.isSetPartitionname()) {
     oprot.writeString(struct.partitionname);
   }
   if (struct.isSetRunas()) {
     oprot.writeString(struct.runas);
   }
 }
Beispiel #8
0
    public void write(org.apache.thrift.protocol.TProtocol oprot, CompactionRequest struct)
        throws org.apache.thrift.TException {
      struct.validate();

      oprot.writeStructBegin(STRUCT_DESC);
      if (struct.dbname != null) {
        oprot.writeFieldBegin(DBNAME_FIELD_DESC);
        oprot.writeString(struct.dbname);
        oprot.writeFieldEnd();
      }
      if (struct.tablename != null) {
        oprot.writeFieldBegin(TABLENAME_FIELD_DESC);
        oprot.writeString(struct.tablename);
        oprot.writeFieldEnd();
      }
      if (struct.partitionname != null) {
        if (struct.isSetPartitionname()) {
          oprot.writeFieldBegin(PARTITIONNAME_FIELD_DESC);
          oprot.writeString(struct.partitionname);
          oprot.writeFieldEnd();
        }
      }
      if (struct.type != null) {
        oprot.writeFieldBegin(TYPE_FIELD_DESC);
        oprot.writeI32(struct.type.getValue());
        oprot.writeFieldEnd();
      }
      if (struct.runas != null) {
        if (struct.isSetRunas()) {
          oprot.writeFieldBegin(RUNAS_FIELD_DESC);
          oprot.writeString(struct.runas);
          oprot.writeFieldEnd();
        }
      }
      oprot.writeFieldStop();
      oprot.writeStructEnd();
    }
  @Test
  public void testMarkCleaned() throws Exception {
    CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR);
    rqst.setPartitionname("ds=today");
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean().size());
    CompactionInfo ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);

    assertEquals(0, txnHandler.findReadyToClean().size());
    txnHandler.markCompacted(ci);
    assertNull(txnHandler.findNextToCompact("fred"));

    List<CompactionInfo> toClean = txnHandler.findReadyToClean();
    assertEquals(1, toClean.size());
    assertNull(txnHandler.findNextToCompact("fred"));
    txnHandler.markCleaned(ci);
    assertNull(txnHandler.findNextToCompact("fred"));
    assertEquals(0, txnHandler.findReadyToClean().size());

    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
    assertEquals(1, rsp.getCompactsSize());
    assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState()));
  }
Beispiel #10
0
 public void read(org.apache.thrift.protocol.TProtocol iprot, CompactionRequest struct)
     throws org.apache.thrift.TException {
   org.apache.thrift.protocol.TField schemeField;
   iprot.readStructBegin();
   while (true) {
     schemeField = iprot.readFieldBegin();
     if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
       break;
     }
     switch (schemeField.id) {
       case 1: // DBNAME
         if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
           struct.dbname = iprot.readString();
           struct.setDbnameIsSet(true);
         } else {
           org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
         break;
       case 2: // TABLENAME
         if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
           struct.tablename = iprot.readString();
           struct.setTablenameIsSet(true);
         } else {
           org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
         break;
       case 3: // PARTITIONNAME
         if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
           struct.partitionname = iprot.readString();
           struct.setPartitionnameIsSet(true);
         } else {
           org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
         break;
       case 4: // TYPE
         if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
           struct.type =
               org.apache.hadoop.hive.metastore.api.CompactionType.findByValue(iprot.readI32());
           struct.setTypeIsSet(true);
         } else {
           org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
         break;
       case 5: // RUNAS
         if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
           struct.runas = iprot.readString();
           struct.setRunasIsSet(true);
         } else {
           org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
         break;
       default:
         org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
     }
     iprot.readFieldEnd();
   }
   iprot.readStructEnd();
   struct.validate();
 }
Beispiel #11
0
  @Override
  public int compareTo(CompactionRequest other) {
    if (!getClass().equals(other.getClass())) {
      return getClass().getName().compareTo(other.getClass().getName());
    }

    int lastComparison = 0;

    lastComparison = Boolean.valueOf(isSetDbname()).compareTo(other.isSetDbname());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetDbname()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dbname, other.dbname);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetTablename()).compareTo(other.isSetTablename());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetTablename()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tablename, other.tablename);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetPartitionname()).compareTo(other.isSetPartitionname());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetPartitionname()) {
      lastComparison =
          org.apache.thrift.TBaseHelper.compareTo(this.partitionname, other.partitionname);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetType()).compareTo(other.isSetType());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetType()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, other.type);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(isSetRunas()).compareTo(other.isSetRunas());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (isSetRunas()) {
      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.runas, other.runas);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    return 0;
  }
  @Test
  public void testMarkCleanedCleansTxnsAndTxnComponents() throws Exception {
    long txnid = openTxn();
    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.INSERT);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    LockResponse res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));

    txnid = openTxn();
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("yourtable");
    comp.setOperationType(DataOperationType.DELETE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));

    txnid = openTxn();
    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("foo");
    comp.setPartitionname("bar");
    comp.setOperationType(DataOperationType.UPDATE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);

    comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
    comp.setTablename("foo");
    comp.setPartitionname("baz");
    comp.setOperationType(DataOperationType.UPDATE);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    assertTrue(res.getState() == LockState.ACQUIRED);
    txnHandler.abortTxn(new AbortTxnRequest(txnid));

    CompactionInfo ci = new CompactionInfo();

    // Now clean them and check that they are removed from the count.
    CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MAJOR);
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean().size());
    ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);
    txnHandler.markCompacted(ci);

    List<CompactionInfo> toClean = txnHandler.findReadyToClean();
    assertEquals(1, toClean.size());
    txnHandler.markCleaned(ci);

    // Check that we are cleaning up the empty aborted transactions
    GetOpenTxnsResponse txnList = txnHandler.getOpenTxns();
    assertEquals(3, txnList.getOpen_txnsSize());
    txnHandler.cleanEmptyAbortedTxns();
    txnList = txnHandler.getOpenTxns();
    assertEquals(2, txnList.getOpen_txnsSize());

    rqst = new CompactionRequest("mydb", "foo", CompactionType.MAJOR);
    rqst.setPartitionname("bar");
    txnHandler.compact(rqst);
    assertEquals(0, txnHandler.findReadyToClean().size());
    ci = txnHandler.findNextToCompact("fred");
    assertNotNull(ci);
    txnHandler.markCompacted(ci);

    toClean = txnHandler.findReadyToClean();
    assertEquals(1, toClean.size());
    txnHandler.markCleaned(ci);

    txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost"));
    txnHandler.cleanEmptyAbortedTxns();
    txnList = txnHandler.getOpenTxns();
    assertEquals(3, txnList.getOpen_txnsSize());
  }