/** * Test that {@link HFileOutputFormat2} RecordWriter amends timestamps if passed a keyvalue whose * timestamp is {@link HConstants#LATEST_TIMESTAMP}. * * @see <a href="https://issues.apache.org/jira/browse/HBASE-2615">HBASE-2615</a> */ @Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test public void test_LATEST_TIMESTAMP_isReplaced() throws Exception { Configuration conf = new Configuration(this.util.getConfiguration()); RecordWriter<ImmutableBytesWritable, Cell> writer = null; TaskAttemptContext context = null; Path dir = util.getDataTestDir("test_LATEST_TIMESTAMP_isReplaced"); try { Job job = new Job(conf); FileOutputFormat.setOutputPath(job, dir); context = createTestTaskAttemptContext(job); HFileOutputFormat2 hof = new HFileOutputFormat2(); writer = hof.getRecordWriter(context); final byte[] b = Bytes.toBytes("b"); // Test 1. Pass a KV that has a ts of LATEST_TIMESTAMP. It should be // changed by call to write. Check all in kv is same but ts. KeyValue kv = new KeyValue(b, b, b); KeyValue original = kv.clone(); writer.write(new ImmutableBytesWritable(), kv); assertFalse(original.equals(kv)); assertTrue(Bytes.equals(CellUtil.cloneRow(original), CellUtil.cloneRow(kv))); assertTrue(Bytes.equals(CellUtil.cloneFamily(original), CellUtil.cloneFamily(kv))); assertTrue(Bytes.equals(CellUtil.cloneQualifier(original), CellUtil.cloneQualifier(kv))); assertNotSame(original.getTimestamp(), kv.getTimestamp()); assertNotSame(HConstants.LATEST_TIMESTAMP, kv.getTimestamp()); // Test 2. Now test passing a kv that has explicit ts. It should not be // changed by call to record write. kv = new KeyValue(b, b, b, kv.getTimestamp() - 1, b); original = kv.clone(); writer.write(new ImmutableBytesWritable(), kv); assertTrue(original.equals(kv)); } finally { if (writer != null && context != null) writer.close(context); dir.getFileSystem(conf).delete(dir, true); } }
@Test(timeOut = 10_000) public void testCorrectMapingOfCellsToShadowCells() throws IOException { // Create the required data final byte[] validShadowCellQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX); final byte[] qualifier2 = Bytes.toBytes("test-qual2"); final byte[] validShadowCellQualifier2 = com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX); final byte[] qualifier3 = Bytes.toBytes("test-qual3"); Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value")); Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value")); Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value")); Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2")); Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2")); Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3")); // Check a list of cells with duplicate values List<Cell> badListWithDups = new ArrayList<>(); badListWithDups.add(cell1); badListWithDups.add(dupCell1WithAnotherValue); // Check dup shadow cell with same MVCC is ignored SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups); assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps"); assertTrue(cellsToShadowCells.containsKey(cell1)); KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey(); KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey(); assertTrue(firstKey.equals(lastKey)); assertTrue( 0 == Bytes.compareTo( firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(), cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()), "Should be equal"); // Modify dup shadow cell to have a greater MVCC and check that is replaced HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1); cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups); assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps"); assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue)); firstKey = (KeyValue) cellsToShadowCells.firstKey(); lastKey = (KeyValue) cellsToShadowCells.lastKey(); assertTrue(firstKey.equals(lastKey)); assertTrue( 0 == Bytes.compareTo( firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(), dupCell1WithAnotherValue.getValueArray(), dupCell1WithAnotherValue.getValueOffset(), dupCell1WithAnotherValue.getValueLength()), "Should be equal"); // Check a list of cells with duplicate values List<Cell> cellListWithDups = new ArrayList<>(); cellListWithDups.add(cell1); cellListWithDups.add(shadowCell1); cellListWithDups.add(dupCell1); // Dup cell cellListWithDups.add(delCell1); // Another Dup cell but with different type cellListWithDups.add(cell2); cellListWithDups.add(cell3); cellListWithDups.add(shadowCell2); cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups); assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps"); assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1)); assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1)); assertFalse( cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved. // The current algo avoids to put the delete cell // as key after the put cell with same value was added assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2)); assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent())); }