@Test public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException { Table table1 = Table.open("Keyspace1"); RowMutation rm; DecoratedKey dk = Util.dk("key"); ColumnFamily cf; for (int i = 0; i < 10; ++i) { rm = new RowMutation("Keyspace1", dk.key); cf = ColumnFamily.create("Keyspace1", "Counter1"); cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L)); rm.add(cf); rm.apply(); } table1.getColumnFamilyStore("Counter1").clearUnsafe(); CommitLog.instance.resetUnsafe(); // disassociate segments from live CL CommitLog.instance.recover(); cf = Util.getColumnFamily(table1, dk, "Counter1"); assert cf.getColumnCount() == 1; Column c = cf.getColumn(ByteBufferUtil.bytes("col")); assert c != null; assert ((CounterColumn) c).total() == 10L; }
@Test(timeout = 5000) public void testTruncateHints() throws Exception { Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF); hintStore.clearUnsafe(); // insert 1 hint RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, ByteBufferUtil.bytes(String.valueOf(COLUMN1)), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .apply(); assert getNoOfHints() == 1; HintedHandOffManager.instance.truncateAllHints(); while (getNoOfHints() > 0) { Thread.sleep(100); } assert getNoOfHints() == 0; }
// Returns every permission on the resource granted to the user. public Set<Permission> authorize(AuthenticatedUser user, IResource resource) { if (user.isSuper()) return Permission.ALL; UntypedResultSet result; try { ResultMessage.Rows rows = authorizeStatement.execute( QueryState.forInternalCalls(), new QueryOptions( ConsistencyLevel.ONE, Lists.newArrayList( ByteBufferUtil.bytes(user.getName()), ByteBufferUtil.bytes(resource.getName())))); result = UntypedResultSet.create(rows.result); } catch (RequestValidationException e) { throw new AssertionError(e); // not supposed to happen } catch (RequestExecutionException e) { logger.warn("CassandraAuthorizer failed to authorize {} for {}", user, resource); return Permission.NONE; } if (result.isEmpty() || !result.one().has(PERMISSIONS)) return Permission.NONE; Set<Permission> permissions = EnumSet.noneOf(Permission.class); for (String perm : result.one().getSet(PERMISSIONS, UTF8Type.instance)) permissions.add(Permission.valueOf(perm)); return permissions; }
// Test compaction of hints column family. It shouldn't remove all columns on compaction. @Test public void testCompactionOfHintsCF() throws Exception { // prepare hints column family Table systemTable = Table.open("system"); ColumnFamilyStore hintStore = systemTable.getColumnFamilyStore(SystemTable.HINTS_CF); hintStore.clearUnsafe(); hintStore.metadata.gcGraceSeconds(36000); // 10 hours hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); hintStore.disableAutoCompaction(); // insert 1 hint RowMutation rm = new RowMutation(TABLE4, ByteBufferUtil.bytes(1)); rm.add( new QueryPath(STANDARD1_CF, null, ByteBufferUtil.bytes(String.valueOf(COLUMN1))), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); RowMutation.hintFor(rm, UUID.randomUUID()).apply(); // flush data to disk hintStore.forceBlockingFlush(); assertEquals(1, hintStore.getSSTables().size()); // submit compaction FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact()); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // single row should not be removed because of gc_grace_seconds // is 10 hours and there are no any tombstones in sstable assertEquals(1, hintStore.getSSTables().size()); }
/** * Test altering the type of a column, including the one in the primary key (#4041) migrated from * cql_tests.py:TestCQL.update_type_test() */ @Test public void testUpdateColumnType() throws Throwable { createTable("CREATE TABLE %s (k text, c text, s set <text>, v text, PRIMARY KEY(k, c))"); // using utf8 character so that we can see the transition to BytesType execute("INSERT INTO %s (k, c, v, s) VALUES ('ɸ', 'ɸ', 'ɸ', {'ɸ'})"); assertRows(execute("SELECT * FROM %s"), row("ɸ", "ɸ", set("ɸ"), "ɸ")); execute("ALTER TABLE %s ALTER v TYPE blob"); assertRows(execute("SELECT * FROM %s"), row("ɸ", "ɸ", set("ɸ"), ByteBufferUtil.bytes("ɸ"))); execute("ALTER TABLE %s ALTER k TYPE blob"); assertRows( execute("SELECT * FROM %s"), row(ByteBufferUtil.bytes("ɸ"), "ɸ", set("ɸ"), ByteBufferUtil.bytes("ɸ"))); execute("ALTER TABLE %s ALTER c TYPE blob"); assertRows( execute("SELECT * FROM %s"), row( ByteBufferUtil.bytes("ɸ"), ByteBufferUtil.bytes("ɸ"), set("ɸ"), ByteBufferUtil.bytes("ɸ"))); execute("ALTER TABLE %s ALTER s TYPE set<blob>"); assertRows( execute("SELECT * FROM %s"), row( ByteBufferUtil.bytes("ɸ"), ByteBufferUtil.bytes("ɸ"), set(ByteBufferUtil.bytes("ɸ")), ByteBufferUtil.bytes("ɸ"))); }
@PooledConnection public void setColumn( String keyspace, String column_family, String key, JSONObject json, ConsistencyLevel consistency_level, boolean index, long timestamp) throws InvalidRequestException, UnavailableException, TimedOutException, TException, HttpException, IOException { List<Mutation> slice = new ArrayList<Mutation>(); for (Object field : json.keySet()) { String name = (String) field; String value = (String) json.get(name); Column c = new Column(); c.setName(ByteBufferUtil.bytes(name)); c.setValue(ByteBufferUtil.bytes(value)); c.setTimestamp(timestamp); Mutation m = new Mutation(); ColumnOrSuperColumn cc = new ColumnOrSuperColumn(); cc.setColumn(c); m.setColumn_or_supercolumn(cc); slice.add(m); } Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap = new HashMap<ByteBuffer, Map<String, List<Mutation>>>(); Map<String, List<Mutation>> cfMutations = new HashMap<String, List<Mutation>>(); cfMutations.put(column_family, slice); mutationMap.put(ByteBufferUtil.bytes(key), cfMutations); getConnection(keyspace).batch_mutate(mutationMap, consistency_level); if (config.isIndexingEnabled() && index) indexer.index(column_family, key, json); }
@PooledConnection public void deleteColumn( String keyspace, String column_family, String key, String column, ConsistencyLevel consistency_level, boolean purgeIndex) throws InvalidRequestException, UnavailableException, TimedOutException, TException, HttpException, IOException { ColumnPath path = new ColumnPath(column_family); path.setColumn(ByteBufferUtil.bytes(column)); getConnection(keyspace) .remove( ByteBufferUtil.bytes(key), path, System.currentTimeMillis() * 1000, consistency_level); // TODO: Revisit deleting a single field because it requires a fetch // first. // Evidently it is impossible to remove just a field from a document in // SOLR // http://stackoverflow.com/questions/4802620/can-you-delete-a-field-from-a-document-in-solr-index if (config.isIndexingEnabled() && purgeIndex) { indexer.delete(column_family, key); JSONObject json = this.getSlice(keyspace, column_family, key, consistency_level); json.remove(column); indexer.index(column_family, key, json); } }
@Test public void testRemoveColumnFamilyWithFlush1() { Keyspace keyspace = Keyspace.open("Keyspace1"); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); Mutation rm; DecoratedKey dk = Util.dk("key1"); // add data rm = new Mutation("Keyspace1", dk.key); rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0); rm.apply(); store.forceBlockingFlush(); // remove rm = new Mutation("Keyspace1", dk.key); rm.delete("Standard1", 1); rm.apply(); ColumnFamily retrieved = store.getColumnFamily( QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); assert retrieved.isMarkedForDelete(); assertNull(retrieved.getColumn(Util.cellname("Column1"))); assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); }
@PooledConnection public JSONArray getRowsWithQuery( String keyspace, String columnFamily, String queryStr, ConsistencyLevel consistencyLevel) throws InvalidRequestException, UnavailableException, TimedOutException, TException, CharacterCodingException { Query query = QueryParser.parse(queryStr); SlicePredicate predicate = new SlicePredicate(); SliceRange range = new SliceRange(ByteBufferUtil.bytes(""), ByteBufferUtil.bytes(""), false, MAX_COLUMNS); predicate.setSlice_range(range); ColumnParent parent = new ColumnParent(columnFamily); IndexClause indexClause = new IndexClause(); indexClause.setCount(MAX_ROWS); indexClause.setStart_key(new byte[0]); for (String keyName : query.getEqStmt().keySet()) { indexClause.addToExpressions( new IndexExpression( ByteBufferUtil.bytes(keyName), IndexOperator.EQ, ByteBufferUtil.bytes(query.getEqStmt().get(keyName)))); } List<KeySlice> rows = getConnection(keyspace) .get_indexed_slices(parent, indexClause, predicate, ConsistencyLevel.QUORUM); return JsonMarshaller.marshallRows(rows, true); }
@Test public void testEchoedRow() throws IOException, ExecutionException, InterruptedException { // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); // disable compaction while flushing cfs.disableAutoCompaction(); // Insert 4 keys in two sstables. We need the sstables to have 2 rows // at least to trigger what was causing CASSANDRA-2653 for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Standard2", ByteBufferUtil.bytes(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.apply(); if (i % 2 == 0) cfs.forceBlockingFlush(); } Collection<SSTableReader> toCompact = cfs.getSSTables(); assert toCompact.size() == 2; // Reinserting the same keys. We will compact only the previous sstable, but we need those new // ones // to make sure we use EchoedRow, otherwise it won't be used because purge can be done. for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Standard2", ByteBufferUtil.bytes(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.apply(); } cfs.forceBlockingFlush(); SSTableReader tmpSSTable = null; for (SSTableReader sstable : cfs.getSSTables()) if (!toCompact.contains(sstable)) tmpSSTable = sstable; assert tmpSSTable != null; // Force compaction on first sstables. Since each row is in only one sstable, we will be using // EchoedRow. Util.compact(cfs, toCompact); assertEquals(2, cfs.getSSTables().size()); // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it // doesn't hide the problem) cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN); assertEquals(1, cfs.getSSTables().size()); // Now assert we do have the 4 keys assertEquals(4, Util.getRangeSlice(cfs).size()); }
@Test public void simpleQueryWithRangeTombstoneTest() throws Exception { Table table = Table.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); // Inserting data String key = "k1"; RowMutation rm; ColumnFamily cf; rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 0; i < 40; i += 2) add(rm, i, 0); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 10, 22, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 1; i < 40; i += 2) add(rm, i, 2); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 19, 27, 3); rm.apply(); // We don't flush to test with both a range tomsbtone in memtable and in sstable QueryPath path = new QueryPath(CFNAME); // Queries by name int[] live = new int[] {4, 9, 11, 17, 28}; int[] dead = new int[] {12, 19, 21, 24, 27}; SortedSet<ByteBuffer> columns = new TreeSet<ByteBuffer>(cfs.getComparator()); for (int i : live) columns.add(b(i)); for (int i : dead) columns.add(b(i)); cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), path, columns)); for (int i : live) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i : dead) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; // Queries by slices cf = cfs.getColumnFamily( QueryFilter.getSliceFilter(dk(key), path, b(7), b(30), false, Integer.MAX_VALUE)); for (int i : new int[] {7, 8, 9, 11, 13, 15, 17, 28, 29, 30}) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i : new int[] {10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; }
@Test public void overlappingRangeTest() throws Exception { CompactionManager.instance.disableAutoCompaction(); Table table = Table.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); // Inserting data String key = "k2"; RowMutation rm; ColumnFamily cf; rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 0; i < 20; i++) add(rm, i, 0); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 15, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 10, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 8, 2); rm.apply(); cfs.forceBlockingFlush(); QueryPath path = new QueryPath(CFNAME); cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path)); for (int i = 0; i < 5; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 16; i < 20; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 5; i <= 15; i++) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; // Compact everything and re-test CompactionManager.instance.performMaximal(cfs); cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path)); for (int i = 0; i < 5; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 16; i < 20; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 5; i <= 15; i++) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; }
private static class Statics { private final String KS = "Keyspace1"; private final ByteBuffer Key = ByteBufferUtil.bytes("Key01"); private final SortedSet<ByteBuffer> NamedCols = new TreeSet<ByteBuffer>(BytesType.instance) { { add(ByteBufferUtil.bytes("AAA")); add(ByteBufferUtil.bytes("BBB")); add(ByteBufferUtil.bytes("CCC")); } }; private final ByteBuffer SC = ByteBufferUtil.bytes("SCName"); private final SortedSet<ByteBuffer> NamedSCCols = new TreeSet<ByteBuffer>(BytesType.instance) { { add(CompositeType.build(SC, ByteBufferUtil.bytes("AAA"))); add(CompositeType.build(SC, ByteBufferUtil.bytes("BBB"))); add(CompositeType.build(SC, ByteBufferUtil.bytes("CCC"))); } }; private final String StandardCF = "Standard1"; private final String SuperCF = "Super1"; private final long readTs = 1369935512292L; private final ColumnFamily StandardCf = TreeMapBackedSortedColumns.factory.create(KS, StandardCF); private final ColumnFamily SuperCf = TreeMapBackedSortedColumns.factory.create(KS, SuperCF); private final Row StandardRow = new Row(Util.dk("key0"), StandardCf); private final Row SuperRow = new Row(Util.dk("key1"), SuperCf); private final Row NullRow = new Row(Util.dk("key2"), null); private Statics() { StandardCf.addColumn(new Column(bb("aaaa"))); StandardCf.addColumn(new Column(bb("bbbb"), bb("bbbbb-value"))); StandardCf.addColumn(new Column(bb("cccc"), bb("ccccc-value"), 1000L)); StandardCf.addColumn(new DeletedColumn(bb("dddd"), 500, 1000)); StandardCf.addColumn(new DeletedColumn(bb("eeee"), bb("eeee-value"), 1001)); StandardCf.addColumn(new ExpiringColumn(bb("ffff"), bb("ffff-value"), 2000, 1000)); StandardCf.addColumn(new ExpiringColumn(bb("gggg"), bb("gggg-value"), 2001, 1000, 2002)); SuperCf.addColumn(new Column(CompositeType.build(SC, bb("aaaa")))); SuperCf.addColumn(new Column(CompositeType.build(SC, bb("bbbb")), bb("bbbbb-value"))); SuperCf.addColumn(new Column(CompositeType.build(SC, bb("cccc")), bb("ccccc-value"), 1000L)); SuperCf.addColumn(new DeletedColumn(CompositeType.build(SC, bb("dddd")), 500, 1000)); SuperCf.addColumn( new DeletedColumn(CompositeType.build(SC, bb("eeee")), bb("eeee-value"), 1001)); SuperCf.addColumn( new ExpiringColumn(CompositeType.build(SC, bb("ffff")), bb("ffff-value"), 2000, 1000)); SuperCf.addColumn( new ExpiringColumn( CompositeType.build(SC, bb("gggg")), bb("gggg-value"), 2001, 1000, 2002)); } }
@Test public void testBytes() { BytesType comparator = new BytesType(); assert comparator.compare(ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.bytes("asdf")) < 0; assert comparator.compare(ByteBufferUtil.bytes("asdf"), ByteBufferUtil.EMPTY_BYTE_BUFFER) > 0; assert comparator.compare(ByteBufferUtil.EMPTY_BYTE_BUFFER, ByteBufferUtil.EMPTY_BYTE_BUFFER) == 0; assert comparator.compare(ByteBufferUtil.bytes("z"), ByteBufferUtil.bytes("a")) > 0; assert comparator.compare(ByteBufferUtil.bytes("a"), ByteBufferUtil.bytes("z")) < 0; assert comparator.compare(ByteBufferUtil.bytes("asdf"), ByteBufferUtil.bytes("asdf")) == 0; assert comparator.compare(ByteBufferUtil.bytes("asdz"), ByteBufferUtil.bytes("asdf")) > 0; }
public static void addMutation( RowMutation rm, String columnFamilyName, String superColumnName, long columnName, String value, long timestamp) { rm.add( new QueryPath( columnFamilyName, ByteBufferUtil.bytes(superColumnName), getBytes(columnName)), ByteBufferUtil.bytes(value), timestamp); }
public static void addMutation( Mutation rm, String columnFamilyName, String superColumnName, long columnName, String value, long timestamp) { CellName cname = superColumnName == null ? CellNames.simpleDense(getBytes(columnName)) : CellNames.compositeDense(ByteBufferUtil.bytes(superColumnName), getBytes(columnName)); rm.add(columnFamilyName, cname, ByteBufferUtil.bytes(value), timestamp); }
private void testDontPurgeAccidentaly(String k, String cfname) throws IOException, ExecutionException, InterruptedException { // This test catches the regression of CASSANDRA-2786 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.clearUnsafe(); cfs.disableAutoCompaction(); // Add test row DecoratedKey key = Util.dk(k); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( cfname, CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); rm.apply(); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesBefore = cfs.getSSTables(); QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis()); assert !(cfs.getColumnFamily(filter).getColumnCount() == 0); // Remove key rm = new RowMutation(KEYSPACE1, key.key); rm.delete(cfname, 2); rm.apply(); ColumnFamily cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0 Thread.sleep(1000); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesAfter = cfs.getSSTables(); Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>(); for (SSTableReader sstable : sstablesAfter) if (!sstablesBefore.contains(sstable)) toCompact.add(sstable); Util.compact(cfs, toCompact); cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; }
private static Mutation getMutation(String colName, String value) { Column c = new Column(); c.setName(ByteBufferUtil.bytes(colName)); c.setValue(ByteBufferUtil.bytes(value)); c.setTimestamp(System.currentTimeMillis()); Mutation m = new Mutation(); m.setColumn_or_supercolumn(new ColumnOrSuperColumn()); m.column_or_supercolumn.setColumn(c); return m; }
public void reduce( Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { ColumnFamily columnFamily; String keyspace = "Keyspace1"; String cfName = "Super1"; Message message; List<ColumnFamily> columnFamilies; columnFamilies = new LinkedList<ColumnFamily>(); String line; /* Create a column family */ columnFamily = ColumnFamily.create(keyspace, cfName); while (values.hasNext()) { // Split the value (line based on your own delimiter) line = values.next().toString(); String[] fields = line.split("\1"); String SuperColumnName = fields[1]; String ColumnName = fields[2]; String ColumnValue = fields[3]; int timestamp = 0; columnFamily.addColumn( new QueryPath( cfName, ByteBufferUtil.bytes(SuperColumnName), ByteBufferUtil.bytes(ColumnName)), ByteBufferUtil.bytes(ColumnValue), timestamp); } columnFamilies.add(columnFamily); /* Get serialized message to send to cluster */ message = createMessage(keyspace, key.getBytes(), cfName, columnFamilies); List<IAsyncResult> results = new ArrayList<IAsyncResult>(); for (InetAddress endpoint : StorageService.instance.getNaturalEndpoints(keyspace, ByteBufferUtil.bytes(key))) { /* Send message to end point */ results.add(MessagingService.instance().sendRR(message, endpoint)); } /* wait for acks */ for (IAsyncResult result : results) { try { result.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { // you should probably add retry logic here throw new RuntimeException(e); } } output.collect(key, new Text(" inserted into Cassandra node(s)")); }
@PooledConnection public String getColumn( String keyspace, String columnFamily, String key, String column, ConsistencyLevel consistencyLevel) throws InvalidRequestException, NotFoundException, UnavailableException, TimedOutException, TException, UnsupportedEncodingException { ColumnPath path = new ColumnPath(columnFamily); path.setColumn(ByteBufferUtil.bytes(column)); ColumnOrSuperColumn column_result = getConnection(keyspace).get(ByteBufferUtil.bytes(key), path, consistencyLevel); return new String(column_result.getColumn().getValue(), "UTF8"); }
@Override protected void applyModifications(List<? extends Modification> mods) throws CacheLoaderException { Cassandra.Client cassandraClient = null; try { cassandraClient = dataSource.getConnection(); Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap = new HashMap<ByteBuffer, Map<String, List<Mutation>>>(); for (Modification m : mods) { switch (m.getType()) { case STORE: store0(((Store) m).getStoredEntry(), mutationMap); break; case CLEAR: clear(); break; case REMOVE: remove0(ByteBufferUtil.bytes(hashKey(((Remove) m).getKey())), mutationMap); break; default: throw new AssertionError(); } } cassandraClient.batch_mutate(mutationMap, writeConsistencyLevel); } catch (Exception e) { throw new CacheLoaderException(e); } finally { dataSource.releaseConnection(cassandraClient); } }
@Test public void test64kColumn() { // a byte buffer more than 64k ByteBuffer buffer = ByteBuffer.allocate(1024 * 65); buffer.clear(); // read more than 64k for (int i = 0; i < 1024 * 64 / 4 + 1; i++) buffer.putInt(0); // for read buffer.flip(); Column column = new Column(ByteBufferUtil.bytes("test"), buffer, 0); SecondaryIndexColumnSizeTest.MockRowIndex mockRowIndex = new SecondaryIndexColumnSizeTest.MockRowIndex(); SecondaryIndexColumnSizeTest.MockColumnIndex mockColumnIndex = new SecondaryIndexColumnSizeTest.MockColumnIndex(); assertTrue(mockRowIndex.validate(column)); assertFalse(mockColumnIndex.validate(column)); // test less than 64k value buffer.flip(); buffer.clear(); buffer.putInt(20); buffer.flip(); assertTrue(mockRowIndex.validate(column)); assertTrue(mockColumnIndex.validate(column)); }
@PooledConnection public JSONObject getSlice( String keyspace, String columnFamily, String key, ConsistencyLevel consistencyLevel) throws InvalidRequestException, UnavailableException, TimedOutException, TException, UnsupportedEncodingException { SlicePredicate predicate = new SlicePredicate(); SliceRange range = new SliceRange(ByteBufferUtil.bytes(""), ByteBufferUtil.bytes(""), false, MAX_COLUMNS); predicate.setSlice_range(range); ColumnParent parent = new ColumnParent(columnFamily); List<ColumnOrSuperColumn> slice = getConnection(keyspace) .get_slice(ByteBufferUtil.bytes(key), parent, predicate, consistencyLevel); if (slice.size() > 0) return JsonMarshaller.marshallSlice(slice); else return null; }
@Test public void testCompactions() throws IOException, ExecutionException, InterruptedException { CompactionManager.instance.disableAutoCompaction(); // this test does enough rows to force multiple block indexes to be used Table table = Table.open(TABLE1); ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); final int ROWS_PER_SSTABLE = 10; Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) { for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i % 2)); RowMutation rm = new RowMutation(TABLE1, key.key); rm.add( new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))), ByteBufferUtil.EMPTY_BYTE_BUFFER, j * ROWS_PER_SSTABLE + i); rm.apply(); inserted.add(key); } store.forceBlockingFlush(); assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size()); } while (true) { Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store); if (ft.get() == 0) break; } if (store.getSSTables().size() > 1) { CompactionManager.instance.performMajor(store); } assertEquals(inserted.size(), Util.getRangeSlice(store).size()); }
public void writeLogEntry(LogEntry logEntry) throws Throwable { List<Mutation> slice = new ArrayList<Mutation>(); slice.add(getMutation(LogEntryColumns.KS.toString(), logEntry.getKeyspace())); slice.add(getMutation(LogEntryColumns.CF.toString(), logEntry.getColumnFamily())); slice.add(getMutation(LogEntryColumns.ROW.toString(), logEntry.getRowKey())); slice.add(getMutation(LogEntryColumns.STATUS.toString(), logEntry.getStatus().toString())); slice.add( getMutation(LogEntryColumns.TIMESTAMP.toString(), Long.toString(logEntry.getTimestamp()))); slice.add(getMutation(LogEntryColumns.HOST.toString(), logEntry.getHost())); if (logEntry.hasErrors()) { for (String errorKey : logEntry.getErrors().keySet()) { slice.add(getMutation(errorKey, logEntry.getErrors().get(errorKey))); } } if (ConfigurationStore.getStore().shouldWriteColumns()) { for (ColumnOperation operation : logEntry.getOperations()) { if (operation.isDelete()) { slice.add(getMutation(operation.getName(), OperationType.DELETE)); } else { slice.add(getMutation(operation.getName(), OperationType.UPDATE)); } } } Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap = new HashMap<ByteBuffer, Map<String, List<Mutation>>>(); Map<String, List<Mutation>> cfMutations = new HashMap<String, List<Mutation>>(); cfMutations.put(COLUMN_FAMILY, slice); ByteBuffer rowKey = ByteBufferUtil.bytes(logEntry.getUuid()); mutationMap.put(rowKey, cfMutations); getConnection(KEYSPACE).batch_mutate(mutationMap, logEntry.getConsistencyLevel()); }
@Override public void start() throws CacheLoaderException { try { dataSource = new DataSource(config.getPoolProperties()); readConsistencyLevel = ConsistencyLevel.valueOf(config.readConsistencyLevel); writeConsistencyLevel = ConsistencyLevel.valueOf(config.writeConsistencyLevel); entryColumnPath = new ColumnPath(config.entryColumnFamily) .setColumn(ENTRY_COLUMN_NAME.getBytes(UTF8Charset)); entryColumnParent = new ColumnParent(config.entryColumnFamily); entryKeyPrefix = ENTRY_KEY_PREFIX + (config.isSharedKeyspace() ? cacheName + "_" : ""); expirationColumnParent = new ColumnParent(config.expirationColumnFamily); expirationKey = ByteBufferUtil.bytes(EXPIRATION_KEY + (config.isSharedKeyspace() ? "_" + cacheName : "")); keyMapper = (TwoWayKey2StringMapper) Util.getInstance(config.getKeyMapper()); } catch (Exception e) { throw new ConfigurationException(e); } log.debug("cleaning up expired entries..."); purgeInternal(); log.debug("started"); super.start(); }
private void addExpiryEntry( String cassandraKey, long expiryTime, Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap) { try { addMutation( mutationMap, expirationKey, config.expirationColumnFamily, ByteBufferUtil.bytes(expiryTime), ByteBufferUtil.bytes(cassandraKey), ByteBufferUtil.EMPTY_BYTE_BUFFER); } catch (Exception e) { // Should not happen } }
@Test public void testCheckForExpiredSSTableBlockers() throws InterruptedException { String KEYSPACE1 = "Keyspace1"; ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); cfs.truncateBlocking(); cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key); rm.add( "Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); rm.applyUnsafe(); cfs.forceBlockingFlush(); SSTableReader blockingSSTable = cfs.getSSTables().iterator().next(); for (int i = 0; i < 10; i++) { rm = new RowMutation(KEYSPACE1, Util.dk("test").key); rm.delete("Standard1", System.currentTimeMillis()); rm.applyUnsafe(); cfs.forceBlockingFlush(); } Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers( cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100); assertEquals(1, blockers.keySet().size()); assertTrue(blockers.keySet().contains(blockingSSTable)); assertEquals(10, blockers.get(blockingSSTable).size()); }
@Test public void testSerializeDeserialize() throws IOException { CounterContext.ContextState state = CounterContext.ContextState.allocate(4, 2); state.writeElement(NodeId.fromInt(1), 4L, 4L); state.writeElement(NodeId.fromInt(2), 4L, 4L, true); state.writeElement(NodeId.fromInt(3), 4L, 4L); state.writeElement(NodeId.fromInt(4), 4L, 4L, true); CounterColumn original = new CounterColumn(ByteBufferUtil.bytes("x"), state.context, 1L); DataOutputBuffer bufOut = new DataOutputBuffer(); Column.serializer().serialize(original, bufOut); byte[] serialized = bufOut.getData(); ByteArrayInputStream bufIn = new ByteArrayInputStream(serialized, 0, serialized.length); CounterColumn deserialized = (CounterColumn) Column.serializer().deserialize(new DataInputStream(bufIn)); assert original.equals(deserialized); bufIn = new ByteArrayInputStream(serialized, 0, serialized.length); CounterColumn deserializedOnRemote = (CounterColumn) Column.serializer().deserialize(new DataInputStream(bufIn), null, true); assert deserializedOnRemote.name().equals(original.name()); assert deserializedOnRemote.total() == original.total(); assert deserializedOnRemote.value().equals(cc.clearAllDelta(original.value())); assert deserializedOnRemote.timestamp() == deserialized.timestamp(); assert deserializedOnRemote.timestampOfLastDelete() == deserialized.timestampOfLastDelete(); }
private static List<ByteBuffer> toByteBufferParams(List<Object> params) { List<ByteBuffer> r = new ArrayList<>(); for (Object param : params) { if (param instanceof ByteBuffer) r.add((ByteBuffer) param); else if (param instanceof Long) r.add(ByteBufferUtil.bytes((Long) param)); else throw new AssertionError(); } return r; }