@Override
public Iterator<MarkedEdge> getEdgesToTargetBySourceType(
final ApplicationScope scope, final SearchByIdType edgeType) {
ValidationUtils.validateApplicationScope(scope);
GraphValidation.validateSearchByIdType(edgeType);
final Id targetId = edgeType.getNode();
final String sourceType = edgeType.getIdType();
final String type = edgeType.getType();
final long maxTimestamp = edgeType.getMaxTimestamp();
final DirectedEdgeMeta directedEdgeMeta =
DirectedEdgeMeta.fromTargetNodeSourceType(targetId, type, sourceType);
final Iterator<ShardEntryGroup> readShards =
edgeShardStrategy.getReadShards(scope, maxTimestamp, directedEdgeMeta);
return new ShardGroupColumnIterator(scope, directedEdgeMeta, shardGroupDeletion, readShards) {
@Override
protected Iterator<MarkedEdge> getIterator(final Collection<Shard> readShards) {
return shardedEdgeSerialization.getEdgesToTargetBySourceType(
edgeColumnFamilies, scope, edgeType, readShards);
}
@Override
protected Iterator<MarkedEdge> getIteratorFullRange(final Collection<Shard> readShards) {
final SearchByIdType edgeTypeFullRange =
new SimpleSearchByIdType(
edgeType.getNode(),
edgeType.getType(),
Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING,
edgeType.getIdType(),
Optional.absent(),
false);
return shardedEdgeSerialization.getEdgesToTargetBySourceType(
edgeColumnFamilies, scope, edgeTypeFullRange, readShards);
}
};
}
@Override
public Iterator<MarkedEdge> getEdgeVersions(
final ApplicationScope scope, final SearchByEdge search) {
ValidationUtils.validateApplicationScope(scope);
GraphValidation.validateSearchByEdge(search);
final Id targetId = search.targetNode();
final Id sourceId = search.sourceNode();
final String type = search.getType();
final long maxTimestamp = search.getMaxTimestamp();
final DirectedEdgeMeta versionMetaData = DirectedEdgeMeta.fromEdge(sourceId, targetId, type);
final Iterator<ShardEntryGroup> readShards =
edgeShardStrategy.getReadShards(scope, maxTimestamp, versionMetaData);
// now create a result iterator with our iterator of read shards
return new ShardGroupColumnIterator(scope, versionMetaData, shardGroupDeletion, readShards) {
@Override
protected Iterator<MarkedEdge> getIterator(final Collection<Shard> readShards) {
return shardedEdgeSerialization.getEdgeVersions(
edgeColumnFamilies, scope, search, readShards);
}
@Override
protected Iterator<MarkedEdge> getIteratorFullRange(final Collection<Shard> readShards) {
final SearchByEdge searchFullRange =
new SimpleSearchByEdge(
search.sourceNode(),
search.getType(),
search.targetNode(),
Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING,
Optional.absent());
return shardedEdgeSerialization.getEdgeVersions(
edgeColumnFamilies, scope, searchFullRange, readShards);
}
};
}
@Override
public MutationBatch writeEdge(
final ApplicationScope scope, final MarkedEdge markedEdge, final UUID timestamp) {
ValidationUtils.validateApplicationScope(scope);
GraphValidation.validateEdge(markedEdge);
ValidationUtils.verifyTimeUuid(timestamp, "timestamp");
final long now = timeService.getCurrentTime();
final Id sourceNode = markedEdge.getSourceNode();
final Id targetNode = markedEdge.getTargetNode();
final String edgeType = markedEdge.getType();
final long edgeTimestamp = markedEdge.getTimestamp();
/** Source write */
final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceNode, edgeType);
final Collection<Shard> sourceWriteShards =
edgeShardStrategy.getWriteShards(scope, edgeTimestamp, sourceEdgeMeta).getWriteShards(now);
final MutationBatch batch =
shardedEdgeSerialization.writeEdgeFromSource(
edgeColumnFamilies, scope, markedEdge, sourceWriteShards, sourceEdgeMeta, timestamp);
/** Source with target type write */
final DirectedEdgeMeta sourceTargetTypeEdgeMeta =
DirectedEdgeMeta.fromSourceNodeTargetType(sourceNode, edgeType, targetNode.getType());
final Collection<Shard> sourceTargetTypeWriteShards =
edgeShardStrategy
.getWriteShards(scope, edgeTimestamp, sourceTargetTypeEdgeMeta)
.getWriteShards(now);
batch.mergeShallow(
shardedEdgeSerialization.writeEdgeFromSourceWithTargetType(
edgeColumnFamilies,
scope,
markedEdge,
sourceTargetTypeWriteShards,
sourceTargetTypeEdgeMeta,
timestamp));
/** Target write */
final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromTargetNode(targetNode, edgeType);
final Collection<Shard> targetWriteShards =
edgeShardStrategy.getWriteShards(scope, edgeTimestamp, targetEdgeMeta).getWriteShards(now);
batch.mergeShallow(
shardedEdgeSerialization.writeEdgeToTarget(
edgeColumnFamilies, scope, markedEdge, targetWriteShards, targetEdgeMeta, timestamp));
/** Target with source type write */
final DirectedEdgeMeta targetSourceTypeEdgeMeta =
DirectedEdgeMeta.fromTargetNodeSourceType(targetNode, edgeType, sourceNode.getType());
final Collection<Shard> targetSourceTypeWriteShards =
edgeShardStrategy
.getWriteShards(scope, edgeTimestamp, targetSourceTypeEdgeMeta)
.getWriteShards(now);
batch.mergeShallow(
shardedEdgeSerialization.writeEdgeToTargetWithSourceType(
edgeColumnFamilies,
scope,
markedEdge,
targetSourceTypeWriteShards,
targetSourceTypeEdgeMeta,
timestamp));
/** Version write */
final DirectedEdgeMeta edgeVersionsMeta =
DirectedEdgeMeta.fromEdge(sourceNode, targetNode, edgeType);
final Collection<Shard> edgeVersionsShards =
edgeShardStrategy
.getWriteShards(scope, edgeTimestamp, edgeVersionsMeta)
.getWriteShards(now);
batch.mergeShallow(
shardedEdgeSerialization.writeEdgeVersions(
edgeColumnFamilies,
scope,
markedEdge,
edgeVersionsShards,
edgeVersionsMeta,
timestamp));
return batch;
}
@Test(timeout = 120000)
@Category(StressTest.class)
public void writeThousandsDelete()
throws InterruptedException, ExecutionException, MigrationException,
UnsupportedEncodingException {
final Id sourceId = IdGenerator.createId("source");
final String edgeType = "test";
final EdgeGenerator generator =
new EdgeGenerator() {
@Override
public Edge newEdge() {
Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target"));
return edge;
}
@Override
public Observable<MarkedEdge> doSearch(final GraphManager manager) {
return manager.loadEdgesFromSource(
new SimpleSearchByEdgeType(
sourceId,
edgeType,
Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent(),
false));
}
};
// final int numInjectors = 2;
final int numInjectors = 1;
/**
* create 3 injectors. This way all the caches are independent of one another. This is the same
* as multiple nodes
*/
final List<Injector> injectors = createInjectors(numInjectors);
final GraphFig graphFig = getInstance(injectors, GraphFig.class);
final long shardSize = graphFig.getShardSize();
// we don't want to starve the cass runtime since it will be on the same box. Only take 50% of
// processing
// power for writes
final int numProcessors = Runtime.getRuntime().availableProcessors() / 2;
final int numWorkersPerInjector = numProcessors / numInjectors;
/** Do 4x shard size so we should have approximately 4 shards */
final long numberOfEdges = shardSize * 4;
final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors;
createExecutor(numWorkersPerInjector);
final AtomicLong writeCounter = new AtomicLong();
// min stop time the min delta + 1 cache cycle timeout
final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout();
logger.info(
"Writing {} edges per worker on {} workers in {} injectors",
workerWriteLimit,
numWorkersPerInjector,
numInjectors);
final List<Future<Boolean>> futures = new ArrayList<>();
for (Injector injector : injectors) {
final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class);
for (int i = 0; i < numWorkersPerInjector; i++) {
Future<Boolean> future =
executor.submit(
new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter));
futures.add(future);
}
}
/** Wait for all writes to complete */
for (Future<Boolean> future : futures) {
future.get();
}
// now get all our shards
final NodeShardCache cache = getInstance(injectors, NodeShardCache.class);
final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType);
// now submit the readers.
final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class);
final long writeCount = writeCounter.get();
final Meter readMeter = registry.meter("readThroughput");
// check our shard state
final Iterator<ShardEntryGroup> existingShardGroups =
cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta);
int shardCount = 0;
while (existingShardGroups.hasNext()) {
final ShardEntryGroup group = existingShardGroups.next();
shardCount++;
logger.info(
"Compaction pending status for group {} is {}", group, group.isCompactionPending());
}
logger.info("found {} shard groups", shardCount);
// now mark and delete all the edges
final GraphManager manager = gmf.createEdgeManager(scope);
// sleep occasionally to stop pushing cassandra over
long count = Long.MAX_VALUE;
while (count != 0) {
// take 10000 then sleep
count =
generator
.doSearch(manager)
.onBackpressureBlock()
.take(1000)
.flatMap(edge -> manager.markEdge(edge))
.flatMap(edge -> manager.deleteEdge(edge))
.countLong()
.toBlocking()
.last();
Thread.sleep(500);
}
// now loop until with a reader until our shards are gone
/** Start reading continuously while we migrate data to ensure our view is always correct */
final ListenableFuture<Long> future =
executor.submit(new ReadWorker(gmf, generator, 0, readMeter));
final List<Throwable> failures = new ArrayList<>();
// add the future
Futures.addCallback(
future,
new FutureCallback<Long>() {
@Override
public void onSuccess(@Nullable final Long result) {
logger.info("Successfully ran the read, re-running");
executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter));
}
@Override
public void onFailure(final Throwable t) {
failures.add(t);
logger.error("Failed test!", t);
}
});
// now start our readers
while (true) {
if (!failures.isEmpty()) {
StringBuilder builder = new StringBuilder();
builder.append("Read runner failed!\n");
for (Throwable t : failures) {
builder.append("Exception is: ");
ByteArrayOutputStream output = new ByteArrayOutputStream();
t.printStackTrace(new PrintWriter(output));
builder.append(output.toString("UTF-8"));
builder.append("\n\n");
}
fail(builder.toString());
}
// reset our count. Ultimately we'll have 4 groups once our compaction completes
shardCount = 0;
// we have to get it from the cache, because this will trigger the compaction process
final Iterator<ShardEntryGroup> groups =
cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta);
ShardEntryGroup group = null;
while (groups.hasNext()) {
group = groups.next();
logger.info("Shard size for group is {}", group.getReadShards());
shardCount += group.getReadShards().size();
}
// we're done, 1 shard remains, we have a group, and it's our default shard
if (shardCount == 1
&& group != null
&& group.getMinShard().getShardIndex() == Shard.MIN_SHARD.getShardIndex()) {
logger.info("All compactions complete,");
break;
}
Thread.sleep(2000);
}
// now that we have finished expanding s
executor.shutdownNow();
}
@Test
public void writeThousandsSingleSource()
throws InterruptedException, ExecutionException, MigrationException,
UnsupportedEncodingException {
final Id sourceId = IdGenerator.createId("source");
final String edgeType = "test";
final EdgeGenerator generator =
new EdgeGenerator() {
@Override
public Edge newEdge() {
Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target"));
return edge;
}
@Override
public Observable<MarkedEdge> doSearch(final GraphManager manager) {
return manager.loadEdgesFromSource(
new SimpleSearchByEdgeType(
sourceId,
edgeType,
Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent()));
}
};
// final int numInjectors = 2;
final int numInjectors = 1;
/**
* create 3 injectors. This way all the caches are independent of one another. This is the same
* as multiple nodes
*/
final List<Injector> injectors = createInjectors(numInjectors);
final GraphFig graphFig = getInstance(injectors, GraphFig.class);
final long shardSize = graphFig.getShardSize();
// we don't want to starve the cass runtime since it will be on the same box. Only take 50% of
// processing
// power for writes
final int numProcessors = Runtime.getRuntime().availableProcessors() / 2;
final int numWorkersPerInjector = numProcessors / numInjectors;
/** Do 4x shard size so we should have approximately 4 shards */
final long numberOfEdges = shardSize * 4;
final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors;
final long expectedShardCount = numberOfEdges / shardSize;
createExecutor(numWorkersPerInjector);
final AtomicLong writeCounter = new AtomicLong();
// min stop time the min delta + 1 cache cycle timeout
final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout();
logger.info(
"Writing {} edges per worker on {} workers in {} injectors",
workerWriteLimit,
numWorkersPerInjector,
numInjectors);
final List<Future<Boolean>> futures = new ArrayList<>();
for (Injector injector : injectors) {
final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class);
for (int i = 0; i < numWorkersPerInjector; i++) {
Future<Boolean> future =
executor.submit(
new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter));
futures.add(future);
}
}
/** Wait for all writes to complete */
for (Future<Boolean> future : futures) {
future.get();
}
// now get all our shards
final NodeShardCache cache = getInstance(injectors, NodeShardCache.class);
final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType);
// now submit the readers.
final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class);
final long writeCount = writeCounter.get();
final Meter readMeter = registry.meter("readThroughput");
final List<Throwable> failures = new ArrayList<>();
for (int i = 0; i < 2; i++) {
/** Start reading continuously while we migrate data to ensure our view is always correct */
final ListenableFuture<Long> future =
executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter));
// add the future
Futures.addCallback(
future,
new FutureCallback<Long>() {
@Override
public void onSuccess(@Nullable final Long result) {
logger.info("Successfully ran the read, re-running");
executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter));
}
@Override
public void onFailure(final Throwable t) {
failures.add(t);
logger.error("Failed test!", t);
}
});
}
int compactedCount;
// now start our readers
while (true) {
if (!failures.isEmpty()) {
StringBuilder builder = new StringBuilder();
builder.append("Read runner failed!\n");
for (Throwable t : failures) {
builder.append("Exception is: ");
ByteArrayOutputStream output = new ByteArrayOutputStream();
t.printStackTrace(new PrintWriter(output));
builder.append(output.toString("UTF-8"));
builder.append("\n\n");
}
fail(builder.toString());
}
// reset our count. Ultimately we'll have 4 groups once our compaction completes
compactedCount = 0;
// we have to get it from the cache, because this will trigger the compaction process
final Iterator<ShardEntryGroup> groups =
cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta);
final Set<ShardEntryGroup> shardEntryGroups = new HashSet<>();
while (groups.hasNext()) {
final ShardEntryGroup group = groups.next();
shardEntryGroups.add(group);
logger.info(
"Compaction pending status for group {} is {}", group, group.isCompactionPending());
if (!group.isCompactionPending()) {
compactedCount++;
}
}
// we're done
if (compactedCount >= expectedShardCount) {
logger.info("All compactions complete, sleeping");
// final Object mutex = new Object();
//
// synchronized ( mutex ){
//
// mutex.wait();
// }
break;
}
Thread.sleep(2000);
}
// now continue reading everything for 30 seconds
Thread.sleep(30000);
executor.shutdownNow();
}