public void testReallocateWrongPlacedReplicasStateOnlyIdToStartInDB() {
final ONodeId idToStart = ONodeId.generateUniqueId();
final ONodeId nodeId = ONodeId.generateUniqueId();
final ONodeAddressStub nodeAddressStub = new ONodeAddressStub(nodeId);
when(nodeLocal.state()).thenReturn(ODHTNode.NodeState.PRODUCTION);
when(nodeLocal.getNodeAddress()).thenReturn(nodeAddressStub);
final ORID startRecordId = new ORecordId(1, new OClusterPositionNodeId(idToStart));
final Iterator<ORecordMetadata> metadataIterator = mock(Iterator.class);
when(nodeLocal.getLocalRingIterator(STORAGE_NAME, startRecordId.nextRid(), startRecordId))
.thenReturn(metadataIterator);
when(metadataIterator.hasNext()).thenReturn(true);
when(metadataIterator.next())
.thenReturn(new ORecordMetadata(startRecordId, new ODistributedVersion(0)));
final ONodeId result =
globalMaintenanceProtocol.reallocateWrongPlacedReplicas(
STORAGE_NAME, CLUSTER_ID, nodeLocal, idToStart, 1, 1);
Assert.assertEquals(result, nodeId);
verify(nodeLocal).state();
verify(nodeLocal).getNodeAddress();
verify(nodeLocal).getLocalRingIterator(STORAGE_NAME, startRecordId.nextRid(), startRecordId);
verifyZeroInteractions(replicaDistributionStrategy, nodeLookup);
verifyNoMoreInteractions(nodeLocal);
}
public void testReallocateWrongPlacedReplicasStateNotInProduction() {
final ONodeId idToStart = ONodeId.generateUniqueId();
final ONodeId nodeId = ONodeId.generateUniqueId();
final ONodeAddressStub nodeAddressStub = new ONodeAddressStub(nodeId);
when(nodeLocal.state()).thenReturn(ODHTNode.NodeState.JOIN);
when(nodeLocal.getNodeAddress()).thenReturn(nodeAddressStub);
final ONodeId result =
globalMaintenanceProtocol.reallocateWrongPlacedReplicas(
STORAGE_NAME, CLUSTER_ID, nodeLocal, idToStart, 1, 1);
Assert.assertEquals(result, nodeId);
verify(nodeLocal).state();
verify(nodeLocal).getNodeAddress();
verifyZeroInteractions(replicaDistributionStrategy, nodeLookup);
verifyNoMoreInteractions(nodeLocal);
}
public void testReallocateWrongPlacedReplicasStateNodeRecordsTenRecordsTwoRecordsIsOutOfDate() {
final ONodeId idToStart = ONodeId.generateUniqueId();
final ONodeId nodeId = ONodeId.generateUniqueId();
final ONodeId successorNodeId = ONodeId.generateUniqueId();
final ONodeAddressStub nodeAddressStub = new ONodeAddressStub(nodeId);
when(nodeLocal.state()).thenReturn(ODHTNode.NodeState.PRODUCTION);
when(nodeLocal.getNodeAddress()).thenReturn(nodeAddressStub);
final ORID startRecordId = new ORecordId(1, new OClusterPositionNodeId(idToStart));
final ORID nextRecordId = startRecordId.nextRid();
final Iterator<ORecordMetadata> metadataIterator = mock(Iterator.class);
when(nodeLocal.getLocalRingIterator(STORAGE_NAME, startRecordId.nextRid(), startRecordId))
.thenReturn(metadataIterator);
when(metadataIterator.hasNext()).thenReturn(true);
when(metadataIterator.next())
.thenReturn(new ORecordMetadata(nextRecordId, new ODistributedVersion(0)));
when(nodeLocal.findSuccessor(
((OClusterPositionNodeId) nextRecordId.getClusterPosition()).getNodeId()))
.thenReturn(new ONodeAddressStub(successorNodeId));
final ODHTNode successorNode = mock(ODHTNode.class);
when(nodeLookup.findById(new ONodeAddressStub(successorNodeId))).thenReturn(successorNode);
final ONodeAddress[] recordSuccessors =
new ONodeAddress[] {
new ONodeAddressStub(ONodeId.generateUniqueId()),
new ONodeAddressStub(ONodeId.generateUniqueId()),
new ONodeAddressStub(ONodeId.generateUniqueId())
};
when(successorNode.getSuccessors()).thenReturn(recordSuccessors);
when(replicaDistributionStrategy.chooseReplicas(recordSuccessors, 2, 1))
.thenReturn(
new Set[] {
new HashSet<ONodeAddress>(Arrays.asList(recordSuccessors[0])),
new HashSet<ONodeAddress>(Arrays.asList(recordSuccessors[1]))
});
final ArrayList<ODocument> missedRecords = new ArrayList<ODocument>();
final List<ORecordMetadata> missedMetadata = new ArrayList<ORecordMetadata>();
for (int i = 0; i < 10; i++) {
final ORecordId missedRid =
new ORecordId(1, new OClusterPositionNodeId(ONodeId.generateUniqueId()));
final ODocument doc = new ODocument();
doc.setIdentity(missedRid);
doc.field("value", "data");
missedRecords.add(doc);
missedMetadata.add(new ORecordMetadata(missedRid, doc.getRecordVersion()));
}
when(nodeLocal.getLocalRingIterator(
STORAGE_NAME,
startRecordId,
new ORecordId(1, new OClusterPositionNodeId(successorNodeId))))
.thenReturn(missedMetadata.iterator());
final ODHTNode firstReplicaHolder = mock(ODHTNode.class);
final ODHTNode secondReplicaHolder = mock(ODHTNode.class);
when(nodeLookup.findById(recordSuccessors[0])).thenReturn(firstReplicaHolder);
when(nodeLookup.findById(recordSuccessors[1])).thenReturn(secondReplicaHolder);
final ORID[] missedIDs = new ORID[10];
for (int i = 0; i < 10; i++) missedIDs[i] = missedMetadata.get(i).getRid();
when(successorNode.findMissedRecords(
STORAGE_NAME, missedMetadata.toArray(new ORecordMetadata[0])))
.thenReturn(missedIDs);
when(firstReplicaHolder.findMissedRecords(
STORAGE_NAME, missedMetadata.toArray(new ORecordMetadata[0])))
.thenReturn(missedIDs);
when(secondReplicaHolder.findMissedRecords(
STORAGE_NAME, missedMetadata.toArray(new ORecordMetadata[0])))
.thenReturn(missedIDs);
for (ODocument missedRecord : missedRecords)
when((ORecordInternal) nodeLocal.readRecordLocal(STORAGE_NAME, missedRecord.getIdentity()))
.thenReturn(missedRecord);
final ODocument outOfDateRecordOne = missedRecords.get(2);
final ODocument outOfDateRecordTwo = missedRecords.get(3);
doThrow(
new OConcurrentModificationException(
outOfDateRecordOne.getIdentity(),
new ODistributedVersion(0),
outOfDateRecordOne.getRecordVersion(),
0))
.when(nodeLocal)
.cleanOutRecord(
STORAGE_NAME, outOfDateRecordOne.getIdentity(), outOfDateRecordOne.getRecordVersion());
doThrow(
new OConcurrentModificationException(
outOfDateRecordTwo.getIdentity(),
new ODistributedVersion(0),
outOfDateRecordTwo.getRecordVersion(),
0))
.when(nodeLocal)
.cleanOutRecord(
STORAGE_NAME, outOfDateRecordTwo.getIdentity(), outOfDateRecordTwo.getRecordVersion());
final ONodeId result =
globalMaintenanceProtocol.reallocateWrongPlacedReplicas(
STORAGE_NAME, CLUSTER_ID, nodeLocal, idToStart, 2, 1);
Assert.assertEquals(result, successorNodeId);
for (ODocument record : missedRecords) {
verify(successorNode).updateReplica(STORAGE_NAME, record, false);
verify(firstReplicaHolder).updateReplica(STORAGE_NAME, record, false);
verify(secondReplicaHolder).updateReplica(STORAGE_NAME, record, false);
verify(nodeLocal)
.cleanOutRecord(STORAGE_NAME, record.getIdentity(), record.getRecordVersion());
}
}
public void testReallocateWrongPlacedReplicasStateNodeRecordsNodeIsAsyncReplicaHolder() {
final ONodeId idToStart = ONodeId.generateUniqueId();
final ONodeId nodeId = ONodeId.generateUniqueId();
final ONodeId successorNodeId = ONodeId.generateUniqueId();
final ONodeAddressStub nodeAddressStub = new ONodeAddressStub(nodeId);
when(nodeLocal.state()).thenReturn(ODHTNode.NodeState.PRODUCTION);
when(nodeLocal.getNodeAddress()).thenReturn(nodeAddressStub);
final ORID startRecordId = new ORecordId(1, new OClusterPositionNodeId(idToStart));
final ORID nextRecordId = startRecordId.nextRid();
final Iterator<ORecordMetadata> metadataIterator = mock(Iterator.class);
when(nodeLocal.getLocalRingIterator(STORAGE_NAME, startRecordId.nextRid(), startRecordId))
.thenReturn(metadataIterator);
when(metadataIterator.hasNext()).thenReturn(true);
when(metadataIterator.next())
.thenReturn(new ORecordMetadata(nextRecordId, new ODistributedVersion(0)));
when(nodeLocal.findSuccessor(
((OClusterPositionNodeId) nextRecordId.getClusterPosition()).getNodeId()))
.thenReturn(new ONodeAddressStub(successorNodeId));
final ODHTNode successorNode = mock(ODHTNode.class);
when(nodeLookup.findById(new ONodeAddressStub(successorNodeId))).thenReturn(successorNode);
final ONodeAddress[] recordSuccessors =
new ONodeAddress[] {
new ONodeAddressStub(nodeId),
new ONodeAddressStub(ONodeId.generateUniqueId()),
new ONodeAddressStub(ONodeId.generateUniqueId())
};
when(successorNode.getSuccessors()).thenReturn(recordSuccessors);
when(replicaDistributionStrategy.chooseReplicas(recordSuccessors, 1, 1))
.thenReturn(
new Set[] {
new HashSet<ONodeAddress>(Arrays.asList(recordSuccessors[1])),
new HashSet<ONodeAddress>(Arrays.asList(recordSuccessors[0]))
});
final ONodeId result =
globalMaintenanceProtocol.reallocateWrongPlacedReplicas(
STORAGE_NAME, CLUSTER_ID, nodeLocal, idToStart, 1, 1);
Assert.assertEquals(result, nodeId);
verify(nodeLocal).state();
verify(nodeLocal).getNodeAddress();
verify(nodeLocal).getLocalRingIterator(STORAGE_NAME, startRecordId.nextRid(), startRecordId);
verify(nodeLocal)
.findSuccessor(((OClusterPositionNodeId) nextRecordId.getClusterPosition()).getNodeId());
verify(nodeLookup).findById(new ONodeAddressStub(successorNodeId));
verifyZeroInteractions(nodeLocal);
verifyNoMoreInteractions(nodeLocal);
verifyNoMoreInteractions(nodeLookup);
}