/**
* Load an out-of-core partition in memory.
*
* @param partitionId Partition id
*/
private void loadPartition(Integer partitionId) {
if (loadedPartition != null) {
if (loadedPartition.getId() == partitionId) {
return;
}
if (LOG.isInfoEnabled()) {
LOG.info("loadPartition: moving partition " + loadedPartition.getId() + " out of core");
}
try {
writePartition(loadedPartition);
onDiskPartitions.put(loadedPartition.getId(), loadedPartition.getVertices().size());
loadedPartition = null;
} catch (IOException e) {
throw new IllegalStateException(
"loadPartition: failed writing " + "partition " + loadedPartition.getId() + " to disk",
e);
}
}
if (LOG.isInfoEnabled()) {
LOG.info("loadPartition: loading partition " + partitionId + " in memory");
}
try {
loadedPartition = readPartition(partitionId);
} catch (IOException e) {
throw new IllegalStateException(
"loadPartition: failed reading " + "partition " + partitionId + " from disk");
}
}
/**
* Adds Partition to and validates Partition is unique. A duplicate Partition results in an
* exception.
*/
private void addPartition(Partition partition) {
if (partitionMap.put(ByteBuffer.wrap(partition.getBytes()), partition) != null) {
throw new IllegalStateException("Duplicate Partition detected: " + partition.toString());
}
if (partition.getId() >= maxPartitionId) {
maxPartitionId = partition.getId() + 1;
}
}
@Override
public void addPartition(Partition<I, V, E, M> partition) {
Lock lock = createLock(partition.getId());
if (lock == null) {
throw new IllegalStateException(
"addPartition: partition " + partition.getId() + " already exists");
}
addPartitionNoLock(partition);
lock.unlock();
}
@Override
public void addPartition(Partition<I, V, E> partition) {
Partition<I, V, E> oldPartition = partitions.get(partition.getId());
if (oldPartition == null) {
oldPartition = partitions.putIfAbsent(partition.getId(), partition);
if (oldPartition == null) {
return;
}
}
oldPartition.addPartition(partition);
}
@Override
public Map<EntityID, Integer> computeNeededAgents(List<Partition> partitions, int agents) {
Map<EntityID, Integer> neededAgentMap = new FastMap<EntityID, Integer>();
int numberOfPartitions = 0;
for (Partition partition : partitions) {
numberOfPartitions++;
if (numberOfPartitions <= agents) {
neededAgentMap.put(partition.getId(), 1);
} else {
neededAgentMap.put(partition.getId(), 0);
}
}
return neededAgentMap;
}
/**
* Add a new partition without requiring a lock.
*
* @param partition Partition to be added
*/
private void addPartitionNoLock(Partition<I, V, E, M> partition) {
synchronized (inMemoryPartitions) {
if (inMemoryPartitions.size() + 1 < maxInMemoryPartitions) {
inMemoryPartitions.put(partition.getId(), partition);
return;
}
}
try {
writePartition(partition);
onDiskPartitions.put(partition.getId(), partition.getVertices().size());
} catch (IOException e) {
throw new IllegalStateException(
"addPartition: failed writing " + "partition " + partition.getId() + "to disk");
}
}
/**
* Write a partition to disk.
*
* @param partition The partition object to write
* @throws java.io.IOException
*/
private void writePartition(Partition<I, V, E, M> partition) throws IOException {
File file = new File(getPartitionPath(partition.getId()));
file.getParentFile().mkdirs();
file.createNewFile();
DataOutputStream outputStream =
new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
for (Vertex<I, V, E, M> vertex : partition.getVertices()) {
vertex.write(outputStream);
}
outputStream.close();
}
protected void validatePartitionIds() {
for (Partition partition : partitionMap.values()) {
long partitionId = partition.getId();
if (partitionId < MinPartitionId) {
throw new IllegalStateException("Partition has invalid ID: Less than " + MinPartitionId);
}
if (partitionId >= maxPartitionId) {
throw new IllegalStateException(
"Partition has invalid ID: Greater than or equal to " + maxPartitionId);
}
}
}
@Override
public void deletePartition(Integer partitionId) {
partitionLocks.remove(partitionId);
if (inMemoryPartitions.containsKey(partitionId)) {
inMemoryPartitions.remove(partitionId);
} else {
if (loadedPartition != null && loadedPartition.getId() == partitionId) {
loadedPartition = null;
} else {
File file = new File(getPartitionPath(partitionId));
file.delete();
}
onDiskPartitions.remove(partitionId);
}
}
@Override
public Partition<I, V, E, M> removePartition(Integer partitionId) {
partitionLocks.remove(partitionId);
if (onDiskPartitions.containsKey(partitionId)) {
Partition<I, V, E, M> partition;
if (loadedPartition != null && loadedPartition.getId() == partitionId) {
partition = loadedPartition;
loadedPartition = null;
} else {
try {
partition = readPartition(partitionId);
} catch (IOException e) {
throw new IllegalStateException(
"removePartition: failed reading " + "partition " + partitionId + " from disk", e);
}
}
onDiskPartitions.remove(partitionId);
return partition;
} else {
return inMemoryPartitions.remove(partitionId);
}
}
@Override
public void addPartitionVertices(Integer partitionId, Collection<Vertex<I, V, E, M>> vertices) {
if (inMemoryPartitions.containsKey(partitionId)) {
Partition<I, V, E, M> partition = inMemoryPartitions.get(partitionId);
partition.putVertices(vertices);
} else if (onDiskPartitions.containsKey(partitionId)) {
Lock lock = getLock(partitionId);
lock.lock();
if (loadedPartition != null && loadedPartition.getId() == partitionId) {
loadedPartition.putVertices(vertices);
} else {
try {
appendVertices(partitionId, vertices);
onDiskPartitions.put(partitionId, onDiskPartitions.get(partitionId) + vertices.size());
} catch (IOException e) {
throw new IllegalStateException(
"addPartitionVertices: failed "
+ "writing vertices to partition "
+ partitionId
+ " on disk",
e);
}
}
lock.unlock();
} else {
Lock lock = createLock(partitionId);
if (lock != null) {
addPartitionNoLock(new Partition<I, V, E, M>(conf, partitionId));
lock.unlock();
} else {
// Another thread is already creating the partition,
// so we make sure it's done before repeating the call.
lock = getLock(partitionId);
lock.lock();
lock.unlock();
}
addPartitionVertices(partitionId, vertices);
}
}
@Test
public void testPartitions() {
try {
// get row count of experiments in the dataset
int expected = getDataSet().getTable("_Partition").getRowCount();
// get number of experiments from the DAO
int actual = getPartitionDAO().count();
// test data contains 2 experiments, check size of returned list
TestCase.assertEquals("Wrong number of partitions", expected, actual);
System.out.println("Expected number of partitions: " + expected + ", actual: " + actual);
for (Partition d : random(getPartitionDAO(), actual, 5)) {
TestCase.assertNotNull(d);
TestCase.assertNotNull(d.getId());
}
} catch (Exception e) {
e.printStackTrace();
TestCase.fail();
}
}
@Test
public void testFailureTracking() throws Exception {
final IPartitionProxy partitionProxy = createNiceMock(IPartitionProxy.class);
final IPartitionProxy partitionProxy2 = createNiceMock(IPartitionProxy.class);
long msgId = 2L;
IFailureTrackingManager failMgr = ManagementContainer.getInstance().getFailureTrackingManager();
StorageImporter importer = new MockStorageImporter();
Customer customer = TestUtils.createTestCustomer();
try {
PartitionStoreStage stage = new PartitionStoreStage(importer);
Pipeline pipeline =
PipelineBuilder.start(
ManagementContainer.getInstance().getConfiguration(),
MessageImporter.PIPELINE_CONFIG_SECTION)
.singleton("store-stage", stage)
.get();
File envelopeFile =
File.createTempFile(
"PartitionStoreTest", "test", ManagementContainer.getInstance().getNfsRoot());
IMessageContext msgCtx = new MessageContext(envelopeFile, null);
msgCtx.setCustomer(customer);
msgCtx.setIndexMessage(true);
msgCtx.setInternalId(msgId);
msgCtx.setPartition(m_part);
msgCtx.setPartitionID(m_part.getId());
Collection<IMessageContext> msgs = Collections.singletonList(msgCtx);
Collection<IMessageContext> emptyResult = Collections.emptyList();
expect(partitionProxy.storeMessages(msgs)).andReturn(emptyResult).anyTimes();
replay(partitionProxy);
PartitionManager pm =
new PartitionManager(
ManagementContainer.getInstance().getPool(ManagementContainer.AM_POOL_NAME)) {
@Override
public IPartitionProxy getContentProxy(int id) {
return partitionProxy;
}
};
PartitionManagerTest.preparePartitionManager(pm);
stage.setPartitionManager(pm);
FailureRecord failureRecord =
FailureTrackingHelper.buildFailureRecord(msgCtx, FailureCategory.STORAGE, null, null);
failureRecord.setStoredOnPartition(false);
failMgr.insertMessageFailure(failureRecord);
msgCtx.setInternalFailureId(msgId);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId);
assertTrue("Should find stored on partition to be true", failureRecord.isStoredOnPartition());
Map<String, String> props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// confirm that transient ID gets set back to real ID of message
// remove the existing failure record
failMgr.removeFailure(msgId);
// change the ID
msgCtx.setInternalFailureId(msgId + 1);
msgCtx.setInternalId(msgId + 1);
// insert the new failure record
failureRecord =
FailureTrackingHelper.buildFailureRecord(msgCtx, FailureCategory.STORAGE, null, null);
failMgr.insertMessageFailure(failureRecord);
// old record should be gone
failureRecord = failMgr.getFailureRecord(msgId);
assertNull("Should no longer find the failure record with permanent ID", failureRecord);
// check that the new record can be fetched
failureRecord = failMgr.getFailureRecord(msgId + 1);
assertNotNull("Should find transient ID failure record", failureRecord);
msgCtx.appendToEnvelope(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID, msgId + 1);
props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId + 1),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// now assign back the permanent ID and process through the pipeline
msgCtx.setInternalId(msgId);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId + 1);
assertNull("Should not find any record based on transient message ID", failureRecord);
failureRecord = failMgr.getFailureRecord(msgId);
assertNotNull("Should find failure record for permanent ID", failureRecord);
assertTrue("Should find stored on partition to be true", failureRecord.isStoredOnPartition());
// envelope should now reflect permanent ID
props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// confirm that state does not change on failure
// force an exception on any call to storeMessages
pm =
new PartitionManager(
ManagementContainer.getInstance().getPool(ManagementContainer.AM_POOL_NAME)) {
@Override
public IPartitionProxy getContentProxy(int id) {
return partitionProxy2;
}
};
PartitionManagerTest.preparePartitionManager(pm);
stage.setPartitionManager(pm);
expect(partitionProxy2.storeMessages(msgs))
.andThrow(new IPartitionProxy.ServiceNotAvailableException(new Exception()))
.anyTimes();
replay(partitionProxy2);
failureRecord.setStoredOnPartition(false);
failMgr.insertMessageFailure(failureRecord);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId);
assertNotNull("Should find the failure record for message", failureRecord);
assertFalse(
"Should find stored on partition to be true", failureRecord.isStoredOnPartition());
} finally {
importer.stop();
TestUtils.quietDeleteCustomer(customer);
}
}
/**
* Create new pod and pass environment variables.
*
* @param memberContext
* @param kubernetesApi
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createPod(
ClusterContext clusterContext,
MemberContext memberContext,
KubernetesApiClient kubernetesApi,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String applicationId = memberContext.getApplicationId();
String cartridgeType = memberContext.getCartridgeType();
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
applicationId, cartridgeType, memberId));
}
Partition partition = memberContext.getPartition();
if (partition == null) {
String message =
String.format(
"Partition not found in member context: [application] %s [cartridge] %s "
+ "[member] %s ",
applicationId, cartridgeType, memberId);
log.error(message);
throw new RuntimeException(message);
}
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message = "Could not find cartridge: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
// Set default values to zero to avoid cpu and memory restrictions
int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
if (cpuProperty != null) {
cpu = Integer.parseInt(cpuProperty.getValue());
}
Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
if (memoryProperty != null) {
memory = Integer.parseInt(memoryProperty.getValue());
}
IaasProvider iaasProvider =
CloudControllerContext.getInstance()
.getIaasProviderOfPartition(cartridge.getType(), partition.getId());
if (iaasProvider == null) {
String message = "Could not find iaas provider: [partition] " + partition.getId();
log.error(message);
throw new RuntimeException(message);
}
// Add dynamic payload to the member context
memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));
// Create pod
long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
String podId = "pod" + "-" + podSeqNo;
String podLabel = KubernetesIaasUtil.fixSpecialCharacters(clusterId);
String dockerImage = iaasProvider.getImage();
List<EnvVar> environmentVariables =
KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);
List<ContainerPort> ports =
KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));
log.info(
String.format(
"Starting pod: [application] %s [cartridge] %s [member] %s "
+ "[cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
cpu,
memory));
kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);
log.info(
String.format(
"Pod started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
podId,
cpu,
memory));
// Add pod id to member context
memberContext.setKubernetesPodId(podId);
memberContext.setKubernetesPodLabel(podLabel);
// Create instance metadata
InstanceMetadata instanceMetadata = new InstanceMetadata();
instanceMetadata.setImageId(dockerImage);
instanceMetadata.setCpu(cpu);
instanceMetadata.setRam(memory);
memberContext.setInstanceMetadata(instanceMetadata);
// Persist cloud controller context
CloudControllerContext.getInstance().persist();
}