/**
* Filter a list of VolumeDescriptors by type(s).
*
* @param descriptors -- Original list.
* @param inclusive -- Types to be included (or null if not used).
* @param exclusive -- Types to be excluded (or null if not used).
* @return List<VolumeDescriptor>
*/
public static List<VolumeDescriptor> filterByType(
List<VolumeDescriptor> descriptors, Type[] inclusive, Type[] exclusive) {
List<VolumeDescriptor> result = new ArrayList<VolumeDescriptor>();
if (descriptors == null) {
return result;
}
HashSet<Type> included = new HashSet<Type>();
if (inclusive != null) {
included.addAll(Arrays.asList(inclusive));
}
HashSet<Type> excluded = new HashSet<Type>();
if (exclusive != null) {
excluded.addAll(Arrays.asList(exclusive));
}
for (VolumeDescriptor desc : descriptors) {
if (excluded.contains(desc.getType())) {
continue;
}
if (included.isEmpty() || included.contains(desc.getType())) {
result.add(desc);
}
}
return result;
}
/**
* Helper method to retrieve the change vpool volume hiding in the volume descriptors and to find
* the new vpool.
*
* @param descriptors list of volumes
* @return Map<URI,URI> of the vpool change volume and the old vpool associated to it.
*/
public static Map<URI, URI> createVolumeToNewVpoolMap(List<VolumeDescriptor> descriptors) {
Map<URI, URI> volumesToNewVpoolMap = new HashMap<URI, URI>();
if (descriptors != null) {
for (VolumeDescriptor volumeDescriptor : descriptors) {
if (volumeDescriptor.getParameters() != null) {
if (volumeDescriptor
.getParameters()
.get(VolumeDescriptor.PARAM_VPOOL_CHANGE_EXISTING_VOLUME_ID)
!= null) {
URI volumeURI =
(URI)
volumeDescriptor
.getParameters()
.get(VolumeDescriptor.PARAM_VPOOL_CHANGE_EXISTING_VOLUME_ID);
URI newVpoolURI =
(URI)
volumeDescriptor
.getParameters()
.get(VolumeDescriptor.PARAM_VPOOL_CHANGE_NEW_VPOOL_ID);
volumesToNewVpoolMap.put(volumeURI, newVpoolURI);
}
}
}
}
return volumesToNewVpoolMap;
}
/**
* Return a List of URIs for the volumes.
*
* @param descriptors List<VolumeDescriptors>
* @return List<URI> of volumes in the input list
*/
public static List<URI> getVolumeURIs(List<VolumeDescriptor> descriptors) {
Set<URI> volumeURIs = new HashSet<>();
for (VolumeDescriptor desc : descriptors) {
volumeURIs.add(desc.getVolumeURI());
}
List<URI> volumeList = new ArrayList<>();
volumeList.addAll(volumeURIs);
return volumeList;
}
/**
* Return a map of pool URI to a list of descriptors in that pool.
*
* @param descriptors List<VolumeDescriptors>
* @return Map of pool URI to List<VolumeDescriptors> in that pool
*/
public static Map<URI, List<VolumeDescriptor>> getPoolMap(List<VolumeDescriptor> descriptors) {
HashMap<URI, List<VolumeDescriptor>> poolMap = new HashMap<URI, List<VolumeDescriptor>>();
for (VolumeDescriptor desc : descriptors) {
if (poolMap.get(desc.getPoolURI()) == null) {
poolMap.put(desc.getPoolURI(), new ArrayList<VolumeDescriptor>());
}
poolMap.get(desc.getPoolURI()).add(desc);
}
return poolMap;
}
/**
* Returns all the descriptors of a given type.
*
* @param descriptors List<VolumeDescriptor> input list
* @param type enum Type
* @return returns list elements matching given type
*/
public static List<VolumeDescriptor> getDescriptors(
List<VolumeDescriptor> descriptors, Type type) {
List<VolumeDescriptor> list = new ArrayList<VolumeDescriptor>();
for (VolumeDescriptor descriptor : descriptors) {
if (descriptor.getType() == type) {
list.add(descriptor);
}
}
return list;
}
private void handleBlockVolumeErrors(DbClient dbClient) {
for (VolumeDescriptor volumeDescriptor :
VolumeDescriptor.getDescriptors(_volumeDescriptors, VolumeDescriptor.Type.BLOCK_DATA)) {
Volume volume = dbClient.queryObject(Volume.class, volumeDescriptor.getVolumeURI());
if (volume != null && (volume.getNativeId() == null || volume.getNativeId().equals(""))) {
_log.info("No native id was present on volume {}, marking inactive", volume.getLabel());
dbClient.markForDeletion(volume);
}
}
}
/**
* Helper that checks any boolean property in the volume descriptor. False is default response.
*
* @param descriptors descriptor list
* @param param string of parameter to check
* @return true if the property is set and true, false otherwise
*/
private static boolean isPropertyEnabled(List<VolumeDescriptor> descriptors, String param) {
if (descriptors != null) {
for (VolumeDescriptor volumeDescriptor : descriptors) {
if (volumeDescriptor.getParameters() != null) {
if (volumeDescriptor.getParameters().get(param) != null) {
return (Boolean) volumeDescriptor.getParameters().get(param);
}
}
}
}
return false;
}
private void handleVplexVolumeErrors(DbClient dbClient) {
List<String> finalMessages = new ArrayList<String>();
for (VolumeDescriptor volumeDescriptor :
VolumeDescriptor.getDescriptors(
_volumeDescriptors, VolumeDescriptor.Type.VPLEX_VIRT_VOLUME)) {
Volume volume = dbClient.queryObject(Volume.class, volumeDescriptor.getVolumeURI());
_log.info("Looking at VPLEX virtual volume {}", volume.getLabel());
boolean deactivateVirtualVolume = true;
List<String> livingVolumeNames = new ArrayList<String>();
_log.info("Its associated volumes are: " + volume.getAssociatedVolumes());
for (String associatedVolumeUri : volume.getAssociatedVolumes()) {
Volume associatedVolume =
dbClient.queryObject(Volume.class, URI.create(associatedVolumeUri));
if (associatedVolume != null && !associatedVolume.getInactive()) {
_log.warn(
"VPLEX virtual volume {} has active associated volume {}",
volume.getLabel(),
associatedVolume.getLabel());
livingVolumeNames.add(associatedVolume.getLabel());
deactivateVirtualVolume = false;
}
}
if (deactivateVirtualVolume) {
_log.info(
"VPLEX virtual volume {} has no active associated volumes, marking for deletion",
volume.getLabel());
dbClient.markForDeletion(volume);
} else {
String message =
"VPLEX virtual volume "
+ volume.getLabel()
+ " will not be marked for deletion "
+ "because it still has active associated volumes (";
message += Joiner.on(",").join(livingVolumeNames) + ")";
finalMessages.add(message);
_log.warn(message);
}
}
if (!finalMessages.isEmpty()) {
String finalMessage = Joiner.on("; ").join(finalMessages) + ".";
_log.error(finalMessage);
}
}
/**
* Returns all descriptors that have the PARAM_DO_NOT_DELETE_VOLUME flag set to true.
*
* @param descriptors List of descriptors to check
* @return all descriptors that have the PARAM_DO_NOT_DELETE_VOLUME flag set to true
*/
public static List<VolumeDescriptor> getDoNotDeleteDescriptors(
List<VolumeDescriptor> descriptors) {
List<VolumeDescriptor> doNotDeleteDescriptors = new ArrayList<VolumeDescriptor>();
if (descriptors != null && !descriptors.isEmpty()) {
for (VolumeDescriptor descriptor : descriptors) {
if (descriptor.getParameters() != null
&& descriptor.getParameters().get(VolumeDescriptor.PARAM_DO_NOT_DELETE_VOLUME) != null
&& descriptor
.getParameters()
.get(VolumeDescriptor.PARAM_DO_NOT_DELETE_VOLUME)
.equals(Boolean.TRUE)) {
doNotDeleteDescriptors.add(descriptor);
}
}
}
return doNotDeleteDescriptors;
}
/**
* Helper method to retrieve the vpool change volume hiding in the volume descriptors
*
* @param descriptors list of volumes
* @return URI of the vpool change volume
*/
public static URI getVirtualPoolChangeVolume(List<VolumeDescriptor> descriptors) {
if (descriptors != null) {
for (VolumeDescriptor volumeDescriptor : descriptors) {
if (volumeDescriptor.getParameters() != null) {
if ((URI)
volumeDescriptor
.getParameters()
.get(VolumeDescriptor.PARAM_VPOOL_CHANGE_VOLUME_ID)
!= null) {
return (URI)
volumeDescriptor.getParameters().get(VolumeDescriptor.PARAM_VPOOL_CHANGE_VOLUME_ID);
}
}
}
}
return null;
}
/**
* Return a map of pool URI to a list of descriptors in that pool of each size.
*
* @param descriptors List<VolumeDescriptors>
* @return Map of pool URI to a map of identical sized volumes to List<VolumeDescriptors> in that
* pool of that size
*/
public static Map<URI, Map<Long, List<VolumeDescriptor>>> getPoolSizeMap(
List<VolumeDescriptor> descriptors) {
Map<URI, Map<Long, List<VolumeDescriptor>>> poolSizeMap =
new HashMap<URI, Map<Long, List<VolumeDescriptor>>>();
for (VolumeDescriptor desc : descriptors) {
// If the outside pool map doesn't exist, create it.
if (poolSizeMap.get(desc._poolURI) == null) {
poolSizeMap.put(desc._poolURI, new HashMap<Long, List<VolumeDescriptor>>());
}
// If the inside size map doesn't exist, create it.
if (poolSizeMap.get(desc._poolURI).get(desc.getVolumeSize()) == null) {
poolSizeMap.get(desc._poolURI).put(desc.getVolumeSize(), new ArrayList<VolumeDescriptor>());
}
// Add volume to the list
poolSizeMap.get(desc._poolURI).get(desc.getVolumeSize()).add(desc);
}
return poolSizeMap;
}
/**
* Helper method to retrieve the source vpool change volume hiding in the volume descriptors.
* Needed primarily for rollback as the old vpool URI will be in the map: Map<Volume URI, Old
* Vpool URI>
*
* @param descriptors list of volumes
* @return Map<URI,URI> of the vpool change volume and the old vpool associated to it.
*/
public static Map<URI, URI> getAllVirtualPoolChangeSourceVolumes(
List<VolumeDescriptor> descriptors) {
Map<URI, URI> sourceVolumes = new HashMap<URI, URI>();
if (descriptors != null) {
for (VolumeDescriptor volumeDescriptor : descriptors) {
if (volumeDescriptor.getParameters() != null) {
if ((URI) volumeDescriptor.getParameters().get(VolumeDescriptor.PARAM_VPOOL_OLD_VPOOL_ID)
!= null) {
URI volumeURI =
(URI)
volumeDescriptor
.getParameters()
.get(VolumeDescriptor.PARAM_VPOOL_CHANGE_VOLUME_ID);
URI oldVpoolURI =
(URI)
volumeDescriptor.getParameters().get(VolumeDescriptor.PARAM_VPOOL_OLD_VPOOL_ID);
sourceVolumes.put(volumeURI, oldVpoolURI);
}
}
}
}
return sourceVolumes;
}
/** {@inheritDoc} */
@Override
public TaskList create(
List<BlockObject> fcSourceObjList,
VirtualArray varray,
String name,
boolean createInactive,
int count,
String taskId) {
// Populate the descriptors list with all volumes required
// to create the VPLEX volume copies.
int sourceCounter = 0;
URI vplexSrcSystemId = null;
TaskList taskList = new TaskList();
List<Volume> vplexCopyVolumes = new ArrayList<Volume>();
List<VolumeDescriptor> volumeDescriptors = new ArrayList<VolumeDescriptor>();
List<BlockObject> sortedSourceObjectList = sortFullCopySourceList(fcSourceObjList);
for (BlockObject fcSourceObj : sortedSourceObjectList) {
URI fcSourceURI = fcSourceObj.getId();
if (URIUtil.isType(fcSourceURI, BlockSnapshot.class)) {
// Full copy of snapshots is not supported for VPLEX.
return super.create(sortedSourceObjectList, varray, name, createInactive, count, taskId);
}
Volume vplexSrcVolume = (Volume) fcSourceObj;
String copyName = name + (sortedSourceObjectList.size() > 1 ? "-" + ++sourceCounter : "");
// Create a volume descriptor for the source VPLEX volume being copied.
// and add it to the descriptors list. Be sure to identify this VPLEX
// volume as the source volume being copied.
vplexSrcSystemId = fcSourceObj.getStorageController();
VolumeDescriptor vplexSrcVolumeDescr =
new VolumeDescriptor(
VolumeDescriptor.Type.VPLEX_VIRT_VOLUME, vplexSrcSystemId, fcSourceURI, null, null);
Map<String, Object> descrParams = new HashMap<String, Object>();
descrParams.put(VolumeDescriptor.PARAM_IS_COPY_SOURCE_ID, Boolean.TRUE);
vplexSrcVolumeDescr.setParameters(descrParams);
volumeDescriptors.add(vplexSrcVolumeDescr);
// Get some info about the VPLEX volume being copied and its storage
// system.
Project vplexSrcProject =
BlockFullCopyUtils.queryFullCopySourceProject(fcSourceObj, _dbClient);
StorageSystem vplexSrcSystem = _dbClient.queryObject(StorageSystem.class, vplexSrcSystemId);
Project vplexSystemProject =
VPlexBlockServiceApiImpl.getVplexProject(vplexSrcSystem, _dbClient, _tenantsService);
// For the VPLEX volume being copied, determine which of the associated
// backend volumes is the primary and, for distributed volumes, which
// is the HA volume. The primary volume will be natively copied and we
// we need to place and prepare a volume to hold the copy. This copy
// will be the primary backend volume for the VPLEX volume copy. For
// a distributed virtual volume, we will need to place and prepare
// a volume to hold the HA volume of the VPLEX volume copy.
Volume vplexSrcPrimaryVolume = null;
Volume vplexSrcHAVolume = null;
StringSet assocVolumeURIs = vplexSrcVolume.getAssociatedVolumes();
Iterator<String> assocVolumeURIsIter = assocVolumeURIs.iterator();
while (assocVolumeURIsIter.hasNext()) {
URI assocVolumeURI = URI.create(assocVolumeURIsIter.next());
Volume assocVolume = _dbClient.queryObject(Volume.class, assocVolumeURI);
if (assocVolume.getVirtualArray().toString().equals(varray.getId().toString())) {
vplexSrcPrimaryVolume = assocVolume;
} else {
vplexSrcHAVolume = assocVolume;
}
}
// Get the capabilities
VirtualPool vpool = BlockFullCopyUtils.queryFullCopySourceVPool(fcSourceObj, _dbClient);
VirtualPoolCapabilityValuesWrapper capabilities =
getCapabilitiesForFullCopyCreate(fcSourceObj, vpool, count);
// Get the number of copies to create and the size of the volumes.
// Note that for the size, we must use the actual provisioned size
// of the source side backend volume. The size passed in the
// capabilities will be the size of the VPLEX volume. When the
// source side backend volume for the copy is provisioned, you
// might not get that actual size. On VMAX, the size will be slightly
// larger while for VNX the size will be exactly what is requested.
// So, if the source side is a VMAX, the source side for the copy
// will be slightly larger than the size in the capabilities. If the HA
// side is VNX and we use the size in the capabilities, then you will
// get exactly that size for the HA backend volume. As a result, source
// side backend volume for the copy will be slightly larger than the
// HA side. Now the way a VPLEX copy is made is it uses native full
// copy to create a native full copy of the source side backend
// volume. It then provisions the HA side volume. The new source side
// backend copy is then imported into VPLEX in the same way as is done
// for a vpool change that imports a volume to VPLEX. This code in the
// VPLEX controller creates a local VPLEX volume using the source side
// copy and for a distributed volume it then attaches as a remote
// mirror the HA backend volume that is provisioned. If the HA volume
// is slightly smaller, then this will fail on the VPLEX. So, we must
// ensure that HA side volume is big enough by using the provisioned
// capacity of the source side backend volume of the VPLEX volume being
// copied.
long size = vplexSrcPrimaryVolume.getProvisionedCapacity();
// Place and prepare a volume for each copy to serve as a native
// copy of a VPLEX backend volume. The VPLEX backend volume that
// is copied is the backend volume in the same virtual array as the
// VPLEX volume i.e, the primary backend volume. Create
// descriptors for these prepared volumes and add them to the list.
List<Volume> vplexCopyPrimaryVolumes =
prepareFullCopyPrimaryVolumes(
copyName, count, vplexSrcPrimaryVolume, capabilities, volumeDescriptors);
// If the VPLEX volume being copied is distributed, then the VPLEX
// HA volume should be non-null. We use the VPLEX scheduler to place
// and then prepare volumes for the HA volumes of the VPLEX volume
// copies. This should be done in the same manner as is done for the
// import volume routine. This is because to form the VPLEX volume
// copy we import the copy of the primary backend volume.
List<Volume> vplexCopyHAVolumes = new ArrayList<Volume>();
if (vplexSrcHAVolume != null) {
vplexCopyHAVolumes.addAll(
prepareFullCopyHAVolumes(
copyName,
count,
size,
vplexSrcSystem,
vplexSystemProject,
varray,
vplexSrcHAVolume,
taskId,
volumeDescriptors));
}
// For each copy to be created, place and prepare a volume for the
// primary backend volume copy. When copying a distributed VPLEX
// volume, we also must place and prepare a volume for the HA
// backend volume copy. Lastly, we must prepare a volume for the
// VPLEX volume copy. Create descriptors for these prepared volumes
// and add them to the volume descriptors list.
for (int i = 0; i < count; i++) {
// Prepare a new VPLEX volume for each copy.
Volume vplexCopyPrimaryVolume = vplexCopyPrimaryVolumes.get(i);
Volume vplexCopyHAVolume = null;
if (vplexCopyHAVolumes.size() != 0) {
vplexCopyHAVolume = vplexCopyHAVolumes.get(i);
}
Volume vplexCopyVolume =
prepareFullCopyVPlexVolume(
copyName,
count,
i,
size,
vplexSrcVolume,
vplexSrcProject,
varray,
vpool,
vplexSrcSystemId,
vplexCopyPrimaryVolume,
vplexCopyHAVolume,
taskId,
volumeDescriptors);
vplexCopyVolumes.add(vplexCopyVolume);
// Create task for each copy.
Operation op = vplexCopyVolume.getOpStatus().get(taskId);
TaskResourceRep task = toTask(vplexCopyVolume, taskId, op);
taskList.getTaskList().add(task);
}
}
// Invoke the VPLEX controller to create the copies.
try {
s_logger.info("Getting VPlex controller {}.", taskId);
VPlexController controller =
getController(VPlexController.class, DiscoveredDataObject.Type.vplex.toString());
// TBD controller needs to be updated to handle CGs.
controller.createFullCopy(vplexSrcSystemId, volumeDescriptors, taskId);
s_logger.info("Successfully invoked controller.");
} catch (InternalException e) {
s_logger.error("Controller error", e);
// Update the status for the VPLEX volume copies and their
// corresponding tasks.
for (Volume vplexCopyVolume : vplexCopyVolumes) {
Operation op = vplexCopyVolume.getOpStatus().get(taskId);
if (op != null) {
op.error(e);
vplexCopyVolume.getOpStatus().updateTaskStatus(taskId, op);
_dbClient.persistObject(vplexCopyVolume);
for (TaskResourceRep task : taskList.getTaskList()) {
if (task.getResource().getId().equals(vplexCopyVolume.getId())) {
task.setState(op.getStatus());
task.setMessage(op.getMessage());
break;
}
}
}
}
// Mark all volumes inactive, except for the VPLEX volume
// we were trying to copy.
for (VolumeDescriptor descriptor : volumeDescriptors) {
if (descriptor.getParameters().get(VolumeDescriptor.PARAM_IS_COPY_SOURCE_ID) == null) {
Volume volume = _dbClient.queryObject(Volume.class, descriptor.getVolumeURI());
volume.setInactive(true);
_dbClient.persistObject(volume);
}
}
}
return taskList;
}
