/**
* Checks the UnManaged Volume's policy with vPool's policy.
*
* @param vPool the vPool
* @param autoTierPolicyId the auto tier policy id on unmanaged volume
* @param system the system
* @return true, if matching, false otherwise
*/
public static boolean checkVPoolValidForUnManagedVolumeAutoTieringPolicy(
VirtualPool vPool, String autoTierPolicyId, StorageSystem system) {
_log.debug("Policy Id: {}, vPool: {}", autoTierPolicyId, vPool);
boolean policyMatching = false;
String policyIdfromVPool = vPool.getAutoTierPolicyName();
if (autoTierPolicyId != null) {
if (policyIdfromVPool != null) {
if (vPool.getUniquePolicyNames()
|| DiscoveredDataObject.Type.vnxblock.name().equalsIgnoreCase(system.getSystemType())) {
// Unique Policy names field will not be set for VNX. vPool will have policy name, not the
// policy's nativeGuid
policyIdfromVPool =
NativeGUIDGenerator.generateAutoTierPolicyNativeGuid(
system.getNativeGuid(),
policyIdfromVPool,
NativeGUIDGenerator.getTieringPolicyKeyForSystem(system));
_log.debug("Policy Id generated: {}", policyIdfromVPool);
}
if (autoTierPolicyId.equalsIgnoreCase(policyIdfromVPool)) {
policyMatching = true;
}
}
} else if ((policyIdfromVPool == null) || (policyIdfromVPool.equalsIgnoreCase("none"))) {
// if policy is not set in both unmanaged volume and vPool. Note
// that the value in the vpool could be set to "none".
policyMatching = true;
}
// Default policy for VNX - match volume with default policy to vPool with no policy as well
if (!policyMatching
&& DiscoveredDataObject.Type.vnxblock.name().equalsIgnoreCase(system.getSystemType())) {
if (autoTierPolicyId != null
&& autoTierPolicyId.contains(VnxFastPolicy.DEFAULT_START_HIGH_THEN_AUTOTIER.name())
&& policyIdfromVPool == null) {
policyMatching = true;
}
}
// Default policy for HDS - match volume with default policy to vPool with no policy as well
if (!policyMatching
&& DiscoveredDataObject.Type.hds.name().equalsIgnoreCase(system.getSystemType())) {
if (autoTierPolicyId != null
&& autoTierPolicyId.contains(HitachiTieringPolicy.All.name())
&& policyIdfromVPool == null) {
policyMatching = true;
}
}
return policyMatching;
}
/**
* find DM or NAS from db using native id
*
* @param system
* @param dbClient
* @param nativeId
* @return
*/
private PhysicalNAS findPhysicalNasByNativeId(
final StorageSystem system, DbClient dbClient, String nativeId) {
URIQueryResultList results = new URIQueryResultList();
PhysicalNAS physicalNas = null;
// Set storage port details to vNas
String nasNativeGuid =
NativeGUIDGenerator.generateNativeGuid(system, nativeId, NativeGUIDGenerator.PHYSICAL_NAS);
dbClient.queryByConstraint(
AlternateIdConstraint.Factory.getPhysicalNasByNativeGuidConstraint(nasNativeGuid), results);
Iterator<URI> iter = results.iterator();
while (iter.hasNext()) {
PhysicalNAS tmpNas = dbClient.queryObject(PhysicalNAS.class, iter.next());
if (tmpNas != null && !tmpNas.getInactive()) {
physicalNas = tmpNas;
_logger.info(
"found physical NAS {}", physicalNas.getNativeGuid() + ":" + physicalNas.getNasName());
break;
}
}
return physicalNas;
}
/**
* Filters supported vPools in UnManaged Volume based on Auto-Tiering Policy.
*
* @param unManagedVolume the UnManaged volume
* @param policyName the policy name associated with UnManaged volume
* @param system the system
* @param dbClient the db client
*/
public static void filterSupportedVpoolsBasedOnTieringPolicy(
UnManagedVolume unManagedVolume, String policyName, StorageSystem system, DbClient dbClient) {
StringSetMap unManagedVolumeInformation = unManagedVolume.getVolumeInformation();
StringSet supportedVpoolURIs =
unManagedVolumeInformation.get(SupportedVolumeInformation.SUPPORTED_VPOOL_LIST.toString());
List<String> vPoolsToRemove = new ArrayList<String>();
if (supportedVpoolURIs != null) {
Iterator<String> itr = supportedVpoolURIs.iterator();
while (itr.hasNext()) {
String uri = itr.next();
VirtualPool vPool = dbClient.queryObject(VirtualPool.class, URI.create(uri));
if (vPool != null && !vPool.getInactive()) {
// generate unmanaged volume's policyId
String autoTierPolicyId =
NativeGUIDGenerator.generateAutoTierPolicyNativeGuid(
system.getNativeGuid(),
policyName,
NativeGUIDGenerator.getTieringPolicyKeyForSystem(system));
if (!checkVPoolValidForUnManagedVolumeAutoTieringPolicy(
vPool, autoTierPolicyId, system)) {
String msg =
"Removing vPool %s from SUPPORTED_VPOOL_LIST in UnManagedVolume %s "
+ "since Auto-tiering Policy %s in UnManaged Volume does not match with vPool's (%s)";
_log.info(
String.format(
msg,
new Object[] {
uri, unManagedVolume.getId(), autoTierPolicyId, vPool.getAutoTierPolicyName()
}));
vPoolsToRemove.add(uri);
}
} else {
// remove Inactive vPool URI
vPoolsToRemove.add(uri);
}
}
}
for (String uri : vPoolsToRemove) { // UnManagedVolume object is persisted by caller
supportedVpoolURIs.remove(uri);
}
}
/**
* Allows the user to remove a storage system from the list of decommisioned resources After that
* corresponding provider should be able to be rescanned and add this system back to the list of
* managed systems.
*
* @param id id the URN of a ViPR Storage provider
* @param param The storage system details.
* @brief removes the storage system from the list of decommissioned systems and rescans the
* provider.
* @return An asynchronous task corresponding to the scan job scheduled for the provider.
* @throws BadRequestException When the system type is not valid or a storage system with the same
* native guid already exists.
* @throws com.emc.storageos.db.exceptions.DatabaseException When an error occurs querying the
* database.
* @throws ControllerException When an error occurs discovering the storage system.
*/
@PUT
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@CheckPermission(roles = {Role.SYSTEM_ADMIN})
@Path("/{id}/storage-systems")
public TaskResourceRep addStorageSystem(
@PathParam("id") URI id, StorageSystemProviderRequestParam param) throws ControllerException {
TaskResourceRep taskRep;
URIQueryResultList list = new URIQueryResultList();
ArgValidator.checkFieldNotEmpty(param.getSystemType(), "system_type");
if (!StorageSystem.Type.isProviderStorageSystem(
DiscoveredDataObject.Type.valueOf(param.getSystemType()))) {
throw APIException.badRequests.cannotAddStorageSystemTypeToStorageProvider(
param.getSystemType());
}
StorageProvider provider = _dbClient.queryObject(StorageProvider.class, id);
ArgValidator.checkEntityNotNull(provider, id, isIdEmbeddedInURL(id));
ArgValidator.checkFieldNotEmpty(param.getSerialNumber(), "serialNumber");
String nativeGuid =
NativeGUIDGenerator.generateNativeGuid(param.getSystemType(), param.getSerialNumber());
// check for duplicate StorageSystem.
List<StorageSystem> systems =
CustomQueryUtility.getActiveStorageSystemByNativeGuid(_dbClient, nativeGuid);
if (systems != null && !systems.isEmpty()) {
throw APIException.badRequests.invalidParameterProviderStorageSystemAlreadyExists(
"nativeGuid", nativeGuid);
}
int cleared =
DecommissionedResource.removeDecommissionedFlag(_dbClient, nativeGuid, StorageSystem.class);
if (cleared == 0) {
log.info("Cleared {} decommissioned systems", cleared);
} else {
log.info("Did not find any decommissioned systems to clear. Continue to scan.");
}
ArrayList<AsyncTask> tasks = new ArrayList<AsyncTask>(1);
String taskId = UUID.randomUUID().toString();
tasks.add(new AsyncTask(StorageProvider.class, provider.getId(), taskId));
BlockController controller = getController(BlockController.class, provider.getInterfaceType());
DiscoveredObjectTaskScheduler scheduler =
new DiscoveredObjectTaskScheduler(_dbClient, new ScanJobExec(controller));
TaskList taskList = scheduler.scheduleAsyncTasks(tasks);
return taskList.getTaskList().listIterator().next();
}
@Override
public void processResult(Operation operation, Object resultObj, Map<String, Object> keyMap)
throws BaseCollectionException {
CloseableIterator<CIMInstance> volumeInstances = null;
try {
_dbClient = (DbClient) keyMap.get(Constants.dbClient);
WBEMClient client = (WBEMClient) keyMap.get(Constants._cimClient);
_unManagedVolumesUpdate = new ArrayList<UnManagedVolume>();
CIMObjectPath storagePoolPath = getObjectPathfromCIMArgument(_args);
String poolNativeGuid = NativeGUIDGenerator.generateNativeGuidForPool(storagePoolPath);
StoragePool pool = checkStoragePoolExistsInDB(poolNativeGuid, _dbClient);
if (pool == null) {
_logger.error(
"Skipping unmanaged volume discovery of Access Sattes as the storage pool with path {} doesn't exist in ViPR",
storagePoolPath.toString());
return;
}
EnumerateResponse<CIMInstance> volumeInstanceChunks =
(EnumerateResponse<CIMInstance>) resultObj;
volumeInstances = volumeInstanceChunks.getResponses();
processVolumes(volumeInstances, keyMap, operation);
while (!volumeInstanceChunks.isEnd()) {
_logger.debug("Processing Next Volume Chunk of size {}", BATCH_SIZE);
volumeInstanceChunks =
client.getInstancesWithPath(
storagePoolPath,
volumeInstanceChunks.getContext(),
new UnsignedInteger32(BATCH_SIZE));
processVolumes(volumeInstanceChunks.getResponses(), keyMap, operation);
}
if (null != _unManagedVolumesUpdate && _unManagedVolumesUpdate.size() > 0) {
_partitionManager.updateInBatches(
_unManagedVolumesUpdate, getPartitionSize(keyMap), _dbClient, "UnManagedVolume");
}
} catch (Exception e) {
_logger.error("Discovering Access States of unManaged Volumes failed", e);
} finally {
volumeInstances.close();
}
}
/**
* Creates an instance of new storage port and associates it with the storage system.
*
* @param portNetworkId
* @param transportType
*/
private void create(String portNetworkId, String transportType) {
logger.info(
"Start create storage port for portNetworkId={}" + " and transportType={}",
portNetworkId,
transportType);
StorageHADomain adapter = CinderUtils.getStorageAdapter(storageSystem, dbClient);
StoragePort port = new StoragePort();
port.setId(URIUtil.createId(StoragePort.class));
port.setStorageDevice(storageSystem.getId());
String portName = generatePortName();
logger.debug("New storage port name is = {}", portName);
String nativeGuid =
NativeGUIDGenerator.generateNativeGuid(storageSystem, portName, NativeGUIDGenerator.PORT);
port.setNativeGuid(nativeGuid);
port.setPortNetworkId(portNetworkId);
port.setStorageHADomain(adapter.getId());
port.setRegistrationStatus(DiscoveredDataObject.RegistrationStatus.REGISTERED.toString());
// always treat it as a frontend port
port.setPortType(PortType.frontend.name());
port.setOperationalStatus(OperationalStatus.OK.toString());
port.setTransportType(transportType);
port.setLabel(portName);
port.setPortName(portName);
port.setPortGroup("Cinder-PortGroup");
port.setCompatibilityStatus(CompatibilityStatus.COMPATIBLE.name());
port.setDiscoveryStatus(DiscoveryStatus.VISIBLE.name());
dbClient.createObject(port);
// Add it to the new ports list
newStoragePortsList.add(port);
// Add it to the local list
allStoragePortsList.add(port);
logger.info(
"End create storage port for portNetworkId={}" + " and transportType={}",
portNetworkId,
transportType);
}
private void setStorageAdapterReference(
StoragePort port, CIMInstance portInstance, StorageSystem device) {
String adapterName = null;
try {
adapterName = portInstance.getObjectPath().getKey(SYSTEMNAME).getValue().toString();
Iterable<String> adapterItr =
Splitter.on(Constants.PATH_DELIMITER_PATTERN).limit(3).split(adapterName);
URIQueryResultList result = new URIQueryResultList();
_dbClient.queryByConstraint(
AlternateIdConstraint.Factory.getStorageHADomainByNativeGuidConstraint(
NativeGUIDGenerator.generateNativeGuid(
device, Iterables.getLast(adapterItr), NativeGUIDGenerator.ADAPTER)),
result);
if (result.iterator().hasNext()) {
URI portGroup = result.iterator().next();
port.setStorageHADomain(portGroup);
StorageHADomain haDomain = _dbClient.queryObject(StorageHADomain.class, portGroup);
port.setPortGroup(haDomain.getAdapterName());
}
} catch (Exception e) {
_logger.warn("Storage Port not found : {}", adapterName);
}
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
public void processResult(Operation operation, Object resultObj, Map<String, Object> keyMap)
throws BaseCollectionException {
try {
final Iterator<CIMInstance> it = (Iterator<CIMInstance>) resultObj;
profile = (AccessProfile) keyMap.get(Constants.ACCESSPROFILE);
Set<String> protocols = (Set<String>) keyMap.get(Constants.PROTOCOLS);
_newPortList = new ArrayList<StoragePort>();
_updatePortList = new ArrayList<StoragePort>();
_dbClient = (DbClient) keyMap.get(Constants.dbClient);
CoordinatorClient coordinator = (CoordinatorClient) keyMap.get(Constants.COORDINATOR_CLIENT);
Map<URI, StoragePool> poolsToMatchWithVpool =
(Map<URI, StoragePool>) keyMap.get(Constants.MODIFIED_STORAGEPOOLS);
Set<URI> systemsToRunRPConnectivity =
(HashSet<URI>) keyMap.get(Constants.SYSTEMS_RUN_RP_CONNECTIVITY);
StorageSystem device = _dbClient.queryObject(StorageSystem.class, profile.getSystemId());
CIMObjectPath storageAdapterPath = getObjectPathfromCIMArgument(args);
Iterable<String> adapterItr =
Splitter.on(Constants.PATH_DELIMITER_PATTERN)
.limit(3)
.split(storageAdapterPath.getKey(Constants.NAME).getValue().toString());
String adapterNativeGuid =
NativeGUIDGenerator.generateNativeGuid(
device, Iterables.getLast(adapterItr), NativeGUIDGenerator.ADAPTER);
StorageHADomain haDomain = getStorageAdapter(_dbClient, adapterNativeGuid);
if (null == haDomain) {
_logger.info("Adapter Not found");
return;
}
while (it.hasNext()) {
CIMInstance portInstance = null;
StoragePort port = null;
try {
portInstance = it.next();
// skip back end ports other than RDF Ports
if (!HADomainType.REMOTE.name().equalsIgnoreCase(haDomain.getAdapterType())
&& "3".equalsIgnoreCase(getCIMPropertyValue(portInstance, USAGERESTRICTION))) {
continue;
}
// only if its an EthernetPort, as protocolEnd point needs
// to run only for Ethernet
// Ports , because SCSI address we don't have it in
// CIM_LogicalPort Class
// 2 - Ethernet Port 4 - FC Port
if ("2".equalsIgnoreCase(getCIMPropertyValue(portInstance, LINKTECHNOLOGY))) {
port = createStoragePort(null, portInstance, profile, haDomain, false, IP, device);
checkProtocolAlreadyExists(protocols, ISCSI);
String deviceId = getCIMPropertyValue(portInstance, DEVICEID);
/*
* For SMI-S 8.x, While getting the iSCSI Port details, we use SystemName property
* (Ex. SYMMETRIX-+-<<SERIAL>>-+-SE-1G-+-0)
* Where this call just add the deviceId to the KeyMap (i.e SE-1G-+-0).
* Hence manually constructing the key.
*/
if (device.getUsingSmis80()) {
String systemName = getCIMPropertyValue(portInstance, SYSTEM_NAME);
StringBuffer deviceIdStrBuf = new StringBuffer(systemName);
deviceIdStrBuf.append(Constants.SMIS80_DELIMITER).append(deviceId);
deviceId = deviceIdStrBuf.toString();
}
_logger.debug("Adding iSCSI Port instance {} to keyMap", deviceId);
keyMap.put(deviceId, port);
addPath(keyMap, operation.getResult(), portInstance.getObjectPath());
} else if ("4".equalsIgnoreCase(getCIMPropertyValue(portInstance, LINKTECHNOLOGY))) {
port = checkStoragePortExistsInDB(portInstance, device, _dbClient);
checkProtocolAlreadyExists(protocols, FC);
createStoragePort(port, portInstance, profile, haDomain, true, FC, device);
} else {
_logger.debug("Unsupported Port : {}", getCIMPropertyValue(portInstance, DEVICEID));
}
} catch (Exception e) {
_logger.warn(
"Port Discovery failed for {}", getCIMPropertyValue(portInstance, DEVICEID), e);
}
}
_dbClient.createObject(_newPortList);
_dbClient.persistObject(_updatePortList);
// ports used later to run Transport Zone connectivity
List<List<StoragePort>> portsUsedToRunTZoneConnectivity =
(List<List<StoragePort>>) keyMap.get(Constants.STORAGE_PORTS);
portsUsedToRunTZoneConnectivity.add(_newPortList);
portsUsedToRunTZoneConnectivity.add(_updatePortList);
List<StoragePool> modifiedPools =
StoragePoolAssociationHelper.getStoragePoolsFromPorts(
_dbClient, _newPortList, _updatePortList);
for (StoragePool pool : modifiedPools) {
// pool matcher will be invoked on this pool
if (!poolsToMatchWithVpool.containsKey(pool.getId())) {
poolsToMatchWithVpool.put(pool.getId(), pool);
}
}
// Systems used to run RP connectivity later after runing pool matcher
systemsToRunRPConnectivity.addAll(
StoragePoolAssociationHelper.getStorageSytemsFromPorts(_newPortList, null));
systemsToRunRPConnectivity.addAll(
StoragePoolAssociationHelper.getStorageSytemsFromPorts(_updatePortList, null));
// discovered ports used later to check for not visible ports
List<StoragePort> discoveredPorts =
(List<StoragePort>) keyMap.get(Constants.DISCOVERED_PORTS);
discoveredPorts.addAll(_newPortList);
discoveredPorts.addAll(_updatePortList);
_logger.debug(
"# Pools used in invoking PoolMatcher during StoragePortProcessor {}",
Joiner.on("\t").join(poolsToMatchWithVpool.keySet()));
} catch (Exception e) {
_logger.error("Port Discovery failed -->{}", getMessage(e));
} finally {
_newPortList = null;
_updatePortList = null;
}
}
/**
* create StoragePort Record, if not present already, else update only the properties.
*
* @param port
* @param portInstance
* @throws URISyntaxException
* @throws IOException
*/
private StoragePort createStoragePort(
StoragePort port,
CIMInstance portInstance,
AccessProfile profile,
StorageHADomain haDomain,
boolean flag,
String transportType,
StorageSystem device)
throws URISyntaxException, IOException {
boolean newPort = false;
if (null == port) {
newPort = true;
port = new StoragePort();
port.setId(URIUtil.createId(StoragePort.class));
// if true, then its FC Port or else its Ethernet, PORTID
// or ethernet will be updated later in ProtocolEndPoint Processor
if (flag) {
port.setPortNetworkId(
WWNUtility.getWWNWithColons(getCIMPropertyValue(portInstance, PORTID)));
}
port.setStorageDevice(profile.getSystemId());
String portNativeGuid = NativeGUIDGenerator.generateNativeGuid(_dbClient, port);
port.setNativeGuid(portNativeGuid);
port.setLabel(portNativeGuid);
port.setPortGroup(haDomain.getAdapterName());
port.setStorageHADomain(haDomain.getId());
}
setPortType(port, portInstance);
port.setTransportType(transportType);
port.setPortName(getCIMPropertyValue(portInstance, PORTNAME));
port.setCompatibilityStatus(DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name());
port.setDiscoveryStatus(DiscoveredDataObject.DiscoveryStatus.VISIBLE.name());
UnsignedInteger16[] operationalStatusCodes =
(UnsignedInteger16[]) portInstance.getPropertyValue(OPERATIONALSTATUS);
OperationalStatus operationalStatus = getPortOperationalStatus(operationalStatusCodes);
if (OperationalStatus.NOT_OK.equals(operationalStatus)) {
_logger.info(
"StoragePort {} operationalStatus is NOT_OK. operationalStatusCodes collected from SMI-S :{}",
port.getId(),
operationalStatusCodes);
} else {
_logger.debug("operationalStatusCodes :{}", operationalStatusCodes);
// there can be multiple statuses. {OK, Online}, {OK, Stopped}
if (operationalStatusCodes != null
&& operationalStatusCodes.length > 1
&& Arrays.asList(operationalStatusCodes).contains(stopped_code)) {
_logger.info(
"StoragePort {} operational status is {OK, Stopped}. operationalStatusCodes :{}",
port.getId(),
operationalStatusCodes);
}
}
port.setOperationalStatus(operationalStatus.name());
String portSpeed = getCIMPropertyValue(portInstance, SPEED);
if (null != portSpeed) {
// SMI returns port speed in bits per sec ?? Is this always true?
Long portSpeedInBitsPerSec = Long.parseLong(portSpeed);
Long portSpeedInGbps = portSpeedInBitsPerSec / GB;
port.setPortSpeed(portSpeedInGbps);
}
setCompatibilityByACLXFlag(device, portInstance, port);
if (flag) {
if (newPort) {
_logger.info("Creating port - {}:{}", port.getLabel(), port.getNativeGuid());
_newPortList.add(port);
} else {
_logger.info("Updating port - {}:{}", port.getLabel(), port.getNativeGuid());
_updatePortList.add(port);
}
}
;
return port;
}
@Override
public void processResult(Operation operation, Object resultObj, Map<String, Object> keyMap)
throws BaseCollectionException {
CloseableIterator<CIMInstance> volumeInstances = null;
EnumerateResponse<CIMInstance> volumeInstanceChunks = null;
CIMObjectPath storagePoolPath = null;
WBEMClient client = null;
try {
_dbClient = (DbClient) keyMap.get(Constants.dbClient);
client = (WBEMClient) keyMap.get(Constants._cimClient);
_profile = (AccessProfile) keyMap.get(Constants.ACCESSPROFILE);
Map<String, VolHostIOObject> exportedVolumes =
(Map<String, VolHostIOObject>) keyMap.get(Constants.EXPORTED_VOLUMES);
Set<String> existingVolumesInCG = (Set<String>) keyMap.get(Constants.VOLUMES_PART_OF_CG);
@SuppressWarnings("unchecked")
Map<String, RemoteMirrorObject> volumeToRAGroupMap =
(Map<String, RemoteMirrorObject>) keyMap.get(Constants.UN_VOLUME_RAGROUP_MAP);
@SuppressWarnings("unchecked")
Map<String, LocalReplicaObject> volumeToLocalReplicaMap =
(Map<String, LocalReplicaObject>) keyMap.get(Constants.UN_VOLUME_LOCAL_REPLICA_MAP);
@SuppressWarnings("unchecked")
Map<String, Set<String>> vmax2ThinPoolToBoundVolumesMap =
(Map<String, Set<String>>) keyMap.get(Constants.VMAX2_THIN_POOL_TO_BOUND_VOLUMES);
Set<String> boundVolumes = null;
storagePoolPath = getObjectPathfromCIMArgument(_args);
String poolNativeGuid = NativeGUIDGenerator.generateNativeGuidForPool(storagePoolPath);
StoragePool pool = checkStoragePoolExistsInDB(poolNativeGuid, _dbClient);
if (pool == null) {
_logger.error(
"Skipping unmanaged volume discovery as the storage pool with path {} doesn't exist in ViPR",
storagePoolPath.toString());
return;
}
StorageSystem system = _dbClient.queryObject(StorageSystem.class, _profile.getSystemId());
_unManagedVolumesInsert = new ArrayList<UnManagedVolume>();
_unManagedVolumesUpdate = new ArrayList<UnManagedVolume>();
_unManagedExportMasksUpdate = new ArrayList<UnManagedExportMask>();
// get bound volumes list for VMAX2 Thin pools
boundVolumes = vmax2ThinPoolToBoundVolumesMap.get(storagePoolPath.toString());
Set<String> poolSupportedSLONames = (Set<String>) keyMap.get(poolNativeGuid);
_logger.debug("Pool Supporting SLO Names:{}", poolSupportedSLONames);
_metaVolumeViewPaths = (List<CIMObjectPath>) keyMap.get(Constants.META_VOLUMES_VIEWS);
if (_metaVolumeViewPaths == null) {
_metaVolumeViewPaths = new ArrayList<CIMObjectPath>();
keyMap.put(Constants.META_VOLUMES_VIEWS, _metaVolumeViewPaths);
}
// create empty place holder list for meta volume paths (cannot
// define this in xml)
_metaVolumePaths = (List<CIMObjectPath>) keyMap.get(Constants.META_VOLUMES);
if (_metaVolumePaths == null) {
_metaVolumePaths = new ArrayList<CIMObjectPath>();
keyMap.put(Constants.META_VOLUMES, _metaVolumePaths);
}
_volumeToSpaceConsumedMap =
(Map<String, String>) keyMap.get(Constants.VOLUME_SPACE_CONSUMED_MAP);
// get VolumeInfo Object and inject Fast Policy Name.
volumeInstanceChunks = (EnumerateResponse<CIMInstance>) resultObj;
volumeInstances = volumeInstanceChunks.getResponses();
processVolumes(
volumeInstances,
keyMap,
operation,
pool,
system,
exportedVolumes,
existingVolumesInCG,
volumeToRAGroupMap,
volumeToLocalReplicaMap,
poolSupportedSLONames,
boundVolumes);
while (!volumeInstanceChunks.isEnd()) {
_logger.info("Processing Next Volume Chunk of size {}", BATCH_SIZE);
volumeInstanceChunks =
client.getInstancesWithPath(
storagePoolPath,
volumeInstanceChunks.getContext(),
new UnsignedInteger32(BATCH_SIZE));
processVolumes(
volumeInstanceChunks.getResponses(),
keyMap,
operation,
pool,
system,
exportedVolumes,
existingVolumesInCG,
volumeToRAGroupMap,
volumeToLocalReplicaMap,
poolSupportedSLONames,
boundVolumes);
}
if (null != _unManagedVolumesUpdate && _unManagedVolumesUpdate.size() > 0) {
_partitionManager.updateInBatches(
_unManagedVolumesUpdate, getPartitionSize(keyMap), _dbClient, UNMANAGED_VOLUME);
}
if (null != _unManagedVolumesInsert && _unManagedVolumesInsert.size() > 0) {
_partitionManager.insertInBatches(
_unManagedVolumesInsert, getPartitionSize(keyMap), _dbClient, UNMANAGED_VOLUME);
}
if (null != _unManagedExportMasksUpdate && _unManagedExportMasksUpdate.size() > 0) {
_partitionManager.updateInBatches(
_unManagedExportMasksUpdate,
getPartitionSize(keyMap),
_dbClient,
UNMANAGED_EXPORT_MASK);
}
performStorageUnManagedVolumeBookKeeping(pool.getId());
} catch (Exception e) {
_logger.error("Processing Storage Volume Information failed :", e);
} finally {
_unManagedVolumesInsert = null;
_unManagedVolumesUpdate = null;
if (null != volumeInstances) {
volumeInstances.close();
}
if (null != volumeInstanceChunks) {
try {
client.closeEnumeration(storagePoolPath, volumeInstanceChunks.getContext());
} catch (Exception e) {
}
}
}
}
/**
* process the FileShareUsage response of the VNX XML API Server.
*
* @param fsUsageList : fileShareUsage map.
* @param keyMap : attribute map.
* @param statList : list of stat objects.
*/
@SuppressWarnings("rawtypes")
private void processFileShareInfo(
final List<Object> fsUsageList,
final Map<String, Object> keyMap,
final List<Stat> statList,
DbClient dbClient)
throws VNXFilePluginException {
final String serialId = keyMap.get(Constants._serialID).toString();
Iterator iterator = fsUsageList.iterator();
keyMap.put(Constants._TimeCollected, System.currentTimeMillis());
Map<String, Long> fsCapacityMap = new HashMap<String, Long>();
while (iterator.hasNext()) {
FileSystemSetUsageStats fsSetUsageStats = (FileSystemSetUsageStats) iterator.next();
List<Item> fsUsageItems = fsSetUsageStats.getItem();
_logger.info(
"Received {} fileShareUsage records at server time {}",
fsUsageItems.size(),
fsSetUsageStats.getTime());
for (Item item : fsUsageItems) {
if (null == item.getFileSystem()) {
continue;
}
final String nativeGuid =
NativeGUIDGenerator.generateNativeGuid(
Type.vnxfile.toString(), serialId, item.getFileSystem());
Stat stat = _zeroRecordGenerator.injectattr(keyMap, nativeGuid, null);
if (null != stat) {
stat.setTimeInMillis((Long) keyMap.get(Constants._TimeCollected));
stat.setTimeCollected((Long) keyMap.get(Constants._TimeCollected));
injectProvisionedCapacity(stat, keyMap);
// The data coming in is in KB. Converting to Bytes
stat.setAllocatedCapacity(item.getSpaceUsed() * 1024);
_statsColumnInjector.injectColumns(stat, dbClient);
statList.add(stat);
// Persists the file system, only if change in used capacity.
DbClient client = (DbClient) keyMap.get(Constants.dbClient);
if (client != null) {
FileShare fileSystem = client.queryObject(FileShare.class, stat.getResourceId());
if (fileSystem != null) {
if (!fileSystem.getInactive()
&& fileSystem.getUsedCapacity() != stat.getAllocatedCapacity()) {
fileSystem.setUsedCapacity(stat.getAllocatedCapacity());
client.persistObject(fileSystem);
}
}
}
}
// filesystem and total capacity in Map
long totalSpace = item.getSpaceTotal();
String fsNativeId = item.getFileSystem();
fsCapacityMap.put(fsNativeId, Long.valueOf(totalSpace));
_logger.info(
"processFileShareInfo - FileSystem native id {} and file system total size{}",
fsNativeId,
String.valueOf(totalSpace));
}
_logger.info("Filesystems found - {} ", fsCapacityMap.size());
keyMap.put(VNXFileConstants.FILE_CAPACITY_MAP, fsCapacityMap);
}
_logger.info("No. of stat objects: {}", statList.size());
}
/** {@inheritDoc} */
@Override
public void updateStatus(JobContext jobContext) throws Exception {
JobStatus jobStatus = getJobStatus();
CloseableIterator<CIMObjectPath> volumeIter = null;
try {
DbClient dbClient = jobContext.getDbClient();
TaskCompleter completer = getTaskCompleter();
BlockSnapshot snapshot = dbClient.queryObject(BlockSnapshot.class, _snapshotURI);
if (jobStatus == JobStatus.IN_PROGRESS) {
return;
}
if (jobStatus == JobStatus.SUCCESS) {
s_logger.info(
"Post-processing successful link snapshot session target {} for task {}",
snapshot.getId(),
completer.getOpId());
// Get the snapshot session to which the target is being linked.
BlockSnapshotSession snapSession =
dbClient.queryObject(BlockSnapshotSession.class, completer.getId());
// Get the snapshot device ID and set it against the BlockSnapshot object.
BlockObject sourceObj = BlockObject.fetch(dbClient, snapshot.getParent().getURI());
CIMConnectionFactory cimConnectionFactory = jobContext.getCimConnectionFactory();
WBEMClient client = getWBEMClient(dbClient, cimConnectionFactory);
volumeIter =
client.associatorNames(getCimJob(), null, SmisConstants.CIM_STORAGE_VOLUME, null, null);
while (volumeIter.hasNext()) {
// Get the sync volume native device id
CIMObjectPath volumePath = volumeIter.next();
s_logger.info("volumePath: {}", volumePath.toString());
CIMInstance volume = client.getInstance(volumePath, false, false, null);
String volumeDeviceId =
volumePath.getKey(SmisConstants.CP_DEVICE_ID).getValue().toString();
s_logger.info("volumeDeviceId: {}", volumeDeviceId);
if (volumeDeviceId.equals(sourceObj.getNativeId())) {
// Don't want the source, we want the linked target.
continue;
}
String volumeElementName =
CIMPropertyFactory.getPropertyValue(volume, SmisConstants.CP_ELEMENT_NAME);
s_logger.info("volumeElementName: {}", volumeElementName);
String volumeWWN = CIMPropertyFactory.getPropertyValue(volume, SmisConstants.CP_WWN_NAME);
s_logger.info("volumeWWN: {}", volumeWWN);
String volumeAltName = CIMPropertyFactory.getPropertyValue(volume, SmisConstants.CP_NAME);
s_logger.info("volumeAltName: {}", volumeAltName);
StorageSystem system = dbClient.queryObject(StorageSystem.class, getStorageSystemURI());
snapshot.setNativeId(volumeDeviceId);
snapshot.setNativeGuid(NativeGUIDGenerator.generateNativeGuid(system, snapshot));
snapshot.setDeviceLabel(volumeElementName);
snapshot.setInactive(false);
snapshot.setIsSyncActive(Boolean.TRUE);
snapshot.setCreationTime(Calendar.getInstance());
snapshot.setWWN(volumeWWN.toUpperCase());
snapshot.setAlternateName(volumeAltName);
snapshot.setSettingsInstance(snapSession.getSessionInstance());
commonSnapshotUpdate(
snapshot,
volume,
client,
system,
sourceObj.getNativeId(),
volumeDeviceId,
false,
dbClient);
s_logger.info(
String.format(
"For target volume path %1$s, going to set blocksnapshot %2$s nativeId to %3$s (%4$s). Associated volume is %5$s (%6$s)",
volumePath.toString(),
snapshot.getId().toString(),
volumeDeviceId,
volumeElementName,
sourceObj.getNativeId(),
sourceObj.getDeviceLabel()));
dbClient.updateObject(snapshot);
}
} else if (jobStatus == JobStatus.FAILED || jobStatus == JobStatus.FATAL_ERROR) {
s_logger.info(
"Failed to link snapshot session target {} for task {}",
snapshot.getId(),
completer.getOpId());
snapshot.setInactive(true);
dbClient.updateObject(snapshot);
}
} catch (Exception e) {
setPostProcessingErrorStatus(
"Encountered an internal error in link snapshot session target job status processing: "
+ e.getMessage());
s_logger.error(
"Encountered an internal error in link snapshot session target job status processing", e);
} finally {
if (volumeIter != null) {
volumeIter.close();
}
super.updateStatus(jobContext);
}
}
/**
* Create StoragePool Record, if not present already, else update only the properties.
*
* @param pool
* @param poolInstance
* @param profile
* @param poolClassName
* @param supportedVolumeTypes
* @param protocols
* @param poolsToMatchWithVpool
* @throws URISyntaxException
* @throws IOException
*/
private void createStoragePool(
StoragePool pool,
CIMInstance poolInstance,
AccessProfile profile,
String poolClassName,
String supportedVolumeTypes,
Set<String> protocols,
Map<URI, StoragePool> poolsToMatchWithVpool,
StorageSystem device)
throws URISyntaxException, IOException {
boolean newPool = false;
boolean modifiedPool = false; // indicates whether to add to modified pools list or not
if (null == pool) {
String instanceID = getCIMPropertyValue(poolInstance, Constants.INSTANCEID);
String nativeIdFromInstance = getNativeIDFromInstance(instanceID);
newPool = true;
pool = new StoragePool();
pool.setId(URIUtil.createId(StoragePool.class));
pool.setPoolName(getCIMPropertyValue(poolInstance, POOL_ID));
pool.setNativeId(nativeIdFromInstance);
pool.setStorageDevice(profile.getSystemId());
pool.setPoolServiceType(PoolServiceType.block.toString());
String poolNativeGuid = NativeGUIDGenerator.generateNativeGuid(_dbClient, pool);
pool.setNativeGuid(poolNativeGuid);
pool.setLabel(poolNativeGuid);
// setting default values on Pool Creation for VMAX and VNX
pool.setMaximumThickVolumeSize(0L);
pool.setMinimumThickVolumeSize(0L);
pool.setMaximumThinVolumeSize(0L);
pool.setMinimumThinVolumeSize(0L);
if (device.getAutoTieringEnabled()) {
pool.setAutoTieringEnabled(Boolean.TRUE);
} else {
pool.setAutoTieringEnabled(Boolean.FALSE);
}
_logger.info(
String.format(
"Maximum default limits for volume capacity in storage pool %s / %s : \n "
+ "max thin volume capacity: %s, max thick volume capacity: %s ",
pool.getPoolName(),
pool.getId(),
pool.getMaximumThinVolumeSize(),
pool.getMaximumThickVolumeSize()));
// set default utilization/subscription limits
double poolSubscriptionPercent =
CapacityMatcher.getMaxPoolSubscriptionPercentage(pool, _coordinator);
double poolUtilizationPercent =
CapacityMatcher.getMaxPoolUtilizationPercentage(pool, _coordinator);
pool.setMaxThinPoolSubscriptionPercentage((int) poolSubscriptionPercent);
pool.setMaxPoolUtilizationPercentage((int) poolUtilizationPercent);
}
String maxSubscriptionPercent =
getCIMPropertyValue(poolInstance, SmisConstants.CP_EMCMAXSUBSCRIPTIONPERCENT);
_logger.info(
String.format(
"Discovered maximum subscription percent of storage pool %s from array : %s ",
pool.getPoolName(), maxSubscriptionPercent));
// null value indicates "not available".
Integer newMaxSubscriptionPercentFromArray =
maxSubscriptionPercent == null ? null : new Integer(maxSubscriptionPercent);
_logger.info(
String.format(
"New maximum subscription percent of storage pool %s from array : %s ",
pool.getPoolName(), newMaxSubscriptionPercentFromArray));
processMaxSubscriptionPercent(
newMaxSubscriptionPercentFromArray, pool, _dbClient, _eventManager);
String subscribedCapacity =
getCIMPropertyValue(poolInstance, SmisConstants.CP_SUBSCRIBEDCAPACITY);
if (null != subscribedCapacity) {
pool.setSubscribedCapacity(ControllerUtils.convertBytesToKBytes(subscribedCapacity));
}
pool.setFreeCapacity(SmisUtils.getFreeCapacity(poolInstance));
pool.setTotalCapacity(SmisUtils.getTotalCapacity(poolInstance));
pool.setPoolClassName(poolClassName);
pool.setSupportedResourceTypes(supportedVolumeTypes);
String operationalStatus = determineOperationalStatus(poolInstance);
if (!newPool
&& (ImplicitPoolMatcher.checkPoolPropertiesChanged(
pool.getOperationalStatus(), operationalStatus)
|| ImplicitPoolMatcher.checkPoolPropertiesChanged(pool.getProtocols(), protocols)
|| ImplicitPoolMatcher.checkPoolPropertiesChanged(
pool.getCompatibilityStatus(),
DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name())
|| ImplicitPoolMatcher.checkPoolPropertiesChanged(
pool.getDiscoveryStatus(), DiscoveredDataObject.DiscoveryStatus.VISIBLE.name()))) {
modifiedPool = true;
}
pool.addProtocols(protocols);
pool.setOperationalStatus(operationalStatus);
pool.setCompatibilityStatus(DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name());
pool.setDiscoveryStatus(DiscoveredDataObject.DiscoveryStatus.VISIBLE.name());
Set<String> diskDrives = new HashSet<String>();
String driveTypes = getCIMPropertyValue(poolInstance, EMC_DRIVE_TYPE);
if (null != driveTypes) {
String driveTypesArr[] = driveTypes.split(SPACE_STR_DELIM);
if (device.checkIfVmax3()
&& driveTypesArr.length == 1
&& driveTypesArr[0].equals(MIXED_DRIVE_TYPE)) {
driveTypesArr = getVMAX3PoolDriveTypes(device, poolInstance);
}
for (String driveType : driveTypesArr) {
String driveDisplayName = SupportedDriveTypeValues.getDiskDriveDisplayName(driveType);
if (null == driveDisplayName) {
_logger.warn(
"UnSupported DiskDrive Type : {} resulting in drives not getting discovered for this pool: {}",
driveType,
getCIMPropertyValue(poolInstance, Constants.INSTANCEID));
continue;
}
diskDrives.add(driveDisplayName);
}
if (!newPool
&& !modifiedPool
&& ImplicitPoolMatcher.checkPoolPropertiesChanged(
pool.getSupportedDriveTypes(), diskDrives)) {
modifiedPool = true;
}
pool.addDriveTypes(diskDrives);
}
_logger.info("Discovered disk drives:[{}] for pool id:{}", driveTypes, pool.getId());
if (newPool) {
_newPoolList.add(pool);
// add new pools to modified pools list to consider them for implicit pool matching.
if (!poolsToMatchWithVpool.containsKey(pool.getId())) {
poolsToMatchWithVpool.put(pool.getId(), pool);
}
} else {
_updatePoolList.add(pool);
// add to modified pool list if pool's property which is required for vPool matcher, has
// changed.
// No need to check whether the pool is already there in the list here
// because this processor is the first to discover pools.
if (modifiedPool && !poolsToMatchWithVpool.containsKey(pool.getId())) {
poolsToMatchWithVpool.put(pool.getId(), pool);
}
}
}
