/** Executes specified fence action on all fence agents concurrently */
protected FenceOperationResult fenceConcurrently(FenceActionType fenceAction) {
List<FenceOperationResult> results = new ArrayList<>(fenceAgents.size());
FenceOperationResult taskResult;
ExecutorCompletionService<FenceOperationResult> tasksExecutor =
ThreadPoolUtil.createCompletionService();
List<Future<FenceOperationResult>> futures =
ThreadPoolUtil.submitTasks(tasksExecutor, createTasks(fenceAction));
for (int i = 0; i < fenceAgents.size(); ++i) {
try {
taskResult = tasksExecutor.take().get();
} catch (ExecutionException | InterruptedException e) {
taskResult = new FenceOperationResult(Status.ERROR, PowerStatus.UNKNOWN, e.getMessage());
log.error("Error getting task result: {}", e.getMessage());
log.debug("Exception", e);
}
results.add(taskResult);
if (tasksProcessor.isGoalReached(taskResult)) {
// action goal is reach, cancel all remaining tasks
for (Future<FenceOperationResult> future : futures) {
future.cancel(true);
}
break;
}
}
return tasksProcessor.createActionResult(results);
}
@Override
public boolean vdsUpEvent(final VDS vds) {
HostStoragePoolParametersBase params = new HostStoragePoolParametersBase(vds);
boolean isSucceeded =
backend.runInternalAction(VdcActionType.InitVdsOnUp, params).getSucceeded();
if (isSucceeded) {
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
try {
// migrate vms that its their default vds and failback
// is on
List<VmStatic> vmsToMigrate =
DbFacade.getInstance().getVmStaticDao().getAllWithFailbackByVds(vds.getId());
if (!vmsToMigrate.isEmpty()) {
CommandContext ctx = new CommandContext(new EngineContext());
ctx.getExecutionContext().setMonitored(true);
backend.runInternalMultipleActions(
VdcActionType.MigrateVmToServer,
new ArrayList<>(createMigrateVmToServerParametersList(vmsToMigrate, vds)),
ctx);
}
} catch (RuntimeException e) {
log.error("Failed to initialize Vds on up: {}", e.getMessage());
log.error("Exception", e);
}
}
});
}
return isSucceeded;
}
// TODO asynch event handler - design infra code to allow async events in segregated thread
public void onMomPolicyChange(@Observes @MomPolicyUpdate final VDSGroup cluster) {
if (cluster == null || cluster.getCompatibilityVersion().compareTo(Version.v3_4) < 0) return;
List<VDS> activeHostsInCluster =
vdsDao.getAllForVdsGroupWithStatus(cluster.getId(), VDSStatus.Up);
// collect all Active hosts into a callable list
List<Callable<Object>> callables = new LinkedList<>();
for (final VDS vds : activeHostsInCluster) {
callables.add(
new Callable<Object>() {
@Override
public Object call() {
try {
resourceManagerProvider
.get()
.runVdsCommand(
VDSCommandType.SetMOMPolicyParameters,
new MomPolicyVDSParameters(
vds,
cluster.isEnableBallooning(),
cluster.isEnableKsm(),
cluster.isKsmMergeAcrossNumaNodes()));
} catch (EngineException e) {
log.error("Could not update MoM policy on host '{}'", vds.getName());
}
return null;
}
});
}
// run all VDSCommands concurrently with executor
if (callables.size() > 0) ThreadPoolUtil.invokeAll(callables);
}
protected void executeHttpMethod(final T method) {
int responseCode = -1;
VdsManager manager = ResourceManager.getInstance().getVdsManager(getParameters().getVdsId());
final HttpClient httpclient = manager.getVdsProxy().getHttpClient();
try {
FutureTask<Integer> futureTask =
new FutureTask(
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return httpclient.executeMethod(method);
}
});
Future<Integer> f = ThreadPoolUtil.execute(futureTask);
if (f.get(Config.<Integer>getValue(getConfigValueTimeLimitForOperation()), TimeUnit.MINUTES)
== null) {
responseCode = futureTask.get();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (Exception e) {
log.debug("Exception", e);
throw createNetworkException(e);
}
if (responseCode == getSuccessCode()) {
Guid createdTask =
Guid.createGuidFromString(processResponseHeaderValue(getMethod(), "Task-Id", null));
getVDSReturnValue()
.setCreationInfo(
new AsyncTaskCreationInfo(
createdTask, getCreatedTaskType(), getParameters().getStoragePoolId()));
handleOkResponse();
getVDSReturnValue().setSucceeded(true);
return;
}
processResponseHeaderValue(getMethod(), "Content-type", "application/json");
String response;
try {
response = getMethod().getResponseBodyAsString();
} catch (Exception e) {
throw createNetworkException(e);
}
Map<String, Object> resultMap = null;
try {
resultMap = new ObjectMapper().readValue(response, HashMap.class);
status = new StatusOnlyReturnForXmlRpc(resultMap);
} catch (Exception e) {
throwVdsErrorException("failed to parse response " + response, EngineError.GeneralException);
}
proceedProxyReturnValue();
}
@Override
public void importHostedEngineVm(final VM vm) {
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
hostedEngineImporterProvider.get().doImport(vm);
}
});
}
/**
* This method is called upon the bean creation as part of the management Service bean life cycle.
*/
@Override
@PostConstruct
public void create() {
try {
// This must be done before starting to sample the hosts status from VDSM since the sampling
// will turn such host from Reboot to NonResponsive
loadService(PmHealthCheckManager.class);
loadService(EngineBackupAwarenessManager.class);
CommandCoordinatorUtil.initAsyncTaskManager();
loadService(ResourceManager.class);
OvfDataUpdater.getInstance().initOvfDataUpdater();
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
MacPoolPerDcSingleton.getInstance().initialize();
}
});
StoragePoolStatusHandler.init();
GlusterJobsManager.init();
try {
log.info("Init VM custom properties utilities");
VmPropertiesUtils.getInstance().init();
} catch (InitializationException e) {
log.error("Initialization of vm custom properties failed.", e);
}
try {
log.info("Init device custom properties utilities");
DevicePropertiesUtils.getInstance().init();
} catch (InitializationException e) {
log.error("Initialization of device custom properties failed.", e);
}
loadService(SchedulingManager.class);
SessionDataContainer.getInstance().cleanupEngineSessionsOnStartup();
loadService(HostDeviceManager.class);
loadService(DwhHeartBeat.class);
if (Config.<Boolean>getValue(ConfigValues.AffinityRulesEnforcementManagerEnabled)) {
loadService(AffinityRulesEnforcementManager.class);
}
loadService(CertificationValidityChecker.class);
} catch (Exception ex) {
log.error("Failed to initialize backend", ex);
throw ex;
}
}
public void onError(@Observes final VDSNetworkException vdsException) {
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
resourceManagerProvider
.get()
.getVdsManager(vdsException.getVdsError().getVdsId())
.handleNetworkException(vdsException);
}
});
}
/**
* Synchronize LUN details comprising the storage domain with the DB
*
* @param storageDomainId
* @param vdsId
*/
@Override
public void syncLunsInfoForBlockStorageDomain(final Guid storageDomainId, final Guid vdsId) {
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
StorageDomainParametersBase parameters =
new StorageDomainParametersBase(storageDomainId);
parameters.setVdsId(vdsId);
backend.runInternalAction(VdcActionType.SyncLunsInfoForBlockStorageDomain, parameters);
}
});
}
@Override
public void refreshHostIfAnyVmHasHostDevices(final List<Guid> vmIds, final Guid hostId) {
if (vmIds.isEmpty()) {
return;
}
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
HostDeviceManager hostDeviceManager = Injector.get(HostDeviceManager.class);
hostDeviceManager.refreshHostIfAnyVmHasHostDevices(vmIds, hostId);
}
});
}
@Override
public void rerun(Guid vmId) {
final IVdsAsyncCommand command = vdsBroker.getAsyncCommandForVm(vmId);
if (command != null) {
// The command will be invoked in a different VDS in its rerun method, so we're calling
// its rerun method from a new thread so that it won't be executed within our current VDSM
// lock
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
command.rerun();
}
});
}
}
@Override
public void processOnVmStop(
final Collection<Guid> vmIds, final Guid hostId, boolean useSeparateThread) {
if (vmIds.isEmpty()) {
return;
}
if (useSeparateThread) {
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
processOnVmStopInternal(vmIds, hostId);
}
});
} else {
processOnVmStopInternal(vmIds, hostId);
}
}
public void handlePartiallyExecuteTasksOfCommand(final List<AsyncTask> tasks) {
ThreadPoolUtil.execute(
new Runnable() {
@SuppressWarnings("synthetic-access")
@Override
public void run() {
TransactionSupport.executeInNewTransaction(
() -> {
try {
for (AsyncTask task : tasks) {
handlePartiallyExecutedTaskOfCommand(task);
}
return null;
} finally {
irsBrokerLatch.countDown();
}
});
}
});
}
@Override
public void updateSlaPolicies(final List<Guid> vmIds, final Guid vdsId) {
if (vmIds.isEmpty()) {
return;
}
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
for (Guid vmId : vmIds) {
CpuQos qos = DbFacade.getInstance().getCpuQosDao().getCpuQosByVmId(vmId);
if (qos != null && qos.getCpuLimit() != null) {
resourceManagerProvider
.get()
.runVdsCommand(
VDSCommandType.UpdateVmPolicy,
new UpdateVmPolicyVDSParams(vdsId, vmId, qos.getCpuLimit().intValue()));
}
}
}
});
}
@Override
public void vdsNotResponding(final VDS vds) {
ExecutionHandler.updateSpecificActionJobCompleted(
vds.getId(), VdcActionType.MaintenanceVds, false);
ThreadPoolUtil.execute(
new Runnable() {
@Override
public void run() {
log.info(
"ResourceManager::vdsNotResponding entered for Host '{}', '{}'",
vds.getId(),
vds.getHostName());
FenceVdsActionParameters params = new FenceVdsActionParameters(vds.getId());
backend.runInternalAction(
VdcActionType.VdsNotRespondingTreatment,
params,
ExecutionHandler.createInternalJobContext());
moveBricksToUnknown(vds);
}
});
}
private void connectAndRefreshAllUpHosts(final boolean commandSucceeded) {
if (isLastMaster || !commandSucceeded) {
log.warn(
"skipping connect and refresh for all hosts, last master '{}', command status '{}'",
isLastMaster,
commandSucceeded);
return;
}
List<Callable<Void>> tasks = new ArrayList<>();
for (final VDS vds : getAllRunningVdssInPool()) {
tasks.add(
() -> {
try {
if (!connectVdsToNewMaster(vds)) {
log.warn(
"failed to connect vds '{}' to the new master '{}'",
vds.getId(),
getNewMasterStorageDomainId());
return null;
}
List<StoragePoolIsoMap> storagePoolIsoMap =
storagePoolIsoMapDao.getAllForStoragePool(getStoragePool().getId());
try {
runVdsCommand(
VDSCommandType.ConnectStoragePool,
new ConnectStoragePoolVDSCommandParameters(
vds,
getStoragePool(),
getNewMasterStorageDomainId(),
storagePoolIsoMap,
true));
} catch (EngineException ex) {
if (EngineError.StoragePoolUnknown == ex.getVdsError().getCode()) {
VDSReturnValue returnVal =
runVdsCommand(
VDSCommandType.ConnectStoragePool,
new ConnectStoragePoolVDSCommandParameters(
vds,
getStoragePool(),
getNewMasterStorageDomainId(),
storagePoolIsoMap));
if (!returnVal.getSucceeded()) {
log.error(
"Post reconstruct actions (connectPool) did not complete on host '{}' in the pool. error {}",
vds.getId(),
returnVal.getVdsError().getMessage());
}
} else {
log.error(
"Post reconstruct actions (refreshPool)"
+ " did not complete on host '{}' in the pool. error {}",
vds.getId(),
ex.getMessage());
}
}
} catch (Exception e) {
log.error(
"Post reconstruct actions (connectPool,refreshPool,disconnect storage)"
+ " did not complete on host '{}' in the pool: {}",
vds.getId(),
e.getMessage());
log.debug("Exception", e);
}
return null;
});
}
ThreadPoolUtil.invokeAll(tasks);
}
@Override
protected void executeCommand() {
Guid oVirtId = getParameters().getVdsForUniqueId();
if (oVirtId != null) {
// if fails to remove deprecated entry, we might attempt to add new oVirt host with an
// existing unique-id.
if (!removeDeprecatedOvirtEntry(oVirtId)) {
log.error(
"Failed to remove duplicated oVirt entry with id '{}'. Abort adding oVirt Host type",
oVirtId);
throw new EngineException(EngineError.HOST_ALREADY_EXISTS);
}
}
TransactionSupport.executeInNewTransaction(
() -> {
addVdsStaticToDb();
addVdsDynamicToDb();
addVdsStatisticsToDb();
getCompensationContext().stateChanged();
return null;
});
if (getParameters().isProvisioned()) {
HostProviderProxy proxy = ProviderProxyFactory.getInstance().create(getHostProvider());
proxy.provisionHost(
getParameters().getvds(),
getParameters().getHostGroup(),
getParameters().getComputeResource(),
getParameters().getHostMac(),
getParameters().getDiscoverName(),
getParameters().getPassword(),
getParameters().getDiscoverIp());
addCustomValue("HostGroupName", getParameters().getHostGroup().getName());
auditLogDirector.log(this, AuditLogType.VDS_PROVISION);
}
// set vds spm id
if (getCluster().getStoragePoolId() != null) {
VdsActionParameters tempVar = new VdsActionParameters(getVdsIdRef());
tempVar.setSessionId(getParameters().getSessionId());
tempVar.setCompensationEnabled(true);
VdcReturnValueBase addVdsSpmIdReturn =
runInternalAction(
VdcActionType.AddVdsSpmId,
tempVar,
cloneContext().withoutLock().withoutExecutionContext());
if (!addVdsSpmIdReturn.getSucceeded()) {
setSucceeded(false);
getReturnValue().setFault(addVdsSpmIdReturn.getFault());
return;
}
}
TransactionSupport.executeInNewTransaction(
() -> {
initializeVds(true);
alertIfPowerManagementNotConfigured(getParameters().getVdsStaticData());
testVdsPowerManagementStatus(getParameters().getVdsStaticData());
setSucceeded(true);
setActionReturnValue(getVdsIdRef());
// If the installation failed, we don't want to compensate for the failure since it will
// remove the
// host, but instead the host should be left in an "install failed" status.
getCompensationContext().cleanupCompensationDataAfterSuccessfulCommand();
return null;
});
// do not install vds's which added in pending mode or for provisioning (currently power
// clients). they are installed as part of the approve process or automatically after provision
if (Config.<Boolean>getValue(ConfigValues.InstallVds)
&& !getParameters().isPending()
&& !getParameters().isProvisioned()) {
final InstallVdsParameters installVdsParameters =
new InstallVdsParameters(getVdsId(), getParameters().getPassword());
installVdsParameters.setAuthMethod(getParameters().getAuthMethod());
installVdsParameters.setOverrideFirewall(getParameters().getOverrideFirewall());
installVdsParameters.setActivateHost(getParameters().getActivateHost());
installVdsParameters.setNetworkProviderId(
getParameters().getVdsStaticData().getOpenstackNetworkProviderId());
installVdsParameters.setNetworkMappings(getParameters().getNetworkMappings());
installVdsParameters.setEnableSerialConsole(getParameters().getEnableSerialConsole());
if (getParameters().getHostedEngineDeployConfiguration() != null) {
Map<String, String> vdsDeployParams =
hostedEngineHelper.createVdsDeployParams(
getVdsId(), getParameters().getHostedEngineDeployConfiguration().getDeployAction());
installVdsParameters.setHostedEngineConfiguration(vdsDeployParams);
}
Map<String, String> values = new HashMap<>();
values.put(VdcObjectType.VDS.name().toLowerCase(), getParameters().getvds().getName());
Step installStep =
executionHandler.addSubStep(
getExecutionContext(),
getExecutionContext().getJob().getStep(StepEnum.EXECUTING),
StepEnum.INSTALLING_HOST,
ExecutionMessageDirector.resolveStepMessage(StepEnum.INSTALLING_HOST, values));
final ExecutionContext installCtx = new ExecutionContext();
installCtx.setJob(getExecutionContext().getJob());
installCtx.setStep(installStep);
installCtx.setMonitored(true);
installCtx.setShouldEndJob(true);
ThreadPoolUtil.execute(
() ->
runInternalAction(
VdcActionType.InstallVdsInternal,
installVdsParameters,
cloneContextAndDetachFromParent().withExecutionContext(installCtx)));
ExecutionHandler.setAsyncJob(getExecutionContext(), true);
}
}
@Override
protected void executeCommand() {
String address = getParameters().getAddress();
String subnet =
StringUtils.isEmpty(getParameters().getSubnet())
? getParameters().getNetwork().getSubnet()
: getParameters().getSubnet();
String gateway =
StringUtils.isEmpty(getParameters().getGateway()) ? "" : getParameters().getGateway();
ArrayList<String> interfaceNames = new ArrayList<>();
Map<String, VdsNetworkInterface> interfaceByName = Entities.entitiesByName(interfaces);
for (VdsNetworkInterface i : getParameters().getInterfaces()) {
VdsNetworkInterface existingIface = interfaceByName.get(i.getName());
if (Boolean.TRUE.equals(existingIface.getBonded())
|| NetworkUtils.isBondVlan(interfaces, existingIface)) {
getParameters().setBondName(NetworkUtils.stripVlan(existingIface));
for (VdsNetworkInterface ix : interfaces) {
if (NetworkUtils.interfaceBasedOn(existingIface, ix.getBondName())) {
interfaceNames.add(NetworkUtils.stripVlan(ix));
}
}
} else {
interfaceNames.add(NetworkUtils.stripVlan(existingIface));
}
}
// updating a vlan over bond if the bond was not provided the bond device should preserve the
// bonding options
String bondingOptions = null;
if (getParameters().getBondingOptions() == null
&& getParameters().getBondName() != null
&& !Entities.entitiesByName(getParameters().getInterfaces())
.containsKey(getParameters().getBondName())) {
VdsNetworkInterface bond =
Entities.entitiesByName(interfaces).get(getParameters().getBondName());
bondingOptions = bond == null ? null : bond.getBondOptions();
} else {
bondingOptions = getParameters().getBondingOptions();
}
NetworkVdsmVDSCommandParameters parameters =
new NetworkVdsmVDSCommandParameters(
getParameters().getVdsId(),
getParameters().getNetwork().getName(),
getParameters().getNetwork().getVlanId(),
getParameters().getBondName(),
interfaceNames.toArray(new String[] {}),
address,
subnet,
gateway,
getParameters().getNetwork().getStp(),
bondingOptions,
getParameters().getBootProtocol());
parameters.setVmNetwork(getParameters().getNetwork().isVmNetwork());
parameters.setOldNetworkName(getParameters().getOldNetworkName());
parameters.setConnectionTimeout(
Config.<Integer>getValue(ConfigValues.NetworkConnectivityCheckTimeoutInSeconds));
parameters.setCheckConnectivity(getParameters().getCheckConnectivity());
IPAddress[] adresses = Dns.getHostAddresses(NetworkUtils.OS_REFERENCE_TO_MACHINE_NAME);
if (adresses != null && adresses.length > 0) {
parameters.setHostAddr(adresses[0].toString());
}
if (getParameters().getCheckConnectivity()) {
ThreadPoolUtil.execute(new EditNetworkThread(parameters));
pollVds(getVds());
} else {
editNetworkThreadCompat(parameters);
}
if (retVal != null && editNetworkDone) {
// update vds network data
retVal =
runVdsCommand(
VDSCommandType.CollectVdsNetworkData,
new CollectHostNetworkDataVdsCommandParameters(getVds()));
if (retVal.getSucceeded()) {
Guid groupId = getVdsDao().get(getParameters().getVdsId()).getVdsGroupId();
NetworkClusterHelper.setStatus(groupId, getParameters().getNetwork());
setSucceeded(true);
}
}
}