/** Executes specified fence action on all fence agents concurrently */
  protected FenceOperationResult fenceConcurrently(FenceActionType fenceAction) {
    List<FenceOperationResult> results = new ArrayList<>(fenceAgents.size());
    FenceOperationResult taskResult;

    ExecutorCompletionService<FenceOperationResult> tasksExecutor =
        ThreadPoolUtil.createCompletionService();
    List<Future<FenceOperationResult>> futures =
        ThreadPoolUtil.submitTasks(tasksExecutor, createTasks(fenceAction));

    for (int i = 0; i < fenceAgents.size(); ++i) {
      try {
        taskResult = tasksExecutor.take().get();
      } catch (ExecutionException | InterruptedException e) {
        taskResult = new FenceOperationResult(Status.ERROR, PowerStatus.UNKNOWN, e.getMessage());
        log.error("Error getting task result: {}", e.getMessage());
        log.debug("Exception", e);
      }

      results.add(taskResult);
      if (tasksProcessor.isGoalReached(taskResult)) {
        // action goal is reach, cancel all remaining tasks
        for (Future<FenceOperationResult> future : futures) {
          future.cancel(true);
        }
        break;
      }
    }
    return tasksProcessor.createActionResult(results);
  }
예제 #2
0
 @Override
 public boolean vdsUpEvent(final VDS vds) {
   HostStoragePoolParametersBase params = new HostStoragePoolParametersBase(vds);
   boolean isSucceeded =
       backend.runInternalAction(VdcActionType.InitVdsOnUp, params).getSucceeded();
   if (isSucceeded) {
     ThreadPoolUtil.execute(
         new Runnable() {
           @Override
           public void run() {
             try {
               // migrate vms that its their default vds and failback
               // is on
               List<VmStatic> vmsToMigrate =
                   DbFacade.getInstance().getVmStaticDao().getAllWithFailbackByVds(vds.getId());
               if (!vmsToMigrate.isEmpty()) {
                 CommandContext ctx = new CommandContext(new EngineContext());
                 ctx.getExecutionContext().setMonitored(true);
                 backend.runInternalMultipleActions(
                     VdcActionType.MigrateVmToServer,
                     new ArrayList<>(createMigrateVmToServerParametersList(vmsToMigrate, vds)),
                     ctx);
               }
             } catch (RuntimeException e) {
               log.error("Failed to initialize Vds on up: {}", e.getMessage());
               log.error("Exception", e);
             }
           }
         });
   }
   return isSucceeded;
 }
예제 #3
0
 // TODO asynch event handler - design infra code to allow async events in segregated thread
 public void onMomPolicyChange(@Observes @MomPolicyUpdate final VDSGroup cluster) {
   if (cluster == null || cluster.getCompatibilityVersion().compareTo(Version.v3_4) < 0) return;
   List<VDS> activeHostsInCluster =
       vdsDao.getAllForVdsGroupWithStatus(cluster.getId(), VDSStatus.Up);
   // collect all Active hosts into a callable list
   List<Callable<Object>> callables = new LinkedList<>();
   for (final VDS vds : activeHostsInCluster) {
     callables.add(
         new Callable<Object>() {
           @Override
           public Object call() {
             try {
               resourceManagerProvider
                   .get()
                   .runVdsCommand(
                       VDSCommandType.SetMOMPolicyParameters,
                       new MomPolicyVDSParameters(
                           vds,
                           cluster.isEnableBallooning(),
                           cluster.isEnableKsm(),
                           cluster.isKsmMergeAcrossNumaNodes()));
             } catch (EngineException e) {
               log.error("Could not update MoM policy on host '{}'", vds.getName());
             }
             return null;
           }
         });
   }
   // run all VDSCommands concurrently with executor
   if (callables.size() > 0) ThreadPoolUtil.invokeAll(callables);
 }
  protected void executeHttpMethod(final T method) {
    int responseCode = -1;
    VdsManager manager = ResourceManager.getInstance().getVdsManager(getParameters().getVdsId());
    final HttpClient httpclient = manager.getVdsProxy().getHttpClient();
    try {
      FutureTask<Integer> futureTask =
          new FutureTask(
              new Callable<Integer>() {
                @Override
                public Integer call() throws Exception {
                  return httpclient.executeMethod(method);
                }
              });
      Future<Integer> f = ThreadPoolUtil.execute(futureTask);
      if (f.get(Config.<Integer>getValue(getConfigValueTimeLimitForOperation()), TimeUnit.MINUTES)
          == null) {
        responseCode = futureTask.get();
      }
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    } catch (Exception e) {
      log.debug("Exception", e);
      throw createNetworkException(e);
    }

    if (responseCode == getSuccessCode()) {
      Guid createdTask =
          Guid.createGuidFromString(processResponseHeaderValue(getMethod(), "Task-Id", null));
      getVDSReturnValue()
          .setCreationInfo(
              new AsyncTaskCreationInfo(
                  createdTask, getCreatedTaskType(), getParameters().getStoragePoolId()));
      handleOkResponse();
      getVDSReturnValue().setSucceeded(true);
      return;
    }

    processResponseHeaderValue(getMethod(), "Content-type", "application/json");

    String response;
    try {
      response = getMethod().getResponseBodyAsString();
    } catch (Exception e) {
      throw createNetworkException(e);
    }

    Map<String, Object> resultMap = null;
    try {
      resultMap = new ObjectMapper().readValue(response, HashMap.class);
      status = new StatusOnlyReturnForXmlRpc(resultMap);
    } catch (Exception e) {
      throwVdsErrorException("failed to parse response " + response, EngineError.GeneralException);
    }

    proceedProxyReturnValue();
  }
예제 #5
0
 @Override
 public void importHostedEngineVm(final VM vm) {
   ThreadPoolUtil.execute(
       new Runnable() {
         @Override
         public void run() {
           hostedEngineImporterProvider.get().doImport(vm);
         }
       });
 }
  /**
   * This method is called upon the bean creation as part of the management Service bean life cycle.
   */
  @Override
  @PostConstruct
  public void create() {

    try {
      // This must be done before starting to sample the hosts status from VDSM since the sampling
      // will turn such host from Reboot to NonResponsive
      loadService(PmHealthCheckManager.class);
      loadService(EngineBackupAwarenessManager.class);
      CommandCoordinatorUtil.initAsyncTaskManager();
      loadService(ResourceManager.class);
      OvfDataUpdater.getInstance().initOvfDataUpdater();
      ThreadPoolUtil.execute(
          new Runnable() {
            @Override
            public void run() {
              MacPoolPerDcSingleton.getInstance().initialize();
            }
          });
      StoragePoolStatusHandler.init();

      GlusterJobsManager.init();

      try {
        log.info("Init VM custom properties utilities");
        VmPropertiesUtils.getInstance().init();
      } catch (InitializationException e) {
        log.error("Initialization of vm custom properties failed.", e);
      }

      try {
        log.info("Init device custom properties utilities");
        DevicePropertiesUtils.getInstance().init();
      } catch (InitializationException e) {
        log.error("Initialization of device custom properties failed.", e);
      }

      loadService(SchedulingManager.class);

      SessionDataContainer.getInstance().cleanupEngineSessionsOnStartup();

      loadService(HostDeviceManager.class);
      loadService(DwhHeartBeat.class);

      if (Config.<Boolean>getValue(ConfigValues.AffinityRulesEnforcementManagerEnabled)) {
        loadService(AffinityRulesEnforcementManager.class);
      }

      loadService(CertificationValidityChecker.class);
    } catch (Exception ex) {
      log.error("Failed to initialize backend", ex);
      throw ex;
    }
  }
예제 #7
0
 public void onError(@Observes final VDSNetworkException vdsException) {
   ThreadPoolUtil.execute(
       new Runnable() {
         @Override
         public void run() {
           resourceManagerProvider
               .get()
               .getVdsManager(vdsException.getVdsError().getVdsId())
               .handleNetworkException(vdsException);
         }
       });
 }
예제 #8
0
 /**
  * Synchronize LUN details comprising the storage domain with the DB
  *
  * @param storageDomainId
  * @param vdsId
  */
 @Override
 public void syncLunsInfoForBlockStorageDomain(final Guid storageDomainId, final Guid vdsId) {
   ThreadPoolUtil.execute(
       new Runnable() {
         @Override
         public void run() {
           StorageDomainParametersBase parameters =
               new StorageDomainParametersBase(storageDomainId);
           parameters.setVdsId(vdsId);
           backend.runInternalAction(VdcActionType.SyncLunsInfoForBlockStorageDomain, parameters);
         }
       });
 }
예제 #9
0
  @Override
  public void refreshHostIfAnyVmHasHostDevices(final List<Guid> vmIds, final Guid hostId) {
    if (vmIds.isEmpty()) {
      return;
    }

    ThreadPoolUtil.execute(
        new Runnable() {
          @Override
          public void run() {
            HostDeviceManager hostDeviceManager = Injector.get(HostDeviceManager.class);
            hostDeviceManager.refreshHostIfAnyVmHasHostDevices(vmIds, hostId);
          }
        });
  }
예제 #10
0
 @Override
 public void rerun(Guid vmId) {
   final IVdsAsyncCommand command = vdsBroker.getAsyncCommandForVm(vmId);
   if (command != null) {
     // The command will be invoked in a different VDS in its rerun method, so we're calling
     // its rerun method from a new thread so that it won't be executed within our current VDSM
     // lock
     ThreadPoolUtil.execute(
         new Runnable() {
           @Override
           public void run() {
             command.rerun();
           }
         });
   }
 }
예제 #11
0
  @Override
  public void processOnVmStop(
      final Collection<Guid> vmIds, final Guid hostId, boolean useSeparateThread) {
    if (vmIds.isEmpty()) {
      return;
    }

    if (useSeparateThread) {
      ThreadPoolUtil.execute(
          new Runnable() {
            @Override
            public void run() {
              processOnVmStopInternal(vmIds, hostId);
            }
          });
    } else {
      processOnVmStopInternal(vmIds, hostId);
    }
  }
예제 #12
0
 public void handlePartiallyExecuteTasksOfCommand(final List<AsyncTask> tasks) {
   ThreadPoolUtil.execute(
       new Runnable() {
         @SuppressWarnings("synthetic-access")
         @Override
         public void run() {
           TransactionSupport.executeInNewTransaction(
               () -> {
                 try {
                   for (AsyncTask task : tasks) {
                     handlePartiallyExecutedTaskOfCommand(task);
                   }
                   return null;
                 } finally {
                   irsBrokerLatch.countDown();
                 }
               });
         }
       });
 }
예제 #13
0
 @Override
 public void updateSlaPolicies(final List<Guid> vmIds, final Guid vdsId) {
   if (vmIds.isEmpty()) {
     return;
   }
   ThreadPoolUtil.execute(
       new Runnable() {
         @Override
         public void run() {
           for (Guid vmId : vmIds) {
             CpuQos qos = DbFacade.getInstance().getCpuQosDao().getCpuQosByVmId(vmId);
             if (qos != null && qos.getCpuLimit() != null) {
               resourceManagerProvider
                   .get()
                   .runVdsCommand(
                       VDSCommandType.UpdateVmPolicy,
                       new UpdateVmPolicyVDSParams(vdsId, vmId, qos.getCpuLimit().intValue()));
             }
           }
         }
       });
 }
예제 #14
0
  @Override
  public void vdsNotResponding(final VDS vds) {
    ExecutionHandler.updateSpecificActionJobCompleted(
        vds.getId(), VdcActionType.MaintenanceVds, false);
    ThreadPoolUtil.execute(
        new Runnable() {
          @Override
          public void run() {
            log.info(
                "ResourceManager::vdsNotResponding entered for Host '{}', '{}'",
                vds.getId(),
                vds.getHostName());

            FenceVdsActionParameters params = new FenceVdsActionParameters(vds.getId());
            backend.runInternalAction(
                VdcActionType.VdsNotRespondingTreatment,
                params,
                ExecutionHandler.createInternalJobContext());

            moveBricksToUnknown(vds);
          }
        });
  }
  private void connectAndRefreshAllUpHosts(final boolean commandSucceeded) {
    if (isLastMaster || !commandSucceeded) {
      log.warn(
          "skipping connect and refresh for all hosts, last master '{}', command status '{}'",
          isLastMaster,
          commandSucceeded);
      return;
    }

    List<Callable<Void>> tasks = new ArrayList<>();
    for (final VDS vds : getAllRunningVdssInPool()) {
      tasks.add(
          () -> {
            try {
              if (!connectVdsToNewMaster(vds)) {
                log.warn(
                    "failed to connect vds '{}' to the new master '{}'",
                    vds.getId(),
                    getNewMasterStorageDomainId());
                return null;
              }

              List<StoragePoolIsoMap> storagePoolIsoMap =
                  storagePoolIsoMapDao.getAllForStoragePool(getStoragePool().getId());
              try {
                runVdsCommand(
                    VDSCommandType.ConnectStoragePool,
                    new ConnectStoragePoolVDSCommandParameters(
                        vds,
                        getStoragePool(),
                        getNewMasterStorageDomainId(),
                        storagePoolIsoMap,
                        true));
              } catch (EngineException ex) {
                if (EngineError.StoragePoolUnknown == ex.getVdsError().getCode()) {
                  VDSReturnValue returnVal =
                      runVdsCommand(
                          VDSCommandType.ConnectStoragePool,
                          new ConnectStoragePoolVDSCommandParameters(
                              vds,
                              getStoragePool(),
                              getNewMasterStorageDomainId(),
                              storagePoolIsoMap));
                  if (!returnVal.getSucceeded()) {
                    log.error(
                        "Post reconstruct actions (connectPool) did not complete on host '{}' in the pool. error {}",
                        vds.getId(),
                        returnVal.getVdsError().getMessage());
                  }
                } else {
                  log.error(
                      "Post reconstruct actions (refreshPool)"
                          + " did not complete on host '{}' in the pool. error {}",
                      vds.getId(),
                      ex.getMessage());
                }
              }
            } catch (Exception e) {
              log.error(
                  "Post reconstruct actions (connectPool,refreshPool,disconnect storage)"
                      + " did not complete on host '{}' in the pool: {}",
                  vds.getId(),
                  e.getMessage());
              log.debug("Exception", e);
            }
            return null;
          });
    }
    ThreadPoolUtil.invokeAll(tasks);
  }
예제 #16
0
  @Override
  protected void executeCommand() {
    Guid oVirtId = getParameters().getVdsForUniqueId();
    if (oVirtId != null) {

      // if fails to remove deprecated entry, we might attempt to add new oVirt host with an
      // existing unique-id.
      if (!removeDeprecatedOvirtEntry(oVirtId)) {
        log.error(
            "Failed to remove duplicated oVirt entry with id '{}'. Abort adding oVirt Host type",
            oVirtId);
        throw new EngineException(EngineError.HOST_ALREADY_EXISTS);
      }
    }

    TransactionSupport.executeInNewTransaction(
        () -> {
          addVdsStaticToDb();
          addVdsDynamicToDb();
          addVdsStatisticsToDb();
          getCompensationContext().stateChanged();
          return null;
        });

    if (getParameters().isProvisioned()) {
      HostProviderProxy proxy = ProviderProxyFactory.getInstance().create(getHostProvider());
      proxy.provisionHost(
          getParameters().getvds(),
          getParameters().getHostGroup(),
          getParameters().getComputeResource(),
          getParameters().getHostMac(),
          getParameters().getDiscoverName(),
          getParameters().getPassword(),
          getParameters().getDiscoverIp());

      addCustomValue("HostGroupName", getParameters().getHostGroup().getName());
      auditLogDirector.log(this, AuditLogType.VDS_PROVISION);
    }

    // set vds spm id
    if (getCluster().getStoragePoolId() != null) {
      VdsActionParameters tempVar = new VdsActionParameters(getVdsIdRef());
      tempVar.setSessionId(getParameters().getSessionId());
      tempVar.setCompensationEnabled(true);
      VdcReturnValueBase addVdsSpmIdReturn =
          runInternalAction(
              VdcActionType.AddVdsSpmId,
              tempVar,
              cloneContext().withoutLock().withoutExecutionContext());
      if (!addVdsSpmIdReturn.getSucceeded()) {
        setSucceeded(false);
        getReturnValue().setFault(addVdsSpmIdReturn.getFault());
        return;
      }
    }
    TransactionSupport.executeInNewTransaction(
        () -> {
          initializeVds(true);
          alertIfPowerManagementNotConfigured(getParameters().getVdsStaticData());
          testVdsPowerManagementStatus(getParameters().getVdsStaticData());
          setSucceeded(true);
          setActionReturnValue(getVdsIdRef());

          // If the installation failed, we don't want to compensate for the failure since it will
          // remove the
          // host, but instead the host should be left in an "install failed" status.
          getCompensationContext().cleanupCompensationDataAfterSuccessfulCommand();
          return null;
        });
    // do not install vds's which added in pending mode or for provisioning (currently power
    // clients). they are installed as part of the approve process or automatically after provision
    if (Config.<Boolean>getValue(ConfigValues.InstallVds)
        && !getParameters().isPending()
        && !getParameters().isProvisioned()) {
      final InstallVdsParameters installVdsParameters =
          new InstallVdsParameters(getVdsId(), getParameters().getPassword());
      installVdsParameters.setAuthMethod(getParameters().getAuthMethod());
      installVdsParameters.setOverrideFirewall(getParameters().getOverrideFirewall());
      installVdsParameters.setActivateHost(getParameters().getActivateHost());
      installVdsParameters.setNetworkProviderId(
          getParameters().getVdsStaticData().getOpenstackNetworkProviderId());
      installVdsParameters.setNetworkMappings(getParameters().getNetworkMappings());
      installVdsParameters.setEnableSerialConsole(getParameters().getEnableSerialConsole());
      if (getParameters().getHostedEngineDeployConfiguration() != null) {
        Map<String, String> vdsDeployParams =
            hostedEngineHelper.createVdsDeployParams(
                getVdsId(), getParameters().getHostedEngineDeployConfiguration().getDeployAction());
        installVdsParameters.setHostedEngineConfiguration(vdsDeployParams);
      }
      Map<String, String> values = new HashMap<>();
      values.put(VdcObjectType.VDS.name().toLowerCase(), getParameters().getvds().getName());
      Step installStep =
          executionHandler.addSubStep(
              getExecutionContext(),
              getExecutionContext().getJob().getStep(StepEnum.EXECUTING),
              StepEnum.INSTALLING_HOST,
              ExecutionMessageDirector.resolveStepMessage(StepEnum.INSTALLING_HOST, values));
      final ExecutionContext installCtx = new ExecutionContext();
      installCtx.setJob(getExecutionContext().getJob());
      installCtx.setStep(installStep);
      installCtx.setMonitored(true);
      installCtx.setShouldEndJob(true);
      ThreadPoolUtil.execute(
          () ->
              runInternalAction(
                  VdcActionType.InstallVdsInternal,
                  installVdsParameters,
                  cloneContextAndDetachFromParent().withExecutionContext(installCtx)));
      ExecutionHandler.setAsyncJob(getExecutionContext(), true);
    }
  }
  @Override
  protected void executeCommand() {
    String address = getParameters().getAddress();
    String subnet =
        StringUtils.isEmpty(getParameters().getSubnet())
            ? getParameters().getNetwork().getSubnet()
            : getParameters().getSubnet();
    String gateway =
        StringUtils.isEmpty(getParameters().getGateway()) ? "" : getParameters().getGateway();
    ArrayList<String> interfaceNames = new ArrayList<>();

    Map<String, VdsNetworkInterface> interfaceByName = Entities.entitiesByName(interfaces);
    for (VdsNetworkInterface i : getParameters().getInterfaces()) {
      VdsNetworkInterface existingIface = interfaceByName.get(i.getName());
      if (Boolean.TRUE.equals(existingIface.getBonded())
          || NetworkUtils.isBondVlan(interfaces, existingIface)) {
        getParameters().setBondName(NetworkUtils.stripVlan(existingIface));
        for (VdsNetworkInterface ix : interfaces) {
          if (NetworkUtils.interfaceBasedOn(existingIface, ix.getBondName())) {
            interfaceNames.add(NetworkUtils.stripVlan(ix));
          }
        }
      } else {
        interfaceNames.add(NetworkUtils.stripVlan(existingIface));
      }
    }

    // updating a vlan over bond if the bond was not provided the bond device should preserve the
    // bonding options
    String bondingOptions = null;
    if (getParameters().getBondingOptions() == null
        && getParameters().getBondName() != null
        && !Entities.entitiesByName(getParameters().getInterfaces())
            .containsKey(getParameters().getBondName())) {
      VdsNetworkInterface bond =
          Entities.entitiesByName(interfaces).get(getParameters().getBondName());
      bondingOptions = bond == null ? null : bond.getBondOptions();
    } else {
      bondingOptions = getParameters().getBondingOptions();
    }

    NetworkVdsmVDSCommandParameters parameters =
        new NetworkVdsmVDSCommandParameters(
            getParameters().getVdsId(),
            getParameters().getNetwork().getName(),
            getParameters().getNetwork().getVlanId(),
            getParameters().getBondName(),
            interfaceNames.toArray(new String[] {}),
            address,
            subnet,
            gateway,
            getParameters().getNetwork().getStp(),
            bondingOptions,
            getParameters().getBootProtocol());
    parameters.setVmNetwork(getParameters().getNetwork().isVmNetwork());
    parameters.setOldNetworkName(getParameters().getOldNetworkName());
    parameters.setConnectionTimeout(
        Config.<Integer>getValue(ConfigValues.NetworkConnectivityCheckTimeoutInSeconds));
    parameters.setCheckConnectivity(getParameters().getCheckConnectivity());

    IPAddress[] adresses = Dns.getHostAddresses(NetworkUtils.OS_REFERENCE_TO_MACHINE_NAME);
    if (adresses != null && adresses.length > 0) {
      parameters.setHostAddr(adresses[0].toString());
    }

    if (getParameters().getCheckConnectivity()) {
      ThreadPoolUtil.execute(new EditNetworkThread(parameters));
      pollVds(getVds());
    } else {
      editNetworkThreadCompat(parameters);
    }

    if (retVal != null && editNetworkDone) {
      // update vds network data
      retVal =
          runVdsCommand(
              VDSCommandType.CollectVdsNetworkData,
              new CollectHostNetworkDataVdsCommandParameters(getVds()));

      if (retVal.getSucceeded()) {
        Guid groupId = getVdsDao().get(getParameters().getVdsId()).getVdsGroupId();
        NetworkClusterHelper.setStatus(groupId, getParameters().getNetwork());
        setSucceeded(true);
      }
    }
  }