private void cleanupUnfinishedWork() {
   Date before = new Date(System.currentTimeMillis() - 2 * _timeBetweenCleanups * 1000l);
   List<SecurityGroupWorkVO> unfinished = _workDao.findUnfinishedWork(before);
   if (unfinished.size() > 0) {
     s_logger.info(
         "Network Group Work cleanup found "
             + unfinished.size()
             + " unfinished work items older than "
             + before.toString());
     ArrayList<Long> affectedVms = new ArrayList<Long>();
     for (SecurityGroupWorkVO work : unfinished) {
       affectedVms.add(work.getInstanceId());
       work.setStep(Step.Error);
       _workDao.update(work.getId(), work);
     }
     scheduleRulesetUpdateToHosts(affectedVms, false, null);
   } else {
     s_logger.debug(
         "Network Group Work cleanup found no unfinished work items older than "
             + before.toString());
   }
 }
  @DB
  public void scheduleRulesetUpdateToHosts(
      List<Long> affectedVms, boolean updateSeqno, Long delayMs) {
    if (affectedVms.size() == 0) {
      return;
    }

    if (delayMs == null) {
      delayMs = new Long(100l);
    }

    Collections.sort(affectedVms);
    if (s_logger.isTraceEnabled()) {
      s_logger.trace(
          "Security Group Mgr: scheduling ruleset updates for " + affectedVms.size() + " vms");
    }
    boolean locked = _workLock.lock(_globalWorkLockTimeout);
    if (!locked) {
      s_logger.warn("Security Group Mgr: failed to acquire global work lock");
      return;
    }

    if (s_logger.isTraceEnabled()) {
      s_logger.trace("Security Group Mgr: acquired global work lock");
    }
    Transaction txn = Transaction.currentTxn();
    try {
      txn.start();
      for (Long vmId : affectedVms) {
        if (s_logger.isTraceEnabled()) {
          s_logger.trace("Security Group Mgr: scheduling ruleset update for " + vmId);
        }
        VmRulesetLogVO log = null;
        SecurityGroupWorkVO work = null;

        log = _rulesetLogDao.findByVmId(vmId);
        if (log == null) {
          log = new VmRulesetLogVO(vmId);
          log = _rulesetLogDao.persist(log);
        }

        if (log != null && updateSeqno) {
          log.incrLogsequence();
          _rulesetLogDao.update(log.getId(), log);
        }
        work = _workDao.findByVmIdStep(vmId, Step.Scheduled);
        if (work == null) {
          work = new SecurityGroupWorkVO(vmId, null, null, SecurityGroupWork.Step.Scheduled, null);
          work = _workDao.persist(work);
          if (s_logger.isTraceEnabled()) {
            s_logger.trace(
                "Security Group Mgr: created new work item for " + vmId + "; id = " + work.getId());
          }
        }

        work.setLogsequenceNumber(log.getLogsequence());
        _workDao.update(work.getId(), work);
      }
      txn.commit();
      for (Long vmId : affectedVms) {
        _executorPool.schedule(new WorkerThread(), delayMs, TimeUnit.MILLISECONDS);
      }
    } finally {
      _workLock.unlock();
      if (s_logger.isTraceEnabled()) {
        s_logger.trace("Security Group Mgr: released global work lock");
      }
    }
  }