@Override
public boolean processAnswers(long agentId, long seq, Answer[] answers) {
List<Long> affectedVms = new ArrayList<Long>();
int commandNum = 0;
for (Answer ans : answers) {
if (ans instanceof SecurityGroupRuleAnswer) {
SecurityGroupRuleAnswer ruleAnswer = (SecurityGroupRuleAnswer) ans;
if (ans.getResult()) {
s_logger.debug(
"Successfully programmed rule " + ruleAnswer.toString() + " into host " + agentId);
_workDao.updateStep(ruleAnswer.getVmId(), ruleAnswer.getLogSequenceNumber(), Step.Done);
recordSuccess(ruleAnswer.getVmId());
} else {
_workDao.updateStep(ruleAnswer.getVmId(), ruleAnswer.getLogSequenceNumber(), Step.Error);
;
s_logger.debug(
"Failed to program rule "
+ ruleAnswer.toString()
+ " into host "
+ agentId
+ " due to "
+ ruleAnswer.getDetails()
+ " and updated jobs");
if (ruleAnswer.getReason() == FailureReason.CANNOT_BRIDGE_FIREWALL) {
s_logger.debug(
"Not retrying security group rules for vm "
+ ruleAnswer.getVmId()
+ " on failure since host "
+ agentId
+ " cannot do bridge firewalling");
} else if (ruleAnswer.getReason() == FailureReason.PROGRAMMING_FAILED) {
if (checkShouldRetryOnFailure(ruleAnswer.getVmId())) {
s_logger.debug(
"Retrying security group rules on failure for vm " + ruleAnswer.getVmId());
affectedVms.add(ruleAnswer.getVmId());
} else {
s_logger.debug(
"Not retrying security group rules for vm "
+ ruleAnswer.getVmId()
+ " on failure: too many retries");
}
}
}
commandNum++;
if (_workTracker != null) _workTracker.processAnswers(agentId, seq, answers);
}
}
if (affectedVms.size() > 0) {
_securityGroupManager.scheduleRulesetUpdateToHosts(affectedVms, false, new Long(10 * 1000l));
}
return true;
}
public void cleanupFinishedWork() {
Date before = new Date(System.currentTimeMillis() - 6 * 3600 * 1000l);
int numDeleted = _workDao.deleteFinishedWork(before);
if (numDeleted > 0) {
s_logger.info(
"Network Group Work cleanup deleted "
+ numDeleted
+ " finished work items older than "
+ before.toString());
}
}
private void processScheduledWork() {
List<SecurityGroupWorkVO> scheduled = _workDao.findScheduledWork();
int numJobs = scheduled.size();
if (numJobs > 0) {
s_logger.debug("Security group work: found scheduled jobs " + numJobs);
Random rand = new Random();
for (int i = 0; i < numJobs; i++) {
long delayMs = 100 + 10 * rand.nextInt(numJobs);
_executorPool.schedule(new WorkerThread(), delayMs, TimeUnit.MILLISECONDS);
}
}
}
private void cleanupUnfinishedWork() {
Date before = new Date(System.currentTimeMillis() - 2 * _timeBetweenCleanups * 1000l);
List<SecurityGroupWorkVO> unfinished = _workDao.findUnfinishedWork(before);
if (unfinished.size() > 0) {
s_logger.info(
"Network Group Work cleanup found "
+ unfinished.size()
+ " unfinished work items older than "
+ before.toString());
ArrayList<Long> affectedVms = new ArrayList<Long>();
for (SecurityGroupWorkVO work : unfinished) {
affectedVms.add(work.getInstanceId());
work.setStep(Step.Error);
_workDao.update(work.getId(), work);
}
scheduleRulesetUpdateToHosts(affectedVms, false, null);
} else {
s_logger.debug(
"Network Group Work cleanup found no unfinished work items older than "
+ before.toString());
}
}
@DB
public void work() {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Checking the database");
}
final SecurityGroupWorkVO work = _workDao.take(_serverId);
if (work == null) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Security Group work: no work found");
}
return;
}
Long userVmId = work.getInstanceId();
if (work.getStep() == Step.Done) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"Security Group work: found a job in done state, rescheduling for vm: " + userVmId);
}
ArrayList<Long> affectedVms = new ArrayList<Long>();
affectedVms.add(userVmId);
scheduleRulesetUpdateToHosts(affectedVms, false, _timeBetweenCleanups * 1000l);
return;
}
UserVm vm = null;
Long seqnum = null;
s_logger.debug("Working on " + work);
final Transaction txn = Transaction.currentTxn();
txn.start();
boolean locked = false;
try {
vm = _userVMDao.acquireInLockTable(work.getInstanceId());
if (vm == null) {
vm = _userVMDao.findById(work.getInstanceId());
if (vm == null) {
s_logger.info("VM " + work.getInstanceId() + " is removed");
locked = true;
return;
}
s_logger.warn("Unable to acquire lock on vm id=" + userVmId);
return;
}
locked = true;
Long agentId = null;
VmRulesetLogVO log = _rulesetLogDao.findByVmId(userVmId);
if (log == null) {
s_logger.warn("Cannot find log record for vm id=" + userVmId);
return;
}
seqnum = log.getLogsequence();
if (vm != null && vm.getState() == State.Running) {
Map<PortAndProto, Set<String>> rules = generateRulesForVM(userVmId);
agentId = vm.getHostId();
if (agentId != null) {
SecurityIngressRulesCmd cmd =
generateRulesetCmd(
vm.getInstanceName(),
vm.getPrivateIpAddress(),
vm.getPrivateMacAddress(),
vm.getId(),
generateRulesetSignature(rules),
seqnum,
rules);
Commands cmds = new Commands(cmd);
try {
_agentMgr.send(agentId, cmds, _answerListener);
} catch (AgentUnavailableException e) {
s_logger.debug(
"Unable to send updates for vm: " + userVmId + "(agentid=" + agentId + ")");
_workDao.updateStep(work.getInstanceId(), seqnum, Step.Done);
}
}
}
} finally {
if (locked) {
_userVMDao.releaseFromLockTable(userVmId);
_workDao.updateStep(work.getId(), Step.Done);
}
txn.commit();
}
}
@DB
public void scheduleRulesetUpdateToHosts(
List<Long> affectedVms, boolean updateSeqno, Long delayMs) {
if (affectedVms.size() == 0) {
return;
}
if (delayMs == null) {
delayMs = new Long(100l);
}
Collections.sort(affectedVms);
if (s_logger.isTraceEnabled()) {
s_logger.trace(
"Security Group Mgr: scheduling ruleset updates for " + affectedVms.size() + " vms");
}
boolean locked = _workLock.lock(_globalWorkLockTimeout);
if (!locked) {
s_logger.warn("Security Group Mgr: failed to acquire global work lock");
return;
}
if (s_logger.isTraceEnabled()) {
s_logger.trace("Security Group Mgr: acquired global work lock");
}
Transaction txn = Transaction.currentTxn();
try {
txn.start();
for (Long vmId : affectedVms) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("Security Group Mgr: scheduling ruleset update for " + vmId);
}
VmRulesetLogVO log = null;
SecurityGroupWorkVO work = null;
log = _rulesetLogDao.findByVmId(vmId);
if (log == null) {
log = new VmRulesetLogVO(vmId);
log = _rulesetLogDao.persist(log);
}
if (log != null && updateSeqno) {
log.incrLogsequence();
_rulesetLogDao.update(log.getId(), log);
}
work = _workDao.findByVmIdStep(vmId, Step.Scheduled);
if (work == null) {
work = new SecurityGroupWorkVO(vmId, null, null, SecurityGroupWork.Step.Scheduled, null);
work = _workDao.persist(work);
if (s_logger.isTraceEnabled()) {
s_logger.trace(
"Security Group Mgr: created new work item for " + vmId + "; id = " + work.getId());
}
}
work.setLogsequenceNumber(log.getLogsequence());
_workDao.update(work.getId(), work);
}
txn.commit();
for (Long vmId : affectedVms) {
_executorPool.schedule(new WorkerThread(), delayMs, TimeUnit.MILLISECONDS);
}
} finally {
_workLock.unlock();
if (s_logger.isTraceEnabled()) {
s_logger.trace("Security Group Mgr: released global work lock");
}
}
}