/**
* Returns control when task is complete.
*
* @param json
* @param logger
*/
private String waitForDeploymentCompletion(JSON json, OctopusApi api, Log logger) {
final long WAIT_TIME = 5000;
final double WAIT_RANDOM_SCALER = 100.0;
JSONObject jsonObj = (JSONObject) json;
String id = jsonObj.getString("TaskId");
Task task = null;
String lastState = "Unknown";
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
logger.info("Task info:");
logger.info("\tId: " + task.getId());
logger.info("\tName: " + task.getName());
logger.info("\tDesc: " + task.getDescription());
logger.info("\tState: " + task.getState());
logger.info("\n\nStarting wait...");
boolean completed = task.getIsCompleted();
while (!completed) {
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
completed = task.getIsCompleted();
lastState = task.getState();
logger.info("Task state: " + lastState);
if (completed) {
break;
}
try {
Thread.sleep(WAIT_TIME + (long) (Math.random() * WAIT_RANDOM_SCALER));
} catch (InterruptedException ex) {
logger.info("Wait interrupted!");
logger.info(ex.getMessage());
completed = true; // bail out of wait loop
}
}
logger.info("Wait complete!");
return lastState;
}
/**
* Update the tags stored in EC2 with the specified information. Re-try 5 times if instances isn't
* up by catchErrorCode - e.g. InvalidSpotInstanceRequestID.NotFound or
* InvalidInstanceRequestID.NotFound
*
* @param ec2
* @param instTags
* @param catchErrorCode
* @param params
* @throws InterruptedException
*/
private void updateRemoteTags(
AmazonEC2 ec2, Collection<Tag> instTags, String catchErrorCode, String... params)
throws InterruptedException {
for (int i = 0; i < 5; i++) {
try {
CreateTagsRequest tagRequest = new CreateTagsRequest();
tagRequest.withResources(params).setTags(instTags);
ec2.createTags(tagRequest);
break;
} catch (AmazonServiceException e) {
if (e.getErrorCode().equals(catchErrorCode)) {
Thread.sleep(5000);
continue;
}
LOGGER.log(Level.SEVERE, e.getErrorMessage(), e);
}
}
}
/**
* Schedules the given configuration.
*
* <p>Copied from the {@link
* DefaultMatrixExecutionStrategyImpl#scheduleConfigurationBuild(hudson.matrix.MatrixBuild.MatrixBuildExecution,
* hudson.matrix.MatrixConfiguration)}
*
* @param execution Contains information about the general build, including the listener used to
* log queue blockage.
* @param configuration The configuration to schedule.
* @param upstreamCause The cause of the build. Will either be an {@link
* hudson.model.Cause.UpstreamCause} or {@link
* com.attask.jenkins.healingmatrixproject.SelfHealingCause}.
*/
private void scheduleConfigurationBuild(
MatrixBuild.MatrixBuildExecution execution,
MatrixConfiguration configuration,
Cause.UpstreamCause upstreamCause)
throws InterruptedException {
MatrixBuild build = (MatrixBuild) execution.getBuild();
execution
.getListener()
.getLogger()
.println(Messages.MatrixBuild_Triggering(ModelHyperlinkNote.encodeTo(configuration)));
// filter the parent actions for those that can be passed to the individual jobs.
List<MatrixChildAction> childActions = Util.filter(build.getActions(), MatrixChildAction.class);
BuildListener listener = execution.getListener();
while (!configuration.scheduleBuild(childActions, upstreamCause)) {
String msg = "Unable to schedule build " + configuration.getFullDisplayName() + ". Retrying.";
listener.error(msg);
Thread.sleep(500);
}
}
@Override
public void onCompleted(AbstractBuild run, TaskListener listener) {
Lamps plugin = Lamps.getInstance();
Set<String> jobs = plugin.getJobs();
String jobName = run.getParent().getFullName();
XfEventMessage xfEventMessage = new XfEventMessage();
if (jobs.contains(jobName)) {
Result result = run.getResult();
Set<Lamp> activeLamps = plugin.getLampsContainingJob(jobName);
for (Lamp lamp : activeLamps) {
Result lampResult = result;
xfEventMessage.sendColorMessage(lamp, lampResult, States.Action.SOLID);
// Create Notification for LCD
StringBuilder infoMsg = new StringBuilder(64);
infoMsg.append(jobName).append(' ').append(run.getDisplayName()).append('\n');
if (Result.FAILURE.equals(result)) {
ArrayList<String> blame = Lists.newArrayList();
if (lamp.isBlame()) {
Set<User> culprits = run.getCulprits();
for (User user : culprits) {
blame.add(user.getDisplayName());
}
}
if (blame.isEmpty()) {
blame.add("Somebody");
}
infoMsg.insert(0, Joiner.on(", ").join(blame) + " broke the build: ");
infoMsg.append(result.toString());
listener.getLogger().println("[XFD] Updating Lamp display: " + infoMsg.toString());
} else if (Result.ABORTED.equals(result)) {
String causeMsg = "BUILD ABORTED";
infoMsg.append(causeMsg);
listener.getLogger().println("[XFD] Updating Lamp display: " + infoMsg.toString());
} else {
infoMsg.append(result.toString());
}
xfEventMessage.sendLCDMessage(lamp, infoMsg.toString());
if (lamp.isSfx()) {
try {
Thread.sleep(1000);
xfEventMessage.sendSfxMessage(lamp, lampResult);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
if (States.resultColorMap.get(lampResult).equals(States.Color.RED) && lamp.isNoisy()) {
try {
Thread.sleep(1000);
xfEventMessage.sendBuzzerMessage(lamp);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
if (lamp.isAggregate()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
plugin.updateAggregateStatus(lamp);
}
}
}
}
/**
* Waits for the given configurations to finish, retrying any that qualify to be rerun.
*
* @param execution Provided by the plugin.
* @param patterns List of regular expression patterns used to scan the log to determine if a
* build should be rerun.
* @param retries Mutable map that tracks the number of times a specific configuration has been
* retried.
* @param configurations The configurations that have already been scheduled to run that should be
* waited for to finish.
* @return The worst result of all the runs. If a build was rerun, only the result of the rerun is
* considered.
* @throws InterruptedException
* @throws IOException
*/
private Result waitForMatrixRuns(
MatrixBuild.MatrixBuildExecution execution,
List<Pattern> patterns,
Map<MatrixConfiguration, Integer> retries,
LinkedList<MatrixConfiguration> configurations)
throws InterruptedException, IOException {
BuildListener listener = execution.getListener();
PrintStream logger = listener.getLogger();
Map<String, String> whyBlockedMap =
new HashMap<
String,
String>(); // keep track of why builds are blocked so we can print unique messages when
// they change.
Result finalResult = Result.SUCCESS;
int iteration = 0;
boolean continueRetrying = true;
while (!configurations.isEmpty()) {
++iteration;
MatrixConfiguration configuration = configurations.removeFirst();
if (isBuilding(execution, configuration, whyBlockedMap)) {
if (iteration >= configurations.size()) {
// Every time we loop through all the configurations, sleep for a bit.
// This is to prevent polling too often while everything is still building.
iteration = 0;
Thread.sleep(1000);
}
configurations.add(configuration);
continue;
}
Run parentBuild = execution.getBuild();
MatrixRun matrixRun = configuration.getBuildByNumber(parentBuild.getNumber());
Result runResult = matrixRun.getResult();
if (continueRetrying
&& runResult.isWorseOrEqualTo(getWorseThanOrEqualTo())
&& runResult.isBetterOrEqualTo(getBetterThanOrEqualTo())) {
if (matchesPattern(matrixRun, patterns)) {
int retriedCount = retries.get(configuration);
if (retriedCount < getMaxRetries()) {
++retriedCount;
retries.put(configuration, retriedCount);
// rerun
String logMessage =
String.format(
"%s was %s. Matched pattern to rerun. Rerunning (%d).",
matrixRun, runResult, retriedCount);
listener.error(logMessage);
HealedAction action = parentBuild.getAction(HealedAction.class);
if (action == null) {
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (parentBuild.getActions()) {
action = parentBuild.getAction(HealedAction.class);
if (action == null) {
action = new HealedAction(matrixRun.getCharset());
parentBuild.addAction(action);
}
}
}
action.addAutoHealedJob(matrixRun);
MatrixConfiguration parent = matrixRun.getParent();
if (parent != null) {
// I'm paranoid about NPEs
parent.removeRun(matrixRun);
matrixRun.delete();
} else {
LOGGER.severe(
"couldn't remove old run, parent was null. This is a Jenkins core bug.");
}
scheduleConfigurationBuild(
execution, configuration, new SelfHealingCause(parentBuild, retriedCount));
configurations.add(configuration);
continue;
} else {
String logMessage =
String.format(
"%s was %s. Matched pattern to rerun, but the max number of retries (%d) has been met.",
matrixRun, runResult, getMaxRetries());
listener.error(logMessage);
if (getStopRetryingAfterOneFails()) {
listener.error("Not retrying any more builds.");
continueRetrying = false;
}
}
} else {
String logMessage =
String.format(
"%s was %s. It did not match the pattern to rerun. Accepting result.",
matrixRun, runResult);
logger.println(logMessage);
}
}
notifyEndRun(matrixRun, execution.getAggregators(), execution.getListener());
finalResult = finalResult.combine(runResult);
}
return finalResult;
}