public int compare(Run record, String key) {
try {
int k = Integer.parseInt(key);
return record.getNumber() - k;
} catch (NumberFormatException nfe) {
return String.valueOf(record.getNumber()).compareTo(key);
}
}
public int compare(Run o1, Run o2) {
if (o1 == null) {
if (o2 == null) {
return 0;
} else {
return -1;
}
}
if (o2 == null) {
return 1;
}
return ((Integer) o2.getNumber()).compareTo(o1.getNumber());
}
public File getRunDir(Run run) {
String runName = run.getName();
if (runName == null) throw new IllegalArgumentException("Run doesn't have a name.");
File runDir = new File(root, runName);
if (!runDir.exists()) runDir.mkdir();
return runDir;
}
public void populate(@NotNull AbstractProject project) {
synchronized (populateLock) {
Run lastBuild = project.getLastBuild();
if (lastBuild == null) {
return;
}
if (lastBuild.getNumber() <= oldest) {
return;
}
for (int number = lastBuild.getNumber(); number > oldest; number--) {
Run build = project.getBuildByNumber(number);
if (build == null) {
continue;
}
String externalizableId = build.getExternalizableId();
size++;
put("projectName", project.getName(), externalizableId);
populateWithChangeInformation(build, externalizableId);
populateWithCauseInformation(build, externalizableId);
populateWithParameters(build, externalizableId);
}
oldest = lastBuild.getNumber();
}
}
/*
* Launches against the agent& main
*/
public void testAgentAndMain() throws Exception {
Project project = workspace.getProject("p1");
Run bndrun = new Run(workspace, project.getBase(), project.getFile("one.bndrun"));
bndrun.setProperty("-runpath", "biz.aQute.remote.launcher");
bndrun.setProperty("-runbundles", "bsn-1,bsn-2");
bndrun.setProperty("-runremote", "agent,main;agent=1090");
final RemoteProjectLauncherPlugin pl =
(RemoteProjectLauncherPlugin) bndrun.getProjectLauncher();
pl.prepare();
List<? extends RunSession> sessions = pl.getRunSessions();
assertEquals(2, sessions.size());
RunSession agent = sessions.get(0);
RunSession main = sessions.get(1);
CountDownLatch agentLatch = launch(agent);
CountDownLatch mainLatch = launch(main);
agent.waitTillStarted(1000);
main.waitTillStarted(1000);
Thread.sleep(500);
agent.cancel();
main.cancel();
agentLatch.await();
mainLatch.await();
assertEquals(-3, agent.getExitCode());
assertEquals(-3, main.getExitCode());
bndrun.close();
}
@Test
public void testRuns() {
try {
// get row count of experiments in the dataset
int expected = getDataSet().getTable("Run").getRowCount();
// get number of experiments from the DAO
int actual = getRunDAO().count();
// test data contains 2 experiments, check size of returned list
TestCase.assertEquals("Wrong number of Run", expected, actual);
System.out.println("Expected number of Run: " + expected + ", actual: " + actual);
for (Run d : random(getRunDAO(), actual, 5)) {
TestCase.assertNotNull(d);
TestCase.assertNotNull(d.getId());
}
} catch (Exception e) {
e.printStackTrace();
TestCase.fail();
}
}
public boolean needsToRun(Run run, int generation) {
Population pop = getPopulationFor(run, generation);
int count = run.getIntProperty(GenetikConstants.POPULATION, Integer.MAX_VALUE, true);
int scoreCount = 0;
for (int i = 0, max = pop.size(); i < max; i++) {
Individual ind = pop.get(i);
if (ind.hasFitness() && ind.hasScores()) {
scoreCount++;
}
}
return scoreCount != count;
}
public Set<Run> search(@NotNull Iterable<String> ids) {
Set<Run> result = new TreeSet<Run>(new RunNumberComparator());
for (String id : ids) {
Set<String> buildIds = map.get(id);
if (buildIds != null) {
for (String buildId : buildIds) {
Run<?, ?> run = Run.fromExternalizableId(buildId);
if (run != null) {
result.add(run);
}
}
}
}
return result;
}
private void populateWithParameters(Run build, String externalizableId) {
ParametersAction action = build.getAction(ParametersAction.class);
if (action != null) {
List<ParameterValue> parameters = action.getParameters();
if (parameters != null) {
for (ParameterValue parameter : parameters) {
if (parameter instanceof StringParameterValue) {
put(
"parameter-" + parameter.getName(),
((StringParameterValue) parameter).value,
externalizableId);
}
}
}
}
}
/*
* Launches against the agent
*/
public void testSimpleLauncher() throws Exception {
Project project = workspace.getProject("p1");
Run bndrun = new Run(workspace, project.getBase(), project.getFile("one.bndrun"));
bndrun.setProperty("-runpath", "biz.aQute.remote.launcher");
bndrun.setProperty("-runbundles", "bsn-1,bsn-2");
bndrun.setProperty("-runremote", "test");
final RemoteProjectLauncherPlugin pl =
(RemoteProjectLauncherPlugin) bndrun.getProjectLauncher();
pl.prepare();
final CountDownLatch latch = new CountDownLatch(1);
final AtomicInteger exitCode = new AtomicInteger(-1);
List<? extends RunSession> sessions = pl.getRunSessions();
assertEquals(1, sessions.size());
final RunSession session = sessions.get(0);
Thread t =
new Thread("test-launch") {
public void run() {
try {
exitCode.set(session.launch());
} catch (Exception e) {
e.printStackTrace();
} finally {
latch.countDown();
}
}
};
t.start();
Thread.sleep(500);
for (Bundle b : context.getBundles()) {
System.out.println(b.getLocation());
}
assertEquals(4, context.getBundles().length);
String p1 = t1.getAbsolutePath();
System.out.println(p1);
assertNotNull(context.getBundle(p1));
assertNotNull(context.getBundle(t2.getAbsolutePath()));
pl.cancel();
latch.await();
assertEquals(-3, exitCode.get());
bndrun.close();
}
private void populateWithCauseInformation(Run build, String externalizableId) {
//noinspection unchecked
List<Cause> causes = (List<Cause>) build.getCauses();
if (causes != null) {
for (Cause cause : causes) {
if (cause instanceof Cause.UserIdCause) {
Cause.UserIdCause userIdCause = (Cause.UserIdCause) cause;
String userId = userIdCause.getUserId();
if (userId != null) {
put("causeId", userId, externalizableId);
}
String userName = userIdCause.getUserName();
if (userName != null) {
put("causeName", userName, externalizableId);
}
}
}
}
}
public String id(Run run) throws UnsupportedEncodingException {
return URLEncoder.encode(run.getParent().getFullDisplayName() + run.getNumber(), "UTF-8");
}
/**
* Waits for the given configurations to finish, retrying any that qualify to be rerun.
*
* @param execution Provided by the plugin.
* @param patterns List of regular expression patterns used to scan the log to determine if a
* build should be rerun.
* @param retries Mutable map that tracks the number of times a specific configuration has been
* retried.
* @param configurations The configurations that have already been scheduled to run that should be
* waited for to finish.
* @return The worst result of all the runs. If a build was rerun, only the result of the rerun is
* considered.
* @throws InterruptedException
* @throws IOException
*/
private Result waitForMatrixRuns(
MatrixBuild.MatrixBuildExecution execution,
List<Pattern> patterns,
Map<MatrixConfiguration, Integer> retries,
LinkedList<MatrixConfiguration> configurations)
throws InterruptedException, IOException {
BuildListener listener = execution.getListener();
PrintStream logger = listener.getLogger();
Map<String, String> whyBlockedMap =
new HashMap<
String,
String>(); // keep track of why builds are blocked so we can print unique messages when
// they change.
Result finalResult = Result.SUCCESS;
int iteration = 0;
boolean continueRetrying = true;
while (!configurations.isEmpty()) {
++iteration;
MatrixConfiguration configuration = configurations.removeFirst();
if (isBuilding(execution, configuration, whyBlockedMap)) {
if (iteration >= configurations.size()) {
// Every time we loop through all the configurations, sleep for a bit.
// This is to prevent polling too often while everything is still building.
iteration = 0;
Thread.sleep(1000);
}
configurations.add(configuration);
continue;
}
Run parentBuild = execution.getBuild();
MatrixRun matrixRun = configuration.getBuildByNumber(parentBuild.getNumber());
Result runResult = matrixRun.getResult();
if (continueRetrying
&& runResult.isWorseOrEqualTo(getWorseThanOrEqualTo())
&& runResult.isBetterOrEqualTo(getBetterThanOrEqualTo())) {
if (matchesPattern(matrixRun, patterns)) {
int retriedCount = retries.get(configuration);
if (retriedCount < getMaxRetries()) {
++retriedCount;
retries.put(configuration, retriedCount);
// rerun
String logMessage =
String.format(
"%s was %s. Matched pattern to rerun. Rerunning (%d).",
matrixRun, runResult, retriedCount);
listener.error(logMessage);
HealedAction action = parentBuild.getAction(HealedAction.class);
if (action == null) {
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (parentBuild.getActions()) {
action = parentBuild.getAction(HealedAction.class);
if (action == null) {
action = new HealedAction(matrixRun.getCharset());
parentBuild.addAction(action);
}
}
}
action.addAutoHealedJob(matrixRun);
MatrixConfiguration parent = matrixRun.getParent();
if (parent != null) {
// I'm paranoid about NPEs
parent.removeRun(matrixRun);
matrixRun.delete();
} else {
LOGGER.severe(
"couldn't remove old run, parent was null. This is a Jenkins core bug.");
}
scheduleConfigurationBuild(
execution, configuration, new SelfHealingCause(parentBuild, retriedCount));
configurations.add(configuration);
continue;
} else {
String logMessage =
String.format(
"%s was %s. Matched pattern to rerun, but the max number of retries (%d) has been met.",
matrixRun, runResult, getMaxRetries());
listener.error(logMessage);
if (getStopRetryingAfterOneFails()) {
listener.error("Not retrying any more builds.");
continueRetrying = false;
}
}
} else {
String logMessage =
String.format(
"%s was %s. It did not match the pattern to rerun. Accepting result.",
matrixRun, runResult);
logger.println(logMessage);
}
}
notifyEndRun(matrixRun, execution.getAggregators(), execution.getListener());
finalResult = finalResult.combine(runResult);
}
return finalResult;
}
private Map<String, Run> processRunJSON(
HealthType ht, JSONArray runs, RequestManager requestManager) {
Map<String, Run> updatedRuns = new HashMap<String, Run>();
List<Run> runsToSave = new ArrayList<Run>();
// 2011-01-25 15:37:27.093
DateFormat logDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
DateFormat simpleLogDateFormat = new SimpleDateFormat("yyyyMMdd");
Pattern simpleDateRegex = Pattern.compile("[0-9]{8}");
StringBuilder sb = new StringBuilder();
for (JSONObject run : (Iterable<JSONObject>) runs) {
String runName = run.getString("runName");
sb.append("Processing " + runName + "\n");
log.debug("Processing " + runName);
if (run.has("status")) {
String xml = run.getString("status");
Status is = new SolidStatus(xml);
is.setHealth(ht);
is.setRunName(runName);
Run r = null;
Matcher m = p.matcher(runName);
if (m.matches()) {
try {
r = requestManager.getRunByAlias(runName);
} catch (IOException ioe) {
log.warn(
"Cannot find run by this alias. This usually means the run hasn't been previously imported. If attemptRunPopulation is false, processing will not take place for this run!");
}
}
try {
if (attemptRunPopulation) {
if (r == null) {
log.debug("Saving new run and status: " + is.getRunName());
r = new SolidRun(xml);
r.getStatus().setHealth(ht);
if (run.has("fullPath")) {
r.setFilePath(run.getString("fullPath"));
}
SequencerReference sr = null;
if (run.has("sequencerName")) {
sr = requestManager.getSequencerReferenceByName(run.getString("sequencerName"));
r.getStatus().setInstrumentName(run.getString("sequencerName"));
r.setSequencerReference(sr);
}
if (r.getSequencerReference() == null) {
sr = requestManager.getSequencerReferenceByName(m.group(1));
r.getStatus().setInstrumentName(m.group(1));
r.setSequencerReference(sr);
}
if (r.getSequencerReference() == null) {
sr = requestManager.getSequencerReferenceByName(r.getStatus().getInstrumentName());
r.setSequencerReference(sr);
}
if (r.getSequencerReference() != null) {
if (run.has("startDate")) {
try {
log.debug("Updating start date:" + run.getString("startDate"));
Matcher m2 = simpleDateRegex.matcher(run.getString("startDate"));
if (m2.matches()) {
r.getStatus()
.setStartDate(simpleLogDateFormat.parse(run.getString("startDate")));
} else {
r.getStatus().setStartDate(logDateFormat.parse(run.getString("startDate")));
}
} catch (ParseException e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
if (run.has("completionDate")) {
try {
if (run.get("completionDate") != null
&& !run.getString("completionDate").equals("null")) {
log.debug("Updating completion date:" + run.getString("completionDate"));
r.getStatus()
.setCompletionDate(logDateFormat.parse(run.getString("completionDate")));
} else {
r.getStatus().setCompletionDate(null);
}
} catch (ParseException e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
}
} else {
log.debug("Updating existing run and status: " + is.getRunName());
r.setAlias(runName);
r.setPlatformType(PlatformType.SOLID);
if (r.getSequencerReference() == null) {
SequencerReference sr = null;
if (run.has("sequencerName")) {
sr = requestManager.getSequencerReferenceByName(run.getString("sequencerName"));
r.getStatus().setInstrumentName(run.getString("sequencerName"));
r.setSequencerReference(sr);
}
if (r.getSequencerReference() == null) {
sr = requestManager.getSequencerReferenceByName(m.group(1));
r.getStatus().setInstrumentName(m.group(1));
r.setSequencerReference(sr);
}
if (r.getSequencerReference() == null) {
sr =
requestManager.getSequencerReferenceByName(r.getStatus().getInstrumentName());
r.setSequencerReference(sr);
}
}
if (r.getSequencerReference() != null) {
if (run.has("startDate")) {
try {
log.debug("Updating start date:" + run.getString("startDate"));
Matcher m2 = simpleDateRegex.matcher(run.getString("startDate"));
if (m2.matches()) {
r.getStatus()
.setStartDate(simpleLogDateFormat.parse(run.getString("startDate")));
} else {
r.getStatus().setStartDate(logDateFormat.parse(run.getString("startDate")));
}
} catch (ParseException e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
if (run.has("completionDate")) {
try {
if (run.get("completionDate") != null
&& !run.getString("completionDate").equals("null")) {
log.debug("Updating completion date:" + run.getString("completionDate"));
r.getStatus()
.setCompletionDate(logDateFormat.parse(run.getString("completionDate")));
} else {
r.getStatus().setCompletionDate(null);
}
} catch (ParseException e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
// update path if changed
if (run.has("fullPath")
&& !"".equals(run.getString("fullPath"))
&& r.getFilePath() != null
&& !"".equals(r.getFilePath())) {
if (!run.getString("fullPath").equals(r.getFilePath())) {
log.debug(
"Updating run file path:"
+ r.getFilePath()
+ " -> "
+ run.getString("fullPath"));
r.setFilePath(run.getString("fullPath"));
}
}
// update status if run isn't completed or failed
if (!r.getStatus().getHealth().equals(HealthType.Completed)
&& !r.getStatus().getHealth().equals(HealthType.Failed)) {
log.debug(
"Saving previously saved status: "
+ is.getRunName()
+ " ("
+ r.getStatus().getHealth().getKey()
+ " -> "
+ is.getHealth().getKey()
+ ")");
r.setStatus(is);
}
}
}
if (r.getSequencerReference() != null) {
List<SequencerPartitionContainer<SequencerPoolPartition>> fs =
((SolidRun) r).getSequencerPartitionContainers();
if (fs.isEmpty()) {
if (run.has("containerId") && !"".equals(run.getString("containerId"))) {
Collection<SequencerPartitionContainer<SequencerPoolPartition>> pfs =
requestManager.listSequencerPartitionContainersByBarcode(
run.getString("containerId"));
if (!pfs.isEmpty()) {
if (pfs.size() == 1) {
SequencerPartitionContainer lf =
new ArrayList<SequencerPartitionContainer<SequencerPoolPartition>>(pfs)
.get(0);
if (lf.getSecurityProfile() != null && r.getSecurityProfile() == null) {
r.setSecurityProfile(lf.getSecurityProfile());
}
if (lf.getPlatform() == null
&& r.getSequencerReference().getPlatform() != null) {
lf.setPlatform(r.getSequencerReference().getPlatform());
}
// else {
// lf.setPlatformType(PlatformType.SOLID);
// }
((RunImpl) r).addSequencerPartitionContainer(lf);
}
} else {
log.debug("No containers linked to run " + r.getId() + ": creating...");
SequencerPartitionContainer f = new SequencerPartitionContainerImpl();
f.setSecurityProfile(r.getSecurityProfile());
f.initEmptyPartitions();
f.setIdentificationBarcode(run.getString("containerNum"));
if (f.getPlatform() == null
&& r.getSequencerReference().getPlatform() != null) {
f.setPlatform(r.getSequencerReference().getPlatform());
}
// else {
// f.setPlatformType(PlatformType.SOLID);
// }
// f.setPaired(r.getPairedEnd());
((RunImpl) r).addSequencerPartitionContainer(f);
}
}
} else {
SequencerPartitionContainer f = fs.iterator().next();
log.debug("Got container " + f.getId());
if (f.getSecurityProfile() == null) {
f.setSecurityProfile(r.getSecurityProfile());
}
if (f.getPlatform() == null && r.getSequencerReference().getPlatform() != null) {
f.setPlatform(r.getSequencerReference().getPlatform());
}
// else {
// f.setPlatformType(PlatformType.SOLID);
// }
if (run.has("containerId") && !"".equals(run.getString("containerId"))) {
f.setIdentificationBarcode(run.getString("containerId"));
}
long flowId = requestManager.saveSequencerPartitionContainer(f);
f.setId(flowId);
}
updatedRuns.put(r.getAlias(), r);
runsToSave.add(r);
}
}
} catch (IOException e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
}
try {
if (runsToSave.size() > 0) {
int[] saved = requestManager.saveRuns(runsToSave);
log.info("Batch saved " + saved.length + " / " + runs.size() + " runs");
}
} catch (IOException e) {
log.error("Couldn't save run batch: " + e.getMessage());
e.printStackTrace();
}
return updatedRuns;
}
public boolean isBuilding(Run record) {
return record.isBuilding();
}
public String getKey(Run record) {
return String.valueOf(record.getNumber());
}
