private Option<SakaiBulkDownloadFolder> findBulkDownloadFolder()
throws NotValidDownloadFolderException {
File[] foundFiles =
folder.listFiles(
new FileFilter() {
public boolean accept(File path) {
return path.isDirectory()
&& path.getName().equals(MergingEnvironment.get().getAssignmentName());
}
});
if (foundFiles.length > 0) {
return Option.apply(new SakaiBulkDownloadFolder(foundFiles[0].getAbsolutePath()));
}
Option<File> zipFile =
DirectoryUtils.find(
folder,
new FileFilter() {
public boolean accept(File pathname) {
return pathname.getName().endsWith(".zip");
}
});
if (zipFile.isEmpty()) {
return Option.empty();
}
// Extract the zip to look for the folder
try {
System.out.println("Extracting bulk downloads...");
ZipFile zip = new ZipFile(zipFile.get());
zip.extractAll(folder.getAbsolutePath());
// Look for a folder, taking the first one found.
Option<File> resultsFolder =
DirectoryUtils.find(
folder,
new FileFilter() {
public boolean accept(File pathname) {
return pathname.isDirectory()
&& pathname.getName().equals(MergingEnvironment.get().getAssignmentName());
}
});
if (resultsFolder.isDefined()) {
try {
return Option.apply(new SakaiBulkDownloadFolder(resultsFolder.get().getAbsolutePath()));
} catch (Exception e) {
return Option.empty();
}
}
System.out.println("done.");
return Option.empty();
} catch (ZipException e) {
return Option.empty();
}
}
@Override
public ContainerRoot pull(String targetNodeName) {
List<String> ips =
KevoreePropertyHelper.getStringNetworkProperties(
this.getModelService().getLastModel(),
targetNodeName,
Constants.KEVOREE_PLATFORM_REMOTE_NODE_IP());
Option<Integer> portOption =
KevoreePropertyHelper.getIntPropertyForGroup(
this.getModelService().getLastModel(), this.getName(), "port", true, targetNodeName);
int PORT = 8000;
if (portOption.isDefined()) {
PORT = portOption.get();
}
for (String ip : ips) {
logger.debug("try to pull model on url=>" + "http://" + ip + ":" + PORT + "/model/current");
ContainerRoot model = pullModel("http://" + ip + ":" + PORT + "/model/current");
if (model != null) {
return model;
}
}
ContainerRoot model = pullModel("http://127.0.0.1:" + PORT + "/model/current");
if (model == null) {
logger.debug("Unable to pull a model on " + targetNodeName);
return null;
} else {
return model;
}
}
@Override
public void push(ContainerRoot model, String targetNodeName) {
List<String> ips =
KevoreePropertyHelper.getStringNetworkProperties(
model, targetNodeName, Constants.KEVOREE_PLATFORM_REMOTE_NODE_IP());
Option<Integer> portOption =
KevoreePropertyHelper.getIntPropertyForGroup(
model, this.getName(), "port", true, targetNodeName);
int PORT = 8000;
if (portOption.isDefined()) {
PORT = portOption.get();
}
boolean sent = false;
for (String ip : ips) {
logger.debug("try to send model on url=>" + "http://" + ip + ":" + PORT + "/model/current");
if (sendModel(model, "http://" + ip + ":" + PORT + "/model/current")) {
sent = true;
break;
}
}
if (!sent) {
logger.debug("try to send model on url=>" + "http://127.0.0.1:" + PORT + "/model/current");
if (!sendModel(model, "http://127.0.0.1:" + PORT + "/model/current")) {
logger.error("Unable to push a model on " + targetNodeName);
}
}
}
public static void addArrayElements(
TreeNode parent, SpreadSheetTable table, List<Integer> rows, LogSupport log) {
int valCol = table.numCols() - 1;
Map<String, String> values = new HashMap<>();
for (int i = 0; i < rows.size(); ++i) {
String name;
if (table.numCols() > 1) {
name = table.get(rows.get(i), 0);
int index = name.lastIndexOf('.');
if (index != -1) {
name = name.substring(index + 1);
}
} else {
name = S.fmt("%s", i);
}
values.put(name, table.get(rows.get(i), valCol));
}
for (Map.Entry<String, String> e : values.entrySet()) {
String name = e.getKey();
String value = e.getValue();
Option<TreeNodeLike> optVN = parent.findChild(name);
TreeNode vn;
if (!optVN.isDefined()) {
vn = ElementHelper.addElement(parent, name);
} else {
vn = (TreeNode) optVN.get();
}
EntryEditorEditingSupport.writeValue(vn, value, log);
}
ElementHelper.removeChildren(parent, values.keySet());
}
/**
* Extract a signed token that was signed by {@link #signToken(String)}.
*
* @param token The signed token to extract.
* @return The verified raw token, or null if the token isn't valid.
*/
public String extractSignedToken(String token) {
scala.Option<String> extracted = crypto.extractSignedToken(token);
if (extracted.isDefined()) {
return extracted.get();
} else {
return null;
}
}
@Override
public void preRestart(Throwable reason, Option<Object> message) {
log.error(
reason,
"Restarting due to [{}] when processing [{}]",
reason.getMessage(),
message.isDefined() ? message.get() : "");
}
@Override
public String getUsername(Context ctx) {
Option<User> user = ContextUtil.getCurrentUser(ctx);
if (user.isEmpty()) {
return null;
}
return user.get().username;
}
public Address getAddressFor(Address remoteAddress) {
final scala.Option<Address> optAddr = system.provider().getExternalAddressFor(remoteAddress);
if (optAddr.isDefined()) {
return optAddr.get();
} else {
throw new UnsupportedOperationException("cannot send to remote address " + remoteAddress);
}
}
public Optional<AssignmentConfig> assign(final RegionStats region, final ServerName server) {
final Option<RegionAssignment> maybeOldAssignment = assignments.get(region);
final ServerName oldServer = maybeOldAssignment.get().getNewServer();
if (Objects.equal(oldServer, server)) return Optional.absent();
final RegionAssignment assignment = new RegionAssignment(region, server);
return Optional.of(
new AssignmentConfig(
assignments.$plus(new Tuple2<RegionStats, RegionAssignment>(region, assignment)),
servers));
}
/**
* Restore an XFormsContainingDocument from XFormsState only.
*
* <p>Used by XFormsStateManager.
*
* @param xformsState XFormsState containing static and dynamic state
* @param disableUpdates whether to disable updates (for recreating initial document upon browser
* back)
*/
public XFormsContainingDocument(XFormsState xformsState, boolean disableUpdates) {
super();
// 1. Restore the static state
{
final scala.Option<String> staticStateDigest = xformsState.staticStateDigest();
if (staticStateDigest.isDefined()) {
final XFormsStaticState cachedState =
XFormsStaticStateCache.instance().getDocument(staticStateDigest.get());
if (cachedState != null) {
// Found static state in cache
indentedLogger().logDebug("", "found static state by digest in cache");
this.staticState = cachedState;
} else {
// Not found static state in cache, create static state from input
indentedLogger().logDebug("", "did not find static state by digest in cache");
indentedLogger().startHandleOperation("initialization", "restoring static state");
this.staticState =
XFormsStaticStateImpl.restore(staticStateDigest, xformsState.staticState());
indentedLogger().endHandleOperation();
// Store in cache
XFormsStaticStateCache.instance().storeDocument(this.staticState);
}
assert this.staticState.isServerStateHandling();
} else {
// Not digest provided, create static state from input
indentedLogger().logDebug("", "did not find static state by digest in cache");
this.staticState =
XFormsStaticStateImpl.restore(staticStateDigest, xformsState.staticState());
assert this.staticState.isClientStateHandling();
}
this.staticOps = new StaticStateGlobalOps(staticState.topLevelPart());
this.xpathDependencies = Version.instance().createUIDependencies(this);
this.supportUpdates = !disableUpdates && !isNoUpdates();
}
// 2. Restore the dynamic state
indentedLogger().startHandleOperation("initialization", "restoring containing document");
try {
restoreDynamicState(xformsState.dynamicState());
} catch (Exception e) {
throw OrbeonLocationException.wrapException(
e, new ExtendedLocationData(null, "re-initializing XForms containing document"));
}
indentedLogger().endHandleOperation();
}
private static ValidationResult checkCompilerLibrary(LibraryDescriptor descriptor) {
if (descriptor == null || descriptor.data().isEmpty()) return ValidationResult.OK;
String libraryName = "Compiler library";
CompilerLibraryData compilerLibraryData = (CompilerLibraryData) descriptor.data().get();
Option<String> compilerLibraryProblem = compilerLibraryData.problem();
if (compilerLibraryProblem.isDefined())
return new ValidationResult(libraryName + ": " + compilerLibraryProblem.get());
return ValidationResult.OK;
}
public SakaiGraderResultFolder(String path)
throws NotValidResultFolderException, NotValidDownloadFolderException {
File topFolder = new File(path);
folder = new File(topFolder, MergingEnvironment.get().getAssignmentName());
if (!(folder.exists() && folder.isDirectory())) {
throw new NotValidResultFolderException("Missing assignment folder: " + path);
}
spreadsheetFile = new File(folder, "grades.xlsx");
Option<SakaiBulkDownloadFolder> bulkDownloadOption = findBulkDownloadFolder();
if (bulkDownloadOption.isEmpty()) {
throw new NotValidResultFolderException("Missing a bulk download folder");
}
bulkDownloadFolder = bulkDownloadOption.get();
}
@Override
public List<String> getNewMessages() {
if (!isConnected) {
throw new IllegalStateException("The cluster has been connected to the ApplicationMaster.");
}
if (hasBeenStopped()) {
throw new RuntimeException("The FlinkYarnCluster has already been stopped");
}
List<String> ret = new ArrayList<String>();
// get messages from ApplicationClient (locally)
while (true) {
Object result = null;
try {
Future<Object> response =
Patterns.ask(
applicationClient, Messages.getLocalGetYarnMessage(), new Timeout(akkaDuration));
result = Await.result(response, akkaDuration);
} catch (Exception ioe) {
LOG.warn("Error retrieving the YARN messages locally", ioe);
break;
}
if (!(result instanceof Option)) {
throw new RuntimeException(
"LocalGetYarnMessage requires a response of type "
+ "Option. Instead the response is of type "
+ result.getClass()
+ ".");
} else {
Option messageOption = (Option) result;
LOG.debug("Received message option {}", messageOption);
if (messageOption.isEmpty()) {
break;
} else {
Object obj = messageOption.get();
if (obj instanceof Messages.YarnMessage) {
Messages.YarnMessage msg = (Messages.YarnMessage) obj;
ret.add("[" + msg.date() + "] " + msg.message());
} else {
LOG.warn("LocalGetYarnMessage returned unexpected type: " + messageOption);
}
}
}
}
return ret;
}
@Override
protected void initialize() throws Exception {
LOG.info("Initializing Mesos resource master");
workerStore.start();
// create the scheduler driver to communicate with Mesos
schedulerCallbackHandler = new SchedulerProxy(self());
// register with Mesos
FrameworkInfo.Builder frameworkInfo = mesosConfig.frameworkInfo().clone().setCheckpoint(true);
Option<Protos.FrameworkID> frameworkID = workerStore.getFrameworkID();
if (frameworkID.isEmpty()) {
LOG.info("Registering as new framework.");
} else {
LOG.info(
"Recovery scenario: re-registering using framework ID {}.", frameworkID.get().getValue());
frameworkInfo.setId(frameworkID.get());
}
MesosConfiguration initializedMesosConfig = mesosConfig.withFrameworkInfo(frameworkInfo);
MesosConfiguration.logMesosConfig(LOG, initializedMesosConfig);
schedulerDriver = initializedMesosConfig.createDriver(schedulerCallbackHandler, false);
// create supporting actors
connectionMonitor = createConnectionMonitor();
launchCoordinator = createLaunchCoordinator();
reconciliationCoordinator = createReconciliationCoordinator();
taskRouter = createTaskRouter();
recoverWorkers();
connectionMonitor.tell(new ConnectionMonitor.Start(), self());
schedulerDriver.start();
}
@Override
public TestCaseResult test(Project project, boolean autoGrade)
throws NotAutomatableException, NotGradableException {
// There should be a setter (editable) for the command
if (project.getClassesManager().isEmpty()) throw new NotGradableException();
Option<ClassDescription> classDescription =
new RootTagFinder(project).findClass("Command Interpreter");
if (classDescription.isEmpty()) {
if (autoGrade) throw new NotAutomatableException();
classDescription = ManualClassFinder.find(project, "Command Interpreter");
}
Class<?> _class = classDescription.get().getJavaClass();
Method[] methods = _class.getMethods();
for (Method method : methods) {
if (method.getName().startsWith("set")) return pass(autoGrade);
}
return fail("Couldn't find an editable property", autoGrade);
}
@Override
public File compile(File templateFile) throws TemplateCompilationException {
File result = null;
String fileName = templateFile.getName();
String ext = fileName.substring(fileName.lastIndexOf('.') + 1);
String importsAsString = getImportsAsString(ext);
int index = getTemplateExtIndex(ext);
if (index >= 0) {
String formatterType = formatterTypes[index];
try {
Option<File> resultOption =
ScalaTemplateCompiler.compile(
templateFile, sourceDirectory, outputDirectory, formatterType, importsAsString);
result = resultOption.isDefined() ? resultOption.get() : null;
} catch (TemplateCompilationError e) {
throw new TemplateCompilationException(e.source(), e.message(), e.line(), e.column());
}
}
return result;
}
@Override
public void preRestart(Throwable reason, scala.Option<Object> message) {
log.debug("preRestart on AskedActor");
log.debug("current value is {}", value);
getSelf().tell(message.get(), getSelf());
}
