/**
* Does the following steps:
*
* <ul>
* <li>Prepares the temporary folders. See {@link #prepareDataFolderAndUpdateJob(Job, String,
* String, String, String)}
* <li>pushes all files from localInputFolderPath to the push_url location (see {@link
* #pushData(Job, String)})
* <li>submits the job to the scheduler
* <li>adds the job to the awaited jobs list in order to download the output data when the job
* is finished
* </ul>
*
* Note: this method is synchronous. The caller will be blocked until the the data is pushed and
* the job submitted.
*
* @param job job object to be submitted to the Scheduler Server for execution
* @param localInputFolderPath path to the folder containing the input data for this job
* @param push_url the url where input data is to be pushed before the job submission
* @param localOutputFolderPath path to the folder where the output data produced by tasks in this
* job should be copied
* @param pull_url the url where the data is to be retrieved after the job is finished
* @param localOutputFolderPath
* @return
* @throws JobCreationException
* @throws SubmissionClosedException
* @throws PermissionException
* @throws NotConnectedException
* @throws FileSystemException
*/
public JobId submit(
Job job,
String localInputFolderPath,
String push_url,
String localOutputFolderPath,
String pull_url)
throws NotConnectedException, PermissionException, SubmissionClosedException,
JobCreationException, FileSystemException {
if (((push_url == null) || (push_url.equals("")))
&& ((pull_url == null) || (pull_url.equals("")))) {
logger.warn(
"For the job "
+ job.getId()
+ " no push or pull urls are defined. No data will be transfered for this job from the local machine ");
return super.submit(job);
}
String newFolderName = createNewFolderName();
String push_Url_update = prepareJobInput(job, localInputFolderPath, push_url, newFolderName);
String pull_url_update = prepareJobOutput(job, localOutputFolderPath, pull_url, newFolderName);
pushData(job, localInputFolderPath);
JobId id = super.submit(job);
AwaitedJob aj =
new AwaitedJob(
id.toString(),
localInputFolderPath,
job.getInputSpace(),
push_Url_update,
localOutputFolderPath,
job.getOutputSpace(),
pull_url_update);
addAwaitedJob(aj);
return id;
}
/**
* This method will create a remote folder for the input data of this job and update the
* inputSpace job property. If the localInputFolder parameter is null, or push_url is null, no
* action will be performed concerning this job's input.
*
* <p>We suppose there is file storage accessible by the client application as well as the tasks
* on the computation nodes.
*
* <p>This storage could be different for input and for output.
*
* <p>The input storage can be accessed, by the client application, using the push_url and by the
* tasks on the nodes using the job's input space url.
*
* <p>Prepare Input Data Transfer
*
* <p>A folder will be created at push_url/newFolderName/input (which, from the nodes side, is the
* job.InputSpace/newFolderName/input) . The InputSpace property of the job will be changed to the
* new location. job.InputSpace = job.InputSpace/NewFolder/input
*
* <p>A generic information will be attached to the job containing the local input folder path.
*
* @param job
* @param localInputFolder path to the input folder on local machine if null, no actions will be
* performed concerning the input data for this job
* @param push_url the url where input data is to be pushed before the job submission
* @param newFolderName name of the new folder to be created
* @return String representing the updated value of the push_url
* @throws FileSystemException
*/
private String prepareJobInput(
Job job, String localInputFolder, String push_url, String newFolderName)
throws FileSystemException {
// if the job defines an input space
// and the localInputFolder is not null
// create a remote folder for the input data
// and update the InputSpace property of the job to reference that
// folder
String inputSpace_url = job.getInputSpace();
String push_url_updated = "";
// the input folder, on the remote input space, relative to the root url
String inputFolder = "";
if ((localInputFolder != null)
&& (inputSpace_url != null)
&& (!inputSpace_url.equals(""))
&& (push_url != null)) {
inputFolder = newFolderName + "/input";
push_url_updated = push_url + "//" + inputFolder;
String inputSpace_url_updated = inputSpace_url + "//" + inputFolder;
createFolder(push_url_updated);
job.setInputSpace(inputSpace_url_updated);
job.addGenericInformation(
GENERIC_INFO_INPUT_FOLDER_PROPERTY_NAME, new File(localInputFolder).getAbsolutePath());
job.addGenericInformation(GENERIC_INFO_PUSH_URL_PROPERTY_NAME, push_url_updated);
}
// if the job defines an output space
// and the localOutputFolder is not null
// create a remote folder for the output data
// and update the OutputSpace property of the job to reference that
// folder
return push_url_updated;
}
