Path genPythonToolkit(String module) throws IOException, InterruptedException {
Path pubsub = Paths.get(getTestRoot().getAbsolutePath(), "python", "pubsub");
System.err.println("Pubsub:" + pubsub);
Path pyTk = Files.createTempDirectory("pytk").toAbsolutePath();
System.err.println("PKTK:" + pyTk);
Path pyPackages =
Paths.get(System.getProperty("topology.toolkit.release"), "opt", "python", "packages")
.toAbsolutePath();
String pythonversion = System.getProperty("topology.test.python");
ProcessBuilder pb = new ProcessBuilder(pythonversion, module, pyTk.toAbsolutePath().toString());
pb.redirectOutput(Redirect.INHERIT);
pb.redirectError(Redirect.INHERIT);
Map<String, String> env = pb.environment();
env.put("PYTHONPATH", pyPackages.toString());
pb.directory(pubsub.toFile());
Process proc = pb.start();
assertEquals(0, proc.waitFor());
return pyTk;
}
private void checkArtifactSignature(S3ArtifactSignature s3ArtifactSignature) {
final Path artifactPath =
Paths.get(
s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getArtifactFilename());
final Path artifactSignaturePath =
Paths.get(s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getFilename());
if (!Files.exists(artifactPath)) {
log.warn("Artifact {} not found for signature {}", artifactPath, s3ArtifactSignature);
return;
}
final List<String> verifyCommand =
new ArrayList<>(executorConfiguration.getArtifactSignatureVerificationCommand().size());
for (String arg : executorConfiguration.getArtifactSignatureVerificationCommand()) {
verifyCommand.add(
arg.replace("{artifactPath}", artifactPath.toString())
.replace("{artifactSignaturePath}", artifactSignaturePath.toString()));
}
try {
final ProcessBuilder processBuilder = new ProcessBuilder(verifyCommand);
processBuilder.directory(taskDefinition.getTaskDirectoryPath().toFile());
processBuilder.redirectError(taskDefinition.getSignatureVerifyOutPath().toFile());
processBuilder.redirectOutput(taskDefinition.getSignatureVerifyOutPath().toFile());
final Process p = processBuilder.start();
p.waitFor(); // TODO: add some sort of timeout?
if (p.exitValue() != 0) {
log.error(
"Failed to validate signature {} for artifact {}",
s3ArtifactSignature.getFilename(),
s3ArtifactSignature.getArtifactFilename());
if (executorConfiguration.isFailTaskOnInvalidArtifactSignature()) {
throw new RuntimeException(
String.format("Failed to validate signature for artifact %s", artifactPath));
}
} else {
log.info(
"Signature {} for artifact {} is valid!",
s3ArtifactSignature.getFilename(),
s3ArtifactSignature.getArtifactFilename());
}
} catch (InterruptedException | IOException e) {
throw Throwables.propagate(e);
}
}
@Override
public Object apply(List<Object> args, Context context) throws ParseException {
File outFile = null;
String editor = getEditor();
try {
outFile = File.createTempFile("stellar_shell", "out");
if (args.size() > 0) {
String arg = (String) args.get(0);
try (PrintWriter pw = new PrintWriter(outFile)) {
IOUtils.write(arg, pw);
}
}
} catch (IOException e) {
String message = "Unable to create temp file: " + e.getMessage();
LOG.error(message, e);
throw new IllegalStateException(message, e);
}
Optional<Object> console = context.getCapability(CONSOLE, false);
try {
PausableInput.INSTANCE.pause();
// shut down the IO for the console
ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath());
processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
try {
Process p = processBuilder.start();
// wait for termination.
p.waitFor();
try (BufferedReader br = new BufferedReader(new FileReader(outFile))) {
String ret = IOUtils.toString(br).trim();
return ret;
}
} catch (Exception e) {
String message = "Unable to read output: " + e.getMessage();
LOG.error(message, e);
return null;
}
} finally {
try {
PausableInput.INSTANCE.unpause();
if (console.isPresent()) {
((Console) console.get()).pushToInputStream("\b\n");
}
} catch (IOException e) {
LOG.error("Unable to unpause: " + e.getMessage(), e);
}
if (outFile.exists()) {
outFile.delete();
}
}
}
public void generatePDF(final List<String> commands) throws IOException {
final ProcessBuilder probuilder = new ProcessBuilder(commands);
probuilder.redirectError();
probuilder.redirectOutput();
probuilder.redirectInput();
final Process process = probuilder.start();
try {
process.waitFor();
} catch (final InterruptedException e) {
e.printStackTrace();
}
process.destroy();
}
@Override
public void run() {
try {
ProcessBuilder pb = new ProcessBuilder("sh", "-c", "kill `pidof mplayer`");
pb.directory(new File(ServerMultimediale.MUSICPATH.toString()));
pb.redirectError(
ProcessBuilder.Redirect.appendTo(new File(ServerMultimediale.LOGFILE.toString())));
pb.redirectOutput(
ProcessBuilder.Redirect.appendTo(new File(ServerMultimediale.LOGFILE.toString())));
Process p = pb.start();
p.waitFor();
System.out.println("killall mplayer");
} catch (InterruptedException | IOException ex) {
Logger.getLogger(Kill.class.getName()).log(Level.SEVERE, null, ex);
}
}
public void mergeFilesOnLocal(String dir, int noOfIndices) throws InterruptedException {
ArrayList<String> ls = new ArrayList();
ls.add("cat");
for (int i = 1; i <= noOfIndices; i++) {
ls.add(dir + "/indexFile" + i + "");
}
try {
ProcessBuilder builder = new ProcessBuilder(ls);
File combineFile = new File("/home/cloudera/Desktop/datasets/IndexFiles/MergedOne");
builder.redirectOutput(combineFile);
builder.redirectError(combineFile);
Process p = builder.start();
p.waitFor();
} catch (IOException e) {
System.out.println(e.getMessage());
}
System.out.println("Newly copied index files are Merged as one index file...");
}
public String getJavaPIDs() {
ProcessBuilder pb =
new ProcessBuilder()
.command(Arrays.asList(getCommandPrefix() + "jvms" + getCommandSuffix(), tag));
pb.redirectError(ProcessBuilder.Redirect.INHERIT);
pb.redirectInput(ProcessBuilder.Redirect.INHERIT);
try {
Process process = pb.start();
StringBuilder sb = new StringBuilder();
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) sb.append(line);
}
return sb.toString().trim();
} catch (IOException e) {
log.error("Failed to read JVM PIDs", e);
return null;
}
}
public void process(PipeCommand pcmd) throws ProcessException, IOException, InterruptedException {
List<CmdCommand> cmd = pcmd.getCommandSet();
String directory = Batch.getWorkDirectory();
List<String> inCommand = new ArrayList<String>();
List<String> outCommand = new ArrayList<String>();
CmdCommand inCmd = null;
CmdCommand outCmd = null;
for (CmdCommand _cmd : cmd) {
List<String> command = new ArrayList<String>();
command.add(_cmd.getPath());
if (_cmd.getArgs() != null && !_cmd.getArgs().isEmpty()) {
for (String args : _cmd.getArgs()) {
command.add(args);
}
}
if (_cmd.getInFile() != null && !_cmd.getInFile().isEmpty()) {
inCommand = command;
inCmd = _cmd;
}
if (_cmd.getOutFile() != null && !_cmd.getOutFile().isEmpty()) {
outCommand = command;
outCmd = _cmd;
}
}
if (inCmd == null || outCmd == null) {
throw new ProcessException("Improper Pipe Configuration");
}
ProcessBuilder builder1 = new ProcessBuilder();
builder1.directory(new File(directory));
File wd = builder1.directory();
builder1.command(inCommand);
String outfilepath = "";
String infilepath = "";
if (Batch.getFiles() != null) {
FileCommand ofcmd = Batch.getFiles().get(outCmd.getOutFile());
FileCommand ifcmd = Batch.getFiles().get(inCmd.getInFile());
if (ofcmd != null) {
outfilepath = ofcmd.getPath();
} else {
System.out.println(
"Error Processing Batch Unable to locate FileCommand with id " + outCmd.getOutFile());
throw new ProcessException(
"Error Processing Batch Unable to locate FileCommand with id: " + outCmd.getOutFile());
}
if (ifcmd != null) {
infilepath = ifcmd.getPath();
} else {
System.out.println(
"Error Processing Batch: Unable to locate FileCommand with id: " + inCmd.getInFile());
throw new ProcessException(
"Error Processing Batch: Unable to locate FileCommand with id: " + inCmd.getInFile());
}
if (infilepath == outfilepath) {
System.out.println("Error Processing Batch: Input and Output File cannot be same ");
throw new ProcessException("Error Processing Batch: Input and Output File cannot be same ");
}
} else {
System.out.println("Error Processing Batch: No Files found to process");
throw new ProcessException("Error Processing Batch: No Files found to process");
}
File infile = new File(wd, infilepath);
builder1.redirectInput(infile);
final Process process1 = builder1.start();
InputStream is1 = process1.getInputStream();
OutputStream os1 = process1.getOutputStream();
os1.close();
ProcessBuilder builder2 = new ProcessBuilder();
builder2.directory(new File(directory));
builder2.command(outCommand);
File outfile = new File(wd, outfilepath);
FileOutputStream fos = new FileOutputStream(outfile);
builder2.redirectError();
final Process process2 = builder2.start();
InputStream is2 = process2.getInputStream();
OutputStream os2 = process2.getOutputStream();
copyStreams(is1, os2);
copyStreams(is2, fos);
process1.waitFor();
process2.waitFor();
fos.close();
System.out.println("Command Terminated! -- ");
}
public AudioInputStream pipe(Attributes attributes) throws EncoderException {
String pipeEnvironment;
String pipeArgument;
File pipeLogFile;
int pipeBuffer;
if (System.getProperty("os.name").indexOf("indows") > 0) {
pipeEnvironment = "cmd.exe";
pipeArgument = "/C";
} else {
pipeEnvironment = "/bin/bash";
pipeArgument = "-c";
}
pipeLogFile = new File("decoder_log.txt");
// buffer 1/4 second of audio.
pipeBuffer = attributes.getSamplingRate() / 4;
AudioFormat audioFormat = Encoder.getTargetAudioFormat(attributes);
String command = toString();
ProcessBuilder pb = new ProcessBuilder(pipeEnvironment, pipeArgument, command);
pb.redirectError(Redirect.appendTo(pipeLogFile));
LOG.fine("Starting piped decoding process");
final Process process;
try {
process = pb.start();
} catch (IOException e1) {
throw new EncoderException("Problem starting piped sub process: " + e1.getMessage());
}
InputStream stdOut = new BufferedInputStream(process.getInputStream(), pipeBuffer);
// read and ignore the 46 byte wav header, only pipe the pcm samples to the audioinputstream
byte[] header = new byte[46];
double sleepSeconds = 0;
double timeoutLimit = 20; // seconds
try {
while (stdOut.available() < header.length) {
try {
Thread.sleep(100);
sleepSeconds += 0.1;
} catch (InterruptedException e) {
e.printStackTrace();
}
if (sleepSeconds > timeoutLimit) {
throw new Error("Could not read from pipe within " + timeoutLimit + " seconds: timeout!");
}
}
int bytesRead = stdOut.read(header);
if (bytesRead != header.length) {
throw new EncoderException(
"Could not read complete WAV-header from pipe. This could result in mis-aligned frames!");
}
} catch (IOException e1) {
throw new EncoderException("Problem reading from piped sub process: " + e1.getMessage());
}
final AudioInputStream audioStream =
new AudioInputStream(stdOut, audioFormat, AudioSystem.NOT_SPECIFIED);
// This thread waits for the end of the subprocess.
new Thread(
new Runnable() {
public void run() {
try {
process.waitFor();
LOG.fine("Finished piped decoding process");
} catch (InterruptedException e) {
LOG.severe("Interrupted while waiting for sub process exit.");
e.printStackTrace();
}
}
},
"Decoding Pipe Reader")
.start();
return audioStream;
}
/**
* This method uses the local mvn installation to resolve and generate the effective POM As a
* consequence, the generated POM contains all dependencies (even nested !)
*
* @param projectRoot
* @param mavenDir
* @return null if a dependency cannot be resolved !!!
* @throws FileNotFoundException
* @throws IOException
* @throws XmlPullParserException
* @throws InterruptedException
*/
@Deprecated
public static String extractClassPathFromPom(String projectRoot, String mavenDir)
throws FileNotFoundException, IOException, XmlPullParserException, InterruptedException {
// First step: we execute package the project in order to get all dependencies in .m2 cache
String outprintlog = "/tmp/" + System.currentTimeMillis();
System.out.println("Packaging the project to obtain all dependencies in meaven cache...");
System.out.println("Mvn console: " + outprintlog);
ProcessBuilder proc1 = new ProcessBuilder("mvn", "package");
proc1.directory(new File(projectRoot));
proc1.redirectOutput(new File(outprintlog));
proc1.redirectError(new File(outprintlog));
proc1.start().waitFor();
System.out.println("Generating a pom in order to get all dependencies...");
ProcessBuilder p2 = new ProcessBuilder("mvn", "help:effective-pom");
p2.directory(new File(projectRoot));
p2.redirectOutput(new File(outprintlog));
p2.redirectError(new File(outprintlog));
Process proc2 = p2.start();
BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc2.getInputStream()));
String s = null;
String effectivePomReaded = "";
boolean recordOutputAsXml = false;
while ((s = stdInput.readLine()) != null) {
String st = s.trim();
if (st.startsWith("<?xml")) recordOutputAsXml = true;
if (recordOutputAsXml) effectivePomReaded += s;
if (st.endsWith("</project>")) recordOutputAsXml = false;
}
StringReader sr = new StringReader(effectivePomReaded);
try {
MavenXpp3Reader reader = new MavenXpp3Reader();
Model model = reader.read(sr);
String classpath = "";
for (Dependency dep : model.getDependencies()) {
String ff =
mavenDir
+ File.separator
+ dep.getGroupId().replaceAll("\\.", File.separator)
+ File.separator
+ dep.getArtifactId()
+ File.separator
+ dep.getVersion()
+ File.separator
+ dep.getArtifactId()
+ "-"
+ dep.getVersion()
+ ".jar";
File fff = new File(ff);
if (!fff.exists()) {
return null;
} else {
classpath += ff + File.pathSeparator;
}
}
return classpath;
} catch (EOFException e) {
System.out.println("EOF reached -- no classpath !");
return "";
}
}
