public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException {
try {
Invocation call = (Invocation) param;
if (verbose) log("Call: " + call);
Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses());
method.setAccessible(true);
int qTime = (int) (System.currentTimeMillis() - receivedTime);
long startNanoTime = System.nanoTime();
Object value = method.invoke(instance, call.getParameters());
long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Served: "
+ call.getMethodName()
+ " queueTime (millisec)= "
+ qTime
+ " procesingTime (microsec)= "
+ processingMicroTime);
}
rpcMetrics.rpcQueueTime.inc(qTime);
rpcMetrics.rpcProcessingTime.inc(processingMicroTime);
MetricsTimeVaryingRate m =
(MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
if (m == null) {
try {
m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry);
} catch (IllegalArgumentException iae) {
// the metrics has been registered; re-fetch the handle
LOG.debug("Error register " + call.getMethodName(), iae);
m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
}
}
// record call time in microseconds
m.inc(processingMicroTime);
if (verbose) log("Return: " + value);
return new ObjectWritable(method.getReturnType(), value);
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException) target;
} else {
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
}
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}
public void configure(JobConf conf) {
numberOfCenters = Integer.valueOf(conf.get("numberOfCenters"));
centersDirectory = conf.get("centersReadDirectory");
try {
Configuration c = new Configuration();
FileSystem fs = FileSystem.get(c);
for (int index = 0; index < numberOfCenters; ++index) {
SequenceFile.Reader reader =
new SequenceFile.Reader(fs, new Path(centersDirectory + "/centers/" + index), c);
LongWritable key = new LongWritable();
Point value = new Point();
reader.next(key, value);
Point center = (Point) value;
centers.add(center);
reader.close();
}
} catch (IOException e) {
// do nothing
// I hope this doesn't happen
System.out.println("well, damn.");
e.printStackTrace();
}
}
/** Used by child copy constructors. */
protected synchronized void copy(Writable other) {
if (other != null) {
try {
DataOutputBuffer out = new DataOutputBuffer();
other.write(out);
DataInputBuffer in = new DataInputBuffer();
in.reset(out.getData(), out.getLength());
readFields(in);
} catch (IOException e) {
throw new IllegalArgumentException("map cannot be copied: " + e.getMessage());
}
} else {
throw new IllegalArgumentException("source map cannot be null");
}
}
/**
* Construct a client-side proxy that implements the named protocol, talking to a server at the
* named address.
*
* @param protocol protocol
* @param clientVersion client's version
* @param addr server address
* @param ticket security ticket
* @param conf configuration
* @param factory socket factory
* @param rpcTimeout max time for each rpc; 0 means no timeout
* @return the proxy
* @throws IOException if any error occurs
*/
@SuppressWarnings("unchecked")
public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
Class<T> protocol,
long clientVersion,
InetSocketAddress addr,
UserGroupInformation ticket,
Configuration conf,
SocketFactory factory,
int rpcTimeout)
throws IOException {
T proxy =
(T)
Proxy.newProxyInstance(
protocol.getClassLoader(),
new Class[] {protocol},
new Invoker(addr, ticket, conf, factory, rpcTimeout, protocol));
String protocolName = protocol.getName();
try {
ProtocolSignature serverInfo =
proxy.getProtocolSignature(
protocolName, clientVersion, ProtocolSignature.getFingerprint(protocol.getMethods()));
return new ProtocolProxy<T>(protocol, proxy, serverInfo.getMethods());
} catch (RemoteException re) {
IOException ioe = re.unwrapRemoteException(IOException.class);
if (ioe.getMessage()
.startsWith(IOException.class.getName() + ": " + NoSuchMethodException.class.getName())) {
// Method getProtocolSignature not supported
long serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion);
if (serverVersion == clientVersion) {
return new ProtocolProxy<T>(protocol, proxy, null);
}
throw new VersionMismatch(protocolName, clientVersion, serverVersion, proxy);
}
throw re;
}
}
public void generateFileChunks(JspWriter out, HttpServletRequest req) throws IOException {
long startOffset = 0;
int chunkSizeToView = 0;
String referrer = req.getParameter("referrer");
boolean noLink = false;
if (referrer == null) {
noLink = true;
}
String filename = req.getParameter("filename");
if (filename == null) {
out.print("Invalid input (file name absent)");
return;
}
String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
int namenodeInfoPort = -1;
if (namenodeInfoPortStr != null) namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
String chunkSizeToViewStr = req.getParameter("chunkSizeToView");
if (chunkSizeToViewStr != null && Integer.parseInt(chunkSizeToViewStr) > 0)
chunkSizeToView = Integer.parseInt(chunkSizeToViewStr);
else chunkSizeToView = jspHelper.defaultChunkSizeToView;
if (!noLink) {
out.print("<h3>Tail of File: ");
JspHelper.printPathWithLinks(filename, out, namenodeInfoPort);
out.print("</h3><hr>");
out.print("<a href=\"" + referrer + "\">Go Back to File View</a><hr>");
} else {
out.print("<h3>" + filename + "</h3>");
}
out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
out.print(
"<input type=\"text\" name=\"chunkSizeToView\" value="
+ chunkSizeToView
+ " size=10 maxlength=10>");
out.print(" <input type=\"submit\" name=\"submit\" value=\"Refresh\"><hr>");
out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename + "\">");
out.print(
"<input type=\"hidden\" name=\"namenodeInfoPort\" value=\"" + namenodeInfoPort + "\">");
if (!noLink) out.print("<input type=\"hidden\" name=\"referrer\" value=\"" + referrer + "\">");
// fetch the block from the datanode that has the last block for this file
DFSClient dfs = new DFSClient(jspHelper.nameNodeAddr, jspHelper.conf);
List<LocatedBlock> blocks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
if (blocks == null || blocks.size() == 0) {
out.print("No datanodes contain blocks of file " + filename);
dfs.close();
return;
}
LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
long blockSize = lastBlk.getBlock().getNumBytes();
long blockId = lastBlk.getBlock().getBlockId();
long genStamp = lastBlk.getBlock().getGenerationStamp();
DatanodeInfo chosenNode;
try {
chosenNode = jspHelper.bestNode(lastBlk);
} catch (IOException e) {
out.print(e.toString());
dfs.close();
return;
}
InetSocketAddress addr = NetUtils.createSocketAddr(chosenNode.getName());
// view the last chunkSizeToView bytes while Tailing
if (blockSize >= chunkSizeToView) startOffset = blockSize - chunkSizeToView;
else startOffset = 0;
out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
jspHelper.streamBlockInAscii(
addr, blockId, genStamp, blockSize, startOffset, chunkSizeToView, out);
out.print("</textarea>");
dfs.close();
}
@Override
public Writable call(
org.apache.hadoop.ipc.RPC.Server server,
String protocolName,
Writable rpcRequest,
long receivedTime)
throws IOException {
try {
Invocation call = (Invocation) rpcRequest;
if (server.verbose) log("Call: " + call);
// Verify rpc version
if (call.getRpcVersion() != writableRpcVersion) {
// Client is using a different version of WritableRpc
throw new IOException(
"WritableRpc version mismatch, client side version="
+ call.getRpcVersion()
+ ", server side version="
+ writableRpcVersion);
}
long clientVersion = call.getProtocolVersion();
final String protoName;
ProtoClassProtoImpl protocolImpl;
if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) {
// VersionProtocol methods are often used by client to figure out
// which version of protocol to use.
//
// Versioned protocol methods should go the protocolName protocol
// rather than the declaring class of the method since the
// the declaring class is VersionedProtocol which is not
// registered directly.
// Send the call to the highest protocol version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protocolName);
}
protocolImpl = highest.protocolTarget;
} else {
protoName = call.declaringClassProtocolName;
// Find the right impl for the protocol based on client version.
ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion);
protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv);
if (protocolImpl == null) { // no match for Protocol AND Version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protoName);
} else { // protocol supported but not the version that client wants
throw new RPC.VersionMismatch(protoName, clientVersion, highest.version);
}
}
}
// Invoke the protocol method
long startTime = Time.now();
Method method =
protocolImpl.protocolClass.getMethod(
call.getMethodName(), call.getParameterClasses());
method.setAccessible(true);
server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters());
int processingTime = (int) (Time.now() - startTime);
int qTime = (int) (startTime - receivedTime);
if (LOG.isDebugEnabled()) {
LOG.debug(
"Served: "
+ call.getMethodName()
+ " queueTime= "
+ qTime
+ " procesingTime= "
+ processingTime);
}
server.rpcMetrics.addRpcQueueTime(qTime);
server.rpcMetrics.addRpcProcessingTime(processingTime);
server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime);
if (server.verbose) log("Return: " + value);
return new ObjectWritable(method.getReturnType(), value);
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException) target;
} else {
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
}
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}
public void generateFileDetails(JspWriter out, HttpServletRequest req, Configuration conf)
throws IOException, InterruptedException {
int chunkSizeToView = 0;
long startOffset = 0;
int datanodePort;
String blockIdStr = null;
long currBlockId = 0;
blockIdStr = req.getParameter("blockId");
if (blockIdStr == null) {
out.print("Invalid input (blockId absent)");
return;
}
currBlockId = Long.parseLong(blockIdStr);
String datanodePortStr = req.getParameter("datanodePort");
if (datanodePortStr == null) {
out.print("Invalid input (datanodePort absent)");
return;
}
datanodePort = Integer.parseInt(datanodePortStr);
String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
int namenodeInfoPort = -1;
if (namenodeInfoPortStr != null) namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
String chunkSizeToViewStr = req.getParameter("chunkSizeToView");
if (chunkSizeToViewStr != null && Integer.parseInt(chunkSizeToViewStr) > 0) {
chunkSizeToView = Integer.parseInt(chunkSizeToViewStr);
} else {
chunkSizeToView = JspHelper.getDefaultChunkSize(conf);
}
String startOffsetStr = req.getParameter("startOffset");
if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0) startOffset = 0;
else startOffset = Long.parseLong(startOffsetStr);
String filename = HtmlQuoting.unquoteHtmlChars(req.getParameter("filename"));
if (filename == null || filename.length() == 0) {
out.print("Invalid input");
return;
}
String blockSizeStr = req.getParameter("blockSize");
long blockSize = 0;
if (blockSizeStr == null || blockSizeStr.length() == 0) {
out.print("Invalid input");
return;
}
blockSize = Long.parseLong(blockSizeStr);
String tokenString = req.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
UserGroupInformation ugi = JspHelper.getUGI(req, conf);
DFSClient dfs = JspHelper.getDFSClient(ugi, jspHelper.nameNodeAddr, conf);
List<LocatedBlock> blocks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
// Add the various links for looking at the file contents
// URL for downloading the full file
String downloadUrl =
"http://"
+ req.getServerName()
+ ":"
+ +req.getServerPort()
+ "/streamFile"
+ URLEncoder.encode(filename, "UTF-8")
+ "?"
+ JspHelper.DELEGATION_PARAMETER_NAME
+ "="
+ tokenString;
out.print("<a name=\"viewOptions\"></a>");
out.print("<a href=\"" + downloadUrl + "\">Download this file</a><br>");
DatanodeInfo chosenNode;
// URL for TAIL
LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
long blockId = lastBlk.getBlock().getBlockId();
try {
chosenNode = jspHelper.bestNode(lastBlk);
} catch (IOException e) {
out.print(e.toString());
dfs.close();
return;
}
String fqdn = InetAddress.getByName(chosenNode.getHost()).getCanonicalHostName();
String tailUrl =
"http://"
+ fqdn
+ ":"
+ chosenNode.getInfoPort()
+ "/tail.jsp?filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ "&chunkSizeToView="
+ chunkSizeToView
+ "&referrer="
+ URLEncoder.encode(req.getRequestURL() + "?" + req.getQueryString(), "UTF-8")
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a href=\"" + tailUrl + "\">Tail this file</a><br>");
out.print("<form action=\"/browseBlock.jsp\" method=GET>");
out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
out.print("<input type=\"hidden\" name=\"blockId\" value=\"" + currBlockId + "\">");
out.print("<input type=\"hidden\" name=\"blockSize\" value=\"" + blockSize + "\">");
out.print("<input type=\"hidden\" name=\"startOffset\" value=\"" + startOffset + "\">");
out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename + "\">");
out.print("<input type=\"hidden\" name=\"datanodePort\" value=\"" + datanodePort + "\">");
out.print(
"<input type=\"hidden\" name=\"namenodeInfoPort\" value=\"" + namenodeInfoPort + "\">");
out.print(
"<input type=\"text\" name=\"chunkSizeToView\" value="
+ chunkSizeToView
+ " size=10 maxlength=10>");
out.print(" <input type=\"submit\" name=\"submit\" value=\"Refresh\">");
out.print("</form>");
out.print("<hr>");
out.print("<a name=\"blockDetails\"></a>");
out.print("<B>Total number of blocks: " + blocks.size() + "</B><br>");
// generate a table and dump the info
out.println("\n<table>");
for (LocatedBlock cur : blocks) {
out.print("<tr>");
blockId = cur.getBlock().getBlockId();
blockSize = cur.getBlock().getNumBytes();
String blk = "blk_" + Long.toString(blockId);
out.print("<td>" + Long.toString(blockId) + ":</td>");
DatanodeInfo[] locs = cur.getLocations();
for (int j = 0; j < locs.length; j++) {
String datanodeAddr = locs[j].getName();
datanodePort =
Integer.parseInt(
datanodeAddr.substring(datanodeAddr.indexOf(':') + 1, datanodeAddr.length()));
fqdn = InetAddress.getByName(locs[j].getHost()).getCanonicalHostName();
String blockUrl =
"http://"
+ fqdn
+ ":"
+ locs[j].getInfoPort()
+ "/browseBlock.jsp?blockId="
+ Long.toString(blockId)
+ "&blockSize="
+ blockSize
+ "&filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&datanodePort="
+ datanodePort
+ "&genstamp="
+ cur.getBlock().getGenerationStamp()
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ "&chunkSizeToView="
+ chunkSizeToView;
out.print(
"<td> </td>" + "<td><a href=\"" + blockUrl + "\">" + datanodeAddr + "</a></td>");
}
out.println("</tr>");
}
out.println("</table>");
out.print("<hr>");
String namenodeHost = jspHelper.nameNodeAddr.getHostName();
out.print(
"<br><a href=\"http://"
+ InetAddress.getByName(namenodeHost).getCanonicalHostName()
+ ":"
+ namenodeInfoPort
+ "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
}
// merge the converted files here
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
System.out.println("I'm in Job2 reduce");
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
try {
String out = "";
for (Text t : values) {
out = t.toString();
out = out.trim();
System.out.println("job2:redInp:-" + out);
break;
}
String[] outl = out.split(" #!# ");
Pattern x = Pattern.compile("(.*)/(.*)\\.(.*)");
Matcher xm = x.matcher(outl[0]);
String prefixPath = "", fnm = "", ext = "";
while (xm.find()) {
prefixPath = xm.group(1);
fnm = xm.group(2);
ext = xm.group(3);
}
String foutname = fnm.split("_")[0];
foutname += "." + ext;
String query = "mencoder -oac copy -ovc copy";
int cnt = 0;
for (String st : outl) {
cnt++;
hdfs.copyToLocalFile(
true,
new Path(st),
new Path("/home/" + fnm.split("_")[0] + "_" + Integer.toString(cnt) + "." + ext));
query += " " + "/home/" + fnm.split("_")[0] + "_" + Integer.toString(cnt) + "." + ext;
}
query += " -o " + "/home/" + foutname;
Process p2 =
Runtime.getRuntime().exec(query); // query for merging the video files is executed here
String ls_str = "";
DataInputStream ls_in = new DataInputStream(p2.getInputStream());
while ((ls_str = ls_in.readLine()) != null) {}
p2.destroy();
hdfs.copyFromLocalFile(
true, true, new Path("/home/" + foutname), new Path(prefixPath + "/" + foutname));
cnt = 0;
String dlt1 = "";
for (String st3 : outl) {
cnt++;
dlt1 += " " + "/home/" + fnm.split("_")[0] + "_" + Integer.toString(cnt) + "." + ext;
}
Runtime rt1 = Runtime.getRuntime();
String[] cmd1 = {"/bin/bash", "-c", "rm" + dlt1}; // delete the files after use
Process pr1 = rt1.exec(cmd1);
pr1.waitFor();
context.write(new Text(""), new Text(prefixPath + "/" + foutname));
} catch (IOException e) {
System.out.println("exception happened - here's what I know: ");
e.printStackTrace();
System.exit(-1);
}
}
// The input video files are split into chunks of 64MB here...
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
System.out.println("job1:mapInp:-" + line);
String[] info = line.split(" ");
info[0] = info[0].trim();
info[1] = info[1].trim();
String lstfnames = "", fname = "";
try {
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
String prefixPath = "", fnm = "";
Pattern x = Pattern.compile("(.*)/(.*)");
Matcher xm = x.matcher(info[0]);
while (xm.find()) {
prefixPath = xm.group(1);
fnm = xm.group(2);
}
String dst = "/home/" + fnm; // dst is path of the file on local system.
hdfs.copyToLocalFile(new Path(info[0]), new Path(dst));
Process p = Runtime.getRuntime().exec("ffmpeg -i " + dst);
String s;
BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));
Pattern D = Pattern.compile("Duration:[ ]*([0-9]+):([0-9]+):([0-9]+)");
long time = 0; // "time" is the duration of the input video file
long sps = 0; // "sps" is the number of seconds(duration) of each video split
while ((s = stdError.readLine()) != null) {
Matcher md = D.matcher(s);
while (md.find()) {
time =
Long.parseLong(md.group(1)) * 3600
+ Long.parseLong(md.group(2)) * 60
+ Long.parseLong(md.group(3));
}
}
Process p1 = Runtime.getRuntime().exec("du -s " + dst);
BufferedReader stdInput1 = new BufferedReader(new InputStreamReader(p1.getInputStream()));
String s1 = "", size = ""; // "size" is the size of input video file
while ((s1 = stdInput1.readLine()) != null) {
String s11[] = s1.split("\t");
size = s11[0];
}
sps = (64 * 1024) * time / (Long.parseLong(size)); // chunk size is 64MB
String hr, min, sc;
hr = Long.toString((sps / 3600));
min = Long.toString((sps % 3600) / 60);
sc = Long.toString(sps % 60);
if (hr.length() < 2) hr = "0" + hr;
if (min.length() < 2) min = "0" + min;
if (sc.length() < 2) sc = "0" + sc;
String splt = hr + ":" + min + ":" + sc;
String query =
"mencoder -oac copy -ovc copy -ss "; // building query to split the input video file
String app = "", inpExt = "";
Pattern xx = Pattern.compile("(.*)\\.(.*)");
Matcher xxm = xx.matcher(dst);
while (xxm.find()) {
fname = xxm.group(1);
inpExt = xxm.group(2);
}
String[] tmpArr = fname.split("/");
String hdfsFname = "";
long stSrt = 0;
int cnt = 0;
while (true) {
if (stSrt > time) break;
if (stSrt + sps > time) {
long t = time - stSrt;
hr = Long.toString((t / 3600));
min = Long.toString((t % 3600) / 60);
sc = Long.toString(t % 60);
if (hr.length() < 2) hr = "0" + hr;
if (min.length() < 2) min = "0" + min;
if (sc.length() < 2) sc = "0" + sc;
splt = hr + ":" + min + ":" + sc;
}
cnt++;
hr = Long.toString((stSrt / 3600));
min = Long.toString((stSrt % 3600) / 60);
sc = Long.toString(stSrt % 60);
if (hr.length() < 2) hr = "0" + hr;
if (min.length() < 2) min = "0" + min;
if (sc.length() < 2) sc = "0" + sc;
app =
hr
+ ":"
+ min
+ ":"
+ sc
+ " -endPos "
+ splt
+ " "
+ dst
+ " -o "
+ fname
+ "_"
+ Integer.toString(cnt)
+ "."
+ inpExt;
Process p2 = Runtime.getRuntime().exec(query + app);
String ls_str = "";
DataInputStream ls_in = new DataInputStream(p2.getInputStream());
while ((ls_str = ls_in.readLine()) != null) {}
p2.destroy();
String[] tmpArr1 = fnm.split("\\.");
hdfs.copyFromLocalFile(
true,
true,
new Path(fname + "_" + Integer.toString(cnt) + "." + inpExt),
new Path(prefixPath + "/" + tmpArr1[0] + "_" + Integer.toString(cnt) + "." + inpExt));
lstfnames +=
prefixPath + "/" + tmpArr1[0] + "_" + Integer.toString(cnt) + "." + inpExt + " #!# ";
stSrt += sps;
}
Runtime rt1 = Runtime.getRuntime();
String[] cmd1 = {"/bin/bash", "-c", "rm " + dst}; // delete the file after use
Process pr1 = rt1.exec(cmd1);
pr1.waitFor();
lstfnames += "*" + info[1];
context.write(
new Text(fname),
new Text(
lstfnames)); // "fname" contains name of the input video file with
// extension(eg.".avi") removed #### "lstfnames" is a string, contains
// all the names of video splits(concatenated)
System.out.println("lstfnames : " + lstfnames);
} catch (IOException e) {
System.out.println("exception happened - here's what I know: ");
e.printStackTrace();
System.exit(-1);
}
}
// Video splts are converted to target format here...
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
try {
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
String st = value.toString();
st = st.trim();
System.out.println("job2:mapInp:-" + st);
String[] fmt = st.split(" #!# \\*");
String[] lst = fmt[0].split(" #!# ");
String out = "", dlt = "";
int flag = 1;
for (String st1 : lst) {
Pattern x = Pattern.compile("(.*)/(.*)");
Matcher xm = x.matcher(st1);
String prefixPath = "", fnm = "", inpExt = "";
while (xm.find()) {
prefixPath = xm.group(1);
fnm = xm.group(2);
}
String[] tmpArr = fnm.split("\\.");
fnm = tmpArr[0];
inpExt = tmpArr[1];
hdfs.copyToLocalFile(true, new Path(st1), new Path("/home/" + fnm + "." + inpExt));
String fname = "/home/" + fnm;
if (flag == 1) {
flag = 0;
out += prefixPath + "/" + fnm + "." + fmt[1];
} else {
out += " #!# " + prefixPath + "/" + fnm + "." + fmt[1];
}
if (fmt[1].equals("mpg") || fmt[1].equals("mpeg") || fmt[1].equals("mp4")) {
Process p =
Runtime.getRuntime()
.exec(
"mencoder -of mpeg -ovc lavc -lavcopts vcodec=mpeg1video -oac copy "
+ "/home/"
+ fnm
+ "."
+ inpExt
+ " -o "
+ fname
+ "."
+ fmt[1]);
String ls_str = "";
DataInputStream ls_in = new DataInputStream(p.getInputStream());
while ((ls_str = ls_in.readLine()) != null) {}
p.destroy();
dlt += " /home/" + fnm + "." + inpExt;
} else if (fmt[1].equals("avi")) {
Process p =
Runtime.getRuntime()
.exec(
"mencoder -ovc lavc -oac mp3lame -o "
+ fname
+ "."
+ fmt[1]
+ " "
+ "/home/"
+ fnm
+ "."
+ inpExt);
String ls_str = "";
DataInputStream ls_in = new DataInputStream(p.getInputStream());
while ((ls_str = ls_in.readLine()) != null) {}
p.destroy();
dlt += " /home/" + fnm + "." + inpExt;
} else {
// TBD
System.out.println("Unsupported target format!!!!!");
}
hdfs.copyFromLocalFile(
true,
true,
new Path(fname + "." + fmt[1]),
new Path(prefixPath + "/" + fnm + "." + fmt[1]));
}
Runtime rt1 = Runtime.getRuntime();
String[] cmd1 = {"/bin/bash", "-c", "rm" + dlt}; // delete the files after use
Process pr1 = rt1.exec(cmd1);
pr1.waitFor();
System.out.println("Job2 mapOut:" + out);
context.write(new Text(lst[0]), new Text(out));
System.out.println(out);
} catch (IOException e) {
System.out.println("exception happened - here's what I know: ");
e.printStackTrace();
System.exit(-1);
}
}
