/** * look for a tag whose text is getStartTag() then read until it closes * * @return true if there is data * @throws java.io.IOException */ public boolean nextKeyValue() throws IOException { String current = m_Sb.toString(); if (current.contains("<scan num=\"67\"")) current = m_Sb.toString(); // break here if (readFromCurrentBuffer()) return true; int newSize; if (m_Current > m_End) { // we are the the end of the split m_Key = null; m_Value = null; m_Sb.setLength(0); return false; } newSize = m_Input.read(m_Buffer); while (newSize > 0) { m_Current += newSize; String read = new String(m_Buffer, 0, newSize); m_Sb.append(read); if (readFromCurrentBuffer()) return true; if (m_Current > m_End) { // we are the the end of the split String s = m_Sb.toString(); if (bufferHasStartTag() == -1) { // not working on a tag m_Key = null; m_Value = null; m_Sb.setLength(0); return false; } if (m_Sb.length() > getMaxTagLength()) { m_Key = null; m_Value = null; m_Sb.setLength(0); return false; } } newSize = m_Input.read(m_Buffer); } // exit because we are at the m_End if (newSize <= 0) { m_Key = null; m_Value = null; m_Sb.setLength(0); return false; } if (m_Current > m_End) { // we are the the end of the split m_Key = null; m_Value = null; m_Sb.setLength(0); return false; } return true; }
/* called once at the beginning of the task */ public void setup(Context context) throws IOException, InterruptedException { BufferedReader br = null; Path[] files = DistributedCache.getLocalCacheFiles(context.getConfiguration()); if (files != null && files.length > 0) br = new BufferedReader(new FileReader(files[0].toString())); String line = null; /* reads the cached file into a hashmap */ try { while ((line = br.readLine()) != null) { String input[] = line.split(","); vector.put(Long.valueOf(input[0]), Double.valueOf(input[1])); } } finally { br.close(); } }
protected void setup(Context context) throws IOException, InterruptedException { FileSystem fs = FileSystem.get(context.getConfiguration()); Path cFile = new Path(context.getConfiguration().get("CFILE")); DataInputStream d = new DataInputStream(fs.open(cFile)); BufferedReader reader = new BufferedReader(new InputStreamReader(d)); String line; while ((line = reader.readLine()) != null) { StringTokenizer tokenizer = new StringTokenizer(line.toString()); if (tokenizer.hasMoreTokens()) { List<Double> centroid = new ArrayList<Double>(58); while (tokenizer.hasMoreTokens()) { centroid.add(Double.parseDouble(tokenizer.nextToken())); } centroids.add(centroid); } } k = centroids.size(); }
public synchronized void close() throws IOException { if (m_Input != null) { m_Input.close(); } }
// The input video files are split into chunks of 64MB here... public void map(Object key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); System.out.println("job1:mapInp:-" + line); String[] info = line.split(" "); info[0] = info[0].trim(); info[1] = info[1].trim(); String lstfnames = "", fname = ""; try { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); String prefixPath = "", fnm = ""; Pattern x = Pattern.compile("(.*)/(.*)"); Matcher xm = x.matcher(info[0]); while (xm.find()) { prefixPath = xm.group(1); fnm = xm.group(2); } String dst = "/home/" + fnm; // dst is path of the file on local system. hdfs.copyToLocalFile(new Path(info[0]), new Path(dst)); Process p = Runtime.getRuntime().exec("ffmpeg -i " + dst); String s; BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream())); Pattern D = Pattern.compile("Duration:[ ]*([0-9]+):([0-9]+):([0-9]+)"); long time = 0; // "time" is the duration of the input video file long sps = 0; // "sps" is the number of seconds(duration) of each video split while ((s = stdError.readLine()) != null) { Matcher md = D.matcher(s); while (md.find()) { time = Long.parseLong(md.group(1)) * 3600 + Long.parseLong(md.group(2)) * 60 + Long.parseLong(md.group(3)); } } Process p1 = Runtime.getRuntime().exec("du -s " + dst); BufferedReader stdInput1 = new BufferedReader(new InputStreamReader(p1.getInputStream())); String s1 = "", size = ""; // "size" is the size of input video file while ((s1 = stdInput1.readLine()) != null) { String s11[] = s1.split("\t"); size = s11[0]; } sps = (64 * 1024) * time / (Long.parseLong(size)); // chunk size is 64MB String hr, min, sc; hr = Long.toString((sps / 3600)); min = Long.toString((sps % 3600) / 60); sc = Long.toString(sps % 60); if (hr.length() < 2) hr = "0" + hr; if (min.length() < 2) min = "0" + min; if (sc.length() < 2) sc = "0" + sc; String splt = hr + ":" + min + ":" + sc; String query = "mencoder -oac copy -ovc copy -ss "; // building query to split the input video file String app = "", inpExt = ""; Pattern xx = Pattern.compile("(.*)\\.(.*)"); Matcher xxm = xx.matcher(dst); while (xxm.find()) { fname = xxm.group(1); inpExt = xxm.group(2); } String[] tmpArr = fname.split("/"); String hdfsFname = ""; long stSrt = 0; int cnt = 0; while (true) { if (stSrt > time) break; if (stSrt + sps > time) { long t = time - stSrt; hr = Long.toString((t / 3600)); min = Long.toString((t % 3600) / 60); sc = Long.toString(t % 60); if (hr.length() < 2) hr = "0" + hr; if (min.length() < 2) min = "0" + min; if (sc.length() < 2) sc = "0" + sc; splt = hr + ":" + min + ":" + sc; } cnt++; hr = Long.toString((stSrt / 3600)); min = Long.toString((stSrt % 3600) / 60); sc = Long.toString(stSrt % 60); if (hr.length() < 2) hr = "0" + hr; if (min.length() < 2) min = "0" + min; if (sc.length() < 2) sc = "0" + sc; app = hr + ":" + min + ":" + sc + " -endPos " + splt + " " + dst + " -o " + fname + "_" + Integer.toString(cnt) + "." + inpExt; Process p2 = Runtime.getRuntime().exec(query + app); String ls_str = ""; DataInputStream ls_in = new DataInputStream(p2.getInputStream()); while ((ls_str = ls_in.readLine()) != null) {} p2.destroy(); String[] tmpArr1 = fnm.split("\\."); hdfs.copyFromLocalFile( true, true, new Path(fname + "_" + Integer.toString(cnt) + "." + inpExt), new Path(prefixPath + "/" + tmpArr1[0] + "_" + Integer.toString(cnt) + "." + inpExt)); lstfnames += prefixPath + "/" + tmpArr1[0] + "_" + Integer.toString(cnt) + "." + inpExt + " #!# "; stSrt += sps; } Runtime rt1 = Runtime.getRuntime(); String[] cmd1 = {"/bin/bash", "-c", "rm " + dst}; // delete the file after use Process pr1 = rt1.exec(cmd1); pr1.waitFor(); lstfnames += "*" + info[1]; context.write( new Text(fname), new Text( lstfnames)); // "fname" contains name of the input video file with // extension(eg.".avi") removed #### "lstfnames" is a string, contains // all the names of video splits(concatenated) System.out.println("lstfnames : " + lstfnames); } catch (IOException e) { System.out.println("exception happened - here's what I know: "); e.printStackTrace(); System.exit(-1); } }