Example #1
1
    /**
     * look for a tag whose text is getStartTag() then read until it closes
     *
     * @return true if there is data
     * @throws java.io.IOException
     */
    public boolean nextKeyValue() throws IOException {
      String current = m_Sb.toString();
      if (current.contains("<scan num=\"67\"")) current = m_Sb.toString(); // break here

      if (readFromCurrentBuffer()) return true;
      int newSize;
      if (m_Current > m_End) { // we are the the end of the split
        m_Key = null;
        m_Value = null;
        m_Sb.setLength(0);
        return false;
      }

      newSize = m_Input.read(m_Buffer);

      while (newSize > 0) {
        m_Current += newSize;
        String read = new String(m_Buffer, 0, newSize);
        m_Sb.append(read);
        if (readFromCurrentBuffer()) return true;
        if (m_Current > m_End) { // we are the the end of the split
          String s = m_Sb.toString();
          if (bufferHasStartTag() == -1) { // not working on a tag
            m_Key = null;
            m_Value = null;
            m_Sb.setLength(0);
            return false;
          }
          if (m_Sb.length() > getMaxTagLength()) {
            m_Key = null;
            m_Value = null;
            m_Sb.setLength(0);
            return false;
          }
        }

        newSize = m_Input.read(m_Buffer);
      }
      // exit because we are at the m_End
      if (newSize <= 0) {
        m_Key = null;
        m_Value = null;
        m_Sb.setLength(0);
        return false;
      }
      if (m_Current > m_End) { // we are the the end of the split
        m_Key = null;
        m_Value = null;
        m_Sb.setLength(0);
        return false;
      }

      return true;
    }
Example #2
0
    protected void setup(Context context) throws IOException, InterruptedException {
      super.setup(context);

      Path pt = new Path("/user/yao/query/query");
      FileSystem fs = FileSystem.get(new Configuration());
      BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(pt)));
      String line = br.readLine();
      String[] keywords = line.split(",");
      k0 = keywords[0];
      k1 = keywords[1];
      k2 = keywords[2];
      br.close();
    }
Example #3
0
 public static List<String> readConfig(Path path, Configuration conf) throws IOException {
   final FileSystem fs = FileSystem.get(conf);
   final BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(path)));
   try {
     final ArrayList<String> addrs = new ArrayList<String>();
     String line;
     while ((line = reader.readLine()) != null) {
       addrs.add(line);
     }
     return addrs;
   } finally {
     reader.close();
   }
 }
    /* called once at the beginning of the task */
    public void setup(Context context) throws IOException, InterruptedException {
      BufferedReader br = null;
      Path[] files = DistributedCache.getLocalCacheFiles(context.getConfiguration());
      if (files != null && files.length > 0)
        br = new BufferedReader(new FileReader(files[0].toString()));
      String line = null;

      /* reads the cached file into a hashmap */
      try {
        while ((line = br.readLine()) != null) {
          String input[] = line.split(",");
          vector.put(Long.valueOf(input[0]), Double.valueOf(input[1]));
        }
      } finally {
        br.close();
      }
    }
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    Path matFile = new Path(args[0]);
    FSDataInputStream matData = fs.open(matFile);
    BufferedReader br = new BufferedReader(new InputStreamReader(matData));

    int i = 0;
    String line;
    while ((line = br.readLine()) != null) {
      StringTokenizer tokenizer = new StringTokenizer(line);
      String iRow = tokenizer.nextToken();
      String iCol = tokenizer.nextToken();
      if (Integer.parseInt(iRow) == Integer.parseInt(iCol)) {
        i++;
      }
    }
    br.close();
    int dimention = i;

    conf.setInt("DIMENTION", dimention);
    Path xFile = new Path("preX/Result");
    FSDataOutputStream xData = fs.create(xFile);
    BufferedWriter iniX = new BufferedWriter(new OutputStreamWriter(xData));
    for (int j = 0; j < dimention; j++) {
      iniX.write(String.valueOf(j) + " 0");
      iniX.newLine();
    }
    iniX.close();

    URI matVec = new URI(args[0]);
    DistributedCache.addCacheFile(matVec, conf);

    int iteration = 0;
    do {
      ToolRunner.run(conf, new Jacobi(), args);
    } while (iteration++ < max_iter && (!stopIteration(conf)));
  }
  public static boolean stopIteration(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path preFile = new Path("preX/Result");
    Path curFile = new Path("curX/part-00000");

    if (!(fs.exists(preFile) && fs.exists(curFile))) {
      System.exit(1);
    }

    boolean stop = true;
    String line1, line2;
    FSDataInputStream in1 = fs.open(preFile);
    FSDataInputStream in2 = fs.open(curFile);
    InputStreamReader isr1 = new InputStreamReader(in1);
    InputStreamReader isr2 = new InputStreamReader(in2);
    BufferedReader br1 = new BufferedReader(isr1);
    BufferedReader br2 = new BufferedReader(isr2);

    while ((line1 = br1.readLine()) != null && (line2 = br2.readLine()) != null) {
      String[] str1 = line1.split("\\s+");
      String[] str2 = line2.split("\\s+");
      double preElem = Double.parseDouble(str1[1]);
      double curElem = Double.parseDouble(str2[1]);
      if (Math.abs(preElem - curElem) > eps) {
        stop = false;
        break;
      }
    }

    if (stop == false) {
      fs.delete(preFile, true);
      if (fs.rename(curFile, preFile) == false) {
        System.exit(1);
      }
    }
    return stop;
  }
Example #7
0
  public BufferedReader loadDataFromFile(String filepath, long offset) {
    try {
      Path pt = new Path(filepath);
      FileSystem fs = FileSystem.get(fsConf);
      InputStreamReader isr;
      if (fs.isDirectory(pt)) { // multiple parts
        isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
      } else { // single file
        FSDataInputStream fileHandler = fs.open(pt);
        if (offset > 0) fileHandler.seek(offset);
        isr = new InputStreamReader(fileHandler);
      }

      BufferedReader reader = new BufferedReader(isr);
      if (offset > 0) reader.readLine(); // skip first line in case of seek
      return reader;
    } catch (FileNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }
    assert false : "Should not reach here!";
    return null;
  }
Example #8
0
    protected void setup(Context context) throws IOException, InterruptedException {
      FileSystem fs = FileSystem.get(context.getConfiguration());

      Path cFile = new Path(context.getConfiguration().get("CFILE"));
      DataInputStream d = new DataInputStream(fs.open(cFile));
      BufferedReader reader = new BufferedReader(new InputStreamReader(d));

      String line;
      while ((line = reader.readLine()) != null) {
        StringTokenizer tokenizer = new StringTokenizer(line.toString());

        if (tokenizer.hasMoreTokens()) {
          List<Double> centroid = new ArrayList<Double>(58);

          while (tokenizer.hasMoreTokens()) {
            centroid.add(Double.parseDouble(tokenizer.nextToken()));
          }

          centroids.add(centroid);
        }
      }

      k = centroids.size();
    }
Example #9
0
 public synchronized void close() throws IOException {
   if (m_Input != null) {
     m_Input.close();
   }
 }