public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String cur_file =
       ((FileSplit) context.getInputSplit()).getPath().getParent().getParent().getName();
   String train_file = context.getConfiguration().get("train_file");
   if (cur_file.equals(train_file)) {
     StringTokenizer st = new StringTokenizer(value.toString());
     String word = st.nextToken();
     String f_id = st.nextToken();
     myKey.set(word);
     myVal.set(f_id);
     context.write(myKey, myVal);
   } else {
     StringTokenizer st = new StringTokenizer(value.toString());
     String word = st.nextToken();
     String f_id = st.nextToken();
     StringBuilder builder = new StringBuilder(dlt);
     while (st.hasMoreTokens()) {
       String filename = st.nextToken();
       String tf_idf = st.nextToken();
       builder.append(filename);
       builder.append(dlt);
       builder.append(tf_idf);
       builder.append("\t");
     }
     myKey.set(word);
     myVal.set(builder.toString());
     context.write(myKey, myVal);
   }
 }
Exemplo n.º 2
1
    public void reduce(Text key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

      String keyS = key.toString();
      if (keyS.startsWith("O") || keyS.startsWith("P") || keyS.startsWith("S")) {
        String sum = new String();

        for (Text val : values) {

          sum += (" " + val.toString());
        }

        // String subKey = keyS.substring(0,keyS.length()-1);

        // Text t = new Text();
        // t.set(subKey);
        result.set(sum);
        context.write(key, result);
      }
      if (keyS.startsWith("L")) {
        //	String [] keyIdS = keyS.substring(1).split("[+]");

        result.set(" ");
        context.write(key, result);

        // String KeyIdS1 = keyIdS[1];
        // result.set(KeyIdS1);
        // context.write(key, result);

        // String KeyIdS2 = keyIdS[2];
        // result.set(KeyIdS2);
        // context.write(key, result);

      }
    }
  @Override
  public void reduce(IntWritable key, Iterable<WriteableData> values, Context context)
      throws IOException, InterruptedException {

    DaalContext daalContext = new DaalContext();

    /* Create an algorithm to compute a sparse variance-covariance matrix on the master node */
    DistributedStep2Master covarianceSparseMaster =
        new DistributedStep2Master(daalContext, Double.class, Method.fastCSR);

    for (WriteableData value : values) {
      PartialResult pr = (PartialResult) value.getObject(daalContext);
      covarianceSparseMaster.input.add(DistributedStep2MasterInputId.partialResults, pr);
    }

    /* Compute a sparse variance-covariance matrix on the master node */
    covarianceSparseMaster.compute();

    /* Finalize computations and retrieve the results */
    Result result = covarianceSparseMaster.finalizeCompute();

    HomogenNumericTable covariance = (HomogenNumericTable) result.get(ResultId.covariance);
    HomogenNumericTable mean = (HomogenNumericTable) result.get(ResultId.mean);

    context.write(new IntWritable(0), new WriteableData(covariance));
    context.write(new IntWritable(1), new WriteableData(mean));

    daalContext.dispose();
  }
Exemplo n.º 4
0
    public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
      int sum = 0;
      boolean filtered = true;
      String keypair = key.toString();
      String[] keys = keypair.split(" ");
      if (keys.length == 1) {
        filtered = false;
      } else {
        if (keys[0].equals("*") || keys[1].equals("*")) {
          filtered = false;
        }
      }

      if (!filtered) {
        for (IntWritable val : values) {
          sum += val.get();
        }
        context.write(key, new IntWritable(sum));
        return;
      }

      for (IntWritable val : values) {
        if (val.get() == -1) {
          filtered = false;
          continue;
        }
        sum += val.get();
      }
      // filter non-needed events
      if (filtered) return;
      context.write(key, new IntWritable(sum));
    }
Exemplo n.º 5
0
 /*
  * Setup gets called exactly once for each mapper, before map() gets called the first time.
  * It's a good place to do configuration or setup that can be shared across many calls to map
  */
 @Override
 public void setup(Context context) {
   targetGram = context.getConfiguration().get("targetWord").toLowerCase();
   try {
     funcNum = Integer.parseInt(context.getConfiguration().get("funcNum"));
   } catch (NumberFormatException e) {
     /* Do nothing. */
   }
 }
Exemplo n.º 6
0
    public void reduce(GFKey key, Iterable<PEIWritable> values, Context context)
        throws IOException, InterruptedException {
      // For a particular key ... process all records and output what we would have expected in this
      // concKnownKeys test
      // Note that we either
      // 1. do a single create
      // 2. create + update
      // 3. create + destroy
      // look at all ops ... and output either
      // 1. create
      // 2. create (with value from update)
      // 3. do nothing (overall result is destroy, so do not create the entry in the gemfire
      // validation region
      String keyStr = (String) key.getKey();
      ValueHolder updateValue = null;
      ValueHolder createValue = null;
      boolean destroyed = false;
      System.out.println("KnownKeysMRv2.reduce() invoked with " + keyStr);
      for (PEIWritable value : values) {
        PersistedEventImpl event = value.getEvent();
        Operation op = event.getOperation();

        ValueHolder vh = null;
        if (op.isDestroy()) {
          destroyed = true;
        } else {
          try {
            vh = (ValueHolder) event.getDeserializedValue();
          } catch (ClassNotFoundException e) {
            System.out.println(
                "KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
          }
          if (op.isUpdate()) {
            updateValue = vh;
          } else {
            createValue = vh;
          }
        }
        System.out.println(
            "KnownKeysMRv2.reduce() record: "
                + op.toString()
                + ": key = "
                + keyStr
                + " and op "
                + op.toString());
      }
      if (!destroyed) {
        if (updateValue != null) {
          context.write(key.getKey(), updateValue);
        } else {
          context.write(key.getKey(), createValue);
        }
      }
    }
Exemplo n.º 7
0
    private static void WriteNewCentroids(Context context) throws IOException {
      FileSystem fs = FileSystem.get(context.getConfiguration());
      Path nextCFile = new Path(context.getConfiguration().get("NEXTCFILE"));
      DataOutputStream d = new DataOutputStream(fs.create(nextCFile, false));
      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(d));

      for (String centroid : newCentroids) {
        writer.write(centroid + "\n");
      }

      writer.close();
    }
  public void reduce(LongWritable key, Iterable<Text> values, Context context)
      throws IOException, InterruptedException {
    LongWritable curNodeId = key;
    double previousPRValue = 1;
    double nextPRValue = 0;
    double localResidual = 0;
    String edgeListOfCurNode = "";
    long localResidualTransformed = 0;

    for (Text value : values) {
      String[] inputInfo = value.toString().split("\\s+");

      // incoming pagerank value
      if (inputInfo.length == 1) {
        nextPRValue += Double.parseDouble(inputInfo[0]);
      }
      // current node info
      else if (inputInfo.length == 3) {
        edgeListOfCurNode = inputInfo[2];
        previousPRValue = Double.parseDouble(inputInfo[1]);
      } else if (inputInfo.length == 2) {
        previousPRValue = Double.parseDouble(inputInfo[1]);
      } else {
        System.out.println("ERROR: received unexpected TEXT in length");
      }
    }
    if (previousPRValue == 1) System.out.println("No node info has been received by a reducer");
    // calculate the pagerank value according to the given formula
    nextPRValue = pagerankFormula(nextPRValue);

    // should also iterate sink nodes list, add the evenly splitted value

    // reducer should store the updated node info(NPR) to output directory
    context.write(null, new Text(curNodeId + " " + nextPRValue + " " + edgeListOfCurNode));

    // then compare PPR with NPR
    try {
      localResidual = Math.abs(previousPRValue - nextPRValue) / nextPRValue;
      localResidualTransformed = (long) (localResidual * 10000);
      // System.out.println("Make sure you got the right transformed residual :
      // "+localResidualTransformed);

    } catch (ArithmeticException e) {
      System.out.println("PPR is zero. Check where you get the value!");
    }

    // assume there is a global counter called residualCounter;

    context.getCounter(myCounter.ResidualCounter.RESIDUAL_SUM).increment(localResidualTransformed);
  }
  @Override
  protected void reduce(Text key, Iterable<Text> vals, Context ctx) {
    String name = null;
    String year = null;

    for (Text xx : vals) {
      String[] parts = xx.toString().split("=");

      if (parts[0].equals("name")) {
        name = parts[1];
      }

      if (parts[0].equals("year")) {
        year = parts[1];
      }
    }

    try {
      if (name != null && year != null) {
        ctx.write(new Text(year), new Text(name));
      }
    } catch (Exception ee) {
      ee.printStackTrace(System.err);
      throw new Error("I give up");
    }
  }
Exemplo n.º 10
0
    @Override
    protected void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      String line = value.toString();
      StringTokenizer tokenizer = new StringTokenizer(line, " ");
      int dim = tokenizer.countTokens();
      double[] coords = new double[dim];
      for (int i = 0; i < dim; i++) {
        coords[i] = Double.valueOf(tokenizer.nextToken());
      }
      Point point = new Point(coords);

      Centroid nearest = null;
      double nearestDistance = Double.MAX_VALUE;
      for (Centroid c : centers) {
        double dist = point.euclideanDistance(c);
        if (nearest == null) {
          nearest = c;
          nearestDistance = dist;
        } else {
          if (dist < nearestDistance) {
            nearest = c;
            nearestDistance = dist;
          }
        }
      }
      context.write(nearest, point);
    }
Exemplo n.º 11
0
    @Override
    public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
        throws IOException, InterruptedException {
      // TODO
      // TextArrayWritable tmp;
      // while(values.hasNext()){
      for (TextArrayWritable value : values) {
        for (Writable w : value.get()) {
          String str = ((Text) w).toString();
          String arrStr[] = str.split("\\|");
          tree.add(new Pair(Integer.parseInt(arrStr[0]), arrStr[1]));
        }
      }

      // reverse tree set
      TreeSet<Pair> treereverse = new TreeSet<Pair>();
      treereverse = (TreeSet) tree.descendingSet();
      Iterator itr = treereverse.iterator();
      int count = 1;
      while (itr.hasNext() && count != N + 1) {
        Pair p = (Pair) itr.next();
        // send top N entry
        context.write(
            new Text(p.second.toString()), new IntWritable(Integer.parseInt(p.first.toString())));
        count++;
      }
    }
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      Configuration c = context.getConfiguration();
      String s = value.toString();
      String input[] = s.split(",");
      Text outputkey = new Text();
      Text outputvalue = new Text();
      double result = 0.0;

      /* multiplies matrix and vector entry with matching column value */

      result = (Double.parseDouble(input[2])) * (vector.get(Long.parseLong(input[1])));
      outputkey.set(input[0]);
      outputvalue.set(Double.toString(result));
      context.write(outputkey, outputvalue);
    }
Exemplo n.º 13
0
 @Override
 protected void cleanup(Context context) throws IOException, InterruptedException {
   super.cleanup(context);
   Configuration conf = context.getConfiguration();
   Path outPath = new Path(conf.get(CENTERS_CONF_KEY));
   FileSystem fs = FileSystem.get(conf);
   // fs.delete(outPath, true);
   SequenceFile.Writer writer =
       SequenceFile.createWriter(
           fs, context.getConfiguration(), outPath, Centroid.class, IntWritable.class);
   final IntWritable mockValue = new IntWritable(0);
   for (Centroid center : centers) {
     writer.append(center, mockValue);
   }
   writer.close();
 }
Exemplo n.º 14
0
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      System.out.println("HELLO" + key + value);

      String line = value.toString();
      String[] components = line.split("\\s+");
      context.write(new Text(components[0]), new Text(components[1]));
    }
Exemplo n.º 15
0
    public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {
      String file = value.toString();
      String[] lines = file.split("\n");

      for (String line : lines) {
        if (line.contains("<author>") && line.contains("</author>")) {
          String author = line.substring(8, line.indexOf("</a"));
          word.set(author);
          context.write(word, one);
        } else if (line.contains("<author>")) {
          String author = line.substring(8);
          word.set(author);
          context.write(word, one);
        }
      }
    }
Exemplo n.º 16
0
 /**
  * Emit all the values for the final solved values table. We want to print this out with the
  * expanded string representation of the game board (to make the output file easily human
  * readable and thus easier to debug).
  */
 @Override
 public void reduce(IntWritable key, Iterable<ByteWritable> values, Context context)
     throws IOException, InterruptedException {
   int keyInt = key.get();
   if (keyInt == 0) return;
   String hash = "'" + Proj2Util.gameUnhasher(keyInt, boardWidth, boardHeight) + "'";
   for (ByteWritable val : values) context.write(new Text(hash), val);
 }
Exemplo n.º 17
0
 // Identity
 public void reduce(Text key, Iterable<Text> values, Context context)
     throws IOException, InterruptedException {
   System.out.println("I'm in Job1 reduce");
   for (Text val : values) {
     System.out.println("job1:redInp:-" + val.toString());
     context.write(new Text(""), val);
   }
 }
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum += val.get();
   }
   context.write(key, new IntWritable(sum));
 }
Exemplo n.º 19
0
 /**
  * Emits provided key-value pair as long as the value field of the MovesWritable is non-zero.
  */
 @Override
 public void map(IntWritable key, MovesWritable val, Context context)
     throws IOException, InterruptedException {
   byte value = val.getValue();
   if (value == 0) {
     return;
   }
   context.write(key, new ByteWritable(value));
 }
Exemplo n.º 20
0
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   Text word = new Text();
   StringTokenizer s = new StringTokenizer(value.toString());
   while (s.hasMoreTokens()) {
     word.set(s.nextToken());
     context.write(word, one);
   }
 }
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   String[] currentUserTuples = line.split("::");
   int currentAgeValue = Integer.parseInt(currentUserTuples[2].trim());
   if (currentUserTuples[1].trim().equalsIgnoreCase("M") && currentAgeValue <= targetAge) {
     context.write(new Text("UserId :: " + currentUserTuples[0].trim()), one);
   }
 }
Exemplo n.º 22
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum = sum + val.get();
   }
   System.out.println("----------inside-----------");
   context.write(key, new Text(Integer.toString(sum)));
 }
Exemplo n.º 23
0
 public void reduce(Text key, Iterable<LongWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (LongWritable val : values) {
     sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
 }
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   StringTokenizer tokenizer = new StringTokenizer(line);
   while (tokenizer.hasMoreTokens()) {
     word.set(tokenizer.nextToken());
     context.write(word, one);
   }
 }
Exemplo n.º 25
0
    /**
     * Maps each number into an initial key/value pair This method will ignore any non-numbers (e.g.
     * letters/grammar)
     *
     * @param lwKey initial key
     * @param tValue initial value
     * @param cContext initial context
     * @throws IOException
     */
    @Override
    public void map(LongWritable lwKey, Text tValue, Context cContext) throws IOException {

      String line = tValue.toString();
      cContext.getConfiguration().getStrings(line);

      String[] lineArr = line.split("\n");
      for (String string : lineArr) {
        if (string.matches("-?\\d+(\\.\\d+)?")) {
          tWord.set(string);
          try {
            cContext.write(tWord, iwOne);
          } catch (InterruptedException ex) {
            Logger.getLogger(FrequencyTest.class.getName()).log(Level.SEVERE, null, ex);
          }
        }
      }
    }
Exemplo n.º 26
0
    public void reduce(Text key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

      for (Text val : values) {
        context.write(key, val);

        System.out.println("Pass2 RED KEY: " + key);
        System.out.println("Pass2 RED VAL: " + val);
      }
    }
Exemplo n.º 27
0
  @Override
  public void setup(Context context) throws IOException {
    Configuration conf = context.getConfiguration();

    int num = Integer.parseInt(conf.get("kvNum"));
    while (--num >= 0) {
      String s = conf.get(String.valueOf(num));
      m_kVectors.add(Kvector.fromString(s));
    }
  }
Exemplo n.º 28
0
  @Override
  /** (K1, V1) = (line#, line) (K2, V2) = (visitor full name, 1) */
  public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {

    parser.parse(value);
    if (parser.isValidRecord()) {
      context.write(new Text(parser.getVisitorFullname()), one);
    }
  }
Exemplo n.º 29
0
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      // read in a document (point in 58-dimensional space)

      List<Double> p = GetPoint(value.toString());

      int idxClosestCentoid = IndexOfClosestCentroid(p);

      context.write(new IntWritable(idxClosestCentoid), value);
    }
Exemplo n.º 30
0
 @Override
 public void map(Object key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   StringTokenizer tokenizer = new StringTokenizer(line);
   int movieId = Integer.parseInt(tokenizer.nextToken());
   while (tokenizer.hasMoreTokens()) {
     String word = tokenizer.nextToken();
     context.write(new Text("1"), new IntWritable(1));
   }
 }