예제 #1
0
파일: UDFAscii.java 프로젝트: Leolh/hive
  public IntWritable evaluate(Text s) {
    if (s == null) {
      return null;
    }

    if (s.getLength() > 0) {
      result.set(s.getBytes()[0]);
    } else {
      result.set(0);
    }

    return result;
  }
예제 #2
0
 public void map(
     LongWritable key,
     Text value,
     OutputCollector<IntWritable, IntWritable> output,
     Reporter reporter)
     throws IOException {
   String line = value.toString();
   IntWritable clave = new IntWritable();
   IntWritable valor = new IntWritable();
   clave.set(Integer.parseInt(line));
   valor.set(Integer.parseInt(line) + 1);
   output.collect(clave, valor);
 }
    public void map(
        Text key, Text value, OutputCollector<IntWritable, IntWritable> output, Reporter reporter)
        throws IOException {

      citationCount.set(Integer.parseInt(value.toString()));
      output.collect(citationCount, uno);
    }
  public void map(
      LongWritable key,
      Text value,
      OutputCollector<IntWritable, DoubleWritable> output,
      Reporter reporter)
      throws IOException {
    /*
     * It implements the mapper. It outputs the numbers of weight and updated weights.
     *
     * Note that the format of intermediate output is <IntWritable, DoubleWritable>,
     * because the key is the number of weight (an integer), and the value is the weight's value (double)
     */
    inputData = value.toString();

    // go through the process
    initialize();
    getposphase();
    getnegphase();
    update();

    // output the intermediate data
    // The <key, value> pairs are <weightID, weightUpdate>
    double[][] vishidinc_array = vishidinc.getArray();
    for (int i = 0; i < numdims; i++) {
      for (int j = 0; j < numhid; j++) {
        weightPos.set(i * numhid + j);
        weightValue.set(vishidinc_array[i][j]);
        output.collect(weightPos, weightValue);
      }
    }
  }
예제 #5
0
    public void run(
        RecordReader<IntWritable, WikipediaPage> input,
        OutputCollector<IntWritable, Text> output,
        Reporter reporter)
        throws IOException {
      IntWritable key = new IntWritable();
      WikipediaPage value = new WikipediaPage();

      long pos = -1;
      long prevPos = -1;

      int prevDocno = 0;

      pos = input.getPos();
      while (input.next(key, value)) {
        if (prevPos != -1 && prevPos != pos) {
          LOG.info(
              "- beginning of block at " + prevPos + ", docno:" + prevDocno + ", file:" + fileno);
          keyOut.set(prevDocno);
          valOut.set(prevPos + "\t" + fileno);
          output.collect(keyOut, valOut);
          reporter.incrCounter(Blocks.Total, 1);
        }

        prevPos = pos;
        pos = input.getPos();
        prevDocno = key.get();
      }
    }
  @Override
  public void reduce(Text key, Iterable<IntWritable> values, Context context)
      throws IOException, InterruptedException {

    int ratingsCount = 0;

    /*
     * For each movieId:
     */
    // Define a counter for bad records.
    for (IntWritable value : values) {

      /*
       * Count the movie ratings.
       */
      ratingsCount += value.get();
    }

    /*
     * Reducer output is the userID and number of ratings.
     */
    context.getCounter(UserIdCounter.NUM_USERS).increment(1);
    rValue.set(ratingsCount);
    context.write(key, rValue);
  }
예제 #7
0
    public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {

      System.out.println("isitlinewise2: " + value.toString());

      StringTokenizer itr = new StringTokenizer(value.toString());
      System.out.println(value);

      // 1st token is the key, you don't want to process it
      itr.nextToken();

      int worth = 0;
      List<String> valList = new ArrayList<String>();
      System.out.println("map1----" + key);
      while (itr.hasMoreTokens()) {
        String val = itr.nextToken();
        System.out.println("map1 vals1: " + val.toString());
        try {
          worth = Integer.parseInt(val.toString());
          System.out.println("map1 worth: " + worth);
        } catch (NumberFormatException e) {
          valList.add(val.toString());
        }
      }
      for (String val : valList) {
        System.out.println("map1 vals2: " + val.toString() + " " + worth);
        word.set(val);
        wordWorth.set(worth);
        context.write(word, wordWorth);
      }
    }
예제 #8
0
    public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {

      String line = value.toString();
      data.set(Integer.parseInt(line));
      context.write(data, new IntWritable(1));
    }
예제 #9
0
  public static void main(String[] args) throws IOException {
    // TODO Auto-generated method stub
    String uri = args[0];

    Configuration conf = new Configuration();

    Path path = new Path(uri);

    FileSystem fs = FileSystem.get(URI.create(uri), conf);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());

      for (int i = 0; i < 100; i++) {
        key.set(100 - i);
        value.set(DATA[i % DATA.length]);

        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
 public IntWritable readInt(IntWritable iw) throws IOException {
   if (iw == null) {
     iw = new IntWritable();
   }
   iw.set(in.readInt());
   return iw;
 }
예제 #11
0
    public void map(
        LongWritable key,
        Text value,
        OutputCollector<IntWritable, HITSNode> output,
        Reporter reporter)
        throws IOException {

      ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
      String line = ((Text) value).toString();
      StringTokenizer itr = new StringTokenizer(line);
      if (itr.hasMoreTokens()) {
        int curr = Integer.parseInt(itr.nextToken());
        if (stopList.contains(curr)) {
          return;
        }
        valOut.setAdjacencyList(links);
        valOut.setHARank((float) 1.0);
        valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
      }
      while (itr.hasMoreTokens()) {
        keyOut.set(Integer.parseInt(itr.nextToken()));
        valOut.setNodeId(keyOut.get());
        // System.out.println(keyOut.toString() + ", " +
        // valOut.toString());
        if (!(stopList.contains(keyOut.get()))) {
          output.collect(keyOut, valOut);
        }
      }
      // emit mentioned mentioner -> mentioned (mentioners) in links
      // emit mentioner mentioned -> mentioner (mentions) outlinks
      // emit mentioned a
      // emit mentioner 1
    }
    public void map(
        LongWritable key,
        Text t,
        OutputCollector<IntWritable, PageRankNode> output,
        Reporter reporter)
        throws IOException {

      String[] arr = t.toString().trim().split("\\s+");

      nid.set(Integer.parseInt(arr[0]));
      if (arr.length == 1) {
        node.setNodeId(Integer.parseInt(arr[0]));
        node.setAdjacencyList(new ArrayListOfIntsWritable());

      } else {
        node.setNodeId(Integer.parseInt(arr[0]));

        int[] neighbors = new int[arr.length - 1];
        for (int i = 1; i < arr.length; i++) {
          neighbors[i - 1] = Integer.parseInt(arr[i]);
        }

        node.setAdjacencyList(new ArrayListOfIntsWritable(neighbors));
      }

      reporter.incrCounter("graph", "numNodes", 1);
      reporter.incrCounter("graph", "numEdges", arr.length - 1);

      if (arr.length > 1) {
        reporter.incrCounter("graph", "numActiveNodes", 1);
      }

      output.collect(nid, node);
    }
  @Before
  public void setUp() throws Exception {
    // create local Pig server
    pigServer = UnitTestUtil.makePigServer();

    // create temp SequenceFile
    File tempFile = File.createTempFile("test", ".txt");
    tempFilename = tempFile.getAbsolutePath();
    Path path = new Path("file:///" + tempFilename);
    Configuration conf = new Configuration();
    FileSystem fs = path.getFileSystem(conf);
    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
      for (int i = 0; i < DATA.length; ++i) {
        key.set(i);
        value.set(DATA[i]);
        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
예제 #14
0
 public IntWritable evaluate(Text s) {
   if (s == null) {
     return null;
   }
   result.set(s.toString().length());
   return result;
 }
예제 #15
0
 /**
  * @param state The final LanczosState to be serialized
  * @param outputPath The path (relative to the current Configuration's FileSystem) to save the
  *     output to.
  */
 public void serializeOutput(LanczosState state, Path outputPath) throws IOException {
   int numEigenVectors = state.getIterationNumber();
   log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath);
   Configuration conf = getConf() != null ? getConf() : new Configuration();
   FileSystem fs = FileSystem.get(outputPath.toUri(), conf);
   SequenceFile.Writer seqWriter =
       new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class);
   try {
     IntWritable iw = new IntWritable();
     for (int i = 0; i < numEigenVectors; i++) {
       // Persist eigenvectors sorted by eigenvalues in descending order\
       NamedVector v =
           new NamedVector(
               state.getRightSingularVector(numEigenVectors - 1 - i),
               "eigenVector"
                   + i
                   + ", eigenvalue = "
                   + state.getSingularValue(numEigenVectors - 1 - i));
       Writable vw = new VectorWritable(v);
       iw.set(i);
       seqWriter.append(iw, vw);
     }
   } finally {
     Closeables.close(seqWriter, false);
   }
 }
예제 #16
0
 public void cleanup(Context context) throws IOException, InterruptedException { // close
   for (Entry<String, Integer> entry : wordMap.entrySet()) {
     word.set(entry.getKey());
     counter.set(entry.getValue());
     context.write(word, counter);
   }
 }
예제 #17
0
 @Override
 public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
   size.value = SIZE;
   IntWritable writable = new IntWritable();
   writable.set(typeConverter.convertTo(Integer.class, value));
   return writable;
 }
예제 #18
0
 /**
  * Convert from long to an integer. This is called for CAST(... AS INT)
  *
  * @param i The long value to convert
  * @return IntWritable
  */
 public IntWritable evaluate(LongWritable i) {
   if (i == null) {
     return null;
   } else {
     intWritable.set((int) i.get());
     return intWritable;
   }
 }
예제 #19
0
 /**
  * Convert from Timestamp to an integer. This is called for CAST(... AS INT)
  *
  * @param i The Timestamp value to convert
  * @return IntWritable
  */
 public IntWritable evaluate(TimestampWritable i) {
   if (i == null) {
     return null;
   } else {
     intWritable.set(i.getSeconds());
     return intWritable;
   }
 }
예제 #20
0
  public IntWritable evaluate(HiveIntervalYearMonthWritable i) {
    if (i == null) {
      return null;
    }

    result.set(i.getHiveIntervalYearMonth().getMonths());
    return result;
  }
예제 #21
0
 /**
  * Convert from boolean to an integer. This is called for CAST(... AS INT)
  *
  * @param i The boolean value to convert
  * @return IntWritable
  */
 public IntWritable evaluate(BooleanWritable i) {
   if (i == null) {
     return null;
   } else {
     intWritable.set(i.get() ? 1 : 0);
     return intWritable;
   }
 }
예제 #22
0
 /**
  * Convert from short to an integer. This is called for CAST(... AS INT)
  *
  * @param i The short value to convert
  * @return IntWritable
  */
 public IntWritable evaluate(ShortWritable i) {
   if (i == null) {
     return null;
   } else {
     intWritable.set(i.get());
     return intWritable;
   }
 }
예제 #23
0
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
      int sum = 0;
      for (IntWritable value : values) sum += value.get();

      writableSum.set(sum);
      context.write(key, writableSum);
    }
예제 #24
0
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {

      String line[] = value.toString().split("[\\s]");

      word.set(line[0].trim());
      freq.set(-1 * Integer.parseInt(line[1].trim()));
      context.write(freq, word);
    }
예제 #25
0
  public IntWritable evaluate(DateWritable d) {
    if (d == null) {
      return null;
    }

    calendar.setTime(d.get());
    result.set(1 + calendar.get(Calendar.MONTH));
    return result;
  }
예제 #26
0
  public IntWritable evaluate(TimestampWritable t) {
    if (t == null) {
      return null;
    }

    calendar.setTime(t.getTimestamp());
    result.set(1 + calendar.get(Calendar.MONTH));
    return result;
  }
예제 #27
0
    @Override
    protected void reduce(IntWritable key, Iterable<Text> value, Context context)
        throws IOException, InterruptedException {

      for (Text val : value) {
        Freq.set(-1 * (key.get()));
        context.write(val, Freq);
      }
    }
예제 #28
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable value : values) {
     sum += value.get();
   }
   reducedValue.set(sum);
   context.write(key, new IntWritable(sum));
 }
예제 #29
0
 public void reduce(Key key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
 }
예제 #30
0
 public void close() throws IOException {
   for (MapIV.Entry<ArrayListOfIntsWritable> e : adjLists.entrySet()) {
     keyOut.set(e.getKey());
     valOut.setNodeId(e.getKey());
     valOut.setHARank((float) 0.0);
     valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
     valOut.setAdjacencyList(e.getValue());
     mOutput.collect(keyOut, valOut);
   }
 }