@Override public int compareTo(KeyPair otherPair) { // TODO Auto-generated method stub int c = ArivalDelay.compareTo(otherPair.ArivalDelay); if (c != 0) return c; else return Flightnumber.compareTo(otherPair.Flightnumber); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; boolean filtered = true; String keypair = key.toString(); String[] keys = keypair.split(" "); if (keys.length == 1) { filtered = false; } else { if (keys[0].equals("*") || keys[1].equals("*")) { filtered = false; } } if (!filtered) { for (IntWritable val : values) { sum += val.get(); } context.write(key, new IntWritable(sum)); return; } for (IntWritable val : values) { if (val.get() == -1) { filtered = false; continue; } sum += val.get(); } // filter non-needed events if (filtered) return; context.write(key, new IntWritable(sum)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } countMap.put(new Text(key), new IntWritable(sum)); }
@Override public int compareTo(PairIntWritable tp) { int cmp = first.compareTo(tp.first); if (cmp != 0) { return cmp; } return second.compareTo(tp.second); }
@Override public boolean equals(Object o) { if (o instanceof PairIntWritable) { PairIntWritable tp = (PairIntWritable) o; return first.equals(tp.first) && second.equals(tp.second); } return false; }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum = sum + val.get(); } System.out.println("----------inside-----------"); context.write(key, new Text(Integer.toString(sum))); }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, IntWritable> output, Reporter reporter) throws IOException { String line = value.toString(); IntWritable clave = new IntWritable(); IntWritable valor = new IntWritable(); clave.set(Integer.parseInt(line)); valor.set(Integer.parseInt(line) + 1); output.collect(clave, valor); }
/** * Reduce the key,value pair into groups. * * @param tKey key to group * @param itrValues values of each key * @param cContext context of Job * @throws IOException */ @Override public void reduce(Text tKey, Iterable<IntWritable> itrValues, Context cContext) throws IOException { int iSum = 0; for (IntWritable value : itrValues) { iSum += value.get(); } try { cContext.write(tKey, new IntWritable(iSum)); } catch (InterruptedException ex) { Logger.getLogger(Reduce.class.getName()).log(Level.SEVERE, null, ex); } }
@Override public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; int count = 0; int delay = 0; for (IntWritable val : values) { delay = val.get(); sum += delay; count++; } float mean = (float) sum / count; context.write(key, new FloatWritable(mean)); }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter) throws IOException { /* * It implements the mapper. It outputs the numbers of weight and updated weights. * * Note that the format of intermediate output is <IntWritable, DoubleWritable>, * because the key is the number of weight (an integer), and the value is the weight's value (double) */ inputData = value.toString(); // go through the process initialize(); getposphase(); getnegphase(); update(); // output the intermediate data // The <key, value> pairs are <weightID, weightUpdate> double[][] vishidinc_array = vishidinc.getArray(); for (int i = 0; i < numdims; i++) { for (int j = 0; j < numhid; j++) { weightPos.set(i * numhid + j); weightValue.set(vishidinc_array[i][j]); output.collect(weightPos, weightValue); } } }
/** * Emit all the values for the final solved values table. We want to print this out with the * expanded string representation of the game board (to make the output file easily human * readable and thus easier to debug). */ @Override public void reduce(IntWritable key, Iterable<ByteWritable> values, Context context) throws IOException, InterruptedException { int keyInt = key.get(); if (keyInt == 0) return; String hash = "'" + Proj2Util.gameUnhasher(keyInt, boardWidth, boardHeight) + "'"; for (ByteWritable val : values) context.write(new Text(hash), val); }
public BucketCache(Configuration conf) throws IOException { bucketCache = new HashMap<IntWritable, Bucket>(); for (String cachePath : PathUtils.getCachePaths(conf)) { String bucketCachePath = cachePath + BUCKET_CACHE_FOLDER; MapFile.Reader reader = new MapFile.Reader(new Path(bucketCachePath), conf); IntWritable key = new IntWritable(); Bucket value = new Bucket(); while (reader.next(key, value)) { bucketCache.put(new IntWritable(key.get()), new Bucket(value)); } } for (IntWritable i : bucketCache.keySet()) { System.out.println("Loaded bucket from cache:" + i.get() + ":" + bucketCache.get(i)); } }
public void testFormat() throws Exception { JobConf job = new JobConf(conf); FileSystem fs = FileSystem.getLocal(conf); Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred"); Path file = new Path(dir, "test.seq"); Reporter reporter = Reporter.NULL; int seed = new Random().nextInt(); // LOG.info("seed = "+seed); Random random = new Random(seed); fs.delete(dir, true); FileInputFormat.setInputPaths(job, dir); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { // LOG.info("creating; entries = " + length); // create a file with length entries SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class); try { for (int i = 0; i < length; i++) { IntWritable key = new IntWritable(i); byte[] data = new byte[random.nextInt(10)]; random.nextBytes(data); BytesWritable value = new BytesWritable(data); writer.append(key, value); } } finally { writer.close(); } // try splitting the file in a variety of sizes InputFormat<IntWritable, BytesWritable> format = new SequenceFileInputFormat<IntWritable, BytesWritable>(); IntWritable key = new IntWritable(); BytesWritable value = new BytesWritable(); for (int i = 0; i < 3; i++) { int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1; // LOG.info("splitting: requesting = " + numSplits); InputSplit[] splits = format.getSplits(job, numSplits); // LOG.info("splitting: got = " + splits.length); // check each split BitSet bits = new BitSet(length); for (int j = 0; j < splits.length; j++) { RecordReader<IntWritable, BytesWritable> reader = format.getRecordReader(splits[j], job, reporter); try { int count = 0; while (reader.next(key, value)) { // if (bits.get(key.get())) { // LOG.info("splits["+j+"]="+splits[j]+" : " + // key.get()); // LOG.info("@"+reader.getPos()); // } assertFalse("Key in multiple partitions.", bits.get(key.get())); bits.set(key.get()); count++; } // LOG.info("splits["+j+"]="+splits[j]+" count=" + // count); } finally { reader.close(); } } assertEquals("Some keys in no partition.", length, bits.cardinality()); } } }
@Override public void readFields(DataInput in) throws IOException { // TODO Auto-generated method stub ArivalDelay.readFields(in); Flightnumber.readFields(in); }
public void testBinary() throws IOException, InterruptedException { Configuration conf = new Configuration(); Job job = new Job(conf); Path outdir = new Path(System.getProperty("test.build.data", "/tmp"), "outseq"); Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); FileOutputFormat.setOutputPath(job, outdir); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class); SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class); SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); BytesWritable bkey = new BytesWritable(); BytesWritable bval = new BytesWritable(); TaskAttemptContext context = MapReduceTestUtil.createDummyMapTaskAttemptContext(job.getConfiguration()); OutputFormat<BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat(); OutputCommitter committer = outputFormat.getOutputCommitter(context); committer.setupJob(job); RecordWriter<BytesWritable, BytesWritable> writer = outputFormat.getRecordWriter(context); IntWritable iwritable = new IntWritable(); DoubleWritable dwritable = new DoubleWritable(); DataOutputBuffer outbuf = new DataOutputBuffer(); LOG.info("Creating data by SequenceFileAsBinaryOutputFormat"); try { for (int i = 0; i < RECORDS; ++i) { iwritable = new IntWritable(r.nextInt()); iwritable.write(outbuf); bkey.set(outbuf.getData(), 0, outbuf.getLength()); outbuf.reset(); dwritable = new DoubleWritable(r.nextDouble()); dwritable.write(outbuf); bval.set(outbuf.getData(), 0, outbuf.getLength()); outbuf.reset(); writer.write(bkey, bval); } } finally { writer.close(context); } committer.commitTask(context); committer.commitJob(job); InputFormat<IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat<IntWritable, DoubleWritable>(); int count = 0; r.setSeed(seed); SequenceFileInputFormat.setInputPaths(job, outdir); LOG.info("Reading data by SequenceFileInputFormat"); for (InputSplit split : iformat.getSplits(job)) { RecordReader<IntWritable, DoubleWritable> reader = iformat.createRecordReader(split, context); MapContext<IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext = new MapContextImpl<IntWritable, DoubleWritable, BytesWritable, BytesWritable>( job.getConfiguration(), context.getTaskAttemptID(), reader, null, null, MapReduceTestUtil.createDummyReporter(), split); reader.initialize(split, mcontext); try { int sourceInt; double sourceDouble; while (reader.nextKeyValue()) { sourceInt = r.nextInt(); sourceDouble = r.nextDouble(); iwritable = reader.getCurrentKey(); dwritable = reader.getCurrentValue(); assertEquals( "Keys don't match: " + "*" + iwritable.get() + ":" + sourceInt + "*", sourceInt, iwritable.get()); assertTrue( "Vals don't match: " + "*" + dwritable.get() + ":" + sourceDouble + "*", Double.compare(dwritable.get(), sourceDouble) == 0); ++count; } } finally { reader.close(); } } assertEquals("Some records not found", RECORDS, count); }
@Override protected void map(Centroid key, IntWritable value, Context context) throws IOException, InterruptedException { context.write(new Text(key.toString()), new Text(value.toString())); }
@Override public void write(DataOutput out) throws IOException { // TODO Auto-generated method stub ArivalDelay.write(out); Flightnumber.write(out); }
@Override public String toString() { return first.toString() + ":" + second.toString(); }
@Override public int hashCode() { return first.hashCode() * 163 + second.hashCode(); }
@Override public void readFields(DataInput in) throws IOException { first.readFields(in); second.readFields(in); }
@Override public void write(DataOutput out) throws IOException { first.write(out); second.write(out); }