@Override public void reduce( Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { String line = ""; String header = ""; TreeMap<String, String> ciudades = new TreeMap<String, String>(); // Obtenemos los datos y los metemos a un treemap para que los ordene por ciudad while (values.hasNext()) { String[] tmp = values.next().toString().split(","); String ciudad = tmp[0]; String mes = tmp[1]; String temperatura = tmp[2]; String fecha = tmp[3]; ciudades.put(ciudad, tmp[1] + "," + tmp[2] + "," + tmp[3]); } // Recorremos las ciudades y vamos imprimiendo for (String ciudad : ciudades.keySet()) { header += ciudad + ",,"; String[] temporal = ciudades.get(ciudad).split(","); line += temporal[2] + "," + temporal[1] + ","; } if (c == 0) { // Imprimimos cabezera output.collect(new Text("Año,"), new Text(header)); c++; } output.collect(new Text(key.toString() + ","), new Text(line)); }
void search( Vector<Star> v1, Vector<Star> v2, BlockIDWritable key, OutputCollector<BlockIDWritable, PairWritable> output) throws IOException { for (int i = 0; i < v1.size(); i++) { for (int j = 0; j < v2.size(); j++) { Star star1 = v1.get(i); Star star2 = v2.get(j); // what is this margin about if (star1.margin && star2.margin) continue; double dist = star1.x * star2.x + star1.y * star2.y + star1.z * star2.z; if (dist > costheta) { p.set(star1, star2, dist); output.collect(key, p); p.set(star2, star1, dist); output.collect(key, p); // num += 2; } } } // end for i,j }
public void run() { // TODO Auto-generated method stub run = true; System.out.println("Running a map"); while (run) { // Figure out how to read input file // Evaluate it Map<String, List<String>> input = reader.getKeyValuePairs(); if (input == null) { map.setStatus(-1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); return; } OutputCollector<String, String> collect = new OutputCollector<String, String>(); Set<String> keySet = input.keySet(); for (String key : keySet) { List<String> values = input.get(key); for (String value : values) { map.getJob().map(key, value, collect); } } List<Pair> results = collect.getResults(); int length = results.size(); FileOutputStream out; try { System.out.println("Trying to Write to File"); out = new FileOutputStream(new File(map.getOutput().get(0))); BufferedWriter dw = new BufferedWriter(new OutputStreamWriter(out)); for (Pair p : results) { dw.append(p.toString()); dw.newLine(); } dw.newLine(); dw.flush(); dw.close(); out.close(); coord.dataNode.addFileToDFS(map.getOutput().get(0), coord.conn.port, false); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); map.setStatus(-1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); return; } map.setStatus(1); Message msg = new Message(); msg.setTask(map); msg.setType('f'); coord.conn.sendMessage(msg); break; } }
@SuppressWarnings("unchecked") public void reduce( Text key, Iterator<Text> values, OutputCollector<NullWritable, Text> output, Reporter reporter) throws IOException { OutputCollector collector = multipleOutputs.getCollector("station", key.toString().replace("-", ""), reporter); while (values.hasNext()) { collector.collect(NullWritable.get(), values.next()); } }
public void map( LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { String line = value.toString(); if (validate(line, reporter)) output.collect(key, value); }
// specify input and out keys public void map( LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { String line = value.toString(); // define new variable to be string ArrayList<Integer> range = new ArrayList<Integer>(); for (int i = 2000; i <= 2010; i++) { range.add(i); } // String[] inputs = line.split(",(?=([^\"]*\"[^\"]*\")*[^\"]*$)"); String[] inputs = line.split(","); try { int year = Integer.parseInt(inputs[165]); if (range.contains(year)) { String dur = inputs[3]; String artist_name = inputs[2]; String song_title = inputs[1]; String final_input = artist_name + ',' + dur + ',' + song_title; Final_Value.set(final_input); output.collect(Final_Value, dummy); } } catch (NumberFormatException e) { // do nothing } }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter) throws IOException { /* * It implements the mapper. It outputs the numbers of weight and updated weights. * * Note that the format of intermediate output is <IntWritable, DoubleWritable>, * because the key is the number of weight (an integer), and the value is the weight's value (double) */ inputData = value.toString(); // go through the process initialize(); getposphase(); getnegphase(); update(); // output the intermediate data // The <key, value> pairs are <weightID, weightUpdate> double[][] vishidinc_array = vishidinc.getArray(); for (int i = 0; i < numdims; i++) { for (int j = 0; j < numhid; j++) { weightPos.set(i * numhid + j); weightValue.set(vishidinc_array[i][j]); output.collect(weightPos, weightValue); } } }
public void map( LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { parser.parse(value); output.collect(new Text(parser.getStationId()), value); }
protected void emit(K key, V val, OutputCollector<K, V> out) throws IOException { ++total; while ((float) kept / total < keep) { ++kept; out.collect(key, val); } }
public void reduce( DoubleWritable key, Iterator<DoubleWritable> values, OutputCollector<DoubleWritable, Text> output, Reporter reporter) throws IOException { output.collect(key, new Text(values.next().toString() + " - ")); }
/** * Takes in (id, node) pairs and emits them right back out * * @param key The Node ID * @param Node The Node object * @param output An Output Collector that collects (id, node) pairs * @param reporter Default reporter object */ public void map( IntWritable key, WikiPage value, OutputCollector<IntWritable, WikiPage> output, Reporter reporter) throws IOException { output.collect(key, value); }
/** * @param flowProcess * @param sinkCall * @throws IOException */ @Override public void sink( FlowProcess<JobConf> flowProcess, SinkCall<BSONWritable[], OutputCollector> sinkCall) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); OutputCollector outputCollector = sinkCall.getOutput(); String keyFieldName = this.fieldMappings.get(this.keyColumnName); Object key; // if fieldMappings doesn't have keyColumnName ("_id") field, then use new ObjectId() as key if (keyFieldName == null) { key = new ObjectId(); } else { key = tupleEntry.selectTuple(new Fields(keyFieldName)).get(0); } // Object key = tupleEntry.selectTuple(new // Fields(this.fieldMappings.get(this.keyColumnName))).get(0); BasicDBObject dbObject = new BasicDBObject(); for (String columnFieldName : columnFieldNames) { String columnFieldMapping = fieldMappings.get(columnFieldName); Object tupleEntryValue = null; try { if (columnFieldMapping != null) { // columnFieldMapping is null if no corresponding field name defined in Mappings. // only write the field value back to mongo if the field also defined in Mappings (ie. not // null) tupleEntryValue = tupleEntry.get(columnFieldMapping); } } catch (FieldsResolverException e) { logger.error("Couldn't resolve field: {}", columnFieldName); } if (tupleEntryValue != null && columnFieldName != keyColumnName) { // logger.info("Putting for output: {} {}", columnFieldName, tupleEntryValue); dbObject.put(columnFieldName, tupleEntryValue); } } logger.info("Putting key for output: {} {}", key, dbObject); // outputCollector.collect(new ObjectId(), dbObject); outputCollector.collect(key, dbObject); }
public void reduce( IntPair key, Iterator<NullWritable> values, OutputCollector<IntPair, NullWritable> output, Reporter reporter) throws IOException { /*[*/ output.collect(key, NullWritable.get()); /*]*/ }
public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException { String line = value.toString(); StringTokenizer tokenizer = new StringTokenizer(line); while (tokenizer.hasMoreTokens()) { word.set(tokenizer.nextToken()); output.collect(word, one); } }
@Override public void reduce( IntWritable key, Iterator<DoubleWritable> values, OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter) throws IOException { output.collect(key, values.next()); }
public void reduce( Text key, Iterator<CrawlDatum> values, OutputCollector<Text, CrawlDatum> output, Reporter reporter) throws IOException { boolean oldSet = false; boolean injectedSet = false; while (values.hasNext()) { CrawlDatum val = values.next(); if (val.getStatus() == CrawlDatum.STATUS_INJECTED) { injected.set(val); injected.setStatus(CrawlDatum.STATUS_DB_UNFETCHED); injectedSet = true; } else { old.set(val); oldSet = true; } } CrawlDatum res = null; /** * Whether to overwrite, ignore or update existing records * * @see https://issues.apache.org/jira/browse/NUTCH-1405 */ // Injected record already exists and overwrite but not update if (injectedSet && oldSet && overwrite) { res = injected; if (update) { LOG.info(key.toString() + " overwritten with injected record but update was specified."); } } // Injected record already exists and update but not overwrite if (injectedSet && oldSet && update && !overwrite) { res = old; old.putAllMetaData(injected); old.setScore(injected.getScore() != scoreInjected ? injected.getScore() : old.getScore()); old.setFetchInterval( injected.getFetchInterval() != interval ? injected.getFetchInterval() : old.getFetchInterval()); } // Old default behaviour if (injectedSet && !oldSet) { res = injected; } else { res = old; } output.collect(key, res); }
@Override public void reduce( Text key, Iterator<Text> iterator, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { long sum = 0; while (iterator.hasNext()) { sum = sum + Long.parseLong(iterator.next().toString()); } output.collect(key, new Text(String.valueOf(sum))); }
@Override public void map( LongWritable longWritable, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { String ip = value.toString(); output.collect(new Text(ip), new Text("1")); }
public void map(Object key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { String[] _allCols = StringUtils.splitPreserveAllTokens(value.toString(), splitChar); StringBuffer stringBuffer = new StringBuffer(); for (int i = 0; i < _allCols.length; i++) { stringBuffer.append(i).append('=').append(_allCols[i]).append("\t"); } output.collect(new Text(_allCols[0]), new Text(stringBuffer.toString())); }
public void map( BytesWritable key, BytesWritable value, OutputCollector<BytesWritable, IntWritable> output, Reporter reporter) throws IOException { // newKey = (key, value) BytesWritable keyValue = new BytesWritable(pair(key, value)); // output (newKey, value) output.collect(keyValue, this.value); }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, Text> output, Reporter reporter) throws IOException { String arr[] = value.toString().split("\\r?\\n"); for (String row : arr) { if (row.startsWith("\"")) { continue; } String parts[] = row.split(","); output.collect(new IntWritable(new Integer(parts[1])), new Text(parts[4])); } }
public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { int sum = 0; while (values.hasNext()) { sum += values.next().get(); } output.collect(key, new IntWritable(sum)); }
public void map( LongWritable key, Text value, OutputCollector<DoubleWritable, DoubleWritable> output, Reporter reporter) throws IOException { String line = value.toString(); DoubleWritable clave = new DoubleWritable(); DoubleWritable valor = new DoubleWritable(); clave.set(Double.parseDouble(line)); valor.set(Math.sqrt(Double.parseDouble(line))); output.collect(clave, valor); }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, IntWritable> output, Reporter reporter) throws IOException { String line = value.toString(); IntWritable clave = new IntWritable(); IntWritable valor = new IntWritable(); clave.set(Integer.parseInt(line)); valor.set(Integer.parseInt(line) + 1); output.collect(clave, valor); }
// reduce method accepts the Key Value pairs from mappers, do the aggregation based on keys and // produce the final out put public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { int sum = 0; /*iterates through all the values available with a key and add them together and give the final result as the key and sum of its values*/ while (values.hasNext()) { sum += values.next().get(); } output.collect(key, new IntWritable(sum)); }
void collectStats(OutputCollector<Text, Text> output, String name, long execTime, Long objSize) throws IOException { long totalSize = objSize.longValue(); float ioRateMbSec = (float) totalSize * 1000 / (execTime * MEGA); LOG.info("Number of bytes processed = " + totalSize); LOG.info("Exec time = " + execTime); LOG.info("IO rate = " + ioRateMbSec); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "tasks"), new Text(String.valueOf(1))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "size"), new Text(String.valueOf(totalSize))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "time"), new Text(String.valueOf(execTime))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "rate"), new Text(String.valueOf(ioRateMbSec * 1000))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "sqrate"), new Text(String.valueOf(ioRateMbSec * ioRateMbSec * 1000))); }
public void reduce( Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { ColumnFamily columnFamily; String keyspace = "Keyspace1"; String cfName = "Super1"; Message message; List<ColumnFamily> columnFamilies; columnFamilies = new LinkedList<ColumnFamily>(); String line; /* Create a column family */ columnFamily = ColumnFamily.create(keyspace, cfName); while (values.hasNext()) { // Split the value (line based on your own delimiter) line = values.next().toString(); String[] fields = line.split("\1"); String SuperColumnName = fields[1]; String ColumnName = fields[2]; String ColumnValue = fields[3]; int timestamp = 0; columnFamily.addColumn( new QueryPath( cfName, ByteBufferUtil.bytes(SuperColumnName), ByteBufferUtil.bytes(ColumnName)), ByteBufferUtil.bytes(ColumnValue), timestamp); } columnFamilies.add(columnFamily); /* Get serialized message to send to cluster */ message = createMessage(keyspace, key.getBytes(), cfName, columnFamilies); List<IAsyncResult> results = new ArrayList<IAsyncResult>(); for (InetAddress endpoint : StorageService.instance.getNaturalEndpoints(keyspace, ByteBufferUtil.bytes(key))) { /* Send message to end point */ results.add(MessagingService.instance().sendRR(message, endpoint)); } /* wait for acks */ for (IAsyncResult result : results) { try { result.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { // you should probably add retry logic here throw new RuntimeException(e); } } output.collect(key, new Text(" inserted into Cassandra node(s)")); }
public void map( LongWritable key, Text value, OutputCollector<IntPair, NullWritable> output, Reporter reporter) throws IOException { parser.parse(value); if (parser.isValidTemperature()) { /*[*/ output.collect( new IntPair(parser.getYearInt(), +parser.getAirTemperature()), NullWritable.get()); /*]*/ } }
@Override public void reduce( Text key, /*[*/ Iterator /*]*/<IntWritable> values, /*[*/ OutputCollector<Text, IntWritable> output, Reporter reporter /*]*/) throws IOException { int maxValue = Integer.MIN_VALUE; while ( /*[*/ values.hasNext() /*]*/) { maxValue = Math.max(maxValue, /*[*/ values.next().get() /*]*/); } /*[*/ output.collect /*]*/(key, new IntWritable(maxValue)); }
/** * Prints the state information if the TLC runs in print-diff-only mode and the last state is set, * it will print the diff only */ public static void printState(TLCStateInfo currentStateInfo, TLCState lastState, int num) { String stateString; /* Added by rjoshi. */ if (lastState != null && TLCGlobals.printDiffsOnly) { stateString = currentStateInfo.state.toString(lastState); } else { stateString = currentStateInfo.state.toString(); } MP.printState( EC.TLC_STATE_PRINT2, new String[] {String.valueOf(num), currentStateInfo.info.toString(), stateString}, currentStateInfo, num); OutputCollector.addStateToTrace(currentStateInfo); }