public void execute(TridentTuple tuple, TridentCollector collector) {
Map<String, Object> event = new HashMap<String, Object>();
if (tuple.get(0) instanceof byte[]) {
byte[] messageBytes = (byte[]) tuple.get(0);
String messageString = new String(messageBytes);
System.out.println("flight delay tweet: " + messageString);
try {
JSONObject json = new JSONObject(messageString);
System.out.println("Created json object");
Iterator keyIt = json.keys();
while (keyIt.hasNext()) {
String key = (String) keyIt.next();
if (!tweetKeyWhiteList.contains(key)) {
continue;
}
Object value = json.get(key);
event.put(key, value);
}
} catch (Exception e) {
System.out.println("Error parsing json from tweet object");
log.error(e);
}
try {
String id = event.get("user_id").toString();
double lat = Double.parseDouble(event.get("lat").toString());
double lon = Double.parseDouble(event.get("lon").toString());
if (lat > 0 || lat < 0) {
MessageDigest md = MessageDigest.getInstance("SHA-256");
md.update(id.getBytes());
byte byteData[] = md.digest();
// convert the byte to hex format method 1
StringBuilder sb = new StringBuilder();
for (int i = 0; i < byteData.length; i++) {
sb.append(Integer.toString((byteData[i] & 0xff) + 0x100, 16).substring(1));
}
id = sb.toString();
GroupByField idField = new GroupByField("TWEET_USER_ID", id);
GroupByField objectType = new GroupByField("OBJECT_TYPE", "TWITTER_USER");
event.put("TWEET_USER_ID", id);
event.put("user_id", id);
collector.emit(new Values(idField, objectType, event));
}
} catch (Exception e) {
log.error("Error parsing tweet info", e);
}
} else {
System.out.println("object at tuple index 0: " + tuple.get(0).getClass().getName());
}
}
private TridentTuple createMockedInstanceTuple(double[] features) {
Instance<Boolean> instance = new Instance<Boolean>(features);
TridentTuple tuple = mock(TridentTuple.class);
given(tuple.get(0)).willReturn(instance);
return tuple;
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
String[] lines = tuple.getString(0).split(",");
if (3 == lines.length) {
collector.emit(new Values(Integer.valueOf(lines[0]), lines[1], lines[2]));
} else {
LOG.error("##### Goods format error message:" + tuple.getString(0));
}
}
public void execute(TridentTuple tuple, TridentCollector collector) {
GenericRecord docEntry = new GenericData.Record(schema);
docEntry.put("docid", tuple.getStringByField("documentId"));
docEntry.put("time", Time.currentTimeMillis());
docEntry.put("line", tuple.getStringByField("document"));
try {
dataFileWriter.append(docEntry);
dataFileWriter.flush();
} catch (IOException e) {
LOG.error("Error writing to document record: " + e);
throw new RuntimeException(e);
}
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
try {
String assetPhysical = tuple.getStringByField("asset_physical");
String beginTime = tuple.getStringByField("begin_time");
String endTime = tuple.getStringByField("end_time");
String clientAddress = tuple.getStringByField("client_address"); // 获取对端IP
String filePath = tuple.getStringByField("file_path");
String fileSize = tuple.getStringByField("file_size"); // 获取文件大小,单位为byte
String ftpAccount = tuple.getStringByField("ftp_account");
Long operateType = tuple.getLongByField("operate_type");
String serverName = tuple.getStringByField("server_name");
String version = tuple.getStringByField("version");
// 由于业务变更，改成属性映射获取业务类型数据与属性字典(暂时只包括businessType和Area编码映射)
Map<String, Object> dictionaryMap = DictionaryMappingUtil.transfer(filePath);
// 获取业务类型编码
String businessType =
dictionaryMap.get(DictionaryMappingUtil.BUSINESSTYPE_KEY).toString(); // "A0001";
LOG.info("**businessType==>" + businessType);
// 获取区域编码
Long area =
Long.parseLong(dictionaryMap.get(DictionaryMappingUtil.PROVINCECODE_KEY).toString());
LOG.info("**area==>" + area);
// 资产名从物理文件名中获取
String assetEnName = DictionaryMappingUtil.getName(assetPhysical, businessType);
LOG.info("**assetEnName==>" + assetEnName);
MonitorDTO monitor = new MonitorDTO();
monitor.setBusiness_type(businessType);
monitor.setAsset_en_name(assetEnName);
monitor.setAsset_physical(assetPhysical);
monitor.setArea(area);
monitor.setOperate_type(operateType);
monitor.setBegin_time(beginTime);
monitor.setEnd_time(endTime);
monitor.setVersion(version);
monitor.setServer_name(serverName);
monitor.setFtp_account(ftpAccount);
monitor.setFile_path(filePath);
monitor.setFile_size(new Long(fileSize));
monitor.setClient_address(clientAddress);
Values values = new Values();
values.add(monitor);
collector.emit(values);
} catch (Exception e) {
e.printStackTrace();
}
}
/*
* Takes the first tuple of the batch and checks the timestamp t0.
* It stores tuples into a state while the timestamp is within [t0 AND t0+timeWindowInSeconds]
* When the timestamp tn of a tuple does not match this condition, THEN t0 = tn, and the previous state is overwritten with new tuples.
* (non-Javadoc)
* @see storm.trident.state.StateUpdater#updateState(storm.trident.state.State, java.util.List, storm.trident.operation.TridentCollector)
*/
@Override
public void updateState(
MemoryMapState<Object> state, List<TridentTuple> tuples, TridentCollector collector) {
List<Object> ids = new ArrayList<Object>();
List<Object> earthquakeGraphs = new ArrayList<Object>();
for (TridentTuple tuple : tuples) {
ids.add((String) tuple.get(0) + (String) tuple.get(1));
earthquakeGraphs.add(tuple.get(2));
}
state.multiPut(Arrays.asList(ids), earthquakeGraphs);
// Tuples emitted to the collector go through the newValuesStream call
// collector.emit(state.multiGet(Arrays.asList(sensorIds)));
}
@Override
public boolean isKeep(TridentTuple tuple) {
// System.out.println(tuple);
_writer.println(tuple.getValues());
_writer.flush();
return true;
}
public void aggregate(Result val, TridentTuple tuple, TridentCollector collector) {
Object v = tuple.getValue(0);
if (val.obj == null) {
val.obj = v;
} else {
val.obj = _agg.combine(val.obj, v);
}
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
Long afmTime = tuple.getLongByField("afmTime");
Long fsmTime = tuple.getLongByField("fsmTime");
if (afmTime.equals(System.currentTimeMillis())) LOG.info("开始执行接入文件监控!");
if (fsmTime.equals(System.currentTimeMillis())) LOG.info("开始执行接入服务器监控!");
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
for (String word : tuple.getString(0).split("[ ;,:\\?\\-\\\"\\!\\.\\r()]+")) {
if (word.length() > size) {
collector.emit(new Values(word.toLowerCase()));
}
}
}
@Override
public boolean isKeep(TridentTuple tuple) {
GameState gameState = (GameState) tuple.get(0);
boolean keep = (gameState.getBoard().isEndState());
if (keep) {
LOG.info("END GAME [" + gameState + "]");
}
return keep;
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
List l = (List) tuple.getValue(0);
if (l != null) {
for (Object o : l) {
collector.emit(new Values(o));
}
}
}
@Override
public List<Map.Entry<Double, double[]>> batchRetrieve(
final MlStormWekaState clustererState, final List<TridentTuple> queryTuples) {
ArrayList<Map.Entry<Double, double[]>> queryResults =
new ArrayList<Map.Entry<Double, double[]>>();
for (TridentTuple query : queryTuples) {
//noinspection unchecked
final Map<Integer, Map.Entry<Integer, double[]>> voteMap =
(Map<Integer, Map.Entry<Integer, double[]>>) query.getValueByField("voteMap");
final double[] fv = new double[numPartitions];
for (Integer key : voteMap.keySet()) {
fv[key] = voteMap.get(key).getKey();
}
try {
Instance testInstance = MlStormFeatureVectorUtils.buildWekaInstance(fv);
double[] distribution = null;
double result = clustererState.predict(testInstance);
queryResults.add(new KeyValuePair<Double, double[]>(result, distribution));
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (Exception e) {
if (e.toString().contains(MlStormWekaState.NOT_READY_TO_PREDICT)) {
System.err.println(
MessageFormat.format(
"Not Ready yet! Continue training with - {0}", Arrays.toString(fv)));
queryResults.add(new KeyValuePair<Double, double[]>(fv[fv.length - 1], null));
} else {
throw new IllegalStateException(e);
}
}
}
return queryResults;
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
// TODO Auto-generated method stub
long flag = tuple.getLong(0);
long time = tuple.getLong(1);
long vid = tuple.getLong(2);
long qid = tuple.getLong(3);
int spd = tuple.getInteger(4);
int xway = tuple.getInteger(5);
int lane = tuple.getInteger(6);
int dir = tuple.getInteger(7);
int seg = tuple.getInteger(8);
int pos = tuple.getInteger(9);
// int part_id = tuple.getInteger(10);
int segbegin = tuple.getInteger(11);
;
int segend = tuple.getInteger(12);
int day = tuple.getInteger(13);
// long tod = tuple.getLong(14);
long tod = time / 60;
// StateFactory mapState = new MemoryMapState.Factory();
// tollsPerVehicleState.newValuesStream()
// .each(new Fields("vid"), new FilterGetVidBolt(vid))
// .project(new Fields("toll"))
// .persistentAggregate(mapState, new Fields("toll"), new Sum(), new
// Fields("tollSum"));
Long tolls = tollsPerVehicleState.sumTolls(new ArrayList<Long>(Arrays.asList(vid)));
Long updatedTime = timestampState.getMaxTS();
// hack to return a string back through a VoltTable
String o =
"2,"
+ time
+ ","
+ LinearRoadConstants.EMIT_TIME_STR
+ ","
+ updatedTime
+ ","
+ qid
+ ","
+ tolls;
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
String pvString = tuple.getString(0);
PageView pageView = PageView.fromString(pvString);
if (null == pageView) {
LOG.error("invalid pageview string '" + pvString + "'");
return;
}
List<Object> values = new ArrayList<Object>();
for (Item field : fields) {
values.add(pageView.getValue(field));
}
collector.emit(values);
}
private double[] getFeatureVectorFromArgs(TridentTuple queryTuple) {
String args =
queryTuple.getStringByField(EnsembleLearnerTopologyBuilder.drpcQueryArgsField.get(0));
try {
return MlStormFeatureVectorUtils.deserializeToFeatureVector(args);
} catch (DecoderException e) {
e.printStackTrace();
throw new RuntimeException(e);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
} catch (ClassNotFoundException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void execute(TridentTuple tuple, TridentCollector collector) {
List<KeyValue> keyValueList = (List<KeyValue>) tuple.getValue(0);
for (KeyValue keyValue : keyValueList) {
Values values = new Values();
if (this.rowKey) {
if (this.rowKeyDeserializer != null) {
values.add(this.rowKeyDeserializer.deserialize(keyValue.key()));
} else {
values.add(new String(keyValue.key()));
}
}
if (this.family) {
if (this.columnFamilyDeserializer != null) {
values.add(this.columnFamilyDeserializer.deserialize(keyValue.family()));
} else {
values.add(new String(keyValue.family()));
}
}
if (this.qualifier) {
if (this.columnQualifierDeserializer != null) {
values.add(this.columnFamilyDeserializer.deserialize(keyValue.qualifier()));
} else {
values.add(new String(keyValue.qualifier()));
}
}
if (this.value) {
if (this.valueDeserializer != null) {
values.add(this.valueDeserializer.deserialize(keyValue.value()));
} else {
values.add(new String(keyValue.value()));
}
}
if (this.timestamp) {
values.add(keyValue.timestamp());
}
collector.emit(values);
}
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
GameState gameState = (GameState) tuple.get(0);
Board currentBoard = gameState.getBoard();
List<Board> history = new ArrayList<Board>();
history.addAll(gameState.getHistory());
history.add(currentBoard);
if (!currentBoard.isEndState()) {
String nextPlayer = Player.next(gameState.getPlayer());
List<Board> boards = gameState.getBoard().nextBoards(nextPlayer);
Log.debug(
"Generated [" + boards.size() + "] children boards for [" + gameState.toString() + "]");
for (Board b : boards) {
GameState newGameState = new GameState(b, history, nextPlayer);
List<Object> values = new ArrayList<Object>();
values.add(newGameState);
collector.emit(values);
}
} else {
Log.debug("End game found! [" + currentBoard + "]");
}
}
@Override
public boolean isKeep(TridentTuple tuple) {
System.out.println("DEBUG: " + tuple.toString());
return true;
}
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
System.out.println("####### tuple = " + tuple.getValues());
collector.emit(tuple.getValues());
}
@Override
public boolean isKeep(TridentTuple tuple) {
int ageBand =
Integer.parseInt(tuple.getStringByField(VehicleFields.AGE_BAND_OF_DRIVER.fieldName));
return ageBand <= 5;
}
