private void sync() {
// note that sometimes the tuples active may be less than max_spout_pending, e.g.
// max_spout_pending = 3
// tx 1, 2, 3 active, tx 2 is acked. there won't be a commit for tx 2 (because tx 1 isn't
// committed yet),
// and there won't be a batch for tx 4 because there's max_spout_pending tx active
TransactionStatus maybeCommit = _activeTx.get(_currTransaction);
if (maybeCommit != null && maybeCommit.status == AttemptStatus.PROCESSED) {
maybeCommit.status = AttemptStatus.COMMITTING;
_collector.emit(
TRANSACTION_COMMIT_STREAM_ID, new Values(maybeCommit.attempt), maybeCommit.attempt);
}
try {
if (_activeTx.size() < _maxTransactionActive) {
BigInteger curr = _currTransaction;
for (int i = 0; i < _maxTransactionActive; i++) {
if ((_coordinatorState.hasCache(curr) || _coordinator.isReady())
&& !_activeTx.containsKey(curr)) {
TransactionAttempt attempt = new TransactionAttempt(curr, _rand.nextLong());
Object state = _coordinatorState.getState(curr, _initializer);
_activeTx.put(curr, new TransactionStatus(attempt));
_collector.emit(
TRANSACTION_BATCH_STREAM_ID,
new Values(attempt, state, previousTransactionId(_currTransaction)),
attempt);
}
curr = nextTransactionId(curr);
}
}
} catch (FailedException e) {
LOG.warn("Failed to get metadata for a transaction", e);
}
}
@Override
public void nextTuple() {
if (delay > 0) {
if (id % sleepFreq == 0) {
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
if (id % latFreq == nextlat) {
time = System.currentTimeMillis();
if (--nextlat <= 0) {
nextlat = 1000;
}
}
ArrayList<String> hosts = new ArrayList<String>();
hosts.add(host);
if (withFt) {
spoutOutputCollector.emit(
new Values(this.id, this.host, this.time, this.payload, hosts), this.id);
} else {
spoutOutputCollector.emit(new Values(this.id, this.host, this.time, this.payload, hosts));
}
time = 0;
this.id++;
}
public void nextTuple() {
collector.emit("signals", new Values("refreshCache"));
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
}
}
@Override
public void nextTuple() {
// emit tweets
Status status = statuses.poll();
if (status == null) Utils.sleep(1000);
else {
ResponseList<User> followers;
ResponseList<User> friends;
try {
Thread.sleep(6000);
followers = twitter.getFollowersList(status.getUser().getScreenName(), -1);
Thread.sleep(6000);
friends = twitter.getFriendsList(status.getUser().getScreenName(), -1);
if (!followers.isEmpty() && !friends.isEmpty()) {
spoutOutputCollector.emit(new Values(status, followers, friends));
}
// followers.clear();
// friends.clear();
} catch (TwitterException ex) {
ex.printStackTrace();
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
}
@Override
public void nextTuple() {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
}
String msgId = name + System.currentTimeMillis() + "---" + rand.nextInt();
_collector.emit(new Values(name + " sent this message msgId = " + msgId), msgId);
// List<LogEntry> list = new ArrayList<LogEntry>();
// LogEntry log = new LogEntry();
// log.setCategory("Storm");
// String message = name + Thread.currentThread().getName() + " "
// + syinfo.getProcessId() + " " + syinfo.getFQDN() + " " + name
// + " sent this message id = " + msgId;
// log.setMessage(message);
// list.add(log);
// try {
// if (!tr.isOpen())
// tr.open();
// client.Log(list);
// } catch (org.apache.thrift.TException e) {
// e.printStackTrace();
// }
}
/** Resend failed tuple */
@Override
public void fail(Object msgId) {
long id = (Long) msgId;
ArrayList<String> hosts = new ArrayList<String>();
hosts.add(host + "-after-failure");
LOG.info("Failed message " + msgId);
spoutOutputCollector.emit(new Values(id, this.host, 0L, this.payload, hosts), id);
}
@Override
public void dispatch(TupleValues tupleValues) {
collector.emit(getSource().getName(), new Values(tupleValues));
if (LOG.isDebugEnabled()) {
LOG.debug("Dispatch {} from {}", tupleValues, getSource().getName());
}
}
@Override
public void fail(Object msgId) {
// Sleep 3 seconds and retry.
// Utils.sleep(3000);
collector.emit(new Values(msgId), msgId);
zkState.updateApplicationStatus((String) msgId, ZKStateConstant.AppStatus.FAILED);
LOG.warn("fail {}", msgId.toString());
}
@Override
public void nextTuple() {
new LatencySimulator(1, 25, 10, 40, 5).simulate(1000);
int numberPart = idGenerator.nextInt(9999999) + 1;
String customerId = "customer-" + Integer.toString(numberPart);
outputCollector.emit(new Values(customerId));
}
/**
* Emits the next message from the queue as a tuple.
*
* <p>Serialization schemes returning null will immediately ack and then emit unanchored on the
* {@link #ERROR_STREAM_NAME} stream for further handling by the consumer.
*
* <p>If no message is ready to emit, this will wait a short time ({@link #WAIT_FOR_NEXT_MESSAGE})
* for one to arrive on the queue, to avoid a tight loop in the spout worker.
*/
@Override
public void nextTuple() {
if (spoutActive && amqpConsumer != null) {
try {
final QueueingConsumer.Delivery delivery = amqpConsumer.nextDelivery(WAIT_FOR_NEXT_MESSAGE);
if (delivery == null) return;
final long deliveryTag = delivery.getEnvelope().getDeliveryTag();
final byte[] message = delivery.getBody();
/////////////////// new rpc
BasicProperties props = delivery.getProperties();
BasicProperties replyProps =
new BasicProperties.Builder().correlationId(props.getCorrelationId()).build();
//////////////////////////////
List<Object> deserializedMessage = serialisationScheme.deserialize(message);
if (deserializedMessage != null && deserializedMessage.size() > 0) {
// let's see what's inside the Object List (checking)
/*System.out.println("Lenght of the list : "+ deserializedMessage.size() +"\n");
for (int i =0; i< deserializedMessage.size(); i++){
Object obj=deserializedMessage.get(i);
System.out.println("Object value: "+ obj + "\n" );
}*/
ArrayList li = new ArrayList();
li.add(deserializedMessage.get(0));
li.add(props);
li.add(replyProps);
deserializedMessage = li;
collector.emit(deserializedMessage, deliveryTag);
/////////// new for AMQPSpout RPC+JSON
try {
String response = "your name was: " + deserializedMessage.get(0) + " ";
this.amqpChannel.basicPublish(
"", props.getReplyTo(), replyProps, response.getBytes("UTF-8"));
// this.amqpChannel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
this.amqpChannel.basicAck(deliveryTag, false);
}
/// publishjson(props, replyProps, delivery.detEnvelope().getDeliveryTag());
// try{
// publishjson(props, replyProps);
// }
catch (IOException e) {
log.error("Error when publishing to the response queue", e);
}
////////////////
} else {
handleMalformedDelivery(deliveryTag, message);
}
} catch (ShutdownSignalException e) {
log.warn("AMQP connection dropped, will attempt to reconnect...");
Utils.sleep(WAIT_AFTER_SHUTDOWN_SIGNAL);
reconnect();
} catch (InterruptedException e) {
// interrupted while waiting for message, big deal
}
}
}
@Override
public void nextTuple() {
Utils.sleep(100);
String[] sentences =
new String[] {
"first sentence", "second sentence", "third sentence", "fourth sentence", "fifth sentence"
};
String sentence = sentences[_rand.nextInt(sentences.length)];
_collector.emit(new Values(sentence));
}
private void sendHbMsg() {
List values = JStormUtils.mk_list(uptime.uptime());
if (spoutOutputCollector != null) {
spoutOutputCollector.emit(Common.TOPOLOGY_MASTER_HB_STREAM_ID, values);
} else if (boltOutputCollector != null) {
boltOutputCollector.emit(Common.TOPOLOGY_MASTER_HB_STREAM_ID, values);
} else {
LOG.warn("Failed to send hearbeat msg. OutputCollector has not been initialized!");
}
}
@Override
public void nextTuple() {
// sleep a second before emitting any word
Utils.sleep(1000);
// generate a random number based on the wordList length
int nextInt = rnd.nextInt(wordList.length);
// emit the word chosen by the random number from wordList
collector.emit(new Values(wordList[nextInt]));
}
@Override
public void nextTuple() {
Utils.sleep(100);
String[] sentences =
new String[] {
"the cow jumped over the moon",
"an apple a day keeps the doctor away",
"four score and seven years ago",
"snow white and the seven dwarfs",
"i am at two with nature"
};
String sentence = sentences[_rand.nextInt(sentences.length)];
_collector.emit(new Values(sentence));
}
@SuppressWarnings("unchecked")
@Override
public void nextTuple() {
// LOG.info("Start to run tuple");
try {
Calendar calendar = Calendar.getInstance();
long fetchTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(this.lastFinishAppTime);
if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) {
LOG.info("Last finished time = {}", calendar.getTime());
List<AppInfo> appInfos =
rmFetch.getResource(
Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime));
if (appInfos != null) {
LOG.info("Get " + appInfos.size() + " from yarn resource manager.");
for (AppInfo app : appInfos) {
String appId = app.getId();
if (!zkState.hasApplication(appId)) {
zkState.addFinishedApplication(
appId,
app.getQueue(),
app.getState(),
app.getFinalStatus(),
app.getUser(),
app.getName());
}
}
}
this.lastFinishAppTime = fetchTime;
zkState.updateLastUpdateTime(fetchTime);
}
List<String> appIds = zkState.loadApplications(10);
for (String appId : appIds) {
collector.emit(new Values(appId), appId);
LOG.info("emit " + appId);
zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE);
}
if (appIds.isEmpty()) {
this.takeRest(5);
} else {
LOG.info("{} apps sent.", appIds.size());
}
} catch (Exception e) {
LOG.error("Fail to run next tuple", e);
}
}
public void nextTuple() {
BatchMessage msg = null;
try {
msg = batchQueue.take();
} catch (InterruptedException e) {
return;
}
if (msg == null) {
return;
}
UUID uuid = msg.getBatchId();
List<MessageExt> msgList = msg.getMsgList();
List<String> msgBodyList = getMessageBody(msgList);
collector.emit(new Values(msgBodyList), uuid);
}
public void nextTuple() {
if (amqpConsumer != null) {
try {
final QueueingConsumer.Delivery delivery = amqpConsumer.nextDelivery(WAIT_FOR_NEXT_MESSAGE);
if (delivery == null) return;
final long deliveryTag = delivery.getEnvelope().getDeliveryTag();
final byte[] message = delivery.getBody();
log.debug("Sending message:" + new String(message));
log.debug("Getting next tuple - " + (messageCount++));
if (messageCount < 0) {
log.debug("You just get a number of messages = max value of long! Wow! Reseting to 0");
this.messageCount = 0;
}
collector.emit(serialisationScheme.deserialize(message), deliveryTag);
} catch (InterruptedException e) {
// interrupted while waiting for message, big deal
}
}
}
@Override
public void nextTuple() {
// next tuple available when _queue contains ids or fillBuffer() is allowed and indicates more
// messages were available
// see class documentation for implementation note on the rationale behind this condition
if (!_queue.isEmpty() || (_inProgress.isEmpty() && fillBuffer())) {
final KafkaMessageId nextId = _queue.poll();
if (nextId != null) {
final byte[] message = _inProgress.get(nextId);
// the next id from buffer should correspond to a message in the pending map
if (message == null) {
throw new IllegalStateException("no pending message for next id " + nextId);
}
// message should be considered a single object from Values' point of view
_collector.emit(new Values((Object) message), nextId);
LOG.debug("emitted kafka message id {} ({} bytes payload)", nextId, message.length);
}
}
}
@Override
public void nextTuple() {
try {
if (numOfReports < maxReports) {
currTime = System.nanoTime();
LOG.info(
"startTime: "
+ startTime
+ " currTime: "
+ currTime
+ " interval: "
+ ((currTime - startTime) * 1e-6));
if (((currTime - startTime) * 1e-6) >= interval) {
if (query == null) query = QueryFactory.create(queryString);
Store store = StoreFactory.getJenaHBaseStore(configFile, iri, isReified);
int solnCount = 0;
ResultSet rs = store.executeSelectQuery(query);
List<Var> listVars = query.getProject().getVars();
while (rs.hasNext()) {
solnCount++;
QuerySolution qs = rs.next();
Object[] results = new String[listVars.size() + 1];
if (solnCount == 1) results[0] = "new";
else results[0] = "cont";
for (int i = 1; i <= listVars.size(); i++) {
Var var = listVars.get(i - 1);
RDFNode value = qs.get(var.toString());
if (value.isResource() || value.isAnon()) results[i] = value.asResource().toString();
else if (value.isLiteral()) results[i] = value.asLiteral().toString();
}
collector.emit(new Values(results));
}
numOfReports += 1;
startTime = currTime;
}
Thread.sleep(30000);
}
} catch (Exception e) {
throw new TopologyException("Exception in query spout:: ", e);
}
}
@Override
public void nextTuple() {
count++;
Event event = new Event();
event.setUser(
new User(
this.getUserId(),
"User Name_" + count,
new Country("" + (count / 10), "c_name_" + (count / 10L))));
event.setTimeStamp(this.getTime());
List<EventElement> elements = event.getElements();
elements.add(
new EventElement(
EventType.INSTALL, "" + JSONBuilder.dateFormatTillMillis.format(new Date())));
elements.add(new EventElement(EventType.CHANGED_LEVEL, "" + 1));
event.setElements(elements);
System.out.println("SPOUT send:" + JSONBuilder.json(event));
Utils.sleep(timeOut);
_collector.emit(new Values(JSONBuilder.json(event)));
}
@Override
public void nextTuple() {
// Commented by Joseph on Oct 2, 2015
// To control the outgoing data rate, we intensionally add 100ms delay here.
// The actual flow rate should be controled by the throghput allocation decision.
Utils.sleep(100);
String[] sentences =
new String[] {
"Hong Kong, traditionally Hongkong officially known as Hong Kong Special Administrative Region of the People's Republic of China is an autonomous territory on the southern coast of China at the Pearl River Estuary and the South China Sea.",
"In the late 1970s, Hong Kong became a major entrept in Asia-Pacific. The territory has developed into a major global trade hub and financial centre,",
"and is regarded as a world city. The 45th-largest economy in the world. Hong Kong ranks top 10 in GDP (PPP) per capita, but also has the most severe income inequality among advanced economies",
"Hong Kong is one of the three most important financial centres alongside New York and London. The territory has a high Human Development Index and is ranked highly in the Global Competitiveness Report.",
"It has been named the freest market economy by the Heritage Foundation Index of Economic Freedom. The service economy, characterised by low taxation and free trade, has been regarded as one of the world's most laissez-faire economic policies,",
"and the currency, the Hong Kong dollar, is the 13th most traded currency in the world. Hong Kong is a member of APEC, ADB, IMF, BIS, WTO, FIFA,",
"and International Olympic Committee, as Hong Kong Basic Law authorizes the territory to develop relations with foreign states on its own in appropriate fields,",
"including the economic, trade, financial and monetary, shipping, communications, tourism, cultural and sports fields. Hong Kong has carried many nicknames: the most famous among those is the Pearl of the Orient",
"which reflected the impressive night-view of the city's light decorations on the skyscrapers along both sides of the Victoria Harbour.",
"The territory is also known as Asia's World City. Archaeological studies support human presence in the Chek Lap Kok area (now Hong Kong International Airport) from 35,000 to 39,000 years ago and on Sai Kung Peninsula from 6,000 years ago.",
"The area of Hong Kong was consolidated under the kingdom of Nanyue (Southern Viet), founded by general Zhao Tuo in 204 BC after the collapse of the short-lived Qin dynasty.",
"When the kingdom of Nanyue was conquered by the Han Dynasty in 111 BC, Hong Kong was assigned to the Jiaozhi commandery. Archaeological evidence indicates that the population increased and early salt production flourished in this time period.",
"Lei Cheng Uk Han Tomb on the Kowloon Peninsula is believed to have been built during the Han dynasty. Hong Kong continued to experience modest growth during the first half of the 20th century.",
"The University of Hong Kong was established in 1911 as the territory's oldest higher education institute. While there was an exodus of 60,000 residents for fear of a German attack on the British colony during the First World War,",
"Hong Kong remained peaceful. Its population increased from 530,000 in 1916 to 725,000 in 1925 and reached 1.6 million by 1941.",
"Since Hong Kong's reunification with China, there has been increasing social tension between Hong Kong residents and mainland Chinese due to cultural and linguistic differences, as well as accusations of unruly behaviour and spending habits of mainland Chinese visitors to the territory.",
"In 2012 Chief Executive elections saw the Beijing backed candidate Leung Chun-Ying elected with 689 votes from a committee panel of 1,200 selected representatives, and assumed office on 1 July 2012.",
"Social conflicts also influenced the mass protests in 2014, primarily caused by the Chinese government's proposal on electoral reform.",
"Hong Kong enjoys a high degree of autonomy, as its political and judicial systems operate independently from those of mainland China. In accordance with the Sino-British Joint Declaration, and the underlying principle of one country, two systems, Hong Kong has a high degree of autonomy as a special administrative region in all areas except defence and foreign affairs.",
"The guarantees over the territory's autonomy and the individual rights and freedoms are enshrined in the Hong Kong Basic Law, the territory's constitutional document, which outlines the system of governance of the Hong Kong Special Administrative Region, but which is subject to the interpretation of the Standing Committee of the National People's Congress (NPCSC)."
};
String sentence = sentences[_rand.nextInt(sentences.length)];
pw.write(
String.valueOf(System.currentTimeMillis())
+ ": "
+ String.valueOf(sentence.length())
+ "\n");
_collector.emit(new Values(sentence));
}
public void nextTuple() {
String word = "Hello";
collector.emit(new Values(word));
}
@Override
public void nextTuple() {
if (emit.get() < maxQueries.get()) {
spoutOutputCollector.emit(new Values(stationID, "38°C"), emit.incrementAndGet());
}
}
/**
* Acks the bad message to avoid retry loops. Also emits the bad message unreliably on the {@link
* #ERROR_STREAM_NAME} stream for consumer handling.
*
* @param deliveryTag AMQP delivery tag
* @param message bytes of the bad message
*/
private void handleMalformedDelivery(long deliveryTag, byte[] message) {
log.debug("Malformed deserialized message, null or zero-length. " + deliveryTag);
ack(deliveryTag);
collector.emit(ERROR_STREAM_NAME, new Values(deliveryTag, message));
}
@Override
public void nextTuple() {
boolean gotRequest = false;
if (_local_drpc_id == null) {
int size = 0;
synchronized (_clients) {
size =
_clients
.size(); // This will only ever grow, so no need to worry about falling off the end
}
for (int i = 0; i < size; i++) {
DRPCInvocationsClient client;
synchronized (_clients) {
client = _clients.get(i);
}
if (!client.isConnected()) {
continue;
}
try {
DRPCRequest req = client.fetchRequest(_function);
if (req.get_request_id().length() > 0) {
Map returnInfo = new HashMap();
returnInfo.put("id", req.get_request_id());
returnInfo.put("host", client.getHost());
returnInfo.put("port", client.getPort());
gotRequest = true;
_collector.emit(
new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)),
new DRPCMessageId(req.get_request_id(), i));
break;
}
} catch (TException e) {
reconnect(client);
LOG.error("Failed to fetch DRPC result from DRPC server", e);
} catch (AuthorizationException aze) {
reconnect(client);
LOG.error("Not authorized to fetch DRPC result from DRPC server", aze);
} catch (Exception e) {
LOG.error("Failed to fetch DRPC result from DRPC server", e);
}
}
checkFutures();
} else {
DistributedRPCInvocations.Iface drpc =
(DistributedRPCInvocations.Iface) ServiceRegistry.getService(_local_drpc_id);
if (drpc != null) { // can happen during shutdown of drpc while topology is still up
try {
DRPCRequest req = drpc.fetchRequest(_function);
if (req.get_request_id().length() > 0) {
Map returnInfo = new HashMap();
returnInfo.put("id", req.get_request_id());
returnInfo.put("host", _local_drpc_id);
returnInfo.put("port", 0);
gotRequest = true;
_collector.emit(
new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)),
new DRPCMessageId(req.get_request_id(), 0));
}
} catch (TException e) {
throw new RuntimeException(e);
} catch (AuthorizationException aze) {
throw new RuntimeException(aze);
}
}
}
if (!gotRequest) {
Utils.sleep(1);
}
}
