/
RSSDriver.java
155 lines (139 loc) · 6.26 KB
/
RSSDriver.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
package dataCollection;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.Date;
public class RSSDriver {
private static final int NTHREDS = 105; //514 in total
RSSDriver(){
}
public static void main(String[] args) throws IOException{
System.out.println((new Date()).toString() + " Starting RSS server on directory " + System.getProperty("user.dir"));
// set time interval between RSS pulls
long minute = 60000;
long sleepFor = minute * 30;
int prevHr = 25;
int hr = 0;
// load the rss feeds list
String inFile = "rssFeedsList.txt";
BufferedReader inputFile = null;
ArrayList<String> rssFeeds = new ArrayList<String>();
String line = "";
inputFile = new BufferedReader (new FileReader (inFile));
while ((line = inputFile.readLine()) != null) {
rssFeeds.add(line);
}
int rssCnt = rssFeeds.size();
inputFile.close();
// kick off all of the RSS feed collector threads using a thread pool and get stock quotes
try{
while (true){
System.out.println((new Date()).toString() + " Generating worker threads");
ExecutorService executor = Executors.newFixedThreadPool(NTHREDS);
List<Future<Long>> list = new ArrayList<Future<Long>>();
String delimiter = ",";
String tblName = "quotes";
HBaseTbl quotesTbl = new HBaseTbl(tblName);
quotesTbl.getTable(tblName);
for(String rf : rssFeeds) {
String[] rssFeed = rf.split(delimiter);
// create and submit an executable thread
Callable<Long> worker = new RSSConsumer(rssFeed[0], rssFeed[1]);
Future<Long> submit = executor.submit(worker);
list.add(submit);
// get a corresponding company stock quote for each rss item
String[] lines = {"",""};
lines = rssFeed[0].split("-");
String index = lines[0];
if (index.equals("S&P500")){
// only get stock quotes in the S&P 500
String quoteName = lines[1];
try {
Quote q = new Quote();
int res = q.get(quoteName);
if (res == 0){
q.insert(quotesTbl);
}
} catch (Exception e) {
System.out.println((new Date()).toString() + " Error inserting quotes " + quoteName + " " + e.getMessage());
// only report, try again on next pass to see if error resolves on its own
}
}
}
// get the corresponding S&P500 index and volatility quotes
try {
Quote q1 = new Quote();
int res = q1.get("SPY");
if (res == 0){
q1.insert(quotesTbl);
}
Quote q2 = new Quote();
res = q2.get("VXX");
if (res == 0){
q2.insert(quotesTbl);
}
} catch (Exception e) {
System.out.println((new Date()).toString() + " Error inserting Hbase quotes table" + e.getMessage());
// only report, try again on next pass to see if error resolves on its own
}
// get the executor thread resulte of the RSS pulls
System.out.println((new Date()).toString() + " Gathering worker results");
long insertCnt = 0;
for (Future<Long> future : list) {
try {
insertCnt += future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
//System.out.println((new Date()).toString() + " " + rssCnt + " rss pulls resulted in " + insertCnt + " newsItems inserts");
// This will make the executor accept no new threads and finish all existing threads in the queue
executor.shutdown();
// Wait for all threads are finished
executor.awaitTermination(1, TimeUnit.SECONDS);
// email results, row counts of the 'newsItems' table
System.out.println((new Date()).toString() + " Emailing current newsItems counts");
String msgTxt = "";
// only send one e-mail per hour
try{
int rowCnt = 0;
HBaseTbl tbl = new HBaseTbl("newsItems");
tbl.getTable("newsItems");
rowCnt = tbl.getRowCount();
msgTxt = (new Date()).toString() + " " + rssCnt + " rss pulls resulted in " + insertCnt + " new rows inserted into Hbase table newsItems. \n" +
(new Date()).toString() + " " + "Current row count of HBase table newsItems is " + rowCnt;
Calendar now = Calendar.getInstance();
hr = now.get(Calendar.MINUTE);
System.out.println("hr = " + hr + " prevhr = " + prevHr);
if(hr != prevHr){
System.out.println((new Date()).toString() + " Emailing current newsItems counts");
Mail sm = new Mail(args[0], args[1], args[2]);
sm.send(msgTxt);
}else{
System.out.println((new Date()).toString() + " Skipping email");
}
prevHr = hr;
} catch (IOException e) {
e.printStackTrace();
}
System.out.println(msgTxt);
System.out.println((new Date()).toString() + " Finished all threads, going to sleep for " + (sleepFor/minute) + " minutes");
Thread.sleep(sleepFor);
System.out.println((new Date()).toString() + " Done sleeping restarting rss pull cycle");
}
}catch (Exception e) {
System.out.println((new Date()).toString() + " Exception in main thread:" + e.getMessage());
}
}
}