@SuppressWarnings("finally")
public JSONObject getKeyWords(JSONObject semiData) {
JSONObject keyWords = new JSONObject();
try {
this.buildIndex(semiData);
this.getIndexInfo(this.indexDir, 4);
this.generateWekaFile(this.termList, this.maxDocNum, this.wekaFile);
JSONArray array = this.Cluster(this.wekaFile, 7);
int totalNum = 0;
for (int i = 0; i < array.length(); i++) {
totalNum += array.getJSONArray(i).length();
}
keyWords.put("maxFreq", this.maxFreq);
keyWords.put("totalNum", totalNum);
keyWords.put("WordList", array);
} catch (WeiboException e) {
System.out.print("getKeyWords++++++weibo\n");
System.out.print("error:" + e.getError() + "toString:" + e.toString());
keyWords.put("error", e.getError());
e.printStackTrace();
} catch (Exception e) {
System.out.print("getKeyWords++++++Exception");
keyWords.put("error", e.toString());
e.printStackTrace();
} finally {
try {
this.myDelete(this.indexDir);
this.myDelete(this.wekaFile);
} catch (Exception e) {
e.printStackTrace();
}
return keyWords;
}
}
public static void main(String[] args) {
String access_token = args[0];
Suggestion suggestion = new Suggestion(access_token);
try {
JSONArray jo = suggestion.suggestionsUsersMayInterested();
System.out.println(jo.toString());
} catch (WeiboException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
String access_token = args[0];
String q = args[1];
Search search = new Search(access_token);
try {
JSONArray jo = search.searchSuggestionsCompanies(q);
System.out.println(jo.toString());
} catch (WeiboException e) {
e.printStackTrace();
}
}
public static String[] constructIds(Response res) throws WeiboException {
try {
JSONArray list = res.asJSONObject().getJSONArray("ids");
String temp = list.toString().substring(1, list.toString().length() - 2);
String[] ids = temp.split(",");
return ids;
} catch (JSONException jsone) {
throw new WeiboException(jsone);
} catch (WeiboException te) {
throw te;
}
}
/**
* @param res
* @return
* @throws WeiboException
*/
static List<User> constructResult(Response res) throws WeiboException {
JSONArray list = res.asJSONArray();
try {
int size = list.length();
List<User> users = new ArrayList<User>(size);
for (int i = 0; i < size; i++) {
users.add(new User(list.getJSONObject(i)));
}
return users;
} catch (JSONException e) {
}
return null;
}
public static void main(String[] args) {
String access_token = args[0];
Weibo weibo = new Weibo();
weibo.setToken(access_token);
String uids = args[1];
Users um = new Users();
try {
JSONArray user = um.getUserCount(uids);
Log.logInfo(user.toString());
} catch (WeiboException e) {
e.printStackTrace();
}
}
public static List<User> constructUsers(Response res) throws WeiboException {
try {
JSONArray list = res.asJSONArray();
int size = list.length();
List<User> users = new ArrayList<User>(size);
for (int i = 0; i < size; i++) {
users.add(new User(list.getJSONObject(i)));
}
return users;
} catch (JSONException jsone) {
throw new WeiboException(jsone);
} catch (WeiboException te) {
throw te;
}
}
public static List<SchoolSearch> constructSchoolSearch(Response res) throws WeiboException {
try {
JSONArray list = res.asJSONArray();
int size = list.length();
List<SchoolSearch> schools = new ArrayList<SchoolSearch>(size);
for (int i = 0; i < size; i++) {
schools.add(new SchoolSearch(list.getJSONObject(i)));
}
return schools;
} catch (JSONException jsone) {
throw new WeiboException(jsone);
} catch (WeiboException te) {
throw te;
}
}
public static List<FavoritesTag> constructTag(Response res) throws WeiboException {
try {
JSONArray list = res.asJSONObject().getJSONArray("tags");
int size = list.length();
List<FavoritesTag> tags = new ArrayList<FavoritesTag>(size);
for (int i = 0; i < size; i++) {
tags.add(new FavoritesTag(list.getJSONObject(i)));
}
return tags;
} catch (JSONException jsone) {
throw new WeiboException(jsone);
} catch (WeiboException te) {
throw te;
}
}
static List<Count> constructCounts(Response res) throws WeiboException {
try {
JSONArray list = res.asJSONArray();
int size = list.length();
List<Count> counts = new ArrayList<Count>(size);
for (int i = 0; i < size; i++) {
counts.add(new Count(list.getJSONObject(i)));
}
return counts;
} catch (JSONException jsone) {
throw new WeiboException(jsone);
} catch (WeiboException te) {
throw te;
}
}
public static TagWapper constructTagWapper(Response res) {
try {
JSONArray tags = res.asJSONArray();
List<Tag> tagList = new ArrayList<Tag>();
for (int i = 0; i < tags.getJSONObject(0).getJSONArray("tags").length(); i++) {
tagList.add(new Tag(tags.getJSONObject(0).getJSONArray("tags").getJSONObject(i)));
}
String id = tags.getJSONObject(0).getString("id");
return new TagWapper(tagList, id);
} catch (JSONException e) {
e.printStackTrace();
} catch (WeiboException e) {
e.printStackTrace();
}
return null;
}
public MentionsIds(JSONObject json) throws WeiboException {
try {
nextCursor = json.getLong("next_cursor");
previousCursor = json.getLong("previous_cursor");
totalNumber = json.getLong("total_number");
hasvisible = json.getBoolean("hasvisible");
interval = json.getLong("interval");
JSONArray list = json.getJSONArray("statuses");
int size = list.length();
statusesIds = new ArrayList<String>(size);
for (int i = 0; i < size; i++) {
statusesIds.add(list.getString(i));
}
} catch (JSONException jsone) {
throw new WeiboException(jsone);
}
}
/**
* @param res
* @return
* @throws WeiboException
*/
public static UserWapper constructWapperUsers(Response res) throws WeiboException {
JSONObject jsonUsers = res.asJSONObject(); // asJSONArray();
try {
JSONArray user = jsonUsers.getJSONArray("users");
int size = user.length();
List<User> users = new ArrayList<User>(size);
for (int i = 0; i < size; i++) {
users.add(new User(user.getJSONObject(i)));
}
long previousCursor = jsonUsers.getLong("previous_curosr");
long nextCursor = jsonUsers.getLong("next_cursor");
if (nextCursor == -1) { // 兼容不同标签名称
nextCursor = jsonUsers.getLong("nextCursor");
}
return new UserWapper(users, previousCursor, nextCursor);
} catch (JSONException jsone) {
throw new WeiboException(jsone);
}
}
public static List<User> constructUser(Response res) throws WeiboException {
JSONObject json = res.asJSONObject();
JSONArray list = null;
try {
if (!json.isNull("users")) {
list = json.getJSONArray("users");
} else {
list = res.asJSONArray();
}
int size = list.length();
List<User> users = new ArrayList<User>(size);
for (int i = 0; i < size; i++) {
users.add(new User(list.getJSONObject(i)));
}
return users;
} catch (JSONException je) {
throw new WeiboException(je);
}
}
public void buildIndex(JSONObject indexData) {
try {
Directory dir = FSDirectory.open(new File(indexDir));
IKAnalyzer analyzer = new IKAnalyzer();
analyzer.setUseSmart(true);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_35, analyzer);
indexWriter = new IndexWriter(dir, iwc);
indexWriter.deleteAll();
JSONArray statusData = indexData.getJSONArray("statusData");
for (int i = 0; i < statusData.length(); i++) {
String text = statusData.getString(i);
Document doc = new Document();
doc.add(
new Field(
"text",
text,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(doc);
}
JSONArray userData = indexData.getJSONArray("userData");
for (int i = 0; i < userData.length(); i++) {
String text = userData.getString(i);
Document doc = new Document();
doc.add(
new Field(
"text",
text,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(doc);
}
// indexWriter.commit();
System.out.println("Index is done");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
indexWriter.close();
} catch (CorruptIndexException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static StatusWapper constructWapperStatus(Response res) throws WeiboException {
JSONObject jsonStatus = res.asJSONObject(); // asJSONArray();
JSONArray statuses = null;
try {
if (!jsonStatus.isNull("statuses")) {
statuses = jsonStatus.getJSONArray("statuses");
}
if (!jsonStatus.isNull("reposts")) {
statuses = jsonStatus.getJSONArray("reposts");
}
int size = statuses.length();
List<Status> status = new ArrayList<Status>(size);
for (int i = 0; i < size; i++) {
status.add(new Status(statuses.getJSONObject(i)));
}
long previousCursor = jsonStatus.getLong("previous_curosr");
long nextCursor = jsonStatus.getLong("next_cursor");
long totalNumber = jsonStatus.getLong("total_number");
String hasvisible = jsonStatus.getString("hasvisible");
return new StatusWapper(status, previousCursor, nextCursor, totalNumber, hasvisible);
} catch (JSONException jsone) {
throw new WeiboException(jsone);
}
}
public JSONArray Cluster(String wekaFilePath, int clusterNum) throws Exception {
File inputFile = new File(wekaFilePath);
ArffLoader arf = new ArffLoader();
arf.setFile(inputFile);
Instances originIns = arf.getDataSet();
Instances insTest = new Instances(originIns);
insTest.deleteStringAttributes();
int totalNum = insTest.numInstances();
// SimpleKMeans sm = new SimpleKMeans();
EM em = new EM();
em.setNumClusters(clusterNum);
MakeDensityBasedClusterer sm = new MakeDensityBasedClusterer();
sm.setClusterer(em);
sm.buildClusterer(insTest);
System.out.println("totalNum:" + insTest.numInstances());
System.out.println("============================");
System.out.println(sm.toString());
Map<Integer, ArrayList<String>> result = new HashMap<Integer, ArrayList<String>>();
for (int i = 0; i < clusterNum; i++) {
result.put(i, new ArrayList<String>());
}
for (int i = 0; i < totalNum; i++) {
Instance ins = originIns.instance(i);
String word = ins.stringValue(0);
Instance tempIns = new Instance(ins);
tempIns.deleteAttributeAt(0);
int cluster = sm.clusterInstance(tempIns);
result.get(cluster).add(word);
}
// print the result
ArrayList<String> words = new ArrayList<String>();
JSONArray keyWords = new JSONArray();
for (int k : result.keySet()) {
words = result.get(k);
PriorityQueue<MyTerm> clusterQueue = new PriorityQueue<MyTerm>(1, MyTermCompare);
for (int i = 0; i < words.size(); i++) {
String s = words.get(i);
assert linkMap.containsKey(s);
int freq = linkMap.get(s).totalFreq;
clusterQueue.add(linkMap.get(s));
words.set(i, "(" + s + ":" + freq + ")");
}
JSONArray clusterArray = new JSONArray();
int num = clusterQueue.size() / 10 + 1; // 5%
int totalFreq = 0;
int totalLength = 0;
for (int i = 0; i < num && !clusterQueue.isEmpty(); ) {
JSONObject mem = new JSONObject();
MyTerm myTerm = clusterQueue.poll();
String word = myTerm.originTrem.text();
if (word.length() == 1) {
continue;
}
mem.put("text", word);
mem.put("freq", myTerm.totalFreq);
clusterArray.put(mem);
i++;
totalFreq += myTerm.totalFreq;
totalLength += word.length();
}
double averFreq = totalFreq * 1.0 / num;
double averLength = totalLength * 1.0 / num;
int count = 0;
while (!clusterQueue.isEmpty() && count < num) {
MyTerm myTerm = clusterQueue.poll();
String word = myTerm.originTrem.text();
int freq = myTerm.totalFreq;
int times = (int) (word.length() / averFreq) + 1;
if (freq > averFreq / times) {
JSONObject mem = new JSONObject();
mem.put("text", word);
mem.put("freq", freq);
mem.put("extra", true);
clusterArray.put(mem);
}
}
keyWords.put(clusterArray);
System.out.println(
"cluster" + k + ":" + words.size() + ":\t" + (int) (words.size() * 1.0 / totalNum * 100));
if (result.get(k).size() < 100) {
System.out.println(result.get(k));
}
}
// System.out.println("errorNum:"+errorNum);
return keyWords;
}
