public static void printRet4Draw() {
String initX = "";
for (int i = 1; i < n; i++) initX += xs[i] + "\t";
initX += xs[n];
LogInfo.logs(initX);
String initY = "";
for (int i = 1; i < n; i++) initY += ys[i] + "\t";
initY += ys[n];
LogInfo.logs(initY);
String finalX = "";
for (int i = 1; i < n; i++) finalX += x[i] + "\t";
finalX += x[n];
LogInfo.logs(finalX);
String finalY = "";
for (int i = 1; i < n; i++) finalY += y[i] + "\t";
finalY += y[n];
LogInfo.logs(finalY);
double totalDis = 0;
for (int i = 1; i <= n; i++) {
totalDis += Math.sqrt((x[i] - xs[i]) * (x[i] - xs[i]) + (y[i] - ys[i]) * (y[i] - ys[i]));
}
LogInfo.logs("Total movements: %.2f", totalDis);
}
@Override
public DoubleMap align() {
Thread t1, t2;
t1 =
new Thread(
new Runnable() {
@Override
public void run() {
LogInfo.logs("wordToPred STARTED!");
wordToPred = aligner1.align();
}
});
t2 =
new Thread(
new Runnable() {
@Override
public void run() {
LogInfo.logs("predToWord STARTED!");
predToWord = aligner2.align();
}
});
LogInfo.begin_threads();
t1.start();
t2.start();
try {
t1.join();
t2.join();
} catch (InterruptedException e) {
e.printStackTrace();
LogInfo.fail(e);
}
LogInfo.end_threads();
return DoubleMap.product(wordToPred, predToWord);
}
/**
* Consider strings of joins such as (relation1 (relation2 (...)))
*
* <p>- forwardBackward: prune (relation (!relation (...))), etc. - doubleNext: prune (next (next
* (...))), (!next (next (...))), etc.
*/
private boolean pruneJoins(Derivation deriv) {
if (!containsStrategy("forwardBackward") && !containsStrategy("doubleNext")) return false;
Formula formula = deriv.formula, current = formula;
String rid1 = null, rid2 = null;
while (current instanceof JoinFormula) {
rid2 = rid1;
rid1 = getRelationIdIfPossible(((JoinFormula) current).relation);
if (rid1 != null && rid2 != null) {
// Prune (!relation (relation (...)))
if (containsStrategy("forwardBackward")) {
if (rid1.equals("!" + rid2) || rid2.equals("!" + rid1)) {
if (DerivationPruner.opts.pruningVerbosity >= 2)
LogInfo.logs("PRUNED [forwardBackward] %s", formula);
return true;
}
}
// Prune (next (next (...)))
if (containsStrategy("doubleNext")) {
if ((rid1.equals("fb:row.row.next") && rid2.equals("fb:row.row.next"))
|| (rid1.equals("!fb:row.row.next") && rid2.equals("!fb:row.row.next"))) {
if (DerivationPruner.opts.pruningVerbosity >= 2)
LogInfo.logs("PRUNED [doubleNext] %s", formula);
return true;
}
}
}
current = ((JoinFormula) current).child;
}
return false;
}
public void run() {
try {
while (!stop) {
if (LogInfo.writeToStdout) readAndProcessCommand();
// Input commands
Execution.inputMap.readEasy(Execution.getFile("input.map"));
boolean killed = Execution.create && new File(Execution.getFile("kill")).exists();
if (killed) Execution.setExecStatus("killed", true);
// Output status
Execution.putOutput("log.note", LogInfo.note);
Execution.putOutput("exec.memory", SysInfoUtils.getUsedMemoryStr());
Execution.putOutput(
"exec.time", new StopWatch(LogInfo.getWatch().getCurrTimeLong()).toString());
Execution.putOutput("exec.errors", "" + LogInfo.getNumErrors());
Execution.putOutput("exec.warnings", "" + LogInfo.getNumWarnings());
Execution.setExecStatus("running", false);
Execution.printOutputMap(Execution.getFile("output.map"));
if (killed) throw new RuntimeException("Killed by 'kill' file");
Utils.sleep(timeInterval);
}
} catch (Exception e) {
e.printStackTrace();
System.exit(1); // Die completely
}
}
public Map<String, String[]> flattenedIndexers() {
Map<String, String[]> out = new HashMap<String, String[]>();
try {
Field[] fields = Class.forName(CLASS_NAME).getDeclaredFields();
for (Field field : fields) {
if (field.getType() == Indexer.class) {
String indexerName = field.getName();
Indexer<String> indexer = (Indexer<String>) field.get(this);
out.put(indexerName, indexer.toArray(new String[0]));
} else if (field.getType() == Map.class) {
String indexerName = field.getName();
Map<Integer, Integer> map = (HashMap) field.get(this);
String[] lines = new String[map.size()];
int i = 0;
for (Entry<Integer, Integer> entry : map.entrySet())
lines[i++] = String.format("%s\t%s", entry.getKey(), entry.getValue());
out.put(indexerName, lines);
}
}
} catch (ClassNotFoundException ex) {
LogInfo.error(ex);
} catch (IllegalArgumentException ex) {
LogInfo.error(ex);
} catch (IllegalAccessException ex) {
LogInfo.error(ex);
}
return out;
}
private String getType1Desc(FormulaGenerationInfo fgInfo) {
if (fgInfo.isUnary) {
return fgInfo.uInfo.getRepresentativeDescrption();
}
if (fgInfo.bInfo.expectedType1.equals(FreebaseInfo.DATE)) return "date";
if (fgInfo.bInfo.expectedType1.equals(FreebaseInfo.FLOAT)) return "number";
if (fgInfo.bInfo.expectedType1.equals(FreebaseInfo.INT)) return "number";
if (fgInfo.bInfo.expectedType1.equals(FreebaseInfo.BOOLEAN)) return "";
if (fgInfo.bInfo.expectedType1.equals(FreebaseInfo.TEXT)) return "description";
Formula type1Formula =
new JoinFormula(
FreebaseInfo.TYPE, new ValueFormula<Value>(new NameValue(fgInfo.bInfo.expectedType1)));
try {
if (fbFormulasInfo.getUnaryInfo(type1Formula) == null) {
LogInfo.logs("No unary info for=%s", type1Formula);
return null;
}
return fbFormulasInfo.getUnaryInfo(type1Formula).getRepresentativeDescrption();
} catch (NullPointerException e) {
if (type1Formula.toString().equals("(fb:type.object.type fb:type.object)")) return "thing";
else {
LogInfo.logs(
"Binfo exType1=%s, exType2=%s", fgInfo.bInfo.expectedType1, fgInfo.bInfo.expectedType2);
throw new RuntimeException(e);
}
}
}
public QuestionGenerator() throws IOException {
Properties props = new Properties();
props.put("annotators", "tokenize,ssplit,pos,parse");
pipeline = new StanfordCoreNLP(props);
LogInfo.begin_track("uploading lexicon");
uploadAlignmentLexicon();
LogInfo.logs("Number of lexicon formulas: %s", formulaToLexemsMap.size());
LogInfo.end_track();
}
public void debugDump() {
LogInfo.logsForce("--------------------");
LogInfo.logsForce("mapType = " + mapType);
LogInfo.logsForce("locked = " + locked);
LogInfo.logsForce("size/capacity = " + size() + "/" + capacity());
LogInfo.logsForce("numCollisions = " + numCollisions);
/*for(int i = 0; i < keys.length; i++) {
System.out.printf("[%d] %s (%d) => %f\n", i, keys[i], (keys[i] == null ? 0 : keys[i].hashCode()), values[i]);
}*/
}
// generate node_dict.tsv & edge_dict.tsv
public static void generateDictFile() throws IOException {
BufferedReader br =
new BufferedReader(new FileReader("/home/xusheng/starry/baike/entity.index"));
BufferedWriter bw =
new BufferedWriter(
new FileWriter("/home/xusheng/pra/examples/graphs/baike/kb_svo" + "/node_dict.tsv"));
String line;
while ((line = br.readLine()) != null) {
String[] spt = line.split("\t");
bw.write(spt[1] + "\t" + spt[0] + "\n");
}
br.close();
bw.close();
Set<String> edges = new HashSet<>();
File f = new File("/home/xusheng/starry/baike/infobox.triple");
br = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
f = new File("/home/xusheng/pra/examples/graphs/baike/kb_svo/edge_dict.tsv");
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8"));
while ((line = br.readLine()) != null) {
String[] spt = line.split("\t");
edges.add(spt[1]);
}
br.close();
int cnt = 0;
for (String edge : edges) {
cnt++;
bw.write(cnt + "\t" + edge + "\n");
}
bw.close();
LogInfo.logs("Total number of eades: %d", edges.size());
}
public DiscriminativeFeatureIndexers(String folder) {
this();
try {
Class cl = Class.forName(CLASS_NAME);
File files[] =
new File(folder)
.listFiles(
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".gz");
}
});
for (File f : files) // for every indexer file
{
String indexerFieldName = f.getName().substring(0, f.getName().lastIndexOf("."));
String values[] = Utils.readLines(f.getAbsolutePath());
if (!indexerFieldName.contains("Map")) {
Indexer<String> indexer =
(Indexer<String>) cl.getDeclaredField(indexerFieldName).get(this);
for (String value : values) {
indexer.getIndex(value);
}
} else {
Map<Integer, Integer> map = (HashMap) cl.getDeclaredField(indexerFieldName).get(this);
for (String value : values) {
String[] ar = value.split("\t");
map.put(Integer.valueOf(ar[0]), Integer.valueOf(ar[1]));
}
}
}
} catch (ClassNotFoundException ex) {
LogInfo.error(ex);
} catch (NoSuchFieldException ex) {
LogInfo.error(ex);
} catch (SecurityException ex) {
LogInfo.error(ex);
} catch (IllegalArgumentException ex) {
LogInfo.error(ex);
} catch (IllegalAccessException ex) {
LogInfo.error(ex);
}
// return featureIndexers;
}
private void logsMatrices(List<ConfusionMatrix> ms) {
final int n = ms.size();
String[] putLine1 = new String[n];
String[] putLine2 = new String[n];
// Figure out width :/
double max = 0.0d;
for (int i = 0; i < n; i++) {
ConfusionMatrix m = ms.get(i);
max = Math.max(max, Math.max(Math.max(m.tp, m.fn), Math.max(m.fp, m.tn)));
}
int w = (int) Math.floor(Math.log10(max)) + 4;
for (int i = 0; i < n; i++) {
putLine1[i] = String.format("[%" + w + ".2f %" + w + ".2f]", ms.get(i).tp, ms.get(i).fn);
putLine2[i] = String.format("[%" + w + ".2f %" + w + ".2f]", ms.get(i).fp, ms.get(i).tn);
}
LogInfo.log(Joiner.on(" ").join(putLine1));
LogInfo.log(Joiner.on(" ").join(putLine2));
}
public static void generate() throws IOException {
File f = new File(rootFp + "/wikipedia/enwiki-20160920-pages-articles-multistream.xml");
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
// f = new File(rootFp + "/nn/data/wikipedia/prior.txt");
f = new File(rootFp + "/nn/data/wikipedia/popularity.txt");
BufferedWriter bw =
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8"));
String line;
int cnt = 0;
while ((line = br.readLine()) != null) {
cnt++;
LogUpgrader.showLine(cnt, 1000000);
Pattern pattern = Pattern.compile("\\[\\[(.*?)\\]\\]");
Matcher matcher = pattern.matcher(line);
while (matcher.find()) {
String raw = matcher.group(1);
String entity = raw.toLowerCase();
String mention = raw; // maintain original case
String[] spt = raw.split("\\|");
if (spt.length > 1) {
entity = spt[0].toLowerCase();
mention = spt[1];
}
// add2apperMap(entity, mention);
add2entTimeMap(entity);
if (cnt < 100) LogInfo.logs("[link]\t" + mention + " : " + entity);
}
/*
pattern = Pattern.compile("<title>(.*?)</title>");
matcher = pattern.matcher(line);
while (matcher.find()) {
String raw = matcher.group(1);
add2apperMap(raw.toLowerCase(), raw);
if (cnt < 100) LogInfo.logs("[title]\t" + raw + " : " + raw.toLowerCase());
}
*/
}
/*
for (Map.Entry<String, HashMap<String, Integer>> entry: appearMap.entrySet()) {
String mention = entry.getKey();
HashMap<String, Integer> entityMap = entry.getValue();
int sum = 0;
for (Map.Entry<String, Integer> entry1: entityMap.entrySet())
sum += entry1.getValue();
ArrayList<Map.Entry<String, Integer>> sorted = MapHelper.sort(entityMap);
for (int i=0; i<sorted.size(); i++)
bw.write(String.format("%s\t%s\t%.4f\n", mention, sorted.get(i).getKey(),
(double) sorted.get(i).getValue() / sum));
}*/
for (Map.Entry<String, Integer> entry : entTimeMap.entrySet()) {
bw.write(entry.getKey() + "\t" + entry.getValue() + "\n");
}
br.close();
bw.close();
}
public static void printRet() {
if (verbose) LogInfo.logs("Final Result:");
String initX = "[";
for (int i = 1; i < n; i++)
initX += ("(" + String.format("%.0f", xs[i]) + "," + String.format("%.0f", ys[i]) + ")\t");
initX += ("(" + String.format("%.0f", xs[n]) + "," + String.format("%.0f", ys[n]) + ")]");
LogInfo.logs(initX);
String finalX = "[";
for (int i = 1; i < n; i++)
finalX += ("(" + String.format("%.0f", x[i]) + "," + String.format("%.0f", y[i]) + ")\t");
finalX += ("(" + String.format("%.0f", x[n]) + "," + String.format("%.0f", y[n]) + ")]");
LogInfo.logs(finalX);
double totalDis = 0;
for (int i = 1; i <= n; i++) {
totalDis += Math.sqrt((x[i] - xs[i]) * (x[i] - xs[i]) + (y[i] - ys[i]) * (y[i] - ys[i]));
}
LogInfo.logs("Total movements: %.2f", totalDis);
}
public static void processFile(String inFile, String outFile) throws Exception {
File f = new File(inFile);
bw = new BufferedWriter(new FileWriter(outFile));
File[] files = f.listFiles();
for (File file : files) {
String path = file.getAbsolutePath();
String rel = file.getName();
if (rel.startsWith("_")) {
LogInfo.logs("Process file %s", path);
extractPos(rel, path);
}
}
}
public boolean logs(int iter, String group) {
List<File> files = Vis.getFilesPerExec(execPaths, iter, group);
if (files == null) return false;
LogInfo.logs("Reading files: %s", files);
final int n = files.size();
List<ConfusionMatrix> softMs = new ArrayList<ConfusionMatrix>(n);
List<ConfusionMatrix> hardMs = new ArrayList<ConfusionMatrix>(n);
for (int i = 0; i < n; i++) {
softMs.add(new ConfusionMatrix());
hardMs.add(new ConfusionMatrix());
}
final double ct = 0.5d;
final double pt = 0.5d;
for (List<Example> row : Vis.zipExamples(files)) {
for (int i = 0; i < n; i++) {
Example ex = row.get(i);
ConfusionMatrix softM = softMs.get(i);
ConfusionMatrix hardM = hardMs.get(i);
updateConfusionMatrix(softMs.get(i), ex, -1.0d, -1.0d);
updateConfusionMatrix(hardMs.get(i), ex, ct, pt);
}
}
LogInfo.begin_track("Soft");
logsMatrices(softMs);
LogInfo.end_track();
LogInfo.begin_track("Hard (compatThresh=%.2f, probThresh=%.2f)", ct, pt);
logsMatrices(hardMs);
LogInfo.end_track();
return true;
}
public DiscriminativeFeatureIndexers(DiscriminativeFeatureIndexers indexersIn) {
this();
try {
Field[] fields = Class.forName(CLASS_NAME).getDeclaredFields();
for (Field field : fields) {
if (field.getType() == Indexer.class) {
Indexer<String> indexerIn = (Indexer<String>) field.get(indexersIn);
Indexer<String> indexerThis = (Indexer<String>) field.get(this);
if (indexerIn != null) indexerThis.addAll(indexerIn);
} else if (field.getType() == HashMap.class) {
Map<Integer, Integer> mapIn = (HashMap) field.get(indexersIn);
Map<Integer, Integer> mapThis = (HashMap) field.get(this);
if (mapIn != null) mapThis.putAll(mapIn);
}
}
} catch (ClassNotFoundException ex) {
LogInfo.error(ex);
} catch (IllegalArgumentException ex) {
LogInfo.error(ex);
} catch (IllegalAccessException ex) {
LogInfo.error(ex);
}
}
public static void readData(String inputFp) throws IOException {
BufferedReader br = new BufferedReader(new FileReader(inputFp));
String line = br.readLine();
String spt[] = line.split("\t");
n = Integer.parseInt(spt[0]);
L = Integer.parseInt(spt[1]);
K = Integer.parseInt(spt[2]);
R = Integer.parseInt(spt[3]);
x = new double[n + 2];
xs = new double[n + 2];
y = new double[n + 2];
ys = new double[n + 2];
line = br.readLine();
spt = line.split("\t");
for (int i = 0; i < spt.length; i++) xs[i + 1] = x[i + 1] = Double.parseDouble(spt[i]);
line = br.readLine();
spt = line.split("\t");
for (int i = 0; i < spt.length; i++) ys[i + 1] = y[i + 1] = Double.parseDouble(spt[i]);
br.close();
LogInfo.begin_track("Input data loaded");
LogInfo.logs("n = %d, L = %d, K = %d, R = %d", n, L, K, R);
printXY();
LogInfo.end_track();
}
public void normalize() {
for (Map.Entry<String, StringDoubleMap> e : entrySet()) {
String s = e.getKey();
StringDoubleMap m = e.getValue();
double sum = m.sum();
if (!NumUtils.isFinite(1.0 / sum)) {
LogInfo.warning(
"normalize(): %s (with %d elements) has sum %f, using uniform", s, m.size(), sum);
m.putAll(1.0 / m.size());
} else {
m.multAll(1.0 / sum);
}
}
}
@Override
public void run() {
builder = new Builder();
builder.build();
dataset = new Dataset();
dataset.read();
params = new Params();
for (String group : dataset.groups()) {
String filename = Execution.getFile("dumped-" + group + ".gz");
out = IOUtils.openOutHard(filename);
processExamples(group, dataset.examples(group));
out.close();
LogInfo.logs("Finished dumping to %s", filename);
StopWatchSet.logStats();
}
}
public static void autoTest(int k, int cals) {
int settingCnt = 0;
R = 20;
K = k;
for (L = 200; L <= 800; L += 200) {
for (double rate = 1.25; rate <= 2; rate += 0.25) {
settingCnt++;
LogInfo.begin_track("Testing data setting %d", settingCnt);
n = (int) (rate * L * K / (2 * R)) + 1;
LogInfo.logs("n = %d, L = %d, K = %d, R = %d, redundancy rate: %.2f", n, L, K, R, rate);
long stTime = System.currentTimeMillis();
for (int numCnt = 1; numCnt <= cals; numCnt++) {
x = new double[n + 2];
xs = new double[n + 2];
y = new double[n + 2];
ys = new double[n + 2];
// Generate x positions
Set<Integer> set = new HashSet<>();
while (set.size() < n) {
int num = (int) (Math.random() * (L - 1)) + 1;
set.add(num);
}
List<Integer> list = new ArrayList<>(set);
Collections.sort(list);
for (int i = 0; i < n; i++) xs[i + 1] = x[i + 1] = list.get(i);
// Generate y positions
set = new HashSet<>();
while (set.size() < n) {
int num = (int) (Math.random() * (L - 1)) + 1;
set.add(num);
}
list = new ArrayList<>(set);
for (int i = 0; i < n; i++) ys[i + 1] = y[i + 1] = list.get(i);
if (verbose) {
LogInfo.begin_track("Data #%d:", numCnt);
printXY();
}
strongDetect();
if (verbose) LogInfo.end_track();
}
long edTime = System.currentTimeMillis();
long time = edTime - stTime;
LogInfo.logs(
"Time: %dms, [n = %d, L = %d, K = %d, R = %d, redundancy rate: %.2f]",
time, n, L, K, R, rate);
LogInfo.end_track();
}
}
}
public static void transform2idx(String inFile, String outFile) throws Exception {
BufferedReader br = new BufferedReader(new FileReader(inFile));
BufferedWriter bw = new BufferedWriter(new FileWriter(outFile));
String line;
while ((line = br.readLine()) != null) {
if (line.startsWith("###")) {
bw.write(line + "\n");
continue;
}
String[] spt = line.split("\t");
String idx1 = EntityIndex.getIdx(spt[0]);
String idx2 = EntityIndex.getIdx(spt[1]);
if (idx1 != null && idx2 != null) bw.write(idx1 + "\t" + idx2 + "\n");
}
br.close();
bw.close();
LogInfo.logs("Job Done.");
}
private void loadFormulaInfo() throws NumberFormatException, IOException {
LogInfo.begin_track("Loading formula info...");
LogInfo.logs("Adding schema properties");
binaryFormulaInfoMap = freebaseInfo.createBinaryFormulaInfoMap();
unaryFormulaInfoMap = freebaseInfo.createUnaryFormulaInfoMap();
LogInfo.logs("Current number of binary formulas: " + binaryFormulaInfoMap.size());
LogInfo.logs("Current number of unary formulas: " + unaryFormulaInfoMap.size());
LogInfo.logs("Compuing reverse for schema formulas");
computeReverseFormulaInfo();
LogInfo.logs("Current number of binary formulas: " + binaryFormulaInfoMap.size());
for (BinaryFormulaInfo info : binaryFormulaInfoMap.values()) {
MapUtils.addToSet(atomicExtype2ToBinaryMap, info.expectedType2, info.formula);
if (!isCvt(info.expectedType1)) {
addMappingFromType2ToFormula(info.expectedType2, info.formula);
}
}
LogInfo.logs("Generate formulas through CVTs");
generateCvtFormulas(); // generate formulas for CVTs
LogInfo.logs("Current number of binary formulas: " + binaryFormulaInfoMap.size());
LogInfo.end_track();
}
public static void strongDetect() {
used = new HashSet<>();
nw = nl = L / (2 * R);
for (int k = 1; k <= K; k++) {
LogInfo.begin_track("Starting round K = %d", k);
// Select the Horizontal Grid Barrier
int tarRow = HGBS();
LogInfo.logs("Target row position: %d @ K = %d", tarRow, k);
// Construct the bipartite graph
edges = new double[nl][n];
if (verbose) LogInfo.logs("Edge Matrix @ K = %d: ", k);
for (int i = 0; i < nl; i++) {
String str = "";
for (int j = 0; j < n; j++) {
if (!used.contains(j + 1))
edges[i][j] = findDist((2 * i + 1) * R, (2 * tarRow + 1) * R, x[j + 1], y[j + 1]);
else edges[i][j] = 2 * L * L;
String tmp = String.format("%.2f", edges[i][j]);
str += (tmp + "\t");
}
if (verbose) LogInfo.logs(str);
}
// naive(edges, tarRow);
Hungarian hungarian = new Hungarian(edges);
int[] ret = hungarian.execute();
LogInfo.logs("Matching result by Hungarian Algo. @ K = %d: ", k);
String str = "";
for (int i = 0; i < ret.length; i++) {
str += ret[i] + "\t";
int sidx = ret[i];
x[sidx + 1] = (2 * i + 1) * R;
y[sidx + 1] = (2 * tarRow + 1) * R;
used.add(sidx + 1);
}
LogInfo.logs("[%s]", str);
LogInfo.end_track();
}
printRet();
// printRet4Draw();
}
private void computeReverseFormulaInfo() {
List<BinaryFormulaInfo> entriesToAdd = new LinkedList<FbFormulasInfo.BinaryFormulaInfo>();
for (Formula formula : binaryFormulaInfoMap.keySet()) {
BinaryFormulaInfo info = binaryFormulaInfoMap.get(formula);
Formula reverseFormula = reverseFormula(formula);
if (!binaryFormulaInfoMap.containsKey(reverseFormula)) {
entriesToAdd.add(
new BinaryFormulaInfo(
reverseFormula,
info.expectedType2,
info.expectedType1,
info.unitId,
info.unitDesc,
info.descriptions,
info.popularity));
}
}
LogInfo.logs("Adding reverse formulas: " + entriesToAdd.size());
for (BinaryFormulaInfo e : entriesToAdd) {
binaryFormulaInfoMap.put(e.formula, e);
}
}
public static void create(String inFile, String outFile, String th) throws Exception {
BufferedReader br = new BufferedReader(new FileReader(inFile));
BufferedWriter bw = new BufferedWriter(new FileWriter(outFile));
Double threshold = Double.parseDouble(th);
LogInfo.logs("Threshold: %f", threshold);
String line = "";
int size = 0;
LogInfo.logs("Start to create Synsets...");
while ((line = br.readLine()) != null) {
LogInfo.begin_track("New Sysnet...");
String[] spt = line.split("\t");
ArrayList<String> list = new ArrayList<>();
for (int i = 1; i < spt.length; i++) {
String[] words = spt[i].split(" ");
StringBuffer phrase = new StringBuffer();
phrase.append(words[0]);
for (int j = 1; j < words.length - 1; j++) phrase.append(" " + words[j]);
String relation = phrase.toString();
Double score = Double.parseDouble(words[words.length - 1]);
// if (!set.contains(relation) && score > threshold) {
if (score > threshold) {
list.add(spt[i]);
set.add(relation);
LogInfo.logs("relation: %s, score: %f", relation, score);
}
}
if (list.size() > 1) {
size++;
bw.write("#" + list.get(0));
for (int j = 1; j < list.size(); j++) bw.write("\t" + list.get(j));
bw.write("\n");
LogInfo.logs("size: %d", list.size());
}
LogInfo.end_track();
}
br.close();
bw.close();
LogInfo.logs("Synsets created! size: %d", size);
}
public static void printXY() {
String str = "[";
for (int j = 1; j <= n - 1; j++) str += ("(" + x[j] + "," + y[j] + ")\t");
str += ("(" + x[n] + "," + y[n] + ")]");
LogInfo.logs(str);
}
public static void main(String[] args) throws Exception {
NERParser instance = NERParser.getInstance();
LogInfo.logs(
instance.getNamedEntity("who is barack obama and it's already 7:20 a.m.").toString());
}
// Print using LogInfo.
public void log() {
LogInfo.logs("%s", toString());
}
private void processExamples(String group, List<Example> examples) {
Evaluation evaluation = new Evaluation();
if (examples.isEmpty()) return;
final String prefix = "iter=0." + group;
Execution.putOutput("group", group);
LogInfo.begin_track_printAll("Processing %s: %s examples", prefix, examples.size());
LogInfo.begin_track("Dumping metadata");
dumpMetadata(group, examples);
LogInfo.end_track();
LogInfo.begin_track("Examples");
for (int e = 0; e < examples.size(); e++) {
Example ex = examples.get(e);
LogInfo.begin_track_printAll("%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
ex.log();
Execution.putOutput("example", e);
StopWatchSet.begin("Parser.parse");
ParserState state = builder.parser.parse(params, ex, false);
StopWatchSet.end();
out.printf("########## Example %s ##########\n", ex.id);
dumpExample(exampleToLispTree(state));
LogInfo.logs("Current: %s", ex.evaluation.summary());
evaluation.add(ex.evaluation);
LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
LogInfo.end_track();
ex.predDerivations.clear(); // To save memory
}
LogInfo.end_track();
LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary());
evaluation.logStats(prefix);
evaluation.putOutput(prefix);
LogInfo.end_track();
}
private String generateCvtQuestion(
FormulaGenerationInfo fgInfo,
String description1,
String description2,
Tree t1,
Tree t2,
List<String> posTags1,
List<String> posTags2) {
boolean isReversed = fbFormulasInfo.isReversed(fgInfo.bInfo.formula);
String type1Desc = getType1Desc(fgInfo);
if (type1Desc == null) return null;
String qWord = fgInfo.getQuestionWord();
String entityDesc = fgInfo.entityInfo1.desc;
description1 = description1.toLowerCase();
description2 = description2.toLowerCase();
String root1 = t1.label().toString();
String root2 = t2.label().toString();
String res;
if (!isReversed) {
// NP NP
if ((isNPorXorSorFRAG(root1)
|| root1.equals("PP")
|| root1.equals("SINV")
|| root1.equals("SBAR"))
&& isNPorXorSorFRAG(root2))
res =
Joiner.on(' ')
.join(new String[] {qWord, type1Desc, "has the", description2, entityDesc, "?"});
// VP NP
else if ((isADJPorVPorUCP(root1) && isNPorXorSorFRAG(root2)))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, description1, "has the", description2, entityDesc, "?"
}); // VP NP non, ADJP NP non
// NP VP or VP VP
else if (isNPorXorSorFRAG(root1) && isADJPorVPorUCP(root2)
|| isADJPorVPorUCP(root1) && isADJPorVPorUCP(root2))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is", description2, entityDesc, "?"
}); // NP VP non, NP ADJP non, S NP non
// other
else if ((root1.equals("VP") && root2.equals("SINV"))
|| (root1.equals("NP") && root2.equals("ADVP"))
|| (root1.equals("NP") && root2.equals("PP"))
|| (root1.equals("NP") && root2.equals("SINV")))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, description1, description2, entityDesc, "?"
}); // NP FRAG non
else
throw new RuntimeException(
"Does not handle: "
+ fgInfo.bInfo
+ ", root1="
+ root1
+ ", root2="
+ root2
+ ", is reversed="
+ isReversed);
} else {
if (isNPorFRAG(root1) && isNPorXorFRAG(root2)) {
if (FreebaseInfo.isPrimitive(fgInfo.bInfo.expectedType1))
res =
Joiner.on(' ')
.join(
new String[] {
qWord,
type1Desc,
"is the",
description1,
"of",
description2,
"of",
entityDesc,
"?"
}); // NP NP reverse, NP FRAG reverse, FRAG S reverse
else
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is the", description1, "of", entityDesc, "?"
}); // NP NP reverse, NP FRAG reverse, FRAG S reverse
} else if (isNPorXorFRAG(root1)
&& root2.equals(
"S")) // the S gives verb that provides information that's why we use description2 and
// not description1
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is the", description2, "by", entityDesc, "?"
}); // NP S reverse
else if (root1.equals("S") && root2.equals("S")
|| root1.equals("S")
&& root2.equals("FRAG")) // maybe should split if its N V or V N - think a bit more
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is the", description1, "by", entityDesc, "?"
}); // S S reverse,
else if (root1.equals("S")
&& root2.equals("NP")) // maybe should think about this one a bit more
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is", entityDesc, "a", description2, "of"
}); // S NP reverse
else if ((isADJPorVPorUCP(root1) && isNPorXorSorFRAG(root2))
|| (root1.equals("SINV") && isNPorXorFRAG(root2)))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is", entityDesc, description1
}); // VP NP reverse, ADJP NP rev, VP S reverse
// added for a case where there were pos errors so this is just so that there is no exception
else if ((root1.equals("VP") && root2.equals("VP")))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "do", entityDesc, description1
}); // VP NP reverse, ADJP NP rev, VP S reverse
else if (isNPorXorSorFRAG(root1) && isADJPorVPorUCP(root2)
|| isNPorXorSorFRAG(root1) && root2.equals("PP"))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is", description1, "that", entityDesc, description2, "?"
}); // NP VP rev, NP ADJP rev
else if (root1.equals("SINV") && root2.equals("VP"))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, description1, "that", entityDesc, description2, "?"
}); // NP VP rev, NP ADJP rev
else if ((isNPorXorSorFRAG(root1) || root1.equals("ADJP")) && root2.equals("SINV"))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, "is the", type1Desc, "for", entityDesc, "that", description2
}); // NP SINV rev
else if ((root1.equals("NP") && root2.equals("SBAR")))
res =
Joiner.on(' ')
.join(
new String[] {qWord, type1Desc, "is the", description2, "of", entityDesc, "?"});
// handle things that start with PP
else if ((root1.equals("ADVP") || root1.equals("PP") || root1.equals("X"))
&& (root2.equals("NP")
|| root2.equals("FRAG")
|| root2.equals("ADJP")
|| root2.equals("X")))
res =
Joiner.on(' ')
.join(
new String[] {
qWord,
type1Desc,
"is the",
description1,
"in",
description2,
"of",
entityDesc,
"?"
});
else if (((root1.equals("ADVP") || root1.equals("PP")) && root2.equals("S")))
res =
Joiner.on(' ')
.join(
new String[] {
qWord,
type1Desc,
"is",
description1,
"of",
description2,
"by",
entityDesc,
"?"
}); // NP VP rev, NP ADJP rev
else if (root1.equals("PP") && root2.equals("VP"))
res =
Joiner.on(' ')
.join(
new String[] {
qWord, type1Desc, "is", description1, "of", entityDesc, description2, "?"
}); // NP VP rev, NP ADJP rev
else
throw new RuntimeException(
"Does not handle: "
+ fgInfo.bInfo
+ ", root1="
+ root1
+ ", root2="
+ root2
+ ", is reversed="
+ isReversed);
}
if (opts.verbose >= 3)
LogInfo.logs(
"QuestionGenration: binary=%s, expType1=%s, exType2=%s, description1=%s, description2=%s, root1=%s, root2=%s, isReverse=%s, res=%s",
fgInfo.bInfo.formula,
fgInfo.bInfo.expectedType1,
fgInfo.bInfo.expectedType2,
description1,
description2,
root1,
root2,
isReversed,
res);
return res;
}
