public static boolean download(
URL url, String file, int prefix, int totalFilesize, IProgressUpdater updater) {
File fFile = new File(new File(file).getParentFile().getPath());
if (!fFile.exists()) fFile.mkdirs();
boolean downloaded = true;
BufferedInputStream in = null;
FileOutputStream out = null;
BufferedOutputStream bout = null;
try {
int count;
int totalCount = 0;
byte data[] = new byte[BUFFER];
in = new BufferedInputStream(url.openStream());
out = new FileOutputStream(file);
bout = new BufferedOutputStream(out);
while ((count = in.read(data, 0, BUFFER)) != -1) {
bout.write(data, 0, count);
totalCount += count;
if (updater != null) updater.update(prefix + totalCount, totalFilesize);
}
} catch (Exception e) {
e.printStackTrace();
Utils.logger.log(Level.SEVERE, "Download error!");
downloaded = false;
} finally {
try {
close(in);
close(bout);
close(out);
} catch (Exception e) {
e.printStackTrace();
}
}
return downloaded;
}
/*
-- This is a helper methid to run the morph files
*/
private static void runMorphDataSet() throws Exception {
String morph_directory =
"../../thesis-datasets/morph/"; // directory where all the morph code is stored
File d = new File(morph_directory);
// get all the files from a directory
File[] fList = d.listFiles();
List<String> dir_list = new ArrayList<String>();
for (File file : fList) {
if (file.isDirectory()) {
dir_list.add(file.getName());
}
}
for (String dir : dir_list) {
directory = morph_directory + dir + "/";
System.out.println("Running TDDD " + directory);
ReadFile.readFile(directory, fileList); // read the two files
System.out.println(fileList.get(0) + " " + fileList.get(1));
preliminaryStep(directory);
startCDC();
fileList.clear();
fileArray.clear();
hashed_File_List.clear();
}
}
public void delete() {
if (delfile != null) {
this.getInsertDB().delShareFile(this.delfile);
String realpath = ServletActionContext.getRequest().getRealPath(inputPath);
File file = new File(realpath + "\\" + delfile);
if (file != null) {
file.delete();
}
}
}
public static String loadStringFromFile(String filename) {
String line = "";
File file = new File(filename);
if (!file.exists()) return line;
try {
BufferedReader reader = new BufferedReader(new FileReader(file));
line = reader.readLine();
reader.close();
} catch (Exception e) {
e.printStackTrace();
}
return line;
}
/** @param args */
public static void main(String[] args) throws Exception {
File file = new File("input.txt");
if (file.exists()) {
System.setIn(new BufferedInputStream(new FileInputStream("input.txt")));
}
out = System.out;
bw = new BufferedWriter(new PrintWriter(out));
// sc = new Scanner(System.in);
br = new BufferedReader(new InputStreamReader(System.in));
C231 t = new C231();
t.solve();
bw.close();
}
/**
* Takes a line and looks for an archive group tag. If one is found, the information from the tag
* is returned as an <CODE>ArchiveGroup</CODE>. If a tag is found, the data in the tag is stored
* in the <CODE>HashMap</CODE> provided.
*
* @param currentLine The line in which to look for the archive tag.
* @param template The <CODE>Template</CODE> to which the <CODE>ArchiveGroup</CODE> belongs.
* @return The archive request tag data as an instance of <CODE>ArchiveGroup</CODE>, <CODE>null
* </CODE> if the line passed in does not contain an arFile tag.
*/
private ArchiveGroup parseArchiveGroupTag(String currentLine, Template template) {
ArchiveGroup archiveTag;
Matcher currentMatcher = archiveRequestPattern.matcher(currentLine);
if (currentMatcher.find()) {
String fileName = currentMatcher.group(1);
archiveTag = template.getArchiveGroup(fileName);
if (archiveTag == null) {
File groupFile = new File(fileName);
archiveTag = new ArchiveGroup(groupFile.getParent(), groupFile.getName());
template.addArchiveGroup(archiveTag);
}
} else archiveTag = null;
return archiveTag;
}
/**
* @param aFile
* @throws IOException
*/
public Elf(final File aFile) throws IOException {
try {
this.efile = new ERandomAccessFile(aFile, "r");
this.ehdr = new ElfHeader(this.efile);
this.file = aFile.getAbsolutePath();
} finally {
if (this.ehdr == null) {
dispose();
}
}
}
public static void extract(String zipFile, String newPath, boolean overwrite)
throws ZipException, IOException {
File file = new File(zipFile);
ZipFile zip = new ZipFile(file);
new File(newPath).mkdir();
Enumeration<? extends ZipEntry> zipFileEntries = zip.entries();
// Process each entry
while (zipFileEntries.hasMoreElements()) {
// grab a zip file entry
ZipEntry entry = (ZipEntry) zipFileEntries.nextElement();
String currentEntry = entry.getName();
File destFile = new File(newPath, currentEntry);
// destFile = new File(newPath, destFile.getName());
File destinationParent = destFile.getParentFile();
// create the parent directory structure if needed
destinationParent.mkdirs();
if (!entry.isDirectory()) {
if (!overwrite && destFile.exists()) continue;
BufferedInputStream is = new BufferedInputStream(zip.getInputStream(entry));
int currentByte;
// establish buffer for writing file
byte data[] = new byte[BUFFER];
// write the current file to disk
FileOutputStream fos = new FileOutputStream(destFile);
BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER);
// read and write until last byte is encountered
while ((currentByte = is.read(data, 0, BUFFER)) != -1) {
dest.write(data, 0, currentByte);
}
close(dest);
close(is);
}
}
}
/*
- This method is used has a helper method to run the algo for the archive dataset
- Note the archive set has multiple directories ( one for each url )
- So Read all of the directories in first and for each directory run the code
*/
private static void runArchiveSet() throws Exception {
System.out.println("Running TDDD archive");
directory = "../../thesis-datasets/datasets2/";
File file = new File(directory);
String[] directory_list =
file.list(
new FilenameFilter() {
@Override
public boolean accept(File current, String name) {
return new File(current, name).isDirectory(); // make sure its a directory
}
});
int totalRuns = 0; // used to avg the runs in the end
int total_iter_count =
0; // this is used check how many times we will iterate through the data so we can make an
// array of that size
for (int i = startBoundary; i <= endBoundary; i += increment) total_iter_count++;
// System.out.println(Arrays.toString(directory_list));
int sets = 0;
// make the arrays to hold the respecitve info for the different verions\
// run it simulateounsly to speed the from the program!
double[] block_size_list_last_year = new double[total_iter_count];
double[] ratio_size_list_last_year = new double[total_iter_count];
double[] block_size_list_six_month = new double[total_iter_count];
double[] ratio_size_list__six_month = new double[total_iter_count];
double[] block_size_list_two_year = new double[total_iter_count];
double[] ratio_size_list_two_year = new double[total_iter_count];
int current = 0;
int six_month = 2;
int last_year = 1;
int two_year = 3;
// loop through and run the cdc for each directory
for (String dir : directory_list) {
ReadFile.readFile(directory + dir, fileList); // read all the files in this directory
preliminaryStep(directory + dir + "/"); // call the preliminaryStep on all the files
totalRuns++;
totalSize =
fileArray.get(current)
.length; // get the length of the file we will be running it against!
// run it against six month
startCDC(
block_size_list_six_month,
ratio_size_list__six_month,
fileArray.get(current),
fileArray.get(six_month),
hashed_File_List.get(current),
hashed_File_List.get(six_month));
// run it against last year
startCDC(
block_size_list_last_year,
ratio_size_list_last_year,
fileArray.get(current),
fileArray.get(last_year),
hashed_File_List.get(current),
hashed_File_List.get(last_year));
// run it against 2
startCDC(
block_size_list_two_year,
ratio_size_list_two_year,
fileArray.get(current),
fileArray.get(two_year),
hashed_File_List.get(current),
hashed_File_List.get(two_year));
// // clear the fileList and hashed_file_list array
fileArray.clear();
hashed_File_List.clear();
fileList.clear();
// if (Double.isNaN(ratio_size_list[0])){
// System.out.println(sets+" "+Arrays.toString(ratio_size_list));
// test = true;
// break;
// }
if (sets % 200 == 0) System.out.println(sets);
++sets;
} // end of directory list for loop
// now output the avged value for all the runs
// System.out.println(Arrays.toString(ratio_size_list));
System.out.println("Printing six_month");
int index = 0;
for (int i = startBoundary; i <= endBoundary; i += increment) {
// avg out the outputs
double blockSize = block_size_list_six_month[index] / (double) totalRuns;
double ratio = ratio_size_list__six_month[index] / (double) totalRuns;
System.out.println(i + " " + i / 2 + 1 + " " + i / 4 + 1 + " " + blockSize + " " + ratio);
index++;
}
System.out.println("Printing last year");
index = 0;
for (int i = startBoundary; i <= endBoundary; i += increment) {
double blockSize = block_size_list_last_year[index] / (double) totalRuns;
double ratio = ratio_size_list_last_year[index] / (double) totalRuns;
System.out.println(i + " " + blockSize + " " + ratio);
index++;
}
System.out.println("Printing two year");
index = 0;
for (int i = startBoundary; i <= endBoundary; i += increment) {
double blockSize = block_size_list_two_year[index] / (double) totalRuns;
double ratio = ratio_size_list_two_year[index] / (double) totalRuns;
System.out.println(i + " " + blockSize + " " + ratio);
index++;
}
}
public static void deleteIfExists(File file) {
if (file.exists()) file.delete();
}
public static void deleteDirectory(File f) throws IOException {
if (f.isDirectory()) {
for (File c : f.listFiles()) deleteDirectory(c);
}
if (!f.delete()) throw new FileNotFoundException("Failed to delete file: " + f);
}
private static String createUnifiedRefFile(String prefix, int numFiles) throws Exception {
if (numFiles < 2) {
println(
"Warning: createUnifiedRefFile called with numFiles = " + numFiles + "; doing nothing.");
return prefix;
} else {
File checker;
checker = new File(prefix + "1");
if (!checker.exists()) {
checker = new File(prefix + ".1");
if (!checker.exists()) {
println("Can't find reference files.");
System.exit(50);
} else {
prefix = prefix + ".";
}
}
String outFileName;
if (prefix.endsWith(".")) {
outFileName = prefix + "all";
} else {
outFileName = prefix + ".all";
}
PrintWriter outFile = new PrintWriter(outFileName);
BufferedReader[] inFile = new BufferedReader[numFiles];
int nextIndex;
checker = new File(prefix + "0");
if (checker.exists()) {
nextIndex = 0;
} else {
nextIndex = 1;
}
int lineCount = countLines(prefix + nextIndex);
for (int r = 0; r < numFiles; ++r) {
if (countLines(prefix + nextIndex) != lineCount) {
println("Line count mismatch in " + (prefix + nextIndex) + ".");
System.exit(60);
}
inFile[r] = new BufferedReader(new FileReader(prefix + nextIndex));
++nextIndex;
}
String line;
for (int i = 0; i < lineCount; ++i) {
for (int r = 0; r < numFiles; ++r) {
line = inFile[r].readLine();
outFile.println(line);
}
}
outFile.close();
for (int r = 0; r < numFiles; ++r) {
inFile[r].close();
}
return outFileName;
}
} // createUnifiedRefFile(String prefix, int numFiles)
private static boolean fileExists(String fileName) {
File checker = new File(fileName);
return checker.exists();
}
private static void processArgsAndInitialize(String[] args) throws Exception {
EvaluationMetric.set_knownMetrics();
// set default values
candFileName = "candidates.txt";
candFileFormat = "plain";
candRank = 1;
refFileName = "references.txt";
refsPerSen = 1;
metricName = "BLEU";
metricOptions = new String[2];
metricOptions[0] = "4";
metricOptions[1] = "closest";
evaluateRefs = false;
verbose = false;
int i = 0;
while (i < args.length) {
String option = args[i];
if (option.equals("-cand")) {
candFileName = args[i + 1];
} else if (option.equals("-format")) {
candFileFormat = args[i + 1];
if (!candFileFormat.equals("plain") && !candFileFormat.equals("nbest")) {
println("candFileFormat must be either plain or nbest.");
System.exit(10);
}
} else if (option.equals("-rank")) {
candRank = Integer.parseInt(args[i + 1]);
if (refsPerSen < 1) {
println("Argument for -rank must be positive.");
System.exit(10);
}
} else if (option.equals("-ref")) {
refFileName = args[i + 1];
} else if (option.equals("-rps")) {
refsPerSen = Integer.parseInt(args[i + 1]);
if (refsPerSen < 1) {
println("refsPerSen must be positive.");
System.exit(10);
}
} else if (option.equals("-m")) {
metricName = args[i + 1];
if (EvaluationMetric.knownMetricName(metricName)) {
int optionCount = EvaluationMetric.metricOptionCount(metricName);
metricOptions = new String[optionCount];
for (int opt = 0; opt < optionCount; ++opt) {
metricOptions[opt] = args[i + opt + 2];
}
i += optionCount;
} else {
println("Unknown metric name " + metricName + ".");
System.exit(10);
}
}
/*
else if (option.equals("-m")) {
metricName = args[i+1];
if (!EvaluationMetric.knownMetricName(metricName)) { println("Unknown metric name " + metricName + "."); System.exit(10); }
if (metricName.equals("BLEU")) {
metricOptions = new String[2];
metricOptions[0] = args[i+2];
metricOptions[1] = args[i+3];
i += 2;
}
}
*/
else if (option.equals("-evr")) {
int evr = Integer.parseInt(args[i + 1]);
if (evr == 1) evaluateRefs = true;
else if (evr == 0) evaluateRefs = false;
else {
println("evalRefs must be either 0 or 1.");
System.exit(10);
}
} else if (option.equals("-v")) {
int v = Integer.parseInt(args[i + 1]);
if (v == 1) verbose = true;
else if (v == 0) verbose = false;
else {
println("verbose must be either 0 or 1.");
System.exit(10);
}
} else {
println("Unknown option " + option);
System.exit(10);
}
i += 2;
} // while (i)
if (refsPerSen > 1) {
// the provided refFileName might be a prefix
File dummy = new File(refFileName);
if (!dummy.exists()) {
refFileName = createUnifiedRefFile(refFileName, refsPerSen);
}
} else {
checkFile(refFileName);
}
// initialize
numSentences = countLines(refFileName) / refsPerSen;
// read in reference sentences
refSentences = new String[numSentences][refsPerSen];
BufferedReader inFile_refs = new BufferedReader(new FileReader(refFileName));
String line;
for (i = 0; i < numSentences; ++i) {
for (int r = 0; r < refsPerSen; ++r) {
// read the rth reference translation for the ith sentence
refSentences[i][r] = inFile_refs.readLine();
}
}
inFile_refs.close();
// set static data members for the EvaluationMetric class
EvaluationMetric.set_numSentences(numSentences);
EvaluationMetric.set_refsPerSen(refsPerSen);
EvaluationMetric.set_refSentences(refSentences);
// do necessary initialization for the evaluation metric
evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
println("Processing " + numSentences + " sentences...");
} // processArgsAndInitialize(String[] args)
