private Object request(
String url, boolean post, Hashtable params, boolean basicAuth, boolean processOutput)
throws Exception {
HttpConnection conn = null;
Writer writer = null;
InputStream is = null;
try {
if (!post && (params != null)) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStreamWriter osw = new OutputStreamWriter(baos, this.encoding);
this.encodeParams(params, osw);
osw.flush();
osw.close();
osw = null;
url += "?" + new String(baos.toByteArray(), this.encoding);
baos = null;
}
conn = (HttpConnection) Connector.open(url);
conn.setRequestMethod(post ? HttpConnection.POST : HttpConnection.GET);
if (basicAuth) {
if (!this.areCredentialsSet()) throw new Exception("Credentials are not set");
String token = base64Encode((this.user + ":" + this.pass).getBytes(this.encoding));
conn.setRequestProperty("Authorization", "Basic " + token);
}
if (post && (params != null)) {
OutputStream os = conn.openOutputStream();
writer = new OutputStreamWriter(os, this.encoding);
this.encodeParams(params, writer);
writer.flush();
writer.close();
os = null;
writer = null;
}
int code = conn.getResponseCode();
if ((code != 200) && (code != 302))
throw new Exception("Unexpected response code " + code + ": " + conn.getResponseMessage());
is = conn.openInputStream();
if (processOutput) {
synchronized (this.json) {
return this.json.parse(is);
}
} else {
this.pump(is, System.out, 1024);
return null;
}
} finally {
if (writer != null) writer.close();
if (is != null) is.close();
if (conn != null) conn.close();
}
}
public static void main(String[] args) throws Exception {
String sourceUrlString = "data/test.html";
if (args.length == 0)
System.err.println("Using default argument of \"" + sourceUrlString + '"');
else sourceUrlString = args[0];
if (sourceUrlString.indexOf(':') == -1) sourceUrlString = "file:" + sourceUrlString;
StreamedSource streamedSource = new StreamedSource(new URL(sourceUrlString));
// streamedSource.setBuffer(new char[65000]); // uncomment this to use a fixed buffer size
Writer writer = null;
try {
writer =
new OutputStreamWriter(
new FileOutputStream("StreamedSourceCopyOutput.html"), streamedSource.getEncoding());
System.out.println("Processing segments:");
int lastSegmentEnd = 0;
for (Segment segment : streamedSource) {
System.out.println(segment.getDebugInfo());
if (segment.getEnd() <= lastSegmentEnd)
continue; // if this tag is inside the previous tag (e.g. a server tag) then ignore it as
// it was already output along with the previous tag.
lastSegmentEnd = segment.getEnd();
if (segment instanceof Tag) {
Tag tag = (Tag) segment;
// HANDLE TAG
// Uncomment the following line to ensure each tag is valid XML:
// writer.write(tag.tidy()); continue;
} else if (segment instanceof CharacterReference) {
CharacterReference characterReference = (CharacterReference) segment;
// HANDLE CHARACTER REFERENCE
// Uncomment the following line to decode all character references instead of copying them
// verbatim:
// characterReference.appendCharTo(writer); continue;
} else {
// HANDLE PLAIN TEXT
}
// unless specific handling has prevented getting to here, simply output the segment as is:
writer.write(segment.toString());
}
writer.close();
System.err.println(
"\nA copy of the source document has been output to StreamedSourceCopyOuput.html");
} catch (Exception ex) {
if (writer != null)
try {
writer.close();
} catch (IOException ex2) {
}
throw ex;
}
}
public void createExcel(String file) {
String filename = createPath(file);
System.out.println(filename);
/*
try {
WorkbookSettings ws = new WorkbookSettings();
ws.setLocale(new Locale("sv", "SE"));
WritableWorkbook workbook = Workbook.createWorkbook(new File(filename), ws);
WritableSheet s = workbook.createSheet("Sheet1", 0);
//writeDataSheet(s);
workbook.write();
workbook.close();
} catch (IOException e) {
e.printStackTrace();
} catch (WriteException e) {
e.printStackTrace();
}**/
try {
File newFile = new File(filename);
Writer out =
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newFile), "UTF8"));
// BufferedWrite output = new BufferedWriter(new OutputStreamReader(new
// FileInputS§tream(filename), "iso-8859-1"));
// FileWriter fw = new FileWriter(newFile.getAbsoluteFile());
// BufferedWriter bw = new BufferedWriter(fw);
// bw.write("Artikel;Antal/st;Pris/st;Total\n");
out.append("Artikel;Antal/st;Pris/st;Total\n");
for (Data d : dataList) {
out.append(d.toString());
out.append("\n");
// bw.write(d.toString());
// bw.write("\n");
}
// bw.close();
out.flush();
out.close();
} catch (UnsupportedEncodingException unsuppEn) {
unsuppEn.printStackTrace();
} catch (IOException ioE) {
ioE.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
public void testFormat() throws Exception {
localFs = FileSystem.getLocal(defaultConf);
localFs.delete(workDir, true);
Job job = new Job(new Configuration(defaultConf));
Path file = new Path(workDir, "test.txt");
int seed = new Random().nextInt();
Random random = new Random(seed);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// create a file with length entries
Writer writer = new OutputStreamWriter(localFs.create(file));
try {
MyClass mc = new MyClass();
for (int i = 0; i < length; i++) {
mc.s = Integer.toString(i);
mc.v = i;
byte[] raw = MessagePack.pack(mc);
byte[] b64e = base64_.encodeBase64(raw);
byte[] b64d = base64_.decode(b64e);
MyClass mc2 = MessagePack.unpack(b64d, mc.getClass());
assertEquals(mc.s, mc2.s);
assertEquals(mc.v, mc2.v);
writer.write(base64_.encodeToString(raw));
}
} finally {
writer.close();
}
checkFormat(job);
}
}
private static void main2(String[] args) {
Config.cmdline(args);
try {
javabughack();
} catch (InterruptedException e) {
return;
}
setupres();
MainFrame f = new MainFrame(null);
if (Utils.getprefb("fullscreen", false)) f.setfs();
f.mt.start();
try {
f.mt.join();
} catch (InterruptedException e) {
f.g.interrupt();
return;
}
dumplist(Resource.remote().loadwaited(), Config.loadwaited);
dumplist(Resource.remote().cached(), Config.allused);
if (ResCache.global != null) {
try {
Writer w = new OutputStreamWriter(ResCache.global.store("tmp/allused"), "UTF-8");
try {
Resource.dumplist(Resource.remote().used(), w);
} finally {
w.close();
}
} catch (IOException e) {
}
}
System.exit(0);
}
private void generateReport(String fileName, Template featureResult, VelocityContext context)
throws Exception {
Writer writer = new FileWriter(new File(reportDirectory, fileName));
featureResult.merge(context, writer);
writer.flush();
writer.close();
}
public static void closeWriterAppend(Writer writer) {
try {
writer.close();
} catch (IOException e) {
logmex("ERR", 0, "FILE WRITER CSV OUTPUT closing", null, e);
}
}
public int dumpNames(String fileName) throws IOException {
File f = new File(fileName);
final int[] entries = {0};
final Writer out = new FileWriter(f);
iterateThru(
new IndexEntriesWalker() {
public void process(String ctxPath, String value, String[] attrs) {
try {
out.write(ctxPath + " " + buildAppendix(value, attrs) + "\n");
entries[0]++;
} catch (IOException e) {
throw new Error(e);
}
}
});
iterateFileNames(
new IndexEntriesWalkerInterruptable() {
public boolean process(String ctxPath, String value) {
try {
out.write(
ctxPath + " " + (value == null || value.length() == 0 ? NO_VAL : value) + "\n");
entries[0]++;
return true;
} catch (IOException e) {
throw new Error(e);
}
}
});
out.close();
return entries[0];
}
public void buildDict(Set<String> dict) throws IOException {
int c = 0;
for (String key : dict) {
if (++c % 1000 == 0) {
writer.flush();
System.err.println(c + "/" + dict.size() + " items processed.");
}
IdTouple touple = getIdTouple(key);
// touple.oid = guessOther(key);
writer
.append(key)
.append("\t")
.append(touple.gid)
.append("\t")
.append(touple.pname) /*.append("\t").append(touple.oid)*/
.append("\n");
writer.flush();
}
writer.close();
if (file != null) {
FileWriter fw = new FileWriter(file);
fw.write(new Date().toString());
fw.close();
}
}
public static void saveFile(final Context context, final File file, final String content)
throws IOException {
context.log("Saving file: " + file.getAbsolutePath());
final FileOutputStream fos = new FileOutputStream(file);
try {
final Writer writer = new OutputStreamWriter(fos, "UTF-8");
writer.write(content);
writer.close();
} finally {
fos.close();
}
}
public DumpResource(String resource_name, String locale_name) {
// Split locale_name into language_country_variant.
String language;
String country;
String variant;
language = locale_name;
{
int i = language.indexOf('_');
if (i >= 0) {
country = language.substring(i + 1);
language = language.substring(0, i);
} else country = "";
}
{
int j = country.indexOf('_');
if (j >= 0) {
variant = country.substring(j + 1);
country = country.substring(0, j);
} else variant = "";
}
Locale locale = new Locale(language, country, variant);
// Get the resource.
ResourceBundle catalog = ResourceBundle.getBundle(resource_name, locale);
// We are only interested in the messsages belonging to the locale
// itself, not in the inherited messages. But catalog.getLocale() exists
// only in Java2 and sometimes differs from the given locale.
try {
Writer w1 = new OutputStreamWriter(System.out, "UTF8");
Writer w2 = new BufferedWriter(w1);
this.out = w2;
this.catalog = catalog;
dump();
w2.close();
w1.close();
System.out.flush();
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
}
}
@Override
@SuppressWarnings("SleepWhileHoldingLock")
public void run() {
try {
// initialize the statusbar
status.removeAll();
JProgressBar progress = new JProgressBar();
progress.setMinimum(0);
progress.setMaximum(doc.getLength());
status.add(progress);
status.revalidate();
// start writing
Writer out = new FileWriter(f);
Segment text = new Segment();
text.setPartialReturn(true);
int charsLeft = doc.getLength();
int offset = 0;
while (charsLeft > 0) {
doc.getText(offset, Math.min(4096, charsLeft), text);
out.write(text.array, text.offset, text.count);
charsLeft -= text.count;
offset += text.count;
progress.setValue(offset);
try {
Thread.sleep(10);
} catch (InterruptedException e) {
Logger.getLogger(FileSaver.class.getName()).log(Level.SEVERE, null, e);
}
}
out.flush();
out.close();
} catch (IOException e) {
final String msg = e.getMessage();
SwingUtilities.invokeLater(
new Runnable() {
public void run() {
JOptionPane.showMessageDialog(
getFrame(),
"Could not save file: " + msg,
"Error saving file",
JOptionPane.ERROR_MESSAGE);
}
});
} catch (BadLocationException e) {
System.err.println(e.getMessage());
}
// we are done... get rid of progressbar
status.removeAll();
status.revalidate();
}
/**
* Create a file that contains the specified message
*
* @param root a git repository root
* @param message a message to write
* @return a file reference
* @throws IOException if file cannot be created
*/
private File createMessageFile(VirtualFile root, final String message) throws IOException {
// filter comment lines
File file = FileUtil.createTempFile(GIT_COMMIT_MSG_FILE_PREFIX, GIT_COMMIT_MSG_FILE_EXT);
file.deleteOnExit();
@NonNls String encoding = GitConfigUtil.getCommitEncoding(myProject, root);
Writer out = new OutputStreamWriter(new FileOutputStream(file), encoding);
try {
out.write(message);
} finally {
out.close();
}
return file;
}
private static void dumplist(Collection<Resource> list, String fn) {
try {
if (fn != null) {
Writer w = new OutputStreamWriter(new FileOutputStream(fn), "UTF-8");
try {
Resource.dumplist(list, w);
} finally {
w.close();
}
}
} catch (IOException e) {
throw (new RuntimeException(e));
}
}
public boolean continueShellProcess(Process proc) {
if (progIndicator.isAborted()) {
try {
Writer stream;
stream = new OutputStreamWriter((BufferedOutputStream) proc.getOutputStream());
stream.write((char) 3);
stream.flush();
stream.close();
} catch (IOException e) {
}
return false;
}
return true;
}
public static int copy(Reader input, Writer output) throws IOException {
char[] buffer = new char[8 * 1024];
int count = 0;
int n = 0;
try {
while (-1 != (n = input.read(buffer))) {
output.write(buffer, 0, n);
count += n;
}
} finally {
output.flush();
output.close();
}
return count;
}
private void writeTransactionsToFile(String transaction) {
try {
writer =
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output, true), "utf-8"));
writer.append(transaction);
writer.append(newLine);
} catch (IOException e) {
System.out.println("DEBUG: -----Start-----\n" + e + "DEBUG: ------End------\n");
}
try {
if (writer != null) writer.close();
} catch (IOException e) {
System.out.println("DEBUG: -----Start-----\n" + e + "DEBUG: ------End------\n");
}
}
public static void main(String[] args) {
String input = args[0];
String code = compiler(input);
String output = args[1];
Writer writer = null;
try {
writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "utf-8"));
writer.write(code);
} catch (IOException ex) {
} finally {
try {
writer.close();
} catch (Exception ex) {
/*ignore*/
}
}
}
protected static void process(File dir) throws IOException {
File file = new File(dir, TESTS);
if (!file.exists()) throw new IOException(file + " not found");
List<TestCase> test_cases = new ArrayList<>();
DataInputStream input = new DataInputStream(new FileInputStream(file));
try {
for (; ; ) {
TestCase test_case = new TestCase();
try {
test_case.readFrom(input);
test_cases.add(test_case);
} catch (Exception e) {
break;
}
}
} finally {
Util.close(input);
}
if (test_cases.isEmpty()) return;
Reader stdout_reader = null, stderr_reader = null;
File tmp = new File(dir, STDOUT);
if (tmp.exists() && tmp.length() > 0) stdout_reader = new FileReader(tmp);
tmp = new File(dir, STDERR);
if (tmp.exists() && tmp.length() > 0) stderr_reader = new FileReader(tmp);
File parent = dir.getParentFile();
File xml_file = new File(parent, "TESTS-" + dir.getName() + "-" + parent.getName() + ".xml");
Writer out = new FileWriter(xml_file);
String classname = dir.getName();
String suffix = parent.getName();
if (suffix != null && !suffix.isEmpty()) classname = classname + "-" + suffix;
try {
generateReport(out, classname, test_cases, stdout_reader, stderr_reader);
} finally {
out.close();
if (stdout_reader != null) stdout_reader.close();
if (stderr_reader != null) stderr_reader.close();
}
}
protected void doExportData() {
FileDialog dialog = new FileDialog(this, "Export Data File...", FileDialog.SAVE);
dialog.setVisible(true);
if (dialog.getFile() != null) {
File file = new File(dialog.getDirectory(), dialog.getFile());
Writer writer = null;
try {
writer = new PrintWriter(file);
treesPanel.writeDataFile(writer);
writer.close();
} catch (IOException ioe) {
JOptionPane.showMessageDialog(
this,
"Error writing data file: " + ioe.getMessage(),
"Export Error",
JOptionPane.ERROR_MESSAGE);
}
}
}
/**
* XMLをファイルに出力する。
*
* @param my_doc XMLの Document Node 。
* @param os 出力先のストリーム。
* @param sortflag ソートするか否か。
*/
public static void output(Document my_doc, OutputStream os, boolean sortflag, UDF_Env env) {
if (sortflag) {
try {
com.udfv.util.UDF_XML.sortDocumentbyEnv(my_doc, env);
} catch (Exception e1) {
e1.printStackTrace();
}
}
try {
OutputFormat format = new OutputFormat(my_doc, "UTF-8", true);
format.setLineWidth(0);
// Writer out = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
Writer out = new OutputStreamWriter(os, "UTF-8");
XMLSerializer serial = new XMLSerializer(out, format);
serial.serialize(my_doc.getDocumentElement());
out.close();
} catch (Exception e2) {
e2.printStackTrace();
}
}
/**
* Write the data in a Graph to a GML OutputStream.
*
* @param gMLOutputStream the GML OutputStream to write the Graph data to
* @throws java.io.IOException thrown if there is an error generating the GML data
*/
public void outputGraph(final OutputStream gMLOutputStream) throws IOException {
// ISO 8859-1 as specified in the GML documentation
Writer writer =
new BufferedWriter(new OutputStreamWriter(gMLOutputStream, Charset.forName("ISO-8859-1")));
List<Vertex> verticies = new ArrayList<Vertex>();
List<Edge> edges = new ArrayList<Edge>();
populateLists(verticies, edges);
if (normalize) {
LexicographicalElementComparator comparator = new LexicographicalElementComparator();
Collections.sort(verticies, comparator);
Collections.sort(edges, comparator);
}
writeGraph(writer, verticies, edges);
writer.flush();
writer.close();
}
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// get the number of workers to run
int count = this.getRequestedCount(request, 4);
// get the executor service
final ServletContext sc = this.getServletContext();
final ExecutorService executorService = (ExecutorService) sc.getAttribute("myExecutorService");
// create work coordinator
CountDownLatch countDownLatch = new CountDownLatch(count);
Long t1 = System.nanoTime();
// create the workers
List<RunnableWorkUnit2> workers = new ArrayList<RunnableWorkUnit2>();
for (int i = 0; i < count; i++) {
RunnableWorkUnit2 wu =
new RunnableWorkUnit2("RunnableTask" + String.valueOf(i + 1), countDownLatch);
workers.add(wu);
}
// run the workers through the executor
for (RunnableWorkUnit2 wu : workers) {
executorService.execute(wu);
}
try {
System.out.println("START WAITING");
countDownLatch.await();
System.out.println("DONE WAITING");
} catch (InterruptedException ex) {
ex.printStackTrace();
}
Long t2 = System.nanoTime();
Writer w = response.getWriter();
w.write(String.format("\n Request processed in %dms!", (t2 - t1) / 1000000));
w.flush();
w.close();
}
public File generateDotFile(String outputDirectory) {
File dotFile = new File(outputDirectory + FILE_SEPARATOR + "dotFile.dot");
Writer out = null;
try {
out = new OutputStreamWriter(new FileOutputStream(dotFile));
out.write("digraph testDependencies {" + NEWLINE);
String failedConfigMethod = "";
if (!failedConfigurations.isEmpty()) {
out.write("compound=true;" + NEWLINE); // otherwise lhead will not work
out.write(NEWLINE + "subgraph clusterFailedConfigs {" + NEWLINE);
out.write("label = \"Failed Configuration Methods\"" + NEWLINE);
// out.write("failedConfig;" + NEWLINE);
// TODO extract CSS
// FIXME does not work - expected border around failedConfigs cluster
out.write("color=blue;" + NEWLINE);
// out.write("node [color=white];" + NEWLINE);
for (ITestNGMethod failedConfig : failedConfigurations) {
out.write(
getClassAndMethodString(failedConfig)
+ " [label=\""
+ failedConfig.getMethodName()
+ "\"];"
+ NEWLINE);
failedConfigMethod = getClassAndMethodString(failedConfig);
}
out.write("}" + NEWLINE + NEWLINE);
}
out.write(" rankdir=BT;" + NEWLINE);
for (ITestNGMethod failedMethod : failedMethods) {
// log("failed method: " + failedMethod.getMethodName());
out.write(
getClassAndMethodString(failedMethod)
+ "[label=\""
+ failedMethod.getMethodName()
+ "\","
+ cssFailedMethod
+ "];"
+ NEWLINE);
for (String depMethod : failedMethod.getMethodsDependedUpon()) {
out.write(
getClassAndMethodString(failedMethod)
+ DEPENDS_UPON
+ getClassAndMethodString(depMethod)
+ NEWLINE);
}
}
for (ITestNGMethod skippedMethod : skippedMethods) {
// log("skipped method: " + skippedMethod.getMethodName());
out.write(
getClassAndMethodString(skippedMethod)
+ "[label=\""
+ skippedMethod.getMethodName()
+ "\","
+ cssSkippedMethod
+ "];"
+ NEWLINE);
if (skippedMethod.getMethodsDependedUpon().length > 0) {
for (String depMethod : skippedMethod.getMethodsDependedUpon()) {
out.write(
getClassAndMethodString(skippedMethod)
+ DEPENDS_UPON
+ getClassAndMethodString(depMethod)
+ NEWLINE);
}
} else {
if (skippedMethod.getGroupsDependedUpon().length > 0) {
for (String depGroup : skippedMethod.getGroupsDependedUpon()) {
out.write(
getClassAndMethodString(skippedMethod)
+ DEPENDS_UPON
+ "Group_"
+ depGroup
+ NEWLINE);
}
} else {
out.write(
getClassAndMethodString(skippedMethod)
+ DEPENDS_UPON
+ failedConfigMethod
+ NEWLINE);
}
}
}
/**
* // let us gather all groups that are for (TestMethod method : methods) {
*
* <p>//log(method.getName() + " no groups: " + method.getGroupsDepUpon().isEmpty());
* //log(method.getName() + " no methods: " + method.getMethodsDepUpon().isEmpty());
*
* <p>//if (method.getGroupsDepUpon().isEmpty() && method.getMethodsDepUpon().isEmpty()) {
* // out.write(method.getName() + DEPENDS_UPON + "failedConfig" + NEWLINE); //}
*
* <p>// // draw dependency edges: method to method // for (String dependedUponMethod :
* method.getMethodsDepUpon()) { // out.write(method.getName() + DEPENDS_UPON +
* dependedUponMethod + NEWLINE); // }
*
* <p>// draw dependency edges: method to group for (String dependedUponGroup :
* method.getGroupsDepUpon()) { out.write(method.getName() + DEPENDS_UPON +
* dependedUponGroup + NEWLINE); } }
*/
// draw group nodes
for (String group : uniqueGroups) {
// failed - red
if (failedGroups.contains(group)) {
out.write(getGroupName(group) + cssFailedGroup + NEWLINE);
}
// skipped - yello
// FIXME to be removed? there is no such thing as skipped group
else if (skippedGroups.contains(group)) {
out.write(getGroupName(group) + cssSkippedGroup + NEWLINE);
}
// normal groups
else {
out.write(getGroupName(group) + cssGroup + NEWLINE);
}
}
// all groups on the same level
if (!uniqueGroups.isEmpty()) {
out.write("{ rank = same;");
for (String group : uniqueGroups) {
out.write(" \"Group_" + group + "\"; ");
}
}
out.write("}" + NEWLINE);
/* for (ISuite suite : suites) {
//System.out.println("isuite: " + suite.getName());
// out.write("isuite: " + suite.getName() + NEWLINE);
for (Map.Entry<String,Collection<ITestNGMethod>> entry : suite.getMethodsByGroups().entrySet()) {
//out.write("entry: " + entry.getKey() + NEWLINE);
for (ITestNGMethod method : entry.getValue()) {
//out.write("\tmethod: " + method.getMethodName() + NEWLINE);
//out.write(method.getMethodName() + " groups: " + Arrays.deepToString(method.getGroups()) + NEWLINE);
//out.write(method.getMethodName() + " dep groups: " + Arrays.deepToString(method.getGroupsDependedUpon()) + NEWLINE);
for (String dependedUponMethod : method.getMethodsDependedUpon()) {
out.write(method.getMethodName() + DEPENDS_UPON + dependedUponMethod.substring(dependedUponMethod.lastIndexOf(".")+1) + NEWLINE);
//out.write("dep upon method: " + dependedUponMethod + NEWLINE);
//String[] tokens = dependedUponMethod.split("\\.");
//out.write(Arrays.deepToString(tokens));
//out.write(method.getMethodName() + DEPENDS_UPON + tokens[tokens.length-2] + "." + tokens[tokens.length-1]);
}
for (String dependedUponGroup : method.getGroupsDependedUpon()) {
out.write(method.getMethodName() + DEPENDS_UPON + dependedUponGroup + NEWLINE);
//out.write("dep upon group: " + dependedUponGroup + NEWLINE);
}
}
}
}*/
out.write("}" + NEWLINE);
// FIXME exception handling
} catch (FileNotFoundException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} catch (IOException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} finally {
try {
out.close();
} catch (IOException e) {
// FIXME
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
return dotFile;
}
public void close() throws IOException {
wrappedWriter.close();
}
/**
* parse sentence and generate .trees file
*
* @param en
* @param align
* @param out
*/
public static void parse(String en, String align, String out, boolean verbose) {
// use alignments?
boolean use_alignments = true;
if (align.startsWith("no_align")) {
use_alignments = false;
System.err.println("Not using alignments.");
} else {
System.err.println("Using alignments from " + align);
}
// setup stanfordparser
String grammar = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
String[] options = {"-outputFormat", "wordsAndTags, typedDependencies"};
LexicalizedParser lp = LexicalizedParser.loadModel(grammar, options);
TreebankLanguagePack tlp = lp.getOp().langpack();
java.util.function.Predicate<java.lang.String> punctuationFilter = x -> true;
GrammaticalStructureFactory gsf =
new edu.stanford.nlp.trees.EnglishGrammaticalStructureFactory(punctuationFilter);
// read document
Iterable<List<? extends HasWord>> sentences;
Reader r = new Reader(en);
String line = null;
List<List<? extends HasWord>> tmp = new ArrayList<List<? extends HasWord>>();
while ((line = r.getNext()) != null) {
Tokenizer<? extends HasWord> token =
tlp.getTokenizerFactory().getTokenizer(new StringReader(line));
List<? extends HasWord> sentence = token.tokenize();
tmp.add(sentence);
}
sentences = tmp;
// set up alignment file reader
Reader alignment = new Reader();
if (use_alignments) {
alignment = new Reader(align);
}
// set up tree file writer
Writer treeWriter = new Writer(out);
// parse
long start = System.currentTimeMillis();
// System.err.print("Parsing sentences ");
int sentID = 0;
for (List<? extends HasWord> sentence : sentences) {
Tree t = new Tree();
// t.setSentID(++sentID);
System.err.println("parse Sentence :" + sentence + "...");
// System.err.print(".");
System.err.println("-----------------------------------------------------------------------");
edu.stanford.nlp.trees.Tree parse = lp.parse(sentence);
// parse.pennPrint();
// List for root node and lexical nodes
List<Node> loneNodes = new LinkedList<Node>();
List<Node> governingNodes = new LinkedList<Node>();
// ROOT node
Node root = new Node(true, true);
root.setTag("ROOT");
t.setRoot(root);
loneNodes.add(root);
governingNodes.add(root);
// tagging
int counter = 0;
String surface = "";
String tag = "";
for (TaggedWord tw : parse.taggedYield()) {
Node n = new Node();
Node governingNode = new Node();
n.setNodeID(++counter);
surface = tw.value();
tag = tw.tag();
if (surface.startsWith("-LRB-")) {
surface = "(";
} else if (surface.startsWith("-RRB-")) {
surface = ")";
// } else if (surface.startsWith("-LSB-")){
// surface = "[";
// } else if (surface.startsWith("-RSB-")){
// surface = "]";
// } else if (surface.startsWith("-LCB-")){
// surface = "{";
// } else if (surface.startsWith("-RCB-")){
// surface = "}";
} else if (surface.startsWith("''")) {
surface = "\"";
}
tag = tag.replaceAll("#", "-NUM-");
surface = surface.replaceAll("&", "-AMP-");
surface = surface.replaceAll("#", "-NUM-");
surface = surface.replaceAll(">", "-GRE-");
surface = surface.replaceAll("=", "-EQU-");
n.setInitialLexicalIndex(counter);
governingNode.setInitialLexicalIndex(counter);
n.setSurface(surface);
// System.out.print("("+tw.value()+" : ");
n.setTag(tag);
governingNode.setTag("_" + tag);
governingNode.setLabel("_gov");
// System.out.print(tw.tag()+")");
loneNodes.add(n);
governingNodes.add(governingNode);
governingNode.setChild(n);
}
// System.out.println("");
// t.setSentLength(t.getNodes().size() - 1);
// List<Node> loneNodes = new LinkedList<Node>();
Node[] nodes = new Node[2000];
// labeling
int depIndex;
int govIndex;
String[] depInfo;
String[] govInfo;
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse);
List<TypedDependency> tdl = gs.typedDependencies(false);
// List<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
for (TypedDependency td : tdl) {
depIndex = td.dep().index();
govIndex = td.gov().index();
// System.out.println("Index1:"+depIndex);
// System.out.println("Index2:"+govIndex);
// if (nodes[depIndex] == null){
// System.out.println("Making node!");
// nodes[depIndex] = new Node();
// }
// if (nodes[govIndex] == null){
// System.out.println("Making node!");
// nodes[govIndex] = new Node();
// }
Node dep = loneNodes.get((depIndex));
Node gov = governingNodes.get((govIndex));
Node depcopy = governingNodes.get((depIndex));
Node govcopy = loneNodes.get((govIndex));
dep.setLabel(td.reln().toString());
depcopy.setLabel(td.reln().toString());
govcopy.setLabel("head");
// System.out.println(td.toString());
govInfo = td.gov().toString().split("/");
depInfo = td.dep().toString().split("/");
// System.out.println(td.gov().toString());
// System.out.println(td.dep().toString());
// dep.setSurface(depInfo[0]);
// dep.setTag(depInfo[1]);
gov.setChild(governingNodes.get(depIndex));
governingNodes.get(depIndex).setParent(gov);
// gov.setChild(dep);
dep.setParent(governingNodes.get(depIndex));
}
// t.setRoot(nodes[0]);
// Collapse tree to remove unneeded governing nodes:
Node gov;
Node dep;
Node parent;
List<Node> children;
for (int i = 1; i < governingNodes.size(); i++) { // start with index 1 to skip root
gov = governingNodes.get(i);
dep = loneNodes.get(i);
if (gov.getChildren().size() <= 1) {
int k = 0;
parent = gov.getParent();
children = parent.getChildren();
for (Node n : children) {
if (n == gov) {
gov.getParent().replaceChild(k, dep);
dep.setParent(gov.getParent());
}
k++;
}
}
}
// Mark head nodes with appropriate label:
int k = 0;
for (Node n : loneNodes) {
if (k != 0) {
if (n.getLabel() == n.getParent().getLabel()) {
n.setLabel("head");
}
} else {
n.setLabel("null");
}
k++;
}
// Sort lexical children of each governing node in lexical order
for (Node n : governingNodes) {
n.sortChildrenByInitialIndex();
}
// combine with alignment
if (use_alignments) {
t.initialize(alignment.readNextAlign());
} else {
t.initializeUnaligned();
}
// write tree to file
treeWriter.write(t);
// print tree to console
System.out.println(t.toSentence());
if (verbose) {
System.err.println(t.toString());
// t.recursivePrint();
}
System.err.println("#######################################################################");
}
long stop = System.currentTimeMillis();
System.err.println("...done! [" + (stop - start) / 1000 + " sec].");
treeWriter.close();
}
public static void main(String args[]) {
double totalTime;
double startTime = System.currentTimeMillis();
parseArguments(args);
// preprocess input files to find the chromosomal boundaries in terms of line number
/*
* All files should be sorted in chromosomal order
*/
System.out.println("Preprocessing input files to find chromosome boundaries...");
System.out.println("\tExons file..");
ArrayList<Integer> exonFileChrBoundaries = getChromosomalBoundaries(exonFileName, 0);
System.out.println("\tExpression file..");
ArrayList<Integer> exprFileChrBoundaries = getChromosomalBoundaries(expressionFileName, 2);
System.out.println("\tMapped reads file..");
ArrayList<Integer> readsFileChrBoundaries = getChromosomalBoundaries(mappedReadsFileName, 2);
try {
BufferedReader inputExons = new BufferedReader(new FileReader(exonFileName));
BufferedReader inputExpr = new BufferedReader(new FileReader(expressionFileName));
BufferedReader inputSAMData = new BufferedReader(new FileReader(mappedReadsFileName));
for (int chromosome : chromosomes) {
int arrayPosition = chromosome - chromosomes.get(0) + 1;
System.out.println("Chromosome " + chromosome);
System.out.println("Reading exons file....");
int numberOfLines =
exonFileChrBoundaries.get(arrayPosition) - exonFileChrBoundaries.get(arrayPosition - 1);
double currentTime = System.currentTimeMillis();
Exons = Exon.readExon(inputExons, chromosome, numberOfLines);
double totalExonReadTime = (System.currentTimeMillis() - currentTime) / 1000F;
Exon.sortExons(Exons);
System.out.println("Reading expression file....");
numberOfLines =
exprFileChrBoundaries.get(arrayPosition) - exprFileChrBoundaries.get(arrayPosition - 1);
ArrayList<Expression> Expressions = new ArrayList<Expression>();
currentTime = System.currentTimeMillis();
Expressions = Expression.readExon(inputExpr, chromosome, numberOfLines);
double totalExprReadTime = (System.currentTimeMillis() - currentTime) / 1000F;
int numberOfExpr = Expressions.size();
System.out.println("Calculating FPKMs....");
currentTime = System.currentTimeMillis();
Exon.getFPKM(Expressions, Exons);
double totalFPKMCalcTime = (System.currentTimeMillis() - currentTime) / 1000F;
Expressions.removeAll(Expressions); // explicitly deleting to free up memory
System.out.println("Reading mapped reads SAM file....");
numberOfLines =
readsFileChrBoundaries.get(arrayPosition)
- readsFileChrBoundaries.get(arrayPosition - 1);
ArrayList<MappedReads> mappedReads = new ArrayList<MappedReads>();
currentTime = System.currentTimeMillis();
mappedReads = MappedReads.readMappedReads(inputSAMData, chromosome, numberOfLines);
double totalReadsReadTime = (System.currentTimeMillis() - currentTime) / 1000F;
MappedReads.sort(mappedReads);
int numberOfReads = mappedReads.size();
System.out.println("Reading reference genome file....");
String referenceFileName = referenceDirectory + "/chr" + chromosome + ".fa";
currentTime = System.currentTimeMillis();
RandomAccessFile inputReference = new RandomAccessFile(referenceFileName, "r");
for (Exon e : Exons) {
e.getReferenceSequence(inputReference);
}
double totalRefCalcTime = (System.currentTimeMillis() - currentTime) / 1000F;
System.out.println("Calculating SNPs....");
currentTime = System.currentTimeMillis();
Exon.getSNPs(Exons, mappedReads);
double totalSNPsCalcTime = (System.currentTimeMillis() - currentTime) / 1000F;
mappedReads.removeAll(mappedReads);
System.out.println("Calculating States....");
currentTime = System.currentTimeMillis();
HiddenMarkovModel.getStates(Exons);
double totalStateCalcTime = (System.currentTimeMillis() - currentTime) / 1000F;
// Print output
if (outputFileName.equals("")) {
// print to stdout
for (Exon e : Exons) System.out.println(e);
} else {
Writer output = new BufferedWriter(new FileWriter(outputFileName));
for (Exon e : Exons) output.write(e + "\n");
output.close();
}
// prints the timing metrics to std out
if (printTimingMetrics) {
double endTime = System.currentTimeMillis();
totalTime = (endTime - startTime) / 1000F;
System.out.println("Total Time: " + totalTime);
System.out.println(
"Time for reading exons file : " + totalExonReadTime + ", " + Exons.size());
System.out.println(
"Time for reading expression file : " + totalExprReadTime + ", " + numberOfExpr);
System.out.println(
"Time for reading mapped reads file: " + totalReadsReadTime + ", " + numberOfReads);
System.out.println("Time for getting reference seq : " + totalRefCalcTime);
System.out.println("Time for calculating FPKM : " + totalFPKMCalcTime);
System.out.println("Time for calculating Num of SNPs : " + totalSNPsCalcTime);
System.out.println("Time for calculating States : " + totalStateCalcTime);
}
}
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
}
}
public static void main(String args[]) {
try {
SeleniumHtmlClient client = new SeleniumHtmlClient();
String testFile = null;
String testSuite = null;
String resultsFilename = null;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--host")) {
i++;
if (i < args.length) {
client.setHost(args[i]);
} else {
throw new BadUsageException("--host must be followed by a hostname");
}
} else if (args[i].equals("--port")) {
i++;
if (i < args.length) {
client.setPort(Integer.parseInt(args[i]));
} else {
throw new BadUsageException("--port must be followed by a port number");
}
} else if (args[i].equals("--browser")) {
i++;
if (i < args.length) {
client.setBrowser(args[i]);
} else {
throw new BadUsageException("--browser must be followed by a browser spec");
}
} else if (args[i].equals("--out")) {
i++;
if (i < args.length) {
resultsFilename = args[i];
} else {
throw new BadUsageException("--out must be followed by a filename");
}
/*
} else if (args[i].equals("--outdir")) {
i++;
if (i < args.length) {
client.setResultsDir(new File(args[i]));
} else {
throw new BadUsageException("--outdir must be followed by a path");
}
*/
} else if (args[i].equals("--baseurl")) {
i++;
if (i < args.length) {
client.setBaseUrl(args[i]);
} else {
throw new BadUsageException("--baseurl must be followed by a URL");
}
} else if (args[i].equals("--test")) {
i++;
if (i < args.length) {
if (testFile == null) {
testFile = args[i];
} else {
throw new BadUsageException("only one test file permitted");
}
} else {
throw new BadUsageException("--test must be followed by a test filepath");
}
} else if (args[i].equals("--testsuite")) {
i++;
if (i < args.length) {
testSuite = args[i];
} else {
throw new BadUsageException("--testsuite must be followed by a testsuite filepath");
}
} else if (args[i].equals("--verbose") || args[i].equals("-v")) {
client.setVerbose(true);
} else if (args[i].equals("--help") || args[i].equals("-h")) {
printUsage();
System.exit(0);
} else {
throw new BadUsageException("Unknown parameter " + args[i]);
}
}
if (testFile == null && testSuite == null) {
throw new BadUsageException("No test or testsuite file specified");
} else if (testFile != null && testSuite != null) {
throw new BadUsageException("A test and testsuite file cannot both be specified");
}
Writer resultsWriter = null;
if (resultsFilename != null) {
resultsWriter = new FileWriter(resultsFilename);
} else /* if (client.resultsDir == null) */ {
resultsWriter = new OutputStreamWriter(System.out);
}
client.setResultsWriter(resultsWriter);
if (testFile != null) {
client.runTest(testFile);
} else {
client.runSuite(testSuite);
}
if (resultsWriter != null) resultsWriter.close();
} catch (BadUsageException e) {
System.err.println("Error: " + e.getMessage());
System.err.println();
printUsage();
System.exit(1);
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
}
public void process() {
boolean exceptionWhenWritingLexerFile = false;
String lexerGrammarFileName = null; // necessary at this scope to have access in the catch below
// Have to be tricky here when Maven or build tools call in and must new Tool()
// before setting options. The banner won't display that way!
if (isVerbose() && showBanner) {
ErrorManager.info("ANTLR Parser Generator Version " + VERSION);
showBanner = false;
}
try {
sortGrammarFiles(); // update grammarFileNames
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
} catch (Error e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
}
for (String grammarFileName : grammarFileNames) {
// If we are in make mode (to support build tools like Maven) and the
// file is already up to date, then we do not build it (and in verbose mode
// we will say so).
if (make) {
try {
if (!buildRequired(grammarFileName)) continue;
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
}
}
if (isVerbose() && !isDepend()) {
System.out.println(grammarFileName);
}
try {
if (isDepend()) {
BuildDependencyGenerator dep = new BuildDependencyGenerator(this, grammarFileName);
/*
List outputFiles = dep.getGeneratedFileList();
List dependents = dep.getDependenciesFileList();
System.out.println("output: "+outputFiles);
System.out.println("dependents: "+dependents);
*/
System.out.println(dep.getDependencies());
continue;
}
Grammar grammar = getRootGrammar(grammarFileName);
// we now have all grammars read in as ASTs
// (i.e., root and all delegates)
grammar.composite.assignTokenTypes();
grammar.composite.defineGrammarSymbols();
grammar.composite.createNFAs();
generateRecognizer(grammar);
if (isPrintGrammar()) {
grammar.printGrammar(System.out);
}
if (isReport()) {
GrammarReport greport = new GrammarReport(grammar);
System.out.println(greport.toString());
// print out a backtracking report too (that is not encoded into log)
System.out.println(greport.getBacktrackingReport());
// same for aborted NFA->DFA conversions
System.out.println(greport.getAnalysisTimeoutReport());
}
if (isProfile()) {
GrammarReport greport = new GrammarReport(grammar);
Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME, greport.toNotifyString());
}
// now handle the lexer if one was created for a merged spec
String lexerGrammarStr = grammar.getLexerGrammar();
// System.out.println("lexer grammar:\n"+lexerGrammarStr);
if (grammar.type == Grammar.COMBINED && lexerGrammarStr != null) {
lexerGrammarFileName = grammar.getImplicitlyGeneratedLexerFileName();
try {
Writer w = getOutputFile(grammar, lexerGrammarFileName);
w.write(lexerGrammarStr);
w.close();
} catch (IOException e) {
// emit different error message when creating the implicit lexer fails
// due to write permission error
exceptionWhenWritingLexerFile = true;
throw e;
}
try {
StringReader sr = new StringReader(lexerGrammarStr);
Grammar lexerGrammar = new Grammar();
lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion;
lexerGrammar.implicitLexer = true;
lexerGrammar.setTool(this);
File lexerGrammarFullFile =
new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName);
lexerGrammar.setFileName(lexerGrammarFullFile.toString());
lexerGrammar.importTokenVocabulary(grammar);
lexerGrammar.parseAndBuildAST(sr);
sr.close();
lexerGrammar.composite.assignTokenTypes();
lexerGrammar.composite.defineGrammarSymbols();
lexerGrammar.composite.createNFAs();
generateRecognizer(lexerGrammar);
} finally {
// make sure we clean up
if (deleteTempLexer) {
File outputDir = getOutputDirectory(lexerGrammarFileName);
File outputFile = new File(outputDir, lexerGrammarFileName);
outputFile.delete();
}
}
}
} catch (IOException e) {
if (exceptionWhenWritingLexerFile) {
ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, lexerGrammarFileName, e);
} else {
ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName);
}
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e);
}
/*
finally {
System.out.println("creates="+ Interval.creates);
System.out.println("hits="+ Interval.hits);
System.out.println("misses="+ Interval.misses);
System.out.println("outOfRange="+ Interval.outOfRange);
}
*/
}
}
protected void writeDOTFile(Grammar g, String name, String dot) throws IOException {
Writer fw = getOutputFile(g, name + ".dot");
fw.write(dot);
fw.close();
}
