/**
* Generate the xml top level element and start streaming the components.
*
* @param components
* @param contentHandler
* @throws SAXException
*/
protected static void generateXML(
Components components, ContentHandler contentHandler, boolean isScrPrivateFile)
throws SAXException {
// detect namespace to use
final String namespace;
if (components.getSpecVersion() == Constants.VERSION_1_0) {
namespace = NAMESPACE_URI_1_0;
} else if (components.getSpecVersion() == Constants.VERSION_1_1) {
namespace = NAMESPACE_URI_1_1;
} else {
namespace = NAMESPACE_URI_1_1_FELIX;
}
contentHandler.startDocument();
contentHandler.startPrefixMapping(PREFIX, namespace);
// wrapper element to generate well formed xml
contentHandler.startElement(
"",
ComponentDescriptorIO.COMPONENTS,
ComponentDescriptorIO.COMPONENTS,
new AttributesImpl());
IOUtils.newline(contentHandler);
for (final Component component : components.getComponents()) {
if (component.isDs()) {
generateXML(namespace, component, contentHandler, isScrPrivateFile);
}
}
// end wrapper element
contentHandler.endElement(
"", ComponentDescriptorIO.COMPONENTS, ComponentDescriptorIO.COMPONENTS);
IOUtils.newline(contentHandler);
contentHandler.endPrefixMapping(PREFIX);
contentHandler.endDocument();
}
public static void outputTopWordsWithProbs(
double[][] topicWordDistr, ArrayList<String> vocab, int numTopWord, String filepath)
throws Exception {
BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
for (int t = 0; t < topicWordDistr.length; t++) {
// sort words
double[] bs = topicWordDistr[t];
ArrayList<RankingItem<Integer>> rankedWords = new ArrayList<RankingItem<Integer>>();
for (int i = 0; i < bs.length; i++) {
rankedWords.add(new RankingItem<Integer>(i, bs[i]));
}
Collections.sort(rankedWords);
// output top words
writer.write("Topic " + (t + 1));
double cumm_prob = 0;
for (int i = 0; i < Math.min(numTopWord, vocab.size()); i++) {
cumm_prob += rankedWords.get(i).getPrimaryValue();
writer.write(
"\t"
+ vocab.get(rankedWords.get(i).getObject())
+ ", "
+ rankedWords.get(i).getPrimaryValue()
+ ", "
+ cumm_prob);
}
writer.write("\n");
}
writer.close();
}
/** @see InputStream#close() */
public void close() throws IOException {
IOUtils.closeQuietly(processInputStream);
IOUtils.closeQuietly(processOutputStream);
if (process != null) {
process.destroy();
}
}
public static void main(String[] args) throws Exception {
String uri = args[0];
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf);
FSDataInputStream in = null;
try {
in = fs.open(new Path(uri));
IOUtils.copyBytes(in, System.out, 4096, false);
in.seek(3); // go back to pos 3 of the file
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
}
/**
* Output top words for each topic with indices
*
* @param topicIndices List of topic indices
* @param topicWordDistr 2D array containing topical word distributions
* @param vocab List of tokens in the vocabulary
* @param numTopWord Number of top words to output
* @param filepath Path to the output file
*/
public static void outputTopWords(
ArrayList<Integer> topicIndices,
double[][] topicWordDistr,
ArrayList<String> vocab,
int numTopWord,
String filepath)
throws Exception {
BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
for (int t = 0; t < topicWordDistr.length; t++) {
// sort words
double[] bs = topicWordDistr[t];
ArrayList<RankingItem<Integer>> rankedWords = new ArrayList<RankingItem<Integer>>();
for (int i = 0; i < bs.length; i++) {
rankedWords.add(new RankingItem<Integer>(i, bs[i]));
}
Collections.sort(rankedWords);
// output top words
writer.write("Topic " + topicIndices.get(t));
for (int i = 0; i < Math.min(numTopWord, vocab.size()); i++) {
writer.write("\t" + vocab.get(rankedWords.get(i).getObject()));
}
writer.write("\n");
}
writer.close();
}
public static void outputDistribution(double[] distr, String filepath) throws Exception {
BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
for (double d : distr) {
writer.write(d + " ");
}
writer.close();
}
// postToRestfulApi -
// Note: params in the addr field need to be URLEncoded
private String postToRestfulApi(
String addr, String data, HttpServletRequest request, HttpServletResponse response) {
if (localCookie) CookieHandler.setDefault(cm);
String result = "";
try {
URLConnection connection = new URL(API_ROOT + addr).openConnection();
String cookieVal = getBrowserInfiniteCookie(request);
if (cookieVal != null) {
connection.addRequestProperty("Cookie", "infinitecookie=" + cookieVal);
connection.setDoInput(true);
}
connection.setDoOutput(true);
connection.setRequestProperty("Accept-Charset", "UTF-8");
// Post JSON string to URL
OutputStream os = connection.getOutputStream();
byte[] b = data.getBytes("UTF-8");
os.write(b);
// Receive results back from API
InputStream is = connection.getInputStream();
result = IOUtils.toString(is, "UTF-8");
String newCookie = getConnectionInfiniteCookie(connection);
if (newCookie != null && response != null) {
setBrowserInfiniteCookie(response, newCookie, request.getServerPort());
}
} catch (Exception e) {
// System.out.println("Exception: " + e.getMessage());
}
return result;
} // TESTED
/**
* Write the xml for a {@link Interface}.
*
* @param interf
* @param contentHandler
* @throws SAXException
*/
protected static void generateXML(Interface interf, ContentHandler contentHandler)
throws SAXException {
final AttributesImpl ai = new AttributesImpl();
IOUtils.addAttribute(ai, "interface", interf.getInterfacename());
IOUtils.indent(contentHandler, 3);
contentHandler.startElement(
INNER_NAMESPACE_URI,
ComponentDescriptorIO.INTERFACE,
ComponentDescriptorIO.INTERFACE_QNAME,
ai);
contentHandler.endElement(
INNER_NAMESPACE_URI,
ComponentDescriptorIO.INTERFACE,
ComponentDescriptorIO.INTERFACE_QNAME);
IOUtils.newline(contentHandler);
}
// callRestfulApi - Calls restful API and returns results as a string
public String callRestfulApi(
String addr, HttpServletRequest request, HttpServletResponse response) {
if (localCookie) CookieHandler.setDefault(cm);
try {
ByteArrayOutputStream output = new ByteArrayOutputStream();
URL url = new URL(API_ROOT + addr);
URLConnection urlConnection = url.openConnection();
String cookieVal = getBrowserInfiniteCookie(request);
if (cookieVal != null) {
urlConnection.addRequestProperty("Cookie", "infinitecookie=" + cookieVal);
urlConnection.setDoInput(true);
urlConnection.setDoOutput(true);
urlConnection.setRequestProperty("Accept-Charset", "UTF-8");
}
IOUtils.copy(urlConnection.getInputStream(), output);
String newCookie = getConnectionInfiniteCookie(urlConnection);
if (newCookie != null && response != null) {
setBrowserInfiniteCookie(response, newCookie, request.getServerPort());
}
return output.toString();
} catch (IOException e) {
System.out.println(e.getMessage());
return null;
}
} // TESTED
/**
* Write the xml for a {@link Implementation}.
*
* @param implementation
* @param contentHandler
* @throws SAXException
*/
protected static void generateXML(Implementation implementation, ContentHandler contentHandler)
throws SAXException {
final AttributesImpl ai = new AttributesImpl();
IOUtils.addAttribute(ai, "class", implementation.getClassame());
IOUtils.indent(contentHandler, 2);
contentHandler.startElement(
INNER_NAMESPACE_URI,
ComponentDescriptorIO.IMPLEMENTATION,
ComponentDescriptorIO.IMPLEMENTATION_QNAME,
ai);
contentHandler.endElement(
INNER_NAMESPACE_URI,
ComponentDescriptorIO.IMPLEMENTATION,
ComponentDescriptorIO.IMPLEMENTATION_QNAME);
IOUtils.newline(contentHandler);
}
public static void main(String[] args) throws IOException {
PrintWriter out;
if (args.length > 1) {
out = new PrintWriter(args[1]);
} else {
out = new PrintWriter(System.out);
}
PrintWriter xmlOut = null;
if (args.length > 2) {
xmlOut = new PrintWriter(args[2]);
}
Properties props = new Properties();
props.put("annotators", "tokenize, ssplit, pos, lemma, ner,parse");
StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
Annotation annotation;
if (args.length > 0) {
annotation = new Annotation(IOUtils.slurpFileNoExceptions(args[0]));
} else {
annotation =
new Annotation(
"Kosgi Santosh sent an email to Stanford University. He didn't get a reply.");
}
pipeline.annotate(annotation);
pipeline.prettyPrint(annotation, out);
}
public static void outputLibSVM(File outputFile, SparseVector[] features, int[][] labels) {
System.out.println("Outputing LIBSVM-formatted data to " + outputFile);
try {
BufferedWriter writer = IOUtils.getBufferedWriter(outputFile);
for (int ii = 0; ii < features.length; ii++) {
if (labels[ii].length == 0) {
continue;
}
// labels
for (int jj = 0; jj < labels[ii].length - 1; jj++) {
writer.write(labels[ii][jj] + ",");
}
writer.write(Integer.toString(labels[ii][labels[ii].length - 1]) + " ");
// features
for (int idx : features[ii].getSortedIndices()) {
double featureVal = features[ii].get(idx);
if (Math.abs(featureVal) < 10E-6) {
continue;
}
writer.write(" " + idx + ":" + features[ii].get(idx));
}
writer.write("\n");
}
writer.close();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(
"Exception while outputing " + "LIBSVM-formatted data to " + outputFile);
}
}
/** Input latent variable assignments */
public static int[][] inputLatentVariableAssignment(String filepath) throws Exception {
ArrayList<int[]> list = new ArrayList<int[]>();
BufferedReader reader = IOUtils.getBufferedReader(filepath);
String line;
String[] sline;
while ((line = reader.readLine()) != null) {
if (line.equals("")) {
list.add(new int[0]);
continue;
}
sline = line.split("\t")[1].split(" ");
int[] assignments = new int[sline.length];
for (int i = 0; i < assignments.length; i++) {
assignments[i] = Integer.parseInt(sline[i]);
}
list.add(assignments);
}
reader.close();
int[][] latentVar = new int[list.size()][];
for (int i = 0; i < latentVar.length; i++) {
latentVar[i] = list.get(i);
}
return latentVar;
}
public static void outputLogLikelihoods(ArrayList<Double> logLhoods, String filepath)
throws Exception {
BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
for (int i = 0; i < logLhoods.size(); i++) {
writer.write(i + "\t" + logLhoods.get(i) + "\n");
}
writer.close();
}
public static Components read(InputStream file) throws SCRDescriptorException {
try {
final XmlHandler xmlHandler = new XmlHandler();
IOUtils.parse(file, xmlHandler);
return xmlHandler.components;
} catch (TransformerException e) {
throw new SCRDescriptorException("Unable to read xml", "[stream]", 0, e);
}
}
private static void extractFromClasspath(String fileName, File folder) {
String[] split = fileName.split("/");
String diskFileName = split[split.length - 1];
try (FileOutputStream in = new FileOutputStream(new File(folder, diskFileName))) {
IOUtils.copy(LWJGLSetup.class.getResourceAsStream(fileName), in);
} catch (Exception e) {
e.printStackTrace();
}
}
public static ArrayList<String> loadVocab(String filepath) throws Exception {
ArrayList<String> voc = new ArrayList<String>();
BufferedReader reader = IOUtils.getBufferedReader(filepath);
String line;
while ((line = reader.readLine()) != null) {
voc.add(line);
}
reader.close();
return voc;
}
public static final boolean receive_file_exists(
Client client, String filePath, long lastModified, long length) throws IOException {
File existsFile = new File(client.getDirectory(), filePath);
if (!existsFile.exists()) return false;
if (existsFile.lastModified() != lastModified) return false;
if (existsFile.length() != length) return false;
File rootDir = getTempReceiveDir(client);
IOUtils.appendLines(new File(rootDir, PATH_TO_MOVE), filePath);
return true;
}
public static double[] inputDistribution(String filepath) throws Exception {
BufferedReader reader = IOUtils.getBufferedReader(filepath);
String[] sline = reader.readLine().split(" ");
reader.close();
double[] distr = new double[sline.length];
for (int i = 0; i < distr.length; i++) {
distr[i] = Double.parseDouble(sline[i]);
}
return distr;
}
public static void outputPerplexity(String outputFile, double perplexity) {
System.out.println("Outputing perplexity to " + outputFile);
try {
BufferedWriter writer = IOUtils.getBufferedWriter(outputFile);
writer.write(perplexity + "\n");
writer.close();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Exception while outputing " + outputFile);
}
}
/**
* Write the xml for a {@link Service}.
*
* @param service
* @param contentHandler
* @throws SAXException
*/
protected static void generateXML(Service service, ContentHandler contentHandler)
throws SAXException {
final AttributesImpl ai = new AttributesImpl();
IOUtils.addAttribute(ai, "servicefactory", String.valueOf(service.isServicefactory()));
IOUtils.indent(contentHandler, 2);
contentHandler.startElement(
INNER_NAMESPACE_URI,
ComponentDescriptorIO.SERVICE,
ComponentDescriptorIO.SERVICE_QNAME,
ai);
if (service.getInterfaces() != null && service.getInterfaces().size() > 0) {
IOUtils.newline(contentHandler);
for (final Interface interf : service.getInterfaces()) {
generateXML(interf, contentHandler);
}
IOUtils.indent(contentHandler, 2);
}
contentHandler.endElement(
INNER_NAMESPACE_URI, ComponentDescriptorIO.SERVICE, ComponentDescriptorIO.SERVICE_QNAME);
IOUtils.newline(contentHandler);
}
/**
* Write the component descriptors to the file.
*
* @param components
* @param file
* @throws SCRDescriptorException
*/
public static void write(Components components, File file, boolean isScrPrivateFile)
throws SCRDescriptorException {
try {
generateXML(components, IOUtils.getSerializer(file), isScrPrivateFile);
} catch (TransformerException e) {
throw new SCRDescriptorException("Unable to write xml", file.toString(), 0, e);
} catch (SAXException e) {
throw new SCRDescriptorException("Unable to generate xml", file.toString(), 0, e);
} catch (IOException e) {
throw new SCRDescriptorException("Unable to write xml", file.toString(), 0, e);
}
}
public static void outputLatentVariables(double[][] vars, String filepath) throws Exception {
BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
StringBuilder line;
for (double[] var : vars) {
line = new StringBuilder();
for (double v : var) {
line.append(Double.toString(v)).append(" ");
}
writer.write(line.toString().trim() + "\n");
}
writer.close();
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
if (configFile == null) {
resp.sendRedirect("cores-config.xml");
return;
}
resp.setContentType("text/xml; charset=UTF-8");
InputStream input = new FileInputStream(configFile);
IOUtils.copy(input, resp.getOutputStream());
}
public static double inputPerplexity(String inputFile) {
double ppx = 0;
try {
BufferedReader reader = IOUtils.getBufferedReader(inputFile);
ppx = Double.parseDouble(reader.readLine());
reader.close();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Exception while inputing " + inputFile);
}
return ppx;
}
private DesignDocument.View loadViewFromFile(
Map<String, DesignDocument.View> views, View input, Class<?> repositoryClass) {
try {
InputStream in = repositoryClass.getResourceAsStream(input.file());
if (in == null) {
throw new FileNotFoundException("Could not load view file with path: " + input.file());
}
String json = IOUtils.toString(in, "UTF-8");
return mapper().readValue(json.replaceAll("\n", ""), DesignDocument.View.class);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
}
private void initLogstash(String testName) {
initLogstashHost();
String pathToLogstash =
SGTestHelper.getSGTestRootDir().replace("\\", "/") + "/src/main/resources/logstash";
String confFilePath = pathToLogstash + "/logstash-shipper-client.conf";
String fixedTestName = testName.substring(0, testName.length() - 2);
String backupFilePath = pathToLogstash + "/logstash-shipper-client-" + fixedTestName + ".conf";
if (process == null) {
try {
LogUtils.log("copying file " + confFilePath + " to " + backupFilePath);
IOUtils.copyFile(confFilePath, backupFilePath);
// backupFilePath = IOUtils.backupFile(confFilePath);
IOUtils.replaceTextInFile(
backupFilePath,
"<path_to_test_folder>",
SGTestHelper.getSGTestRootDir().replace("\\", "/")
+ "/../"
+ suiteName
+ "/"
+ testName);
IOUtils.replaceTextInFile(backupFilePath, "<suite_name>", suiteName);
IOUtils.replaceTextInFile(backupFilePath, "<test_name>", testName);
IOUtils.replaceTextInFile(backupFilePath, "<build_number>", buildNumber);
IOUtils.replaceTextInFile(backupFilePath, "<version>", version);
IOUtils.replaceTextInFile(backupFilePath, "<host>", logstashHost);
String logstashJarPath =
DeploymentUtils.getLocalRepository() + "net/logstash/1.2.2/logstash-1.2.2.jar";
logstashLogPath = pathToLogstash + "/logstash-" + fixedTestName + ".txt";
String cmdLine =
"java -jar "
+ logstashJarPath
+ " agent -f "
+ backupFilePath
+ " -l "
+ logstashLogPath;
final String[] parts = cmdLine.split(" ");
final ProcessBuilder pb = new ProcessBuilder(parts);
LogUtils.log("Executing Command line: " + cmdLine);
TimeUnit.SECONDS.sleep(1);
process = pb.start();
} catch (Exception e) {
e.printStackTrace();
}
}
}
/**
* Create a new font data element
*
* @param ttf The TTF file to read
* @param size The size of the new font
* @throws java.io.IOException Indicates a failure to
*/
private FontData(InputStream ttf, float size) throws IOException {
if (ttf.available() > MAX_FILE_SIZE) {
throw new IOException("Can't load font - too big");
}
byte[] data = IOUtils.toByteArray(ttf);
if (data.length > MAX_FILE_SIZE) {
throw new IOException("Can't load font - too big");
}
this.size = size;
try {
javaFont = Font.createFont(Font.TRUETYPE_FONT, new ByteArrayInputStream(data));
TTFFile rawFont = new TTFFile();
if (!rawFont.readFont(new FontFileReader(data))) {
throw new IOException("Invalid font file");
}
upem = rawFont.getUPEM();
ansiKerning = rawFont.getAnsiKerning();
charWidth = rawFont.getAnsiWidth();
fontName = rawFont.getPostScriptName();
familyName = rawFont.getFamilyName();
String name = getName();
System.err.println("Loaded: " + name + " (" + data.length + ")");
boolean bo = false;
boolean it = false;
if (name.indexOf(',') >= 0) {
name = name.substring(name.indexOf(','));
if (name.indexOf("Bold") >= 0) {
bo = true;
}
if (name.indexOf("Italic") >= 0) {
it = true;
}
}
if ((bo & it)) {
javaFont = javaFont.deriveFont(Font.BOLD | Font.ITALIC);
} else if (bo) {
javaFont = javaFont.deriveFont(Font.BOLD);
} else if (it) {
javaFont = javaFont.deriveFont(Font.ITALIC);
}
} catch (FontFormatException e) {
IOException x = new IOException("Failed to read font");
x.initCause(e);
throw x;
}
}
private void initLogstashHost() {
if (logstashHost != null) {
return;
}
Properties props;
try {
props = IOUtils.readPropertiesFromFile(propsFile);
} catch (final Exception e) {
throw new IllegalStateException("Failed reading properties file : " + e.getMessage());
}
logstashHost = props.getProperty("logstash_server_host");
}
@Override
public void run() {
builder = new Builder();
builder.build();
dataset = new Dataset();
dataset.read();
params = new Params();
for (String group : dataset.groups()) {
String filename = Execution.getFile("dumped-" + group + ".gz");
out = IOUtils.openOutHard(filename);
processExamples(group, dataset.examples(group));
out.close();
LogInfo.logs("Finished dumping to %s", filename);
StopWatchSet.logStats();
}
}