/**
   * Generate the xml top level element and start streaming the components.
   *
   * @param components
   * @param contentHandler
   * @throws SAXException
   */
  protected static void generateXML(
      Components components, ContentHandler contentHandler, boolean isScrPrivateFile)
      throws SAXException {
    // detect namespace to use
    final String namespace;
    if (components.getSpecVersion() == Constants.VERSION_1_0) {
      namespace = NAMESPACE_URI_1_0;
    } else if (components.getSpecVersion() == Constants.VERSION_1_1) {
      namespace = NAMESPACE_URI_1_1;
    } else {
      namespace = NAMESPACE_URI_1_1_FELIX;
    }
    contentHandler.startDocument();
    contentHandler.startPrefixMapping(PREFIX, namespace);

    // wrapper element to generate well formed xml
    contentHandler.startElement(
        "",
        ComponentDescriptorIO.COMPONENTS,
        ComponentDescriptorIO.COMPONENTS,
        new AttributesImpl());
    IOUtils.newline(contentHandler);

    for (final Component component : components.getComponents()) {
      if (component.isDs()) {
        generateXML(namespace, component, contentHandler, isScrPrivateFile);
      }
    }
    // end wrapper element
    contentHandler.endElement(
        "", ComponentDescriptorIO.COMPONENTS, ComponentDescriptorIO.COMPONENTS);
    IOUtils.newline(contentHandler);
    contentHandler.endPrefixMapping(PREFIX);
    contentHandler.endDocument();
  }
Example #2
1
  public static void outputTopWordsWithProbs(
      double[][] topicWordDistr, ArrayList<String> vocab, int numTopWord, String filepath)
      throws Exception {
    BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
    for (int t = 0; t < topicWordDistr.length; t++) {
      // sort words
      double[] bs = topicWordDistr[t];
      ArrayList<RankingItem<Integer>> rankedWords = new ArrayList<RankingItem<Integer>>();
      for (int i = 0; i < bs.length; i++) {
        rankedWords.add(new RankingItem<Integer>(i, bs[i]));
      }
      Collections.sort(rankedWords);

      // output top words
      writer.write("Topic " + (t + 1));
      double cumm_prob = 0;
      for (int i = 0; i < Math.min(numTopWord, vocab.size()); i++) {
        cumm_prob += rankedWords.get(i).getPrimaryValue();
        writer.write(
            "\t"
                + vocab.get(rankedWords.get(i).getObject())
                + ", "
                + rankedWords.get(i).getPrimaryValue()
                + ", "
                + cumm_prob);
      }
      writer.write("\n");
    }
    writer.close();
  }
  /** @see InputStream#close() */
  public void close() throws IOException {
    IOUtils.closeQuietly(processInputStream);
    IOUtils.closeQuietly(processOutputStream);

    if (process != null) {
      process.destroy();
    }
  }
Example #4
0
 public static void main(String[] args) throws Exception {
   String uri = args[0];
   Configuration conf = new Configuration();
   FileSystem fs = FileSystem.get(URI.create(uri), conf);
   FSDataInputStream in = null;
   try {
     in = fs.open(new Path(uri));
     IOUtils.copyBytes(in, System.out, 4096, false);
     in.seek(3); // go back to pos 3 of the file
     IOUtils.copyBytes(in, System.out, 4096, false);
   } finally {
     IOUtils.closeStream(in);
   }
 }
Example #5
0
  /**
   * Output top words for each topic with indices
   *
   * @param topicIndices List of topic indices
   * @param topicWordDistr 2D array containing topical word distributions
   * @param vocab List of tokens in the vocabulary
   * @param numTopWord Number of top words to output
   * @param filepath Path to the output file
   */
  public static void outputTopWords(
      ArrayList<Integer> topicIndices,
      double[][] topicWordDistr,
      ArrayList<String> vocab,
      int numTopWord,
      String filepath)
      throws Exception {

    BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
    for (int t = 0; t < topicWordDistr.length; t++) {
      // sort words
      double[] bs = topicWordDistr[t];
      ArrayList<RankingItem<Integer>> rankedWords = new ArrayList<RankingItem<Integer>>();
      for (int i = 0; i < bs.length; i++) {
        rankedWords.add(new RankingItem<Integer>(i, bs[i]));
      }
      Collections.sort(rankedWords);

      // output top words
      writer.write("Topic " + topicIndices.get(t));
      for (int i = 0; i < Math.min(numTopWord, vocab.size()); i++) {
        writer.write("\t" + vocab.get(rankedWords.get(i).getObject()));
      }
      writer.write("\n");
    }
    writer.close();
  }
Example #6
0
 public static void outputDistribution(double[] distr, String filepath) throws Exception {
   BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
   for (double d : distr) {
     writer.write(d + " ");
   }
   writer.close();
 }
Example #7
0
  // postToRestfulApi -
  // Note: params in the addr field need to be URLEncoded
  private String postToRestfulApi(
      String addr, String data, HttpServletRequest request, HttpServletResponse response) {
    if (localCookie) CookieHandler.setDefault(cm);
    String result = "";
    try {
      URLConnection connection = new URL(API_ROOT + addr).openConnection();
      String cookieVal = getBrowserInfiniteCookie(request);
      if (cookieVal != null) {
        connection.addRequestProperty("Cookie", "infinitecookie=" + cookieVal);
        connection.setDoInput(true);
      }
      connection.setDoOutput(true);
      connection.setRequestProperty("Accept-Charset", "UTF-8");

      // Post JSON string to URL
      OutputStream os = connection.getOutputStream();
      byte[] b = data.getBytes("UTF-8");
      os.write(b);

      // Receive results back from API
      InputStream is = connection.getInputStream();
      result = IOUtils.toString(is, "UTF-8");

      String newCookie = getConnectionInfiniteCookie(connection);
      if (newCookie != null && response != null) {
        setBrowserInfiniteCookie(response, newCookie, request.getServerPort());
      }
    } catch (Exception e) {
      // System.out.println("Exception: " + e.getMessage());
    }
    return result;
  } // TESTED
 /**
  * Write the xml for a {@link Interface}.
  *
  * @param interf
  * @param contentHandler
  * @throws SAXException
  */
 protected static void generateXML(Interface interf, ContentHandler contentHandler)
     throws SAXException {
   final AttributesImpl ai = new AttributesImpl();
   IOUtils.addAttribute(ai, "interface", interf.getInterfacename());
   IOUtils.indent(contentHandler, 3);
   contentHandler.startElement(
       INNER_NAMESPACE_URI,
       ComponentDescriptorIO.INTERFACE,
       ComponentDescriptorIO.INTERFACE_QNAME,
       ai);
   contentHandler.endElement(
       INNER_NAMESPACE_URI,
       ComponentDescriptorIO.INTERFACE,
       ComponentDescriptorIO.INTERFACE_QNAME);
   IOUtils.newline(contentHandler);
 }
Example #9
0
  // callRestfulApi - Calls restful API and returns results as a string
  public String callRestfulApi(
      String addr, HttpServletRequest request, HttpServletResponse response) {
    if (localCookie) CookieHandler.setDefault(cm);

    try {
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      URL url = new URL(API_ROOT + addr);
      URLConnection urlConnection = url.openConnection();
      String cookieVal = getBrowserInfiniteCookie(request);
      if (cookieVal != null) {
        urlConnection.addRequestProperty("Cookie", "infinitecookie=" + cookieVal);
        urlConnection.setDoInput(true);
        urlConnection.setDoOutput(true);
        urlConnection.setRequestProperty("Accept-Charset", "UTF-8");
      }
      IOUtils.copy(urlConnection.getInputStream(), output);
      String newCookie = getConnectionInfiniteCookie(urlConnection);
      if (newCookie != null && response != null) {
        setBrowserInfiniteCookie(response, newCookie, request.getServerPort());
      }
      return output.toString();
    } catch (IOException e) {
      System.out.println(e.getMessage());
      return null;
    }
  } // TESTED
Example #10
0
 /**
  * Write the xml for a {@link Implementation}.
  *
  * @param implementation
  * @param contentHandler
  * @throws SAXException
  */
 protected static void generateXML(Implementation implementation, ContentHandler contentHandler)
     throws SAXException {
   final AttributesImpl ai = new AttributesImpl();
   IOUtils.addAttribute(ai, "class", implementation.getClassame());
   IOUtils.indent(contentHandler, 2);
   contentHandler.startElement(
       INNER_NAMESPACE_URI,
       ComponentDescriptorIO.IMPLEMENTATION,
       ComponentDescriptorIO.IMPLEMENTATION_QNAME,
       ai);
   contentHandler.endElement(
       INNER_NAMESPACE_URI,
       ComponentDescriptorIO.IMPLEMENTATION,
       ComponentDescriptorIO.IMPLEMENTATION_QNAME);
   IOUtils.newline(contentHandler);
 }
Example #11
0
  public static void main(String[] args) throws IOException {
    PrintWriter out;
    if (args.length > 1) {
      out = new PrintWriter(args[1]);
    } else {
      out = new PrintWriter(System.out);
    }
    PrintWriter xmlOut = null;
    if (args.length > 2) {
      xmlOut = new PrintWriter(args[2]);
    }
    Properties props = new Properties();
    props.put("annotators", "tokenize, ssplit, pos, lemma, ner,parse");
    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
    Annotation annotation;
    if (args.length > 0) {
      annotation = new Annotation(IOUtils.slurpFileNoExceptions(args[0]));
    } else {
      annotation =
          new Annotation(
              "Kosgi Santosh sent an email to Stanford University. He didn't get a reply.");
    }

    pipeline.annotate(annotation);
    pipeline.prettyPrint(annotation, out);
  }
Example #12
0
  public static void outputLibSVM(File outputFile, SparseVector[] features, int[][] labels) {
    System.out.println("Outputing LIBSVM-formatted data to " + outputFile);
    try {
      BufferedWriter writer = IOUtils.getBufferedWriter(outputFile);
      for (int ii = 0; ii < features.length; ii++) {
        if (labels[ii].length == 0) {
          continue;
        }
        // labels
        for (int jj = 0; jj < labels[ii].length - 1; jj++) {
          writer.write(labels[ii][jj] + ",");
        }
        writer.write(Integer.toString(labels[ii][labels[ii].length - 1]) + " ");

        // features
        for (int idx : features[ii].getSortedIndices()) {
          double featureVal = features[ii].get(idx);
          if (Math.abs(featureVal) < 10E-6) {
            continue;
          }
          writer.write(" " + idx + ":" + features[ii].get(idx));
        }
        writer.write("\n");
      }
      writer.close();
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(
          "Exception while outputing " + "LIBSVM-formatted data to " + outputFile);
    }
  }
Example #13
0
  /** Input latent variable assignments */
  public static int[][] inputLatentVariableAssignment(String filepath) throws Exception {
    ArrayList<int[]> list = new ArrayList<int[]>();
    BufferedReader reader = IOUtils.getBufferedReader(filepath);
    String line;
    String[] sline;
    while ((line = reader.readLine()) != null) {
      if (line.equals("")) {
        list.add(new int[0]);
        continue;
      }

      sline = line.split("\t")[1].split(" ");
      int[] assignments = new int[sline.length];
      for (int i = 0; i < assignments.length; i++) {
        assignments[i] = Integer.parseInt(sline[i]);
      }
      list.add(assignments);
    }
    reader.close();

    int[][] latentVar = new int[list.size()][];
    for (int i = 0; i < latentVar.length; i++) {
      latentVar[i] = list.get(i);
    }
    return latentVar;
  }
Example #14
0
 public static void outputLogLikelihoods(ArrayList<Double> logLhoods, String filepath)
     throws Exception {
   BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
   for (int i = 0; i < logLhoods.size(); i++) {
     writer.write(i + "\t" + logLhoods.get(i) + "\n");
   }
   writer.close();
 }
Example #15
0
 public static Components read(InputStream file) throws SCRDescriptorException {
   try {
     final XmlHandler xmlHandler = new XmlHandler();
     IOUtils.parse(file, xmlHandler);
     return xmlHandler.components;
   } catch (TransformerException e) {
     throw new SCRDescriptorException("Unable to read xml", "[stream]", 0, e);
   }
 }
Example #16
0
 private static void extractFromClasspath(String fileName, File folder) {
   String[] split = fileName.split("/");
   String diskFileName = split[split.length - 1];
   try (FileOutputStream in = new FileOutputStream(new File(folder, diskFileName))) {
     IOUtils.copy(LWJGLSetup.class.getResourceAsStream(fileName), in);
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
Example #17
0
 public static ArrayList<String> loadVocab(String filepath) throws Exception {
   ArrayList<String> voc = new ArrayList<String>();
   BufferedReader reader = IOUtils.getBufferedReader(filepath);
   String line;
   while ((line = reader.readLine()) != null) {
     voc.add(line);
   }
   reader.close();
   return voc;
 }
 public static final boolean receive_file_exists(
     Client client, String filePath, long lastModified, long length) throws IOException {
   File existsFile = new File(client.getDirectory(), filePath);
   if (!existsFile.exists()) return false;
   if (existsFile.lastModified() != lastModified) return false;
   if (existsFile.length() != length) return false;
   File rootDir = getTempReceiveDir(client);
   IOUtils.appendLines(new File(rootDir, PATH_TO_MOVE), filePath);
   return true;
 }
Example #19
0
 public static double[] inputDistribution(String filepath) throws Exception {
   BufferedReader reader = IOUtils.getBufferedReader(filepath);
   String[] sline = reader.readLine().split(" ");
   reader.close();
   double[] distr = new double[sline.length];
   for (int i = 0; i < distr.length; i++) {
     distr[i] = Double.parseDouble(sline[i]);
   }
   return distr;
 }
Example #20
0
 public static void outputPerplexity(String outputFile, double perplexity) {
   System.out.println("Outputing perplexity to " + outputFile);
   try {
     BufferedWriter writer = IOUtils.getBufferedWriter(outputFile);
     writer.write(perplexity + "\n");
     writer.close();
   } catch (Exception e) {
     e.printStackTrace();
     throw new RuntimeException("Exception while outputing " + outputFile);
   }
 }
Example #21
0
 /**
  * Write the xml for a {@link Service}.
  *
  * @param service
  * @param contentHandler
  * @throws SAXException
  */
 protected static void generateXML(Service service, ContentHandler contentHandler)
     throws SAXException {
   final AttributesImpl ai = new AttributesImpl();
   IOUtils.addAttribute(ai, "servicefactory", String.valueOf(service.isServicefactory()));
   IOUtils.indent(contentHandler, 2);
   contentHandler.startElement(
       INNER_NAMESPACE_URI,
       ComponentDescriptorIO.SERVICE,
       ComponentDescriptorIO.SERVICE_QNAME,
       ai);
   if (service.getInterfaces() != null && service.getInterfaces().size() > 0) {
     IOUtils.newline(contentHandler);
     for (final Interface interf : service.getInterfaces()) {
       generateXML(interf, contentHandler);
     }
     IOUtils.indent(contentHandler, 2);
   }
   contentHandler.endElement(
       INNER_NAMESPACE_URI, ComponentDescriptorIO.SERVICE, ComponentDescriptorIO.SERVICE_QNAME);
   IOUtils.newline(contentHandler);
 }
Example #22
0
 /**
  * Write the component descriptors to the file.
  *
  * @param components
  * @param file
  * @throws SCRDescriptorException
  */
 public static void write(Components components, File file, boolean isScrPrivateFile)
     throws SCRDescriptorException {
   try {
     generateXML(components, IOUtils.getSerializer(file), isScrPrivateFile);
   } catch (TransformerException e) {
     throw new SCRDescriptorException("Unable to write xml", file.toString(), 0, e);
   } catch (SAXException e) {
     throw new SCRDescriptorException("Unable to generate xml", file.toString(), 0, e);
   } catch (IOException e) {
     throw new SCRDescriptorException("Unable to write xml", file.toString(), 0, e);
   }
 }
Example #23
0
 public static void outputLatentVariables(double[][] vars, String filepath) throws Exception {
   BufferedWriter writer = IOUtils.getBufferedWriter(filepath);
   StringBuilder line;
   for (double[] var : vars) {
     line = new StringBuilder();
     for (double v : var) {
       line.append(Double.toString(v)).append(" ");
     }
     writer.write(line.toString().trim() + "\n");
   }
   writer.close();
 }
Example #24
0
  @Override
  protected void doGet(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    if (configFile == null) {
      resp.sendRedirect("cores-config.xml");
      return;
    }
    resp.setContentType("text/xml; charset=UTF-8");

    InputStream input = new FileInputStream(configFile);
    IOUtils.copy(input, resp.getOutputStream());
  }
Example #25
0
 public static double inputPerplexity(String inputFile) {
   double ppx = 0;
   try {
     BufferedReader reader = IOUtils.getBufferedReader(inputFile);
     ppx = Double.parseDouble(reader.readLine());
     reader.close();
   } catch (Exception e) {
     e.printStackTrace();
     throw new RuntimeException("Exception while inputing " + inputFile);
   }
   return ppx;
 }
Example #26
0
 private DesignDocument.View loadViewFromFile(
     Map<String, DesignDocument.View> views, View input, Class<?> repositoryClass) {
   try {
     InputStream in = repositoryClass.getResourceAsStream(input.file());
     if (in == null) {
       throw new FileNotFoundException("Could not load view file with path: " + input.file());
     }
     String json = IOUtils.toString(in, "UTF-8");
     return mapper().readValue(json.replaceAll("\n", ""), DesignDocument.View.class);
   } catch (Exception e) {
     throw Exceptions.propagate(e);
   }
 }
  private void initLogstash(String testName) {

    initLogstashHost();

    String pathToLogstash =
        SGTestHelper.getSGTestRootDir().replace("\\", "/") + "/src/main/resources/logstash";
    String confFilePath = pathToLogstash + "/logstash-shipper-client.conf";
    String fixedTestName = testName.substring(0, testName.length() - 2);
    String backupFilePath = pathToLogstash + "/logstash-shipper-client-" + fixedTestName + ".conf";

    if (process == null) {

      try {

        LogUtils.log("copying file " + confFilePath + " to " + backupFilePath);
        IOUtils.copyFile(confFilePath, backupFilePath);
        //                backupFilePath = IOUtils.backupFile(confFilePath);
        IOUtils.replaceTextInFile(
            backupFilePath,
            "<path_to_test_folder>",
            SGTestHelper.getSGTestRootDir().replace("\\", "/")
                + "/../"
                + suiteName
                + "/"
                + testName);
        IOUtils.replaceTextInFile(backupFilePath, "<suite_name>", suiteName);
        IOUtils.replaceTextInFile(backupFilePath, "<test_name>", testName);
        IOUtils.replaceTextInFile(backupFilePath, "<build_number>", buildNumber);
        IOUtils.replaceTextInFile(backupFilePath, "<version>", version);
        IOUtils.replaceTextInFile(backupFilePath, "<host>", logstashHost);

        String logstashJarPath =
            DeploymentUtils.getLocalRepository() + "net/logstash/1.2.2/logstash-1.2.2.jar";
        logstashLogPath = pathToLogstash + "/logstash-" + fixedTestName + ".txt";
        String cmdLine =
            "java -jar "
                + logstashJarPath
                + " agent -f "
                + backupFilePath
                + " -l "
                + logstashLogPath;

        final String[] parts = cmdLine.split(" ");
        final ProcessBuilder pb = new ProcessBuilder(parts);
        LogUtils.log("Executing Command line: " + cmdLine);

        TimeUnit.SECONDS.sleep(1);
        process = pb.start();

      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }
Example #28
0
  /**
   * Create a new font data element
   *
   * @param ttf The TTF file to read
   * @param size The size of the new font
   * @throws java.io.IOException Indicates a failure to
   */
  private FontData(InputStream ttf, float size) throws IOException {
    if (ttf.available() > MAX_FILE_SIZE) {
      throw new IOException("Can't load font - too big");
    }
    byte[] data = IOUtils.toByteArray(ttf);
    if (data.length > MAX_FILE_SIZE) {
      throw new IOException("Can't load font - too big");
    }

    this.size = size;
    try {
      javaFont = Font.createFont(Font.TRUETYPE_FONT, new ByteArrayInputStream(data));
      TTFFile rawFont = new TTFFile();
      if (!rawFont.readFont(new FontFileReader(data))) {
        throw new IOException("Invalid font file");
      }
      upem = rawFont.getUPEM();
      ansiKerning = rawFont.getAnsiKerning();
      charWidth = rawFont.getAnsiWidth();
      fontName = rawFont.getPostScriptName();
      familyName = rawFont.getFamilyName();

      String name = getName();
      System.err.println("Loaded: " + name + " (" + data.length + ")");
      boolean bo = false;
      boolean it = false;
      if (name.indexOf(',') >= 0) {
        name = name.substring(name.indexOf(','));

        if (name.indexOf("Bold") >= 0) {
          bo = true;
        }
        if (name.indexOf("Italic") >= 0) {
          it = true;
        }
      }

      if ((bo & it)) {
        javaFont = javaFont.deriveFont(Font.BOLD | Font.ITALIC);
      } else if (bo) {
        javaFont = javaFont.deriveFont(Font.BOLD);
      } else if (it) {
        javaFont = javaFont.deriveFont(Font.ITALIC);
      }
    } catch (FontFormatException e) {
      IOException x = new IOException("Failed to read font");
      x.initCause(e);
      throw x;
    }
  }
  private void initLogstashHost() {

    if (logstashHost != null) {
      return;
    }

    Properties props;
    try {
      props = IOUtils.readPropertiesFromFile(propsFile);
    } catch (final Exception e) {
      throw new IllegalStateException("Failed reading properties file : " + e.getMessage());
    }
    logstashHost = props.getProperty("logstash_server_host");
  }
Example #30
0
 @Override
 public void run() {
   builder = new Builder();
   builder.build();
   dataset = new Dataset();
   dataset.read();
   params = new Params();
   for (String group : dataset.groups()) {
     String filename = Execution.getFile("dumped-" + group + ".gz");
     out = IOUtils.openOutHard(filename);
     processExamples(group, dataset.examples(group));
     out.close();
     LogInfo.logs("Finished dumping to %s", filename);
     StopWatchSet.logStats();
   }
 }