Ejemplo n.º 1
0
  // ## operation runReactor()
  public void runReactor() {
    // #[ operation runReactor()
    // run reactor
    String dir = System.getProperty("RMG.workingDirectory");

    try {
      // system call for reactor
      String[] command = {dir + "/software/reactorModel/reactor.exe"};
      File runningDir = new File("chemkin");
      Process reactor = Runtime.getRuntime().exec(command, null, runningDir);
      InputStream ips = reactor.getInputStream();
      InputStreamReader is = new InputStreamReader(ips);
      BufferedReader br = new BufferedReader(is);
      String line = null;
      while ((line = br.readLine()) != null) {
        // System.out.println(line);
      }
      int exitValue = reactor.waitFor();
    } catch (Exception e) {
      System.out.println("Error in running reactor!");
      System.out.println(e.getMessage());
      System.exit(0);
    }

    // #]
  }
Ejemplo n.º 2
0
  // ## operation checkChemkinMessage()
  public void checkChemkinMessage() {
    // #[ operation checkChemkinMessage()
    try {
      String dir = System.getProperty("RMG.workingDirectory");
      String filename = "chemkin/chem.message";
      FileReader fr = new FileReader(filename);
      BufferedReader br = new BufferedReader(fr);

      String line = br.readLine().trim();
      if (line.startsWith("NO ERRORS FOUND ON INPUT")) {
        return;
      } else if (line.startsWith("WARNING...THERE IS AN ERROR IN THE LINKING FILE")) {
        System.out.println("Error in chemkin linking to reactor!");
        System.exit(0);
      } else {
        System.out.println("Unknown message in chem.message!");
        System.exit(0);
      }
    } catch (Exception e) {
      System.out.println("Can't read chem.message!");
      System.out.println(e.getMessage());
      System.exit(0);
    }

    // #]
  }
Ejemplo n.º 3
0
  public ArrayList<String> collectLinks(String p) {
    ArrayList<String> PageLinks = new ArrayList<String>();
    try {

      URL url = new URL(p);
      BufferedReader br3 = new BufferedReader(new InputStreamReader(url.openStream()));
      String str = "";
      while (null != (str = br3.readLine())) {
        Pattern link =
            Pattern.compile(
                "<a target=\"_top\" href=\"/m/.*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
        Matcher match = link.matcher(str);
        while (match.find()) {
          String tmp = match.group();
          int start = tmp.indexOf('/');
          tmp = tmp.substring(start + 1, tmp.indexOf('\"', start + 1));
          if (Crawl.contains("http://www.rottentomatoes.com/" + tmp)
              || ToCrawl.contains("http://www.rottentomatoes.com/" + tmp)
              || PageLinks.contains("http://www.rottentomatoes.com/" + tmp)) continue;
          PageLinks.add("http://www.rottentomatoes.com/" + tmp);
          // bw4.write("http://www.rottentomatoes.com/"+tmp+"\r\n");
        }
      }

      br3.close();
    } catch (Exception ex) {
      ex.printStackTrace();
    }

    return PageLinks;
  }
Ejemplo n.º 4
0
  private void processTxt(Node operation) {
    List<Node> targets = getChildNodes(operation, "target");
    List<Node> optionNodes = getChildNodes(operation, "opt");
    List<Node> separatorNode = getChildNodes(operation, "separator");
    if (targets.isEmpty() || optionNodes.isEmpty()) {
      return;
    }
    String defaultSeparator = "=";
    String globalSeparator = defaultSeparator;
    if (!separatorNode.isEmpty()) {
      globalSeparator = separatorNode.get(0).getTextContent();
      if (globalSeparator.length() != 1) {
        globalSeparator = defaultSeparator;
      }
    }
    Map<String, String> options = new HashMap<String, String>();
    Map<String, String> processedOptions = new HashMap<String, String>();
    for (int i = 0; i < optionNodes.size(); i++) {
      Node option = optionNodes.get(i);
      String name = option.getAttributes().getNamedItem("name").getNodeValue();
      String value = option.getTextContent();
      if (options.containsKey(name)) {
        options.remove(name);
      }
      options.put(name, value);
    }
    for (int t = 0; t < targets.size(); t++) {
      File target = new File(absolutePath(targets.get(t).getTextContent()));
      File tmpFile = new File(Utils.timestamp());
      BufferedWriter bw = null;
      BufferedReader br = null;
      try {
        Node separatorAttr = targets.get(t).getAttributes().getNamedItem("separator");
        String separator = (separatorAttr == null) ? globalSeparator : separatorAttr.getNodeValue();
        if (separator.length() != 1) {
          separator = globalSeparator;
        }
        bw = new BufferedWriter(new FileWriter(tmpFile));
        if (target.exists()) {
          br = new BufferedReader(new FileReader(target));
          for (String line; (line = br.readLine()) != null; ) {
            String[] parts = line.split(separator);
            if (parts.length < 2) {
              bw.write(line);
              bw.newLine();
              continue;
            }

            String optName = parts[0].trim();
            if (options.containsKey(optName)) {
              String optValue = options.get(optName);
              bw.write(optName + " " + separator + " " + optValue);
              bw.newLine();
              processedOptions.put(optName, optValue);
              options.remove(optName);
            } else if (processedOptions.containsKey(optName)) {
              bw.write(optName + " " + separator + " " + processedOptions.get(optName));
              bw.newLine();
            } else {
              bw.write(line);
              bw.newLine();
            }
          }
          br.close();
        }
        for (Map.Entry<String, String> entry : options.entrySet()) {
          bw.write(entry.getKey() + " " + separator + " " + entry.getValue());
          bw.newLine();
        }
        bw.close();
        FileUtils.copyFile(tmpFile, target);
        FileUtils.forceDelete(tmpFile);
      } catch (IOException ex) {
        Utils.onError(new Error.WriteTxtConfig(target.getPath()));
      }
    }
  }
Ejemplo n.º 5
0
  public void CrawlRT(String RTPage) throws IOException {
    ArrayList<String> t = new ArrayList<String>();
    String crawlData;
    String crawlData2;
    String crawlData3;

    FileReader freader = new FileReader("Crawl.txt");
    BufferedReader br = new BufferedReader(freader);
    FileReader freader2 = new FileReader("Tocrawl.txt");
    BufferedReader br2 = new BufferedReader(freader2);
    FileWriter fwriter2 = new FileWriter("Tocrawl.txt", true);
    BufferedWriter bw2 = new BufferedWriter(fwriter2);
    FileWriter fwriter = new FileWriter("Crawl.txt", true);
    BufferedWriter bw = new BufferedWriter(fwriter);

    /*while(null != (crawlData2 = br.readLine()))
    {
    	if(crawlData2 !=null)
    		Crawl.add(crawlData2);
    }
    t = collectLinks(RTPage);
    Iterator<String> e3= t.iterator();
    while(e3.hasNext())
    {
    	String ee = e3.next();

    		if(!Crawl.contains(ee))
    		{
    			bw2.write(ee+"\r\n");
    		}



    }
    br.close();
    br2.close();
    bw.close();
    bw2.close();*/

    if (null == (crawlData = br.readLine()))
    // if(true)
    {
      // initial iteration
      bw.write(RTPage + "\r\n");
      Crawl.add(RTPage);
      t = collectLinks(RTPage);
      ToCrawl.addAll(t);
    } else {
      // collect data from files and load to array lists
      while (null != (crawlData2 = br.readLine())) {
        if (crawlData2 != null) Crawl.add(crawlData2);
      }

      while (null != (crawlData3 = br2.readLine())) {
        if (crawlData3 != null) ToCrawl.add(crawlData3);
      }
    }
    System.out.println("Crawlled");

    // Number of movies to be crawled
    for (int i = 0; i < 1000; i++) {
      if (ToCrawl.size() > 0) {
        Crawl.removeAll(Collections.singleton(null));
        ToCrawl.removeAll(Collections.singleton(null));
        String c = ToCrawl.get(0);
        if (Crawl.contains(c)) ToCrawl.remove(c);
        else {
          // collect links and collect data from a particular link
          Crawl.add(c);
          t = collectLinks(c);
          CollectData(c);
          ToCrawl.remove(c);
          Iterator<String> e3 = t.iterator();
          while (e3.hasNext()) {
            String ee = e3.next();
            if (!ToCrawl.contains(ee)) {
              if (!Crawl.contains(ee)) {
                ToCrawl.add(ee);
              }
            }
          }
          bw.write(c + "\r\n");
        }
      }
    }

    System.out.println("To Be Crawlled");
    Iterator<String> e2 = ToCrawl.iterator();
    while (e2.hasNext()) {
      // write to file the movies still to be crawled.
      bw2.write(e2.next() + "\r\n");
    }

    prop.setProperty("Id", Integer.toString(n));
    prop.store(new FileOutputStream("config.properties"), null);
    br.close();
    br2.close();
    bw.close();
    bw2.close();
  }
Ejemplo n.º 6
0
  public void CollectData(String link) {

    try {
      // Creating an empty XML Document

      DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance();
      DocumentBuilder docBuilder = dbfac.newDocumentBuilder();
      Document doc = docBuilder.newDocument();
      int flag = 0;
      // create the root element and add it to the document
      Element movie = doc.createElement("movie");
      doc.appendChild(movie);
      movie.setAttribute("id", String.valueOf(n));
      n++;
      // create sub elements
      Element genres = doc.createElement("genres");
      Element actors = doc.createElement("actors");
      Element reviews = doc.createElement("reviews");

      URL movieUrl = new URL(link);
      URL reviewsURL = new URL(link + "reviews/#type=top_critics");
      BufferedWriter bw3 = new BufferedWriter(new FileWriter("movies.xml", true));
      int count = -1;
      String auth = "";
      BufferedReader br3 = new BufferedReader(new InputStreamReader(movieUrl.openStream()));
      String str2 = "";
      String info = "";
      while (null != (str2 = br3.readLine())) {
        // start reading the html document
        if (str2.isEmpty()) continue;
        if (count == 14) break;
        if (count == 12) {
          if (!str2.contains("<h3>Cast</h3>")) continue;
          else count++;
        }
        if (count == 13) {
          if (str2.contains(">ADVERTISEMENT</p>")) {
            count++;
            movie.appendChild(actors);
            continue;
          } else {
            if (str2.contains("itemprop=\"name\">")) {
              Element actor = doc.createElement("actor");
              actors.appendChild(actor);
              Text text = doc.createTextNode(Jsoup.parse(str2.toString()).text());
              actor.appendChild(text);
            } else continue;
          }
        }

        if (count <= 11) {
          switch (count) {
            case -1:
              {
                if (!str2.contains("property=\"og:image\"")) continue;
                else {
                  Pattern image =
                      Pattern.compile("http://.*.jpg", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
                  Matcher match = image.matcher(str2);
                  while (match.find()) {

                    Element imageLink = doc.createElement("imageLink");
                    movie.appendChild(imageLink);
                    Text text = doc.createTextNode(match.group());
                    imageLink.appendChild(text);
                    count++;
                  }
                }
                break;
              }
            case 0:
              {
                if (str2.contains("<title>")) {

                  Element name = doc.createElement("name");
                  movie.appendChild(name);
                  Text text =
                      doc.createTextNode(
                          Jsoup.parse(str2.toString().replace(" - Rotten Tomatoes", "")).text());
                  name.appendChild(text);
                  count++;
                }
                break;
              }
            case 1:
              {
                if (!str2.contains("itemprop=\"ratingValue\"")) break;
                else {
                  Element score = doc.createElement("score");
                  movie.appendChild(score);
                  Text text = doc.createTextNode(Jsoup.parse(str2.toString()).text());
                  score.appendChild(text);
                  count++;
                }
                break;
              }
            case 2:
              {
                if (!str2.contains("itemprop=\"description\">")) continue;
                else count++;
                break;
              }
            case 3:
              {
                if (!str2.contains("itemprop=\"duration\"")) info = info.concat(str2);
                else {
                  Element MovieInfo = doc.createElement("MovieInfo");
                  movie.appendChild(MovieInfo);
                  Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                  MovieInfo.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 4:
              {
                if (!str2.contains("itemprop=\"genre\"")) info = info.concat(str2);
                else {
                  Element duration = doc.createElement("duration");
                  movie.appendChild(duration);
                  Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                  duration.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 5:
              {
                if (info.contains("itemprop=\"genre\"")) {
                  Element genre = doc.createElement("genre");
                  genres.appendChild(genre);
                  Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                  genre.appendChild(text);
                  info = "";
                }
                if (str2.contains(">Directed By:<")) {
                  count++;
                  movie.appendChild(genres);
                  continue;
                } else {

                  if (str2.contains("itemprop=\"genre\"")) {
                    Element genre = doc.createElement("genre");
                    genres.appendChild(genre);
                    Text text = doc.createTextNode(Jsoup.parse(str2.toString()).text());
                    genre.appendChild(text);
                  } else continue;
                }
                break;
              }
            case 6:
              {
                if (!str2.contains(">Written By:<")) {
                  if (str2.contains(">In Theaters:<")) {
                    Element director = doc.createElement("director");
                    movie.appendChild(director);
                    Text text =
                        doc.createTextNode(
                            Jsoup.parse(info.toString().replace("Directed By: ", "")).text());
                    director.appendChild(text);
                    info = str2;
                    count += 2;
                    break;
                  }
                  info = info.concat(str2);
                } else {
                  Element director = doc.createElement("director");
                  movie.appendChild(director);
                  Text text =
                      doc.createTextNode(
                          Jsoup.parse(info.toString().replace("Directed By: ", "")).text());
                  director.appendChild(text);
                  info = "";
                  count++;
                }
                break;
              }
            case 7:
              {
                if (!str2.contains(">In Theaters:<")) {
                  if (str2.contains(">On DVD:<")) {
                    Element writer = doc.createElement("writer");
                    movie.appendChild(writer);
                    Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                    writer.appendChild(text);
                    info = str2;
                    count += 2;
                    break;
                  }
                  info = info.concat(str2);
                } else {
                  Element writer = doc.createElement("writer");
                  movie.appendChild(writer);
                  Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                  writer.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 8:
              {
                if (!str2.contains(">On DVD:<")) info = info.concat(str2);
                else {
                  Element TheatreRelease = doc.createElement("TheatreRelease");
                  movie.appendChild(TheatreRelease);
                  Text text =
                      doc.createTextNode(
                          Jsoup.parse(info.toString().replace("In Theaters:", "")).text());
                  TheatreRelease.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 9:
              {
                if (!str2.contains(">US Box Office:<")) {
                  if (str2.contains("itemprop=\"productionCompany\"")) {
                    Element DvdRelease = doc.createElement("DvdRelease");
                    movie.appendChild(DvdRelease);
                    Text text =
                        doc.createTextNode(
                            Jsoup.parse(info.toString().replace("On DVD:", "")).text());
                    DvdRelease.appendChild(text);
                    info = str2;
                    count += 2;
                    break;
                  }
                  info = info.concat(str2);
                } else {
                  Element DvdRelease = doc.createElement("DvdRelease");
                  movie.appendChild(DvdRelease);
                  Text text =
                      doc.createTextNode(
                          Jsoup.parse(info.toString().replace("On DVD:", "")).text());
                  DvdRelease.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 10:
              {
                if (!str2.contains("itemprop=\"productionCompany\"")) info = info.concat(str2);
                else {
                  Element BOCollection = doc.createElement("BOCollection");
                  movie.appendChild(BOCollection);
                  Text text =
                      doc.createTextNode(
                          Jsoup.parse(info.toString().replace("US Box Office:", "")).text());
                  BOCollection.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }
            case 11:
              {
                if (!str2.contains(">Official Site")) info = info.concat(str2);
                else {
                  Element Production = doc.createElement("Production");
                  movie.appendChild(Production);
                  Text text = doc.createTextNode(Jsoup.parse(info.toString()).text());
                  Production.appendChild(text);
                  info = str2;
                  count++;
                }
                break;
              }

            default:
              break;
          }
        }
      }
      BufferedReader br4 = new BufferedReader(new InputStreamReader(reviewsURL.openStream()));
      String str3 = "";
      String info2 = "";
      int count2 = 0;
      while (null != (str3 = br4.readLine())) {
        if (count2 == 0) {

          if (!str3.contains("<div class=\"reviewsnippet\">")) continue;
          else count2++;
        }
        if (count2 == 1) {
          if (!str3.contains("<p class=\"small subtle\">")) info2 = info2.concat(str3);
          else {
            Element review = doc.createElement("review");
            reviews.appendChild(review);
            Text text = doc.createTextNode(Jsoup.parse(info2.toString()).text());
            review.appendChild(text);
            info2 = "";
            count2 = 0;
          }
        }
      }
      movie.appendChild(reviews);
      TransformerFactory transfac = TransformerFactory.newInstance();
      Transformer trans = transfac.newTransformer();
      trans.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
      trans.setOutputProperty(OutputKeys.INDENT, "yes");

      // create string from xml tree
      StringWriter sw = new StringWriter();
      StreamResult result = new StreamResult(sw);
      DOMSource source = new DOMSource(doc);
      trans.transform(source, result);
      String xmlString = sw.toString();
      bw3.write(xmlString);
      br3.close();
      br4.close();
      bw3.close();
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }