private static void analyzeNext() {
    ArrayList previous = new ArrayList();
    String token = nextToken();
    analysis.add("", token, 0); // Insert 1st token directly.
    previous.add(token);

    while (true) {
      int nWords = Math.min(previous.size(), MAX);
      token = nextToken();
      if (token == PERIOD) break;

      for (int j = 1; j <= nWords; j++) {
        String s1 = concat(previous, j); // ({"a", "b", "c"}, 2) -> "b c"
        analysis.add(s1, token, j);
      }

      previous.add(token);
      if (previous.size() == MAX) previous.remove(0);
    }

    int nWords = Math.min(previous.size(), MAX);
    for (int j = 1; j < nWords; j++) {
      String s1 = concat(previous, j); // ({"a", "b", "c"}, 2) -> "b c"
      analysis.add(s1, PERIOD, j);
    }
  }
 public void calculateStatistics(Analysis analysis) { // Build frequency charts
   if (word.startsWith("re")) {
     analysis.inc("re");
   }
   if (word.startsWith("pre")) {
     analysis.inc("pre");
   }
   if (word.startsWith("pro")) {
     analysis.inc("pro");
     word = "*" + word + "*"; // Mark special words
   }
   if (word.length() > 4) {
     if (word.startsWith("de") && word.endsWith("ed")) {
       analysis.inc("de-ed");
       // word = word.upperCase();
     }
     if (word.startsWith("Vine")) {
       analysis.inc("markerWord");
       analysis.getMarkerWord(this);
     }
     if (word.startsWith("marker")) {
       analysis.inc("markerWord");
       analysis.getMarkerWord(this);
     }
   }
 }
  public static void main(String[] args) {
    //      if (args.length > 0) nSentencesToGenerate = Integer.parseInt(args[0]);
    if (args.length > 0) fileNames = args;

    long start = System.currentTimeMillis();

    if (fileNames.length == 0) {
      String s = RewriteManuscript.manuscriptString;
      System.out.println("******** Reading and analyzing manuscript string ********");
      analyze(s);
    }

    for (int i = 0; i < fileNames.length; i++) {
      String s = readFile(fileNames[i]);
      System.out.println("******** Reading and analyzing file: " + fileNames[i] + "********");
      analyze(s);
    }
    analysis.calculateStatistics();
    System.out.println("******** Creating a new masterpiece of literature... *******\n");
    generateSentences();

    long end = System.currentTimeMillis();
    long total = end - start;
    System.out.println("" + total + "ms");
  }
  public static String generateSentence() {
    ArrayList al = new ArrayList();
    String s;

    s = analysis.choose(al, 0);
    al.add(s);
    s = analysis.choose(al, 1);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 1);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 2);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 3);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 4);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 3);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 3);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 1);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 4);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 4);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 3);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 2);
    al.add(s);
    if (s == PERIOD) return concat(al);
    s = analysis.choose(al, 2);
    al.add(s);
    if (s == PERIOD) return concat(al);

    al.add(PERIOD);
    return concat(al);
  }