public int count(int b1, int q1, int n1, int b2, int q2, int n2) {
    if (b2 == 0 || q2 <= 1) {
      int tb = b1;
      int tq = q1;
      int tn = n1;
      b1 = b2;
      q1 = q2;
      n1 = n2;
      b2 = tb;
      q2 = tq;
      n2 = tn;
    }

    if (b1 == 0 || q1 <= 1) {
      HashSet<Integer> set = new HashSet<Integer>();
      set.add(b1);
      if (n1 > 1) {
        set.add(b1 * q1);
      }

      long curr = b2;
      for (int i = 0; i < n2; i++) {
        set.add((int) curr);
        curr *= q2;
        if (curr > 500000000) {
          return (n2 - i - 1) + set.size();
        }
      }

      return set.size();
    } else {
      HashSet<String> set = new HashSet<String>();

      int factors[] = findFactors(b1, q1, b2, q2);
      int repb1[] = decompose(b1, factors);
      int repb2[] = decompose(b2, factors);
      int repq1[] = decompose(q1, factors);
      int repq2[] = decompose(q2, factors);

      for (int i = 0; i < n1; i++) {
        set.add(Arrays.toString(repb1));
        for (int j = 0; j < repb1.length; j++) {
          repb1[j] += repq1[j];
        }
      }

      for (int i = 0; i < n2; i++) {
        set.add(Arrays.toString(repb2));
        for (int j = 0; j < repb2.length; j++) {
          repb2[j] += repq2[j];
        }
      }

      return set.size();
    }
  }
  /** Find all methods in this framework which raise SQLFeatureNotSupportedException. */
  public void testSupportedMethods() throws Exception {
    getTestConfiguration().setVerbosity(true);

    HashSet<String> vanishedMethodList = new HashSet<String>();
    HashSet<String> unsupportedList = new HashSet<String>();
    HashSet<String> notUnderstoodList = new HashSet<String>();

    // Build map of interfaces to their methods which may raise SQLFeatureNotSupportedException.
    initializeExcludableMap(vanishedMethodList);

    vetDataSource(unsupportedList, notUnderstoodList);
    vetConnectionPooledDataSource(unsupportedList, notUnderstoodList);
    vetXADataSource(unsupportedList, notUnderstoodList);

    //
    // Print methods which behave unexpectedly.
    //
    printVanishedMethodList(vanishedMethodList);
    printUnsupportedList(unsupportedList);
    printNotUnderstoodList(notUnderstoodList);

    int actualErrorCount =
        vanishedMethodList.size() + unsupportedList.size() + notUnderstoodList.size();

    assertEquals("Unexpected discrepancies.", 0, actualErrorCount);
  }
Beispiel #3
0
  static void runTest(HashSet<Integer> caseSet) {
    int cases = 0, passed = 0;
    while (true) {
      String label = Reader.nextLine();
      if (label == null || !label.startsWith("--")) break;

      int N = Integer.parseInt(Reader.nextLine());
      Reader.nextLine();
      double __answer = Double.parseDouble(Reader.nextLine());

      cases++;
      if (caseSet.size() > 0 && !caseSet.contains(cases - 1)) continue;
      System.err.print(String.format("  Testcase #%d ... ", cases - 1));

      if (doTest(N, __answer)) passed++;
    }
    if (caseSet.size() > 0) cases = caseSet.size();
    System.err.println(String.format("%nPassed : %d/%d cases", passed, cases));

    int T = (int) (System.currentTimeMillis() / 1000) - 1398529188;
    double PT = T / 60.0, TT = 75.0;
    System.err.println(
        String.format(
            "Time   : %d minutes %d secs%nScore  : %.2f points",
            T / 60, T % 60, 950 * (0.3 + (0.7 * TT * TT) / (10.0 * PT * PT + TT * TT))));
  }
  public static void main(String[] args) throws Exception {
    String graphName = args[0];
    int nShards = Integer.parseInt(args[1]);

    CircleOfTrustSalsa csalsa = new CircleOfTrustSalsa(new VertexQuery(graphName, nShards), 10000);

    VertexIdTranslate vertexTrans =
        VertexIdTranslate.fromFile(
            new File(ChiFilenames.getVertexTranslateDefFile(graphName, nShards)));

    BufferedReader cmdIn = new BufferedReader(new InputStreamReader(System.in));
    while (true) {
      System.out.print("Enter vertex id to query >> :: ");
      String ln = cmdIn.readLine();
      int vertex = Integer.parseInt(ln);

      // Circle of trust is just the vertex's followers for now
      HashSet<Integer> circle = csalsa.queryService.queryOutNeighbors(vertexTrans.forward(vertex));

      int maxCircleSize = 300;
      // max 500
      if (circle.size() > maxCircleSize) {
        int[] all = new int[circle.size()];
        int i = 0;
        for (Integer v : circle) all[i++] = v;
        HashSet<Integer> filteredCircle = new HashSet<Integer>();
        Random r = new Random(260379);
        for (i = 0; i < maxCircleSize; i++)
          filteredCircle.add(all[Math.abs(r.nextInt()) % all.length]);
        circle = filteredCircle;
      }

      csalsa.initializeGraph(circle);

      long t = System.currentTimeMillis();
      csalsa.computeSALSA(3);
      logger.info("SALSA computation took " + (System.currentTimeMillis() - t) + "ms");

      circle.add(vertexTrans.forward(vertex));
      ArrayList<SalsaVertex> top = csalsa.topAuthorities(20, circle);
      int j = 1;
      for (SalsaVertex sv : top) {
        int originalId = vertexTrans.backward(sv.id);
        logger.info(
            "Top "
                + (j++)
                + " = "
                + originalId
                + " "
                + csalsa.namify(originalId)
                + " ("
                + sv.value
                + ")");
      }
    }
  }
  public void testMostCommonCharIn() {
    FileResource fr = new FileResource("dictionaries/English");
    HashSet<String> dict = readDictionary(fr);
    System.out.println("English dict has " + dict.size() + " entries");

    char mostCommon = mostCommonCharIn(dict);
    System.out.println(" most common char: " + mostCommon);

    FileResource fr1 = new FileResource("dictionaries/Spanish");
    HashSet<String> dict1 = readDictionary(fr1);
    System.out.println("Spanish dict has " + dict1.size() + " entries");

    mostCommon = mostCommonCharIn(dict1);
    System.out.println(" most common char: " + mostCommon);
  }
Beispiel #6
0
 /**
  * Returns the list of friends
  *
  * @return The list of friends
  */
 public static Player[] getFriends() {
   synchronized (monitor) {
     final Player[] rc = new Player[friends.size()];
     friends.toArray(rc);
     return rc;
   }
 }
 public static double[] printPrecRec(String file, String gold) throws IOException {
   Scanner in = new Scanner(new FileReader(gold));
   HashSet<String> g = new HashSet<String>();
   while (in.hasNextLine()) g.add(in.nextLine());
   in.close();
   in = new Scanner(new FileReader(file));
   HashSet<String> f = new HashSet<String>();
   while (in.hasNextLine()) {
     f.add(in.nextLine());
   }
   in.close();
   int correct = JaccardFromTF.intersect(g, f).size();
   double[] res = new double[3];
   res[0] = 100.0 * correct / f.size();
   res[1] = 100.0 * correct / g.size();
   if (res[0] + res[1] == 0) res[2] = 0;
   else res[2] = 2.0 * res[0] * res[1] / (res[0] + res[1]);
   System.out.println("Precision \t Recall \t FM");
   System.out.println(res[0] + "\t " + res[1] + "\t" + res[2]);
   return res;
 }
Beispiel #8
0
  @Override
  public void execute() {
    Map<String, Set<CortexLinksMap>> links = new HashMap<String, Set<CortexLinksMap>>();
    Map<String, Set<String>> linkFilenames = new HashMap<String, Set<String>>();
    if (LINKS != null && !LINKS.isEmpty()) {
      for (CortexLinksMap cl : LINKS) {
        String name = cl.getCortexLinks().getColor(0).getSampleName();

        if (!links.containsKey(name)) {
          links.put(name, new HashSet<CortexLinksMap>());
          linkFilenames.put(name, new HashSet<String>());
        }

        links.get(name).add(cl);
        linkFilenames.get(name).add(cl.getCortexLinks().getCortexLinksFile().getAbsolutePath());
      }
      log.info("Loaded {} links", LINKS.size());

      for (CortexLinksMap clm : LINKS) {
        log.info(
            "  {}: {}",
            clm.getCortexLinks().getCortexLinksFile().getAbsolutePath(),
            clm.getCortexLinks().getNumLinks());
      }
    }

    log.info("Loaded {} graphs", GRAPHS.size());
    for (CortexGraph cg : GRAPHS) {
      String name = cg.getColor(0).getSampleName();
      graphs.put(cg.getCortexFile().getName(), cg);

      log.info(
          "  {}: {} {}", cg.getCortexFile().getName(), cg.getKmerSize(), linkFilenames.get(name));
    }

    loadContigs();
    log.info("Loaded {} contigs", contigs.size());

    log.info("Listening on port {}", PORT);

    try {
      HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
      server.createContext("/", new PageHandler());
      server.createContext("/graphlist", new GraphList());
      server.createContext("/search", new SearchHandler());
      server.createContext("/cr", new CtxRecordHandler());
      server.createContext("/links", new LinksHandler());
      server.setExecutor(null);
      server.start();
    } catch (IOException e) {
      throw new IndianaException("Unable to start server", e);
    }
  }
  public void testReadDictionary() {

    FileResource fr = new FileResource("dictionaries/English");
    HashSet<String> dict = readDictionary(fr);
    System.out.println("dict has " + dict.size() + " entries");

    Iterator<String> it = dict.iterator();
    for (int i = 0; i < 25; i++) {
      if (it.hasNext()) {
        System.out.println(i + "\t" + it.next());
      }
    }
  }
  int[] findFactors(int x, int y, int w, int z) {
    HashSet<Integer> factors = new HashSet<Integer>();

    addFactors(factors, x);
    addFactors(factors, y);
    addFactors(factors, w);
    addFactors(factors, z);

    int res[] = new int[factors.size()];
    int i = 0;
    for (int f : factors) {
      res[i++] = f;
    }
    return res;
  }
Beispiel #11
0
  public int count(int[] seq, int bound) {
    HashSet<String> set = new HashSet<String>();
    HashSet<Integer> nums = new HashSet<Integer>();

    int n = seq.length;

    for (int i = 0; i < n; i++) {
      nums.add(seq[i]);
    }

    int t = nums.size();
    int total = 0;

    nums.add(-1);

    for (int i = 0; i < n; i++) {
      for (int j = 0; j < n; j++) {
        for (int x : nums) {
          int seq2[] = seq.clone();

          if (i < j) {
            for (int k = i + 1; k < j + 1; k++) {
              seq2[k - 1] = seq2[k];
            }
          } else if (i > j) {
            for (int k = i; k > j; k--) {
              seq2[k] = seq2[k - 1];
            }
          }
          seq2[j] = x;

          String key = Arrays.toString(seq2);
          if (!set.contains(key)) {
            if (x == -1) {
              total += bound - t;
            } else {
              total++;
            }
            total %= MOD;
            set.add(key);
          }
        }
      }
    }

    return total;
  }
Beispiel #12
0
 void solve() throws IOException {
   int[] gr = new int[5001];
   for (int i = 1; i < 5001; i++) {
     HashSet<Integer> s = new HashSet<Integer>();
     for (int j = 1; j + j <= i; j++) {
       s.add(gr[i - j]);
     }
     for (int t = 0; t <= s.size(); t++) {
       if (!s.contains(t)) {
         gr[i] = t;
         break;
       }
     }
   }
   /*for (int i = 0;i < 1000; i++) {
   	out.println(i + " " + gr[i]);
   }*/
   int a = nextInt();
   int b = nextInt();
   int c = nextInt();
   int ans = gr[a] ^ gr[b] ^ gr[c];
   if (ans == 0) {
     out.println("NO");
   } else {
     out.println("YES");
     for (int i = 1; i + i <= a; i++) {
       if ((gr[a - i] ^ gr[b] ^ gr[c]) == 0) {
         out.println((a - i) + " " + b + " " + c);
         return;
       }
     }
     for (int i = 1; i + i <= b; i++) {
       if ((gr[a] ^ gr[b - i] ^ gr[c]) == 0) {
         out.println(a + " " + (b - i) + " " + c);
         return;
       }
     }
     for (int i = 1; i + i <= c; i++) {
       if ((gr[a] ^ gr[b] ^ gr[c - i]) == 0) {
         out.println(a + " " + b + " " + (c - i));
         return;
       }
     }
   }
 }
  // debug print the list of methods which throw SQLFeatureNotSupportedException
  private void printUnsupportedList(HashSet<String> unsupportedList) {
    int count = unsupportedList.size();

    if (count == 0) {
      return;
    }

    println("--------------- UNSUPPORTED METHODS ------------------");
    println("--");

    String[] result = new String[count];

    unsupportedList.toArray(result);
    Arrays.sort(result);

    for (int i = 0; i < count; i++) {
      println(result[i]);
    }
  }
  // debug print the list of methods which have disappeared from the SQL interface
  private void printVanishedMethodList(HashSet<String> vanishedMethodList) {
    int count = vanishedMethodList.size();

    if (count == 0) {
      return;
    }

    println("--------------- VANISHED METHODS ------------------");
    println("--");

    String[] result = new String[count];

    vanishedMethodList.toArray(result);
    Arrays.sort(result);

    for (int i = 0; i < count; i++) {
      println(result[i]);
    }
  }
  // Debug print the list of method failures which we don't understand
  private void printNotUnderstoodList(HashSet<String> notUnderstoodList) {
    int count = notUnderstoodList.size();

    if (count == 0) {
      return;
    }

    println("\n\n");
    println("--------------- NOT UNDERSTOOD METHODS ------------------");
    println("--");

    String[] result = new String[count];

    notUnderstoodList.toArray(result);
    Arrays.sort(result);

    for (int i = 0; i < count; i++) {
      println(result[i]);
    }
  }
 public void run() {
   // do it ...
   try {
     IndexWriter indexWriter =
         LuceneUtils.createIndexWriter(
             indexPath, overwriteIndex, LuceneUtils.AnalyzerType.WhitespaceAnalyzer);
     for (Iterator<File> iterator = inputFiles.iterator(); iterator.hasNext(); ) {
       File inputFile = iterator.next();
       if (verbose) System.out.println("Processing " + inputFile.getPath() + ".");
       if (verbose) System.out.println("Counting images.");
       run = 0;
       readFile(indexWriter, inputFile);
       if (verbose) System.out.printf("%d images found in the data file.\n", docCount);
       int numberOfRepresentatives = 1000; // TODO: clever selection.
       // select a number of representative "fixed stars" randomly from file
       if (numberOfRepresentatives > Math.sqrt(docCount))
         numberOfRepresentatives = (int) Math.sqrt(docCount);
       if (verbose)
         System.out.printf(
             "Selecting %d representative images for hashing.\n", numberOfRepresentatives);
       representativesID = new HashSet<Integer>(numberOfRepresentatives);
       while (representativesID.size() < numberOfRepresentatives) {
         representativesID.add((int) Math.floor(Math.random() * (docCount - 1)));
       }
       representatives = new ArrayList<LireFeature>(numberOfRepresentatives);
       docCount = 0;
       run = 1;
       if (verbose) System.out.println("Now getting representatives from the data file.");
       readFile(indexWriter, inputFile);
       docCount = 0;
       run = 2;
       if (verbose) System.out.println("Finally we start the indexing process, please wait ...");
       readFile(indexWriter, inputFile);
       if (verbose) System.out.println("Indexing finished.");
     }
     indexWriter.commit();
     indexWriter.close();
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 Map<String, Set<Integer>> findCandidateUnionDecisions() {
   HashSet<Integer> curDecisions = new HashSet<Integer>();
   for (int i = 0; i < unionTypes.size(); i++) {
     for (int j = i + 1; j < unionTypes.size(); j++) {
       List<String> iBases = unionTypes.get(i).getBases();
       List<String> jBases = unionTypes.get(j).getBases();
       if (isPrefixOf(iBases, jBases) || isPrefixOf(jBases, iBases)) {
         curDecisions.add(i);
         curDecisions.add(j);
       }
     }
   }
   Map<String, Set<Integer>> candidateUnionDecisions = new HashMap<String, Set<Integer>>();
   for (InferredType subelt : unionTypes) {
     candidateUnionDecisions.putAll(subelt.findCandidateUnionDecisions());
   }
   if (curDecisions.size() > 0) {
     candidateUnionDecisions.put(name, curDecisions);
   }
   return candidateUnionDecisions;
 }
Beispiel #18
0
  /**
   * Implements {@link ReceiveStreamListener#update(ReceiveStreamEvent)}.
   *
   * <p>{@link #rtpManager} will use this to notify us of <tt>ReceiveStreamEvent</tt>s.
   */
  @Override
  public void update(ReceiveStreamEvent event) {
    if (event == null) return;
    ReceiveStream receiveStream = event.getReceiveStream();

    if (event instanceof NewReceiveStreamEvent) {
      if (receiveStream == null) {
        logger.warn("NewReceiveStreamEvent: null");
        return;
      }

      final long ssrc = getReceiveStreamSSRC(receiveStream);

      ReceiveStreamDesc receiveStreamDesc = findReceiveStream(ssrc);

      if (receiveStreamDesc != null) {
        String s = "NewReceiveStreamEvent for an existing SSRC. ";
        if (receiveStream != receiveStreamDesc.receiveStream)
          s += "(but different ReceiveStream object)";
        logger.warn(s);
        return;
      } else receiveStreamDesc = new ReceiveStreamDesc(receiveStream);

      if (logger.isInfoEnabled()) logger.info("New ReceiveStream, ssrc=" + ssrc);

      // Find the format of the ReceiveStream
      DataSource dataSource = receiveStream.getDataSource();
      if (dataSource instanceof PushBufferDataSource) {
        Format format = null;
        PushBufferDataSource pbds = (PushBufferDataSource) dataSource;
        for (PushBufferStream pbs : pbds.getStreams()) {
          if ((format = pbs.getFormat()) != null) break;
        }

        if (format == null) {
          logger.error("Failed to handle new ReceiveStream: " + "Failed to determine format");
          return;
        }

        receiveStreamDesc.format = format;
      } else {
        logger.error("Failed to handle new ReceiveStream: " + "Unsupported DataSource");
        return;
      }

      int rtpClockRate = -1;
      if (receiveStreamDesc.format instanceof AudioFormat)
        rtpClockRate = (int) ((AudioFormat) receiveStreamDesc.format).getSampleRate();
      else if (receiveStreamDesc.format instanceof VideoFormat) rtpClockRate = 90000;
      getSynchronizer().setRtpClockRate(ssrc, rtpClockRate);

      // create a Processor and configure it
      Processor processor = null;
      try {
        processor = Manager.createProcessor(receiveStream.getDataSource());
      } catch (NoProcessorException npe) {
        logger.error("Failed to create Processor: ", npe);
        return;
      } catch (IOException ioe) {
        logger.error("Failed to create Processor: ", ioe);
        return;
      }

      if (logger.isInfoEnabled()) logger.info("Created processor for SSRC=" + ssrc);

      processor.addControllerListener(this);
      receiveStreamDesc.processor = processor;

      final int streamCount;
      synchronized (receiveStreams) {
        receiveStreams.add(receiveStreamDesc);
        streamCount = receiveStreams.size();
      }

      /*
       * XXX TODO IRBABOON
       * This is a terrible hack which works around a failure to realize()
       * some of the Processor-s for audio streams, when multiple streams
       * start nearly simultaneously. The cause of the problem is currently
       * unknown (and synchronizing all FMJ calls in RecorderRtpImpl
       * does not help).
       * XXX TODO NOOBABRI
       */
      if (receiveStreamDesc.format instanceof AudioFormat) {
        final Processor p = processor;
        new Thread() {
          @Override
          public void run() {
            // delay configuring the processors for the different
            // audio streams to decrease the probability that they
            // run together.
            try {
              int ms = 450 * (streamCount - 1);
              logger.warn(
                  "Sleeping for "
                      + ms
                      + "ms before"
                      + " configuring processor for SSRC="
                      + ssrc
                      + " "
                      + System.currentTimeMillis());
              Thread.sleep(ms);
            } catch (Exception e) {
            }

            p.configure();
          }
        }.run();
      } else {
        processor.configure();
      }
    } else if (event instanceof TimeoutEvent) {
      if (receiveStream == null) {
        // TODO: we might want to get the list of ReceiveStream-s from
        // rtpManager and compare it to our list, to see if we should
        // remove a stream.
        logger.warn("TimeoutEvent: null.");
        return;
      }

      // FMJ silently creates new ReceiveStream instances, so we have to
      // recognize them by the SSRC.
      ReceiveStreamDesc receiveStreamDesc = findReceiveStream(getReceiveStreamSSRC(receiveStream));
      if (receiveStreamDesc != null) {
        if (logger.isInfoEnabled()) {
          logger.info("ReceiveStream timeout, ssrc=" + receiveStreamDesc.ssrc);
        }

        removeReceiveStream(receiveStreamDesc, true);
      }
    } else if (event != null && logger.isInfoEnabled()) {
      logger.info("Unhandled ReceiveStreamEvent (" + event.getClass().getName() + "): " + event);
    }
  }
Beispiel #19
0
  /**
   * The Main-Class for the Pig Jar that will provide a shell and setup a classpath appropriate for
   * executing Jar files.
   *
   * @param args -jar can be used to add additional jar files (colon separated). - will start a
   *     shell. -e will execute the rest of the command line as if it was input to the shell.
   * @throws IOException
   */
  public static void main(String args[]) {
    int rc = 1;
    Properties properties = new Properties();
    PropertiesUtil.loadPropertiesFromFile(properties);

    boolean verbose = false;
    boolean gruntCalled = false;
    String logFileName = null;

    try {
      BufferedReader pin = null;
      boolean debug = false;
      boolean dryrun = false;
      ArrayList<String> params = new ArrayList<String>();
      ArrayList<String> paramFiles = new ArrayList<String>();
      HashSet<String> optimizerRules = new HashSet<String>();

      CmdLineParser opts = new CmdLineParser(args);
      opts.registerOpt('4', "log4jconf", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('b', "brief", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('c', "cluster", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('d', "debug", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('e', "execute", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('f', "file", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('h', "help", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
      opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('l', "logfile", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('m', "param_file", CmdLineParser.ValueExpected.OPTIONAL);
      opts.registerOpt('o', "hod", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
      opts.registerOpt('r', "dryrun", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('t', "optimizer_off", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('v', "verbose", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('w', "warning", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('x', "exectype", CmdLineParser.ValueExpected.REQUIRED);
      opts.registerOpt('F', "stop_on_failure", CmdLineParser.ValueExpected.NOT_ACCEPTED);
      opts.registerOpt('M', "no_multiquery", CmdLineParser.ValueExpected.NOT_ACCEPTED);

      ExecMode mode = ExecMode.UNKNOWN;
      String file = null;
      ExecType execType = ExecType.MAPREDUCE;
      String execTypeString = properties.getProperty("exectype");
      if (execTypeString != null && execTypeString.length() > 0) {
        execType = PigServer.parseExecType(execTypeString);
      }
      String cluster = "local";
      String clusterConfigured = properties.getProperty("cluster");
      if (clusterConfigured != null && clusterConfigured.length() > 0) {
        cluster = clusterConfigured;
      }

      // by default warning aggregation is on
      properties.setProperty("aggregate.warning", "" + true);

      // by default multiquery optimization is on
      properties.setProperty("opt.multiquery", "" + true);

      // by default we keep going on error on the backend
      properties.setProperty("stop.on.failure", "" + false);

      char opt;
      while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
        switch (opt) {
          case '4':
            String log4jconf = opts.getValStr();
            if (log4jconf != null) {
              properties.setProperty(LOG4J_CONF, log4jconf);
            }
            break;

          case 'b':
            properties.setProperty(BRIEF, "true");
            break;

          case 'c':
            // Needed away to specify the cluster to run the MR job on
            // Bug 831708 - fixed
            String clusterParameter = opts.getValStr();
            if (clusterParameter != null && clusterParameter.length() > 0) {
              cluster = clusterParameter;
            }
            break;

          case 'd':
            String logLevel = opts.getValStr();
            if (logLevel != null) {
              properties.setProperty(DEBUG, logLevel);
            }
            debug = true;
            break;

          case 'e':
            mode = ExecMode.STRING;
            break;

          case 'f':
            mode = ExecMode.FILE;
            file = opts.getValStr();
            break;

          case 'F':
            properties.setProperty("stop.on.failure", "" + true);
            break;

          case 'h':
            usage();
            return;

          case 'i':
            System.out.println(getVersionString());
            return;

          case 'j':
            String jarsString = opts.getValStr();
            if (jarsString != null) {
              properties.setProperty(JAR, jarsString);
            }
            break;

          case 'l':
            // call to method that validates the path to the log file
            // and sets up the file to store the client side log file
            String logFileParameter = opts.getValStr();
            if (logFileParameter != null && logFileParameter.length() > 0) {
              logFileName = validateLogFile(logFileParameter, null);
            } else {
              logFileName = validateLogFile(logFileName, null);
            }
            properties.setProperty("pig.logfile", logFileName);
            break;

          case 'm':
            paramFiles.add(opts.getValStr());
            break;

          case 'M':
            // turns off multiquery optimization
            properties.setProperty("opt.multiquery", "" + false);
            break;

          case 'o':
            // TODO sgroschupf using system properties is always a very bad idea
            String gateway = System.getProperty("ssh.gateway");
            if (gateway == null || gateway.length() == 0) {
              properties.setProperty("hod.server", "local");
            } else {
              properties.setProperty("hod.server", System.getProperty("ssh.gateway"));
            }
            break;

          case 'p':
            String val = opts.getValStr();
            params.add(opts.getValStr());
            break;

          case 'r':
            // currently only used for parameter substitution
            // will be extended in the future
            dryrun = true;
            break;

          case 't':
            optimizerRules.add(opts.getValStr());
            break;

          case 'v':
            properties.setProperty(VERBOSE, "" + true);
            verbose = true;
            break;

          case 'w':
            properties.setProperty("aggregate.warning", "" + false);
            break;

          case 'x':
            try {
              execType = PigServer.parseExecType(opts.getValStr());
            } catch (IOException e) {
              throw new RuntimeException("ERROR: Unrecognized exectype.", e);
            }
            break;
          default:
            {
              Character cc = new Character(opt);
              throw new AssertionError("Unhandled option " + cc.toString());
            }
        }
      }
      // configure logging
      configureLog4J(properties);
      // create the context with the parameter
      PigContext pigContext = new PigContext(execType, properties);

      if (logFileName == null) {
        logFileName = validateLogFile(null, null);
      }

      pigContext.getProperties().setProperty("pig.logfile", logFileName);

      if (optimizerRules.size() > 0) {
        pigContext
            .getProperties()
            .setProperty("pig.optimizer.rules", ObjectSerializer.serialize(optimizerRules));
      }

      LogicalPlanBuilder.classloader = pigContext.createCl(null);

      // construct the parameter substitution preprocessor
      Grunt grunt = null;
      BufferedReader in;
      String substFile = null;
      switch (mode) {
        case FILE:
          {
            // Run, using the provided file as a pig file
            in = new BufferedReader(new FileReader(file));

            // run parameter substitution preprocessor first
            substFile = file + ".substituted";
            pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
            if (dryrun) {
              log.info("Dry run completed. Substituted pig script is at " + substFile);
              return;
            }

            logFileName = validateLogFile(logFileName, file);
            pigContext.getProperties().setProperty("pig.logfile", logFileName);

            // Set job name based on name of the script
            pigContext
                .getProperties()
                .setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(file).getName());

            if (!debug) {
              new File(substFile).deleteOnExit();
            }

            grunt = new Grunt(pin, pigContext);
            gruntCalled = true;
            int results[] = grunt.exec();
            rc = getReturnCodeForStats(results);
            return;
          }

        case STRING:
          {
            // Gather up all the remaining arguments into a string and pass them into
            // grunt.
            StringBuffer sb = new StringBuffer();
            String remainders[] = opts.getRemainingArgs();
            for (int i = 0; i < remainders.length; i++) {
              if (i != 0) sb.append(' ');
              sb.append(remainders[i]);
            }
            in = new BufferedReader(new StringReader(sb.toString()));
            grunt = new Grunt(in, pigContext);
            gruntCalled = true;
            int results[] = grunt.exec();
            rc = getReturnCodeForStats(results);
            return;
          }

        default:
          break;
      }

      // If we're here, we don't know yet what they want.  They may have just
      // given us a jar to execute, they might have given us a pig script to
      // execute, or they might have given us a dash (or nothing) which means to
      // run grunt interactive.
      String remainders[] = opts.getRemainingArgs();
      if (remainders == null) {
        // Interactive
        mode = ExecMode.SHELL;
        ConsoleReader reader = new ConsoleReader(System.in, new OutputStreamWriter(System.out));
        reader.setDefaultPrompt("grunt> ");
        final String HISTORYFILE = ".pig_history";
        String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE;
        reader.setHistory(new History(new File(historyFile)));
        ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
        grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
        grunt.setConsoleReader(reader);
        gruntCalled = true;
        grunt.run();
        rc = 0;
        return;
      } else {
        // They have a pig script they want us to run.
        if (remainders.length > 1) {
          throw new RuntimeException(
              "You can only run one pig script " + "at a time from the command line.");
        }
        mode = ExecMode.FILE;
        in = new BufferedReader(new FileReader(remainders[0]));

        // run parameter substitution preprocessor first
        substFile = remainders[0] + ".substituted";
        pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
        if (dryrun) {
          log.info("Dry run completed. Substituted pig script is at " + substFile);
          return;
        }

        logFileName = validateLogFile(logFileName, remainders[0]);
        pigContext.getProperties().setProperty("pig.logfile", logFileName);

        if (!debug) {
          new File(substFile).deleteOnExit();
        }

        // Set job name based on name of the script
        pigContext
            .getProperties()
            .setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(remainders[0]).getName());

        grunt = new Grunt(pin, pigContext);
        gruntCalled = true;
        int[] results = grunt.exec();
        rc = getReturnCodeForStats(results);
        return;
      }

      // Per Utkarsh and Chris invocation of jar file via pig depricated.
    } catch (ParseException e) {
      usage();
      rc = 2;
    } catch (NumberFormatException e) {
      usage();
      rc = 2;
    } catch (PigException pe) {
      if (pe.retriable()) {
        rc = 1;
      } else {
        rc = 2;
      }

      if (!gruntCalled) {
        LogUtils.writeLog(pe, logFileName, log, verbose);
      }
    } catch (Throwable e) {
      rc = 2;
      if (!gruntCalled) {
        LogUtils.writeLog(e, logFileName, log, verbose);
      }
    } finally {
      // clear temp files
      FileLocalizer.deleteTempFiles();
      PerformanceTimerFactory.getPerfTimerFactory().dumpTimers();
      System.exit(rc);
    }
  }
Beispiel #20
0
 public static synchronized int getConnectionCount() {
   return m_notUsedConnection.size() + m_usedUsedConnection.size();
 }
  /**
   * Preform a merge commit
   *
   * @param project a project
   * @param root a vcs root
   * @param added added files
   * @param removed removed files
   * @param messageFile a message file for commit
   * @param author an author
   * @param exceptions the list of exceptions to report
   * @param partialOperation
   * @return true if merge commit was successful
   */
  private static boolean mergeCommit(
      final Project project,
      final VirtualFile root,
      final Set<FilePath> added,
      final Set<FilePath> removed,
      final File messageFile,
      final String author,
      List<VcsException> exceptions,
      @NotNull final PartialOperation partialOperation) {
    HashSet<FilePath> realAdded = new HashSet<FilePath>();
    HashSet<FilePath> realRemoved = new HashSet<FilePath>();
    // perform diff
    GitSimpleHandler diff = new GitSimpleHandler(project, root, GitCommand.DIFF);
    diff.setSilent(true);
    diff.setStdoutSuppressed(true);
    diff.addParameters("--diff-filter=ADMRUX", "--name-status", "HEAD");
    diff.endOptions();
    String output;
    try {
      output = diff.run();
    } catch (VcsException ex) {
      exceptions.add(ex);
      return false;
    }
    String rootPath = root.getPath();
    for (StringTokenizer lines = new StringTokenizer(output, "\n", false);
        lines.hasMoreTokens(); ) {
      String line = lines.nextToken().trim();
      if (line.length() == 0) {
        continue;
      }
      String[] tk = line.split("\t");
      switch (tk[0].charAt(0)) {
        case 'M':
        case 'A':
          realAdded.add(VcsUtil.getFilePath(rootPath + "/" + tk[1]));
          break;
        case 'D':
          realRemoved.add(VcsUtil.getFilePathForDeletedFile(rootPath + "/" + tk[1], false));
          break;
        default:
          throw new IllegalStateException("Unexpected status: " + line);
      }
    }
    realAdded.removeAll(added);
    realRemoved.removeAll(removed);
    if (realAdded.size() != 0 || realRemoved.size() != 0) {

      final List<FilePath> files = new ArrayList<FilePath>();
      files.addAll(realAdded);
      files.addAll(realRemoved);
      final Ref<Boolean> mergeAll = new Ref<Boolean>();
      try {
        GuiUtils.runOrInvokeAndWait(
            new Runnable() {
              public void run() {
                String message =
                    GitBundle.message("commit.partial.merge.message", partialOperation.getName());
                SelectFilePathsDialog dialog =
                    new SelectFilePathsDialog(
                        project,
                        files,
                        message,
                        null,
                        "Commit All Files",
                        CommonBundle.getCancelButtonText(),
                        false);
                dialog.setTitle(GitBundle.getString("commit.partial.merge.title"));
                dialog.show();
                mergeAll.set(dialog.isOK());
              }
            });
      } catch (RuntimeException ex) {
        throw ex;
      } catch (Exception ex) {
        throw new RuntimeException("Unable to invoke a message box on AWT thread", ex);
      }
      if (!mergeAll.get()) {
        return false;
      }
      // update non-indexed files
      if (!updateIndex(project, root, realAdded, realRemoved, exceptions)) {
        return false;
      }
      for (FilePath f : realAdded) {
        VcsDirtyScopeManager.getInstance(project).fileDirty(f);
      }
      for (FilePath f : realRemoved) {
        VcsDirtyScopeManager.getInstance(project).fileDirty(f);
      }
    }
    // perform merge commit
    try {
      GitSimpleHandler handler = new GitSimpleHandler(project, root, GitCommand.COMMIT);
      handler.setStdoutSuppressed(false);
      handler.addParameters("-F", messageFile.getAbsolutePath());
      if (author != null) {
        handler.addParameters("--author=" + author);
      }
      handler.endOptions();
      handler.run();
      GitRepositoryManager manager = GitUtil.getRepositoryManager(project);
      manager.updateRepository(root);
    } catch (VcsException ex) {
      exceptions.add(ex);
      return false;
    }
    return true;
  }
  @Test
  public void ortest3() {
    final HashSet<Integer> V1 = new HashSet<Integer>();
    final HashSet<Integer> V2 = new HashSet<Integer>();

    final MutableRoaringBitmap rr = new MutableRoaringBitmap();
    final MutableRoaringBitmap rr2 = new MutableRoaringBitmap();
    // For the first 65536: rr2 has a bitmap container, and rr has
    // an array container.
    // We will check the union between a BitmapCintainer and an
    // arrayContainer
    for (int k = 0; k < 4000; ++k) {
      rr2.add(k);
      V1.add(k);
    }
    for (int k = 3500; k < 4500; ++k) {
      rr.add(k);
      V1.add(k);
    }
    for (int k = 4000; k < 65000; ++k) {
      rr2.add(k);
      V1.add(k);
    }

    // In the second node of each roaring bitmap, we have two bitmap
    // containers.
    // So, we will check the union between two BitmapContainers
    for (int k = 65536; k < 65536 + 10000; ++k) {
      rr.add(k);
      V1.add(k);
    }

    for (int k = 65536; k < 65536 + 14000; ++k) {
      rr2.add(k);
      V1.add(k);
    }

    // In the 3rd node of each Roaring Bitmap, we have an
    // ArrayContainer, so, we will try the union between two
    // ArrayContainers.
    for (int k = 4 * 65535; k < 4 * 65535 + 1000; ++k) {
      rr.add(k);
      V1.add(k);
    }

    for (int k = 4 * 65535; k < 4 * 65535 + 800; ++k) {
      rr2.add(k);
      V1.add(k);
    }

    // For the rest, we will check if the union will take them in
    // the result
    for (int k = 6 * 65535; k < 6 * 65535 + 1000; ++k) {
      rr.add(k);
      V1.add(k);
    }

    for (int k = 7 * 65535; k < 7 * 65535 + 2000; ++k) {
      rr2.add(k);
      V1.add(k);
    }

    final MutableRoaringBitmap rror = MutableRoaringBitmap.or(rr, rr2);
    boolean valide = true;

    // Si tous les elements de rror sont dans V1 et que tous les
    // elements de
    // V1 sont dans rror(V2)
    // alors V1 == rror

    final Object[] tab = V1.toArray();
    final Vector<Integer> vector = new Vector<Integer>();
    for (Object aTab : tab) vector.add((Integer) aTab);

    for (final int i : rror.toArray()) {
      if (!vector.contains(i)) {
        valide = false;
      }
      V2.add(i);
    }
    for (int i = 0; i < V1.size(); i++)
      if (!V2.contains(vector.elementAt(i))) {
        valide = false;
      }

    Assert.assertEquals(valide, true);
  }
  /**
   * This does the work of main, but it never calls System.exit, so it is appropriate to be called
   * progrmmatically. Termination of the program with a message to the user is indicated by throwing
   * Daikon.TerminationMessage.
   *
   * @see #main(String[])
   * @see daikon.Daikon.TerminationMessage
   */
  public static void mainHelper(final String[] args)
      throws FileNotFoundException, StreamCorruptedException, OptionalDataException, IOException,
          ClassNotFoundException {
    daikon.LogHelper.setupLogs(daikon.LogHelper.INFO);

    LongOpt[] longopts =
        new LongOpt[] {
          new LongOpt(Daikon.config_option_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
          new LongOpt(output_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
          new LongOpt(dir_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
          new LongOpt(conf_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
          new LongOpt(filter_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
          new LongOpt(Daikon.debugAll_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
          new LongOpt(Daikon.debug_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
          new LongOpt(Daikon.ppt_regexp_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
          new LongOpt(Daikon.track_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
        };
    Getopt g = new Getopt("daikon.tools.InvariantChecker", args, "h", longopts);
    int c;
    while ((c = g.getopt()) != -1) {
      switch (c) {
        case 0:
          // got a long option
          String option_name = longopts[g.getLongind()].getName();
          if (Daikon.help_SWITCH.equals(option_name)) {
            System.out.println(usage);
            throw new Daikon.TerminationMessage();
          } else if (conf_SWITCH.equals(option_name)) {
            doConf = true;
          } else if (filter_SWITCH.equals(option_name)) {
            doFilter = true;
          } else if (dir_SWITCH.equals(option_name)) {
            dir_file = new File(g.getOptarg());
            if (!dir_file.exists() || !dir_file.isDirectory())
              throw new Daikon.TerminationMessage("Error reading the directory " + dir_file);

          } else if (output_SWITCH.equals(option_name)) {
            output_file = new File(g.getOptarg());
            output_stream = new PrintStream(new FileOutputStream(output_file));
          } else if (Daikon.config_option_SWITCH.equals(option_name)) {
            String item = g.getOptarg();
            daikon.config.Configuration.getInstance().apply(item);
            break;
          } else if (Daikon.debugAll_SWITCH.equals(option_name)) {
            Global.debugAll = true;
          } else if (Daikon.debug_SWITCH.equals(option_name)) {
            LogHelper.setLevel(g.getOptarg(), LogHelper.FINE);
          } else if (Daikon.track_SWITCH.equals(option_name)) {
            LogHelper.setLevel("daikon.Debug", LogHelper.FINE);
            String error = Debug.add_track(g.getOptarg());
            if (error != null) {
              throw new Daikon.TerminationMessage(
                  "Error parsing track argument '" + g.getOptarg() + "' - " + error);
            }
          } else {
            throw new RuntimeException("Unknown long option received: " + option_name);
          }
          break;
        case 'h':
          System.out.println(usage);
          throw new Daikon.TerminationMessage();
        case '?':
          break; // getopt() already printed an error
        default:
          System.out.println("getopt() returned " + c);
          break;
      }
    }

    // Loop through each filename specified
    for (int i = g.getOptind(); i < args.length; i++) {

      // Get the file and make sure it exists
      File file = new File(args[i]);
      if (!file.exists()) {
        throw new Error("File " + file + " not found.");
      }

      // These aren't "endsWith()" because there might be a suffix on the end
      // (eg, a date).
      String filename = file.toString();
      if (filename.indexOf(".inv") != -1) {
        if (inv_file != null) {
          throw new Daikon.TerminationMessage("multiple inv files specified", usage);
        }
        inv_file = file;
      } else if (filename.indexOf(".dtrace") != -1) {
        dtrace_files.add(filename);
      } else {
        throw new Error("Unrecognized argument: " + file);
      }
    }
    if (dir_file == null) {
      checkInvariants();
      return;
    }

    // Yoav additions:
    File[] filesInDir = dir_file.listFiles();
    if (filesInDir == null || filesInDir.length == 0)
      throw new Daikon.TerminationMessage("The directory " + dir_file + " is empty", usage);
    ArrayList<File> invariants = new ArrayList<File>();
    for (File f : filesInDir) if (f.toString().indexOf(".inv") != -1) invariants.add(f);
    if (invariants.size() == 0)
      throw new Daikon.TerminationMessage(
          "Did not find any invariant files in the directory " + dir_file, usage);
    ArrayList<File> dtraces = new ArrayList<File>();
    for (File f : filesInDir) if (f.toString().indexOf(".dtrace") != -1) dtraces.add(f);
    if (dtraces.size() == 0)
      throw new Daikon.TerminationMessage(
          "Did not find any dtrace files in the directory " + dir_file, usage);

    System.out.println(
        "Collecting data for invariants files " + invariants + " and dtrace files " + dtraces);

    dtrace_files.clear();
    for (File dtrace : dtraces) {
      dtrace_files.add(dtrace.toString());
    }

    String commaLine = "";
    for (File inFile : invariants) {
      String name = inFile.getName().replace(".inv", "").replace(".gz", "");
      commaLine += "," + name;
    }
    outputComma.add(commaLine);

    commaLine = "";
    for (File inFile : invariants) {
      inv_file = inFile;
      failedInvariants.clear();
      testedInvariants.clear();
      error_cnt = 0;

      output_stream =
          new PrintStream(
              new FileOutputStream(
                  inFile.toString().replace(".inv", "").replace(".gz", "")
                      + ".false-positives.txt"));
      checkInvariants();
      output_stream.close();

      int failedCount = failedInvariants.size();
      int testedCount = testedInvariants.size();
      String percent = toPercentage(failedCount, testedCount);
      commaLine += "," + percent;
    }
    outputComma.add(commaLine);

    System.out.println();
    for (String output : outputComma) System.out.println(output);
  }
  private static void checkInvariants() throws IOException {
    // Read the invariant file
    PptMap ppts = FileIO.read_serialized_pptmap(inv_file, true);

    // Yoav: make sure we have unique invariants
    InvariantFilters fi = InvariantFilters.defaultFilters();
    // Set<String> allInvariantsStr = new HashSet<String>();
    Set<Invariant> allInvariants = new HashSet<Invariant>();
    for (PptTopLevel ppt : ppts.all_ppts())
      for (Iterator<PptSlice> i = ppt.views_iterator(); i.hasNext(); ) {
        PptSlice slice = i.next();
        for (Invariant inv : slice.invs) {
          if (doConf && inv.getConfidence() < Invariant.dkconfig_confidence_limit) {
            // System.out.printf ("inv ignored (conf): %s:%s\n", inv.ppt.name(),
            //                   inv.format());
            continue;
          }

          if (doFilter && fi.shouldKeep(inv) == null) {
            // System.out.printf ("inv ignored (filter): %s:%s\n",
            //                     inv.ppt.name(), inv.format());
            continue;
          }
          activeInvariants.add(inv);

          // String n = invariant2str(ppt, inv);
          // if (!allInvariants.contains(inv) && allInvariantsStr.contains(n)) throw new
          // Daikon.TerminationMessage("Two invariants have the same ppt.name+inv.rep:"+n);
          allInvariants.add(inv);
          // allInvariantsStr.add(n);
        }
      }

    // Read and process the data trace files
    FileIO.Processor processor = new InvariantCheckProcessor();

    Daikon.FileIOProgress progress = new Daikon.FileIOProgress();
    progress.start();
    progress.clear();
    FileIO.read_data_trace_files(dtrace_files, ppts, processor, false);
    progress.shouldStop = true;
    System.out.println();
    System.out.printf(
        "%s: %,d errors found in %,d samples (%s)\n",
        inv_file, error_cnt, sample_cnt, toPercentage(error_cnt, sample_cnt));
    int failedCount = failedInvariants.size();
    int testedCount = testedInvariants.size();
    String percent = toPercentage(failedCount, testedCount);
    System.out.println(
        inv_file
            + ": "
            + failedCount
            + " false positives, out of "
            + testedCount
            + ", which is "
            + percent
            + ".");
    if (false) {
      for (Invariant inv : failedInvariants) {
        System.out.printf("+%s:%s\n", inv.ppt.name(), inv.format());
      }
    }
  }