public int count(int b1, int q1, int n1, int b2, int q2, int n2) {
if (b2 == 0 || q2 <= 1) {
int tb = b1;
int tq = q1;
int tn = n1;
b1 = b2;
q1 = q2;
n1 = n2;
b2 = tb;
q2 = tq;
n2 = tn;
}
if (b1 == 0 || q1 <= 1) {
HashSet<Integer> set = new HashSet<Integer>();
set.add(b1);
if (n1 > 1) {
set.add(b1 * q1);
}
long curr = b2;
for (int i = 0; i < n2; i++) {
set.add((int) curr);
curr *= q2;
if (curr > 500000000) {
return (n2 - i - 1) + set.size();
}
}
return set.size();
} else {
HashSet<String> set = new HashSet<String>();
int factors[] = findFactors(b1, q1, b2, q2);
int repb1[] = decompose(b1, factors);
int repb2[] = decompose(b2, factors);
int repq1[] = decompose(q1, factors);
int repq2[] = decompose(q2, factors);
for (int i = 0; i < n1; i++) {
set.add(Arrays.toString(repb1));
for (int j = 0; j < repb1.length; j++) {
repb1[j] += repq1[j];
}
}
for (int i = 0; i < n2; i++) {
set.add(Arrays.toString(repb2));
for (int j = 0; j < repb2.length; j++) {
repb2[j] += repq2[j];
}
}
return set.size();
}
}
/** Find all methods in this framework which raise SQLFeatureNotSupportedException. */
public void testSupportedMethods() throws Exception {
getTestConfiguration().setVerbosity(true);
HashSet<String> vanishedMethodList = new HashSet<String>();
HashSet<String> unsupportedList = new HashSet<String>();
HashSet<String> notUnderstoodList = new HashSet<String>();
// Build map of interfaces to their methods which may raise SQLFeatureNotSupportedException.
initializeExcludableMap(vanishedMethodList);
vetDataSource(unsupportedList, notUnderstoodList);
vetConnectionPooledDataSource(unsupportedList, notUnderstoodList);
vetXADataSource(unsupportedList, notUnderstoodList);
//
// Print methods which behave unexpectedly.
//
printVanishedMethodList(vanishedMethodList);
printUnsupportedList(unsupportedList);
printNotUnderstoodList(notUnderstoodList);
int actualErrorCount =
vanishedMethodList.size() + unsupportedList.size() + notUnderstoodList.size();
assertEquals("Unexpected discrepancies.", 0, actualErrorCount);
}
static void runTest(HashSet<Integer> caseSet) {
int cases = 0, passed = 0;
while (true) {
String label = Reader.nextLine();
if (label == null || !label.startsWith("--")) break;
int N = Integer.parseInt(Reader.nextLine());
Reader.nextLine();
double __answer = Double.parseDouble(Reader.nextLine());
cases++;
if (caseSet.size() > 0 && !caseSet.contains(cases - 1)) continue;
System.err.print(String.format(" Testcase #%d ... ", cases - 1));
if (doTest(N, __answer)) passed++;
}
if (caseSet.size() > 0) cases = caseSet.size();
System.err.println(String.format("%nPassed : %d/%d cases", passed, cases));
int T = (int) (System.currentTimeMillis() / 1000) - 1398529188;
double PT = T / 60.0, TT = 75.0;
System.err.println(
String.format(
"Time : %d minutes %d secs%nScore : %.2f points",
T / 60, T % 60, 950 * (0.3 + (0.7 * TT * TT) / (10.0 * PT * PT + TT * TT))));
}
public static void main(String[] args) throws Exception {
String graphName = args[0];
int nShards = Integer.parseInt(args[1]);
CircleOfTrustSalsa csalsa = new CircleOfTrustSalsa(new VertexQuery(graphName, nShards), 10000);
VertexIdTranslate vertexTrans =
VertexIdTranslate.fromFile(
new File(ChiFilenames.getVertexTranslateDefFile(graphName, nShards)));
BufferedReader cmdIn = new BufferedReader(new InputStreamReader(System.in));
while (true) {
System.out.print("Enter vertex id to query >> :: ");
String ln = cmdIn.readLine();
int vertex = Integer.parseInt(ln);
// Circle of trust is just the vertex's followers for now
HashSet<Integer> circle = csalsa.queryService.queryOutNeighbors(vertexTrans.forward(vertex));
int maxCircleSize = 300;
// max 500
if (circle.size() > maxCircleSize) {
int[] all = new int[circle.size()];
int i = 0;
for (Integer v : circle) all[i++] = v;
HashSet<Integer> filteredCircle = new HashSet<Integer>();
Random r = new Random(260379);
for (i = 0; i < maxCircleSize; i++)
filteredCircle.add(all[Math.abs(r.nextInt()) % all.length]);
circle = filteredCircle;
}
csalsa.initializeGraph(circle);
long t = System.currentTimeMillis();
csalsa.computeSALSA(3);
logger.info("SALSA computation took " + (System.currentTimeMillis() - t) + "ms");
circle.add(vertexTrans.forward(vertex));
ArrayList<SalsaVertex> top = csalsa.topAuthorities(20, circle);
int j = 1;
for (SalsaVertex sv : top) {
int originalId = vertexTrans.backward(sv.id);
logger.info(
"Top "
+ (j++)
+ " = "
+ originalId
+ " "
+ csalsa.namify(originalId)
+ " ("
+ sv.value
+ ")");
}
}
}
public void testMostCommonCharIn() {
FileResource fr = new FileResource("dictionaries/English");
HashSet<String> dict = readDictionary(fr);
System.out.println("English dict has " + dict.size() + " entries");
char mostCommon = mostCommonCharIn(dict);
System.out.println(" most common char: " + mostCommon);
FileResource fr1 = new FileResource("dictionaries/Spanish");
HashSet<String> dict1 = readDictionary(fr1);
System.out.println("Spanish dict has " + dict1.size() + " entries");
mostCommon = mostCommonCharIn(dict1);
System.out.println(" most common char: " + mostCommon);
}
/**
* Returns the list of friends
*
* @return The list of friends
*/
public static Player[] getFriends() {
synchronized (monitor) {
final Player[] rc = new Player[friends.size()];
friends.toArray(rc);
return rc;
}
}
public static double[] printPrecRec(String file, String gold) throws IOException {
Scanner in = new Scanner(new FileReader(gold));
HashSet<String> g = new HashSet<String>();
while (in.hasNextLine()) g.add(in.nextLine());
in.close();
in = new Scanner(new FileReader(file));
HashSet<String> f = new HashSet<String>();
while (in.hasNextLine()) {
f.add(in.nextLine());
}
in.close();
int correct = JaccardFromTF.intersect(g, f).size();
double[] res = new double[3];
res[0] = 100.0 * correct / f.size();
res[1] = 100.0 * correct / g.size();
if (res[0] + res[1] == 0) res[2] = 0;
else res[2] = 2.0 * res[0] * res[1] / (res[0] + res[1]);
System.out.println("Precision \t Recall \t FM");
System.out.println(res[0] + "\t " + res[1] + "\t" + res[2]);
return res;
}
@Override
public void execute() {
Map<String, Set<CortexLinksMap>> links = new HashMap<String, Set<CortexLinksMap>>();
Map<String, Set<String>> linkFilenames = new HashMap<String, Set<String>>();
if (LINKS != null && !LINKS.isEmpty()) {
for (CortexLinksMap cl : LINKS) {
String name = cl.getCortexLinks().getColor(0).getSampleName();
if (!links.containsKey(name)) {
links.put(name, new HashSet<CortexLinksMap>());
linkFilenames.put(name, new HashSet<String>());
}
links.get(name).add(cl);
linkFilenames.get(name).add(cl.getCortexLinks().getCortexLinksFile().getAbsolutePath());
}
log.info("Loaded {} links", LINKS.size());
for (CortexLinksMap clm : LINKS) {
log.info(
" {}: {}",
clm.getCortexLinks().getCortexLinksFile().getAbsolutePath(),
clm.getCortexLinks().getNumLinks());
}
}
log.info("Loaded {} graphs", GRAPHS.size());
for (CortexGraph cg : GRAPHS) {
String name = cg.getColor(0).getSampleName();
graphs.put(cg.getCortexFile().getName(), cg);
log.info(
" {}: {} {}", cg.getCortexFile().getName(), cg.getKmerSize(), linkFilenames.get(name));
}
loadContigs();
log.info("Loaded {} contigs", contigs.size());
log.info("Listening on port {}", PORT);
try {
HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
server.createContext("/", new PageHandler());
server.createContext("/graphlist", new GraphList());
server.createContext("/search", new SearchHandler());
server.createContext("/cr", new CtxRecordHandler());
server.createContext("/links", new LinksHandler());
server.setExecutor(null);
server.start();
} catch (IOException e) {
throw new IndianaException("Unable to start server", e);
}
}
public void testReadDictionary() {
FileResource fr = new FileResource("dictionaries/English");
HashSet<String> dict = readDictionary(fr);
System.out.println("dict has " + dict.size() + " entries");
Iterator<String> it = dict.iterator();
for (int i = 0; i < 25; i++) {
if (it.hasNext()) {
System.out.println(i + "\t" + it.next());
}
}
}
int[] findFactors(int x, int y, int w, int z) {
HashSet<Integer> factors = new HashSet<Integer>();
addFactors(factors, x);
addFactors(factors, y);
addFactors(factors, w);
addFactors(factors, z);
int res[] = new int[factors.size()];
int i = 0;
for (int f : factors) {
res[i++] = f;
}
return res;
}
public int count(int[] seq, int bound) {
HashSet<String> set = new HashSet<String>();
HashSet<Integer> nums = new HashSet<Integer>();
int n = seq.length;
for (int i = 0; i < n; i++) {
nums.add(seq[i]);
}
int t = nums.size();
int total = 0;
nums.add(-1);
for (int i = 0; i < n; i++) {
for (int j = 0; j < n; j++) {
for (int x : nums) {
int seq2[] = seq.clone();
if (i < j) {
for (int k = i + 1; k < j + 1; k++) {
seq2[k - 1] = seq2[k];
}
} else if (i > j) {
for (int k = i; k > j; k--) {
seq2[k] = seq2[k - 1];
}
}
seq2[j] = x;
String key = Arrays.toString(seq2);
if (!set.contains(key)) {
if (x == -1) {
total += bound - t;
} else {
total++;
}
total %= MOD;
set.add(key);
}
}
}
}
return total;
}
void solve() throws IOException {
int[] gr = new int[5001];
for (int i = 1; i < 5001; i++) {
HashSet<Integer> s = new HashSet<Integer>();
for (int j = 1; j + j <= i; j++) {
s.add(gr[i - j]);
}
for (int t = 0; t <= s.size(); t++) {
if (!s.contains(t)) {
gr[i] = t;
break;
}
}
}
/*for (int i = 0;i < 1000; i++) {
out.println(i + " " + gr[i]);
}*/
int a = nextInt();
int b = nextInt();
int c = nextInt();
int ans = gr[a] ^ gr[b] ^ gr[c];
if (ans == 0) {
out.println("NO");
} else {
out.println("YES");
for (int i = 1; i + i <= a; i++) {
if ((gr[a - i] ^ gr[b] ^ gr[c]) == 0) {
out.println((a - i) + " " + b + " " + c);
return;
}
}
for (int i = 1; i + i <= b; i++) {
if ((gr[a] ^ gr[b - i] ^ gr[c]) == 0) {
out.println(a + " " + (b - i) + " " + c);
return;
}
}
for (int i = 1; i + i <= c; i++) {
if ((gr[a] ^ gr[b] ^ gr[c - i]) == 0) {
out.println(a + " " + b + " " + (c - i));
return;
}
}
}
}
// debug print the list of methods which throw SQLFeatureNotSupportedException
private void printUnsupportedList(HashSet<String> unsupportedList) {
int count = unsupportedList.size();
if (count == 0) {
return;
}
println("--------------- UNSUPPORTED METHODS ------------------");
println("--");
String[] result = new String[count];
unsupportedList.toArray(result);
Arrays.sort(result);
for (int i = 0; i < count; i++) {
println(result[i]);
}
}
// debug print the list of methods which have disappeared from the SQL interface
private void printVanishedMethodList(HashSet<String> vanishedMethodList) {
int count = vanishedMethodList.size();
if (count == 0) {
return;
}
println("--------------- VANISHED METHODS ------------------");
println("--");
String[] result = new String[count];
vanishedMethodList.toArray(result);
Arrays.sort(result);
for (int i = 0; i < count; i++) {
println(result[i]);
}
}
// Debug print the list of method failures which we don't understand
private void printNotUnderstoodList(HashSet<String> notUnderstoodList) {
int count = notUnderstoodList.size();
if (count == 0) {
return;
}
println("\n\n");
println("--------------- NOT UNDERSTOOD METHODS ------------------");
println("--");
String[] result = new String[count];
notUnderstoodList.toArray(result);
Arrays.sort(result);
for (int i = 0; i < count; i++) {
println(result[i]);
}
}
public void run() {
// do it ...
try {
IndexWriter indexWriter =
LuceneUtils.createIndexWriter(
indexPath, overwriteIndex, LuceneUtils.AnalyzerType.WhitespaceAnalyzer);
for (Iterator<File> iterator = inputFiles.iterator(); iterator.hasNext(); ) {
File inputFile = iterator.next();
if (verbose) System.out.println("Processing " + inputFile.getPath() + ".");
if (verbose) System.out.println("Counting images.");
run = 0;
readFile(indexWriter, inputFile);
if (verbose) System.out.printf("%d images found in the data file.\n", docCount);
int numberOfRepresentatives = 1000; // TODO: clever selection.
// select a number of representative "fixed stars" randomly from file
if (numberOfRepresentatives > Math.sqrt(docCount))
numberOfRepresentatives = (int) Math.sqrt(docCount);
if (verbose)
System.out.printf(
"Selecting %d representative images for hashing.\n", numberOfRepresentatives);
representativesID = new HashSet<Integer>(numberOfRepresentatives);
while (representativesID.size() < numberOfRepresentatives) {
representativesID.add((int) Math.floor(Math.random() * (docCount - 1)));
}
representatives = new ArrayList<LireFeature>(numberOfRepresentatives);
docCount = 0;
run = 1;
if (verbose) System.out.println("Now getting representatives from the data file.");
readFile(indexWriter, inputFile);
docCount = 0;
run = 2;
if (verbose) System.out.println("Finally we start the indexing process, please wait ...");
readFile(indexWriter, inputFile);
if (verbose) System.out.println("Indexing finished.");
}
indexWriter.commit();
indexWriter.close();
} catch (Exception e) {
e.printStackTrace();
}
}
Map<String, Set<Integer>> findCandidateUnionDecisions() {
HashSet<Integer> curDecisions = new HashSet<Integer>();
for (int i = 0; i < unionTypes.size(); i++) {
for (int j = i + 1; j < unionTypes.size(); j++) {
List<String> iBases = unionTypes.get(i).getBases();
List<String> jBases = unionTypes.get(j).getBases();
if (isPrefixOf(iBases, jBases) || isPrefixOf(jBases, iBases)) {
curDecisions.add(i);
curDecisions.add(j);
}
}
}
Map<String, Set<Integer>> candidateUnionDecisions = new HashMap<String, Set<Integer>>();
for (InferredType subelt : unionTypes) {
candidateUnionDecisions.putAll(subelt.findCandidateUnionDecisions());
}
if (curDecisions.size() > 0) {
candidateUnionDecisions.put(name, curDecisions);
}
return candidateUnionDecisions;
}
/**
* Implements {@link ReceiveStreamListener#update(ReceiveStreamEvent)}.
*
* <p>{@link #rtpManager} will use this to notify us of <tt>ReceiveStreamEvent</tt>s.
*/
@Override
public void update(ReceiveStreamEvent event) {
if (event == null) return;
ReceiveStream receiveStream = event.getReceiveStream();
if (event instanceof NewReceiveStreamEvent) {
if (receiveStream == null) {
logger.warn("NewReceiveStreamEvent: null");
return;
}
final long ssrc = getReceiveStreamSSRC(receiveStream);
ReceiveStreamDesc receiveStreamDesc = findReceiveStream(ssrc);
if (receiveStreamDesc != null) {
String s = "NewReceiveStreamEvent for an existing SSRC. ";
if (receiveStream != receiveStreamDesc.receiveStream)
s += "(but different ReceiveStream object)";
logger.warn(s);
return;
} else receiveStreamDesc = new ReceiveStreamDesc(receiveStream);
if (logger.isInfoEnabled()) logger.info("New ReceiveStream, ssrc=" + ssrc);
// Find the format of the ReceiveStream
DataSource dataSource = receiveStream.getDataSource();
if (dataSource instanceof PushBufferDataSource) {
Format format = null;
PushBufferDataSource pbds = (PushBufferDataSource) dataSource;
for (PushBufferStream pbs : pbds.getStreams()) {
if ((format = pbs.getFormat()) != null) break;
}
if (format == null) {
logger.error("Failed to handle new ReceiveStream: " + "Failed to determine format");
return;
}
receiveStreamDesc.format = format;
} else {
logger.error("Failed to handle new ReceiveStream: " + "Unsupported DataSource");
return;
}
int rtpClockRate = -1;
if (receiveStreamDesc.format instanceof AudioFormat)
rtpClockRate = (int) ((AudioFormat) receiveStreamDesc.format).getSampleRate();
else if (receiveStreamDesc.format instanceof VideoFormat) rtpClockRate = 90000;
getSynchronizer().setRtpClockRate(ssrc, rtpClockRate);
// create a Processor and configure it
Processor processor = null;
try {
processor = Manager.createProcessor(receiveStream.getDataSource());
} catch (NoProcessorException npe) {
logger.error("Failed to create Processor: ", npe);
return;
} catch (IOException ioe) {
logger.error("Failed to create Processor: ", ioe);
return;
}
if (logger.isInfoEnabled()) logger.info("Created processor for SSRC=" + ssrc);
processor.addControllerListener(this);
receiveStreamDesc.processor = processor;
final int streamCount;
synchronized (receiveStreams) {
receiveStreams.add(receiveStreamDesc);
streamCount = receiveStreams.size();
}
/*
* XXX TODO IRBABOON
* This is a terrible hack which works around a failure to realize()
* some of the Processor-s for audio streams, when multiple streams
* start nearly simultaneously. The cause of the problem is currently
* unknown (and synchronizing all FMJ calls in RecorderRtpImpl
* does not help).
* XXX TODO NOOBABRI
*/
if (receiveStreamDesc.format instanceof AudioFormat) {
final Processor p = processor;
new Thread() {
@Override
public void run() {
// delay configuring the processors for the different
// audio streams to decrease the probability that they
// run together.
try {
int ms = 450 * (streamCount - 1);
logger.warn(
"Sleeping for "
+ ms
+ "ms before"
+ " configuring processor for SSRC="
+ ssrc
+ " "
+ System.currentTimeMillis());
Thread.sleep(ms);
} catch (Exception e) {
}
p.configure();
}
}.run();
} else {
processor.configure();
}
} else if (event instanceof TimeoutEvent) {
if (receiveStream == null) {
// TODO: we might want to get the list of ReceiveStream-s from
// rtpManager and compare it to our list, to see if we should
// remove a stream.
logger.warn("TimeoutEvent: null.");
return;
}
// FMJ silently creates new ReceiveStream instances, so we have to
// recognize them by the SSRC.
ReceiveStreamDesc receiveStreamDesc = findReceiveStream(getReceiveStreamSSRC(receiveStream));
if (receiveStreamDesc != null) {
if (logger.isInfoEnabled()) {
logger.info("ReceiveStream timeout, ssrc=" + receiveStreamDesc.ssrc);
}
removeReceiveStream(receiveStreamDesc, true);
}
} else if (event != null && logger.isInfoEnabled()) {
logger.info("Unhandled ReceiveStreamEvent (" + event.getClass().getName() + "): " + event);
}
}
/**
* The Main-Class for the Pig Jar that will provide a shell and setup a classpath appropriate for
* executing Jar files.
*
* @param args -jar can be used to add additional jar files (colon separated). - will start a
* shell. -e will execute the rest of the command line as if it was input to the shell.
* @throws IOException
*/
public static void main(String args[]) {
int rc = 1;
Properties properties = new Properties();
PropertiesUtil.loadPropertiesFromFile(properties);
boolean verbose = false;
boolean gruntCalled = false;
String logFileName = null;
try {
BufferedReader pin = null;
boolean debug = false;
boolean dryrun = false;
ArrayList<String> params = new ArrayList<String>();
ArrayList<String> paramFiles = new ArrayList<String>();
HashSet<String> optimizerRules = new HashSet<String>();
CmdLineParser opts = new CmdLineParser(args);
opts.registerOpt('4', "log4jconf", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('b', "brief", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('c', "cluster", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('d', "debug", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('e', "execute", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('f', "file", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('h', "help", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('i', "version", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('j', "jar", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('l', "logfile", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('m', "param_file", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('o', "hod", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('p', "param", CmdLineParser.ValueExpected.OPTIONAL);
opts.registerOpt('r', "dryrun", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('t', "optimizer_off", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('v', "verbose", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('w', "warning", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('x', "exectype", CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('F', "stop_on_failure", CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('M', "no_multiquery", CmdLineParser.ValueExpected.NOT_ACCEPTED);
ExecMode mode = ExecMode.UNKNOWN;
String file = null;
ExecType execType = ExecType.MAPREDUCE;
String execTypeString = properties.getProperty("exectype");
if (execTypeString != null && execTypeString.length() > 0) {
execType = PigServer.parseExecType(execTypeString);
}
String cluster = "local";
String clusterConfigured = properties.getProperty("cluster");
if (clusterConfigured != null && clusterConfigured.length() > 0) {
cluster = clusterConfigured;
}
// by default warning aggregation is on
properties.setProperty("aggregate.warning", "" + true);
// by default multiquery optimization is on
properties.setProperty("opt.multiquery", "" + true);
// by default we keep going on error on the backend
properties.setProperty("stop.on.failure", "" + false);
char opt;
while ((opt = opts.getNextOpt()) != CmdLineParser.EndOfOpts) {
switch (opt) {
case '4':
String log4jconf = opts.getValStr();
if (log4jconf != null) {
properties.setProperty(LOG4J_CONF, log4jconf);
}
break;
case 'b':
properties.setProperty(BRIEF, "true");
break;
case 'c':
// Needed away to specify the cluster to run the MR job on
// Bug 831708 - fixed
String clusterParameter = opts.getValStr();
if (clusterParameter != null && clusterParameter.length() > 0) {
cluster = clusterParameter;
}
break;
case 'd':
String logLevel = opts.getValStr();
if (logLevel != null) {
properties.setProperty(DEBUG, logLevel);
}
debug = true;
break;
case 'e':
mode = ExecMode.STRING;
break;
case 'f':
mode = ExecMode.FILE;
file = opts.getValStr();
break;
case 'F':
properties.setProperty("stop.on.failure", "" + true);
break;
case 'h':
usage();
return;
case 'i':
System.out.println(getVersionString());
return;
case 'j':
String jarsString = opts.getValStr();
if (jarsString != null) {
properties.setProperty(JAR, jarsString);
}
break;
case 'l':
// call to method that validates the path to the log file
// and sets up the file to store the client side log file
String logFileParameter = opts.getValStr();
if (logFileParameter != null && logFileParameter.length() > 0) {
logFileName = validateLogFile(logFileParameter, null);
} else {
logFileName = validateLogFile(logFileName, null);
}
properties.setProperty("pig.logfile", logFileName);
break;
case 'm':
paramFiles.add(opts.getValStr());
break;
case 'M':
// turns off multiquery optimization
properties.setProperty("opt.multiquery", "" + false);
break;
case 'o':
// TODO sgroschupf using system properties is always a very bad idea
String gateway = System.getProperty("ssh.gateway");
if (gateway == null || gateway.length() == 0) {
properties.setProperty("hod.server", "local");
} else {
properties.setProperty("hod.server", System.getProperty("ssh.gateway"));
}
break;
case 'p':
String val = opts.getValStr();
params.add(opts.getValStr());
break;
case 'r':
// currently only used for parameter substitution
// will be extended in the future
dryrun = true;
break;
case 't':
optimizerRules.add(opts.getValStr());
break;
case 'v':
properties.setProperty(VERBOSE, "" + true);
verbose = true;
break;
case 'w':
properties.setProperty("aggregate.warning", "" + false);
break;
case 'x':
try {
execType = PigServer.parseExecType(opts.getValStr());
} catch (IOException e) {
throw new RuntimeException("ERROR: Unrecognized exectype.", e);
}
break;
default:
{
Character cc = new Character(opt);
throw new AssertionError("Unhandled option " + cc.toString());
}
}
}
// configure logging
configureLog4J(properties);
// create the context with the parameter
PigContext pigContext = new PigContext(execType, properties);
if (logFileName == null) {
logFileName = validateLogFile(null, null);
}
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (optimizerRules.size() > 0) {
pigContext
.getProperties()
.setProperty("pig.optimizer.rules", ObjectSerializer.serialize(optimizerRules));
}
LogicalPlanBuilder.classloader = pigContext.createCl(null);
// construct the parameter substitution preprocessor
Grunt grunt = null;
BufferedReader in;
String substFile = null;
switch (mode) {
case FILE:
{
// Run, using the provided file as a pig file
in = new BufferedReader(new FileReader(file));
// run parameter substitution preprocessor first
substFile = file + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, file);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(file).getName());
if (!debug) {
new File(substFile).deleteOnExit();
}
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
case STRING:
{
// Gather up all the remaining arguments into a string and pass them into
// grunt.
StringBuffer sb = new StringBuffer();
String remainders[] = opts.getRemainingArgs();
for (int i = 0; i < remainders.length; i++) {
if (i != 0) sb.append(' ');
sb.append(remainders[i]);
}
in = new BufferedReader(new StringReader(sb.toString()));
grunt = new Grunt(in, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
default:
break;
}
// If we're here, we don't know yet what they want. They may have just
// given us a jar to execute, they might have given us a pig script to
// execute, or they might have given us a dash (or nothing) which means to
// run grunt interactive.
String remainders[] = opts.getRemainingArgs();
if (remainders == null) {
// Interactive
mode = ExecMode.SHELL;
ConsoleReader reader = new ConsoleReader(System.in, new OutputStreamWriter(System.out));
reader.setDefaultPrompt("grunt> ");
final String HISTORYFILE = ".pig_history";
String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE;
reader.setHistory(new History(new File(historyFile)));
ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
grunt.setConsoleReader(reader);
gruntCalled = true;
grunt.run();
rc = 0;
return;
} else {
// They have a pig script they want us to run.
if (remainders.length > 1) {
throw new RuntimeException(
"You can only run one pig script " + "at a time from the command line.");
}
mode = ExecMode.FILE;
in = new BufferedReader(new FileReader(remainders[0]));
// run parameter substitution preprocessor first
substFile = remainders[0] + ".substituted";
pin = runParamPreprocessor(in, params, paramFiles, substFile, debug || dryrun);
if (dryrun) {
log.info("Dry run completed. Substituted pig script is at " + substFile);
return;
}
logFileName = validateLogFile(logFileName, remainders[0]);
pigContext.getProperties().setProperty("pig.logfile", logFileName);
if (!debug) {
new File(substFile).deleteOnExit();
}
// Set job name based on name of the script
pigContext
.getProperties()
.setProperty(PigContext.JOB_NAME, "PigLatin:" + new File(remainders[0]).getName());
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
int[] results = grunt.exec();
rc = getReturnCodeForStats(results);
return;
}
// Per Utkarsh and Chris invocation of jar file via pig depricated.
} catch (ParseException e) {
usage();
rc = 2;
} catch (NumberFormatException e) {
usage();
rc = 2;
} catch (PigException pe) {
if (pe.retriable()) {
rc = 1;
} else {
rc = 2;
}
if (!gruntCalled) {
LogUtils.writeLog(pe, logFileName, log, verbose);
}
} catch (Throwable e) {
rc = 2;
if (!gruntCalled) {
LogUtils.writeLog(e, logFileName, log, verbose);
}
} finally {
// clear temp files
FileLocalizer.deleteTempFiles();
PerformanceTimerFactory.getPerfTimerFactory().dumpTimers();
System.exit(rc);
}
}
public static synchronized int getConnectionCount() {
return m_notUsedConnection.size() + m_usedUsedConnection.size();
}
/**
* Preform a merge commit
*
* @param project a project
* @param root a vcs root
* @param added added files
* @param removed removed files
* @param messageFile a message file for commit
* @param author an author
* @param exceptions the list of exceptions to report
* @param partialOperation
* @return true if merge commit was successful
*/
private static boolean mergeCommit(
final Project project,
final VirtualFile root,
final Set<FilePath> added,
final Set<FilePath> removed,
final File messageFile,
final String author,
List<VcsException> exceptions,
@NotNull final PartialOperation partialOperation) {
HashSet<FilePath> realAdded = new HashSet<FilePath>();
HashSet<FilePath> realRemoved = new HashSet<FilePath>();
// perform diff
GitSimpleHandler diff = new GitSimpleHandler(project, root, GitCommand.DIFF);
diff.setSilent(true);
diff.setStdoutSuppressed(true);
diff.addParameters("--diff-filter=ADMRUX", "--name-status", "HEAD");
diff.endOptions();
String output;
try {
output = diff.run();
} catch (VcsException ex) {
exceptions.add(ex);
return false;
}
String rootPath = root.getPath();
for (StringTokenizer lines = new StringTokenizer(output, "\n", false);
lines.hasMoreTokens(); ) {
String line = lines.nextToken().trim();
if (line.length() == 0) {
continue;
}
String[] tk = line.split("\t");
switch (tk[0].charAt(0)) {
case 'M':
case 'A':
realAdded.add(VcsUtil.getFilePath(rootPath + "/" + tk[1]));
break;
case 'D':
realRemoved.add(VcsUtil.getFilePathForDeletedFile(rootPath + "/" + tk[1], false));
break;
default:
throw new IllegalStateException("Unexpected status: " + line);
}
}
realAdded.removeAll(added);
realRemoved.removeAll(removed);
if (realAdded.size() != 0 || realRemoved.size() != 0) {
final List<FilePath> files = new ArrayList<FilePath>();
files.addAll(realAdded);
files.addAll(realRemoved);
final Ref<Boolean> mergeAll = new Ref<Boolean>();
try {
GuiUtils.runOrInvokeAndWait(
new Runnable() {
public void run() {
String message =
GitBundle.message("commit.partial.merge.message", partialOperation.getName());
SelectFilePathsDialog dialog =
new SelectFilePathsDialog(
project,
files,
message,
null,
"Commit All Files",
CommonBundle.getCancelButtonText(),
false);
dialog.setTitle(GitBundle.getString("commit.partial.merge.title"));
dialog.show();
mergeAll.set(dialog.isOK());
}
});
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new RuntimeException("Unable to invoke a message box on AWT thread", ex);
}
if (!mergeAll.get()) {
return false;
}
// update non-indexed files
if (!updateIndex(project, root, realAdded, realRemoved, exceptions)) {
return false;
}
for (FilePath f : realAdded) {
VcsDirtyScopeManager.getInstance(project).fileDirty(f);
}
for (FilePath f : realRemoved) {
VcsDirtyScopeManager.getInstance(project).fileDirty(f);
}
}
// perform merge commit
try {
GitSimpleHandler handler = new GitSimpleHandler(project, root, GitCommand.COMMIT);
handler.setStdoutSuppressed(false);
handler.addParameters("-F", messageFile.getAbsolutePath());
if (author != null) {
handler.addParameters("--author=" + author);
}
handler.endOptions();
handler.run();
GitRepositoryManager manager = GitUtil.getRepositoryManager(project);
manager.updateRepository(root);
} catch (VcsException ex) {
exceptions.add(ex);
return false;
}
return true;
}
@Test
public void ortest3() {
final HashSet<Integer> V1 = new HashSet<Integer>();
final HashSet<Integer> V2 = new HashSet<Integer>();
final MutableRoaringBitmap rr = new MutableRoaringBitmap();
final MutableRoaringBitmap rr2 = new MutableRoaringBitmap();
// For the first 65536: rr2 has a bitmap container, and rr has
// an array container.
// We will check the union between a BitmapCintainer and an
// arrayContainer
for (int k = 0; k < 4000; ++k) {
rr2.add(k);
V1.add(k);
}
for (int k = 3500; k < 4500; ++k) {
rr.add(k);
V1.add(k);
}
for (int k = 4000; k < 65000; ++k) {
rr2.add(k);
V1.add(k);
}
// In the second node of each roaring bitmap, we have two bitmap
// containers.
// So, we will check the union between two BitmapContainers
for (int k = 65536; k < 65536 + 10000; ++k) {
rr.add(k);
V1.add(k);
}
for (int k = 65536; k < 65536 + 14000; ++k) {
rr2.add(k);
V1.add(k);
}
// In the 3rd node of each Roaring Bitmap, we have an
// ArrayContainer, so, we will try the union between two
// ArrayContainers.
for (int k = 4 * 65535; k < 4 * 65535 + 1000; ++k) {
rr.add(k);
V1.add(k);
}
for (int k = 4 * 65535; k < 4 * 65535 + 800; ++k) {
rr2.add(k);
V1.add(k);
}
// For the rest, we will check if the union will take them in
// the result
for (int k = 6 * 65535; k < 6 * 65535 + 1000; ++k) {
rr.add(k);
V1.add(k);
}
for (int k = 7 * 65535; k < 7 * 65535 + 2000; ++k) {
rr2.add(k);
V1.add(k);
}
final MutableRoaringBitmap rror = MutableRoaringBitmap.or(rr, rr2);
boolean valide = true;
// Si tous les elements de rror sont dans V1 et que tous les
// elements de
// V1 sont dans rror(V2)
// alors V1 == rror
final Object[] tab = V1.toArray();
final Vector<Integer> vector = new Vector<Integer>();
for (Object aTab : tab) vector.add((Integer) aTab);
for (final int i : rror.toArray()) {
if (!vector.contains(i)) {
valide = false;
}
V2.add(i);
}
for (int i = 0; i < V1.size(); i++)
if (!V2.contains(vector.elementAt(i))) {
valide = false;
}
Assert.assertEquals(valide, true);
}
/**
* This does the work of main, but it never calls System.exit, so it is appropriate to be called
* progrmmatically. Termination of the program with a message to the user is indicated by throwing
* Daikon.TerminationMessage.
*
* @see #main(String[])
* @see daikon.Daikon.TerminationMessage
*/
public static void mainHelper(final String[] args)
throws FileNotFoundException, StreamCorruptedException, OptionalDataException, IOException,
ClassNotFoundException {
daikon.LogHelper.setupLogs(daikon.LogHelper.INFO);
LongOpt[] longopts =
new LongOpt[] {
new LongOpt(Daikon.config_option_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
new LongOpt(output_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
new LongOpt(dir_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
new LongOpt(conf_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
new LongOpt(filter_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
new LongOpt(Daikon.debugAll_SWITCH, LongOpt.NO_ARGUMENT, null, 0),
new LongOpt(Daikon.debug_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
new LongOpt(Daikon.ppt_regexp_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
new LongOpt(Daikon.track_SWITCH, LongOpt.REQUIRED_ARGUMENT, null, 0),
};
Getopt g = new Getopt("daikon.tools.InvariantChecker", args, "h", longopts);
int c;
while ((c = g.getopt()) != -1) {
switch (c) {
case 0:
// got a long option
String option_name = longopts[g.getLongind()].getName();
if (Daikon.help_SWITCH.equals(option_name)) {
System.out.println(usage);
throw new Daikon.TerminationMessage();
} else if (conf_SWITCH.equals(option_name)) {
doConf = true;
} else if (filter_SWITCH.equals(option_name)) {
doFilter = true;
} else if (dir_SWITCH.equals(option_name)) {
dir_file = new File(g.getOptarg());
if (!dir_file.exists() || !dir_file.isDirectory())
throw new Daikon.TerminationMessage("Error reading the directory " + dir_file);
} else if (output_SWITCH.equals(option_name)) {
output_file = new File(g.getOptarg());
output_stream = new PrintStream(new FileOutputStream(output_file));
} else if (Daikon.config_option_SWITCH.equals(option_name)) {
String item = g.getOptarg();
daikon.config.Configuration.getInstance().apply(item);
break;
} else if (Daikon.debugAll_SWITCH.equals(option_name)) {
Global.debugAll = true;
} else if (Daikon.debug_SWITCH.equals(option_name)) {
LogHelper.setLevel(g.getOptarg(), LogHelper.FINE);
} else if (Daikon.track_SWITCH.equals(option_name)) {
LogHelper.setLevel("daikon.Debug", LogHelper.FINE);
String error = Debug.add_track(g.getOptarg());
if (error != null) {
throw new Daikon.TerminationMessage(
"Error parsing track argument '" + g.getOptarg() + "' - " + error);
}
} else {
throw new RuntimeException("Unknown long option received: " + option_name);
}
break;
case 'h':
System.out.println(usage);
throw new Daikon.TerminationMessage();
case '?':
break; // getopt() already printed an error
default:
System.out.println("getopt() returned " + c);
break;
}
}
// Loop through each filename specified
for (int i = g.getOptind(); i < args.length; i++) {
// Get the file and make sure it exists
File file = new File(args[i]);
if (!file.exists()) {
throw new Error("File " + file + " not found.");
}
// These aren't "endsWith()" because there might be a suffix on the end
// (eg, a date).
String filename = file.toString();
if (filename.indexOf(".inv") != -1) {
if (inv_file != null) {
throw new Daikon.TerminationMessage("multiple inv files specified", usage);
}
inv_file = file;
} else if (filename.indexOf(".dtrace") != -1) {
dtrace_files.add(filename);
} else {
throw new Error("Unrecognized argument: " + file);
}
}
if (dir_file == null) {
checkInvariants();
return;
}
// Yoav additions:
File[] filesInDir = dir_file.listFiles();
if (filesInDir == null || filesInDir.length == 0)
throw new Daikon.TerminationMessage("The directory " + dir_file + " is empty", usage);
ArrayList<File> invariants = new ArrayList<File>();
for (File f : filesInDir) if (f.toString().indexOf(".inv") != -1) invariants.add(f);
if (invariants.size() == 0)
throw new Daikon.TerminationMessage(
"Did not find any invariant files in the directory " + dir_file, usage);
ArrayList<File> dtraces = new ArrayList<File>();
for (File f : filesInDir) if (f.toString().indexOf(".dtrace") != -1) dtraces.add(f);
if (dtraces.size() == 0)
throw new Daikon.TerminationMessage(
"Did not find any dtrace files in the directory " + dir_file, usage);
System.out.println(
"Collecting data for invariants files " + invariants + " and dtrace files " + dtraces);
dtrace_files.clear();
for (File dtrace : dtraces) {
dtrace_files.add(dtrace.toString());
}
String commaLine = "";
for (File inFile : invariants) {
String name = inFile.getName().replace(".inv", "").replace(".gz", "");
commaLine += "," + name;
}
outputComma.add(commaLine);
commaLine = "";
for (File inFile : invariants) {
inv_file = inFile;
failedInvariants.clear();
testedInvariants.clear();
error_cnt = 0;
output_stream =
new PrintStream(
new FileOutputStream(
inFile.toString().replace(".inv", "").replace(".gz", "")
+ ".false-positives.txt"));
checkInvariants();
output_stream.close();
int failedCount = failedInvariants.size();
int testedCount = testedInvariants.size();
String percent = toPercentage(failedCount, testedCount);
commaLine += "," + percent;
}
outputComma.add(commaLine);
System.out.println();
for (String output : outputComma) System.out.println(output);
}
private static void checkInvariants() throws IOException {
// Read the invariant file
PptMap ppts = FileIO.read_serialized_pptmap(inv_file, true);
// Yoav: make sure we have unique invariants
InvariantFilters fi = InvariantFilters.defaultFilters();
// Set<String> allInvariantsStr = new HashSet<String>();
Set<Invariant> allInvariants = new HashSet<Invariant>();
for (PptTopLevel ppt : ppts.all_ppts())
for (Iterator<PptSlice> i = ppt.views_iterator(); i.hasNext(); ) {
PptSlice slice = i.next();
for (Invariant inv : slice.invs) {
if (doConf && inv.getConfidence() < Invariant.dkconfig_confidence_limit) {
// System.out.printf ("inv ignored (conf): %s:%s\n", inv.ppt.name(),
// inv.format());
continue;
}
if (doFilter && fi.shouldKeep(inv) == null) {
// System.out.printf ("inv ignored (filter): %s:%s\n",
// inv.ppt.name(), inv.format());
continue;
}
activeInvariants.add(inv);
// String n = invariant2str(ppt, inv);
// if (!allInvariants.contains(inv) && allInvariantsStr.contains(n)) throw new
// Daikon.TerminationMessage("Two invariants have the same ppt.name+inv.rep:"+n);
allInvariants.add(inv);
// allInvariantsStr.add(n);
}
}
// Read and process the data trace files
FileIO.Processor processor = new InvariantCheckProcessor();
Daikon.FileIOProgress progress = new Daikon.FileIOProgress();
progress.start();
progress.clear();
FileIO.read_data_trace_files(dtrace_files, ppts, processor, false);
progress.shouldStop = true;
System.out.println();
System.out.printf(
"%s: %,d errors found in %,d samples (%s)\n",
inv_file, error_cnt, sample_cnt, toPercentage(error_cnt, sample_cnt));
int failedCount = failedInvariants.size();
int testedCount = testedInvariants.size();
String percent = toPercentage(failedCount, testedCount);
System.out.println(
inv_file
+ ": "
+ failedCount
+ " false positives, out of "
+ testedCount
+ ", which is "
+ percent
+ ".");
if (false) {
for (Invariant inv : failedInvariants) {
System.out.printf("+%s:%s\n", inv.ppt.name(), inv.format());
}
}
}