public static void main(String[] args) {
Pair<String, Integer> grade1 = new Pair<String, Integer>("Williams", 90);
Pair<String, Integer> grade2 = new Pair<String, Integer>("Brown", 44);
System.out.println(grade1.first());
System.out.println(grade1.second());
System.out.println(grade2.first());
System.out.println(grade2.second());
}
public static InputStream getInputStreamFromUrl(String url, String user, String password) {
try {
Pair<String, Integer> hostAndPort = validateUrl(url);
HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
if ((user != null) && (password != null)) {
httpclient.getParams().setAuthenticationPreemptive(true);
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
httpclient
.getState()
.setCredentials(
new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM),
defaultcreds);
s_logger.info(
"Added username="******", password="******"for host "
+ hostAndPort.first()
+ ":"
+ hostAndPort.second());
}
// Execute the method.
GetMethod method = new GetMethod(url);
int statusCode = httpclient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
s_logger.error("Failed to read from URL: " + url);
return null;
}
return method.getResponseBodyAsStream();
} catch (Exception ex) {
s_logger.error("Failed to read from URL: " + url);
return null;
}
}
/**
* {@inheritDoc} Determines whether two pairs are essentially the same. That is that the two pairs
* have the same label and the (second) values are also equal.
*/
@Override
public boolean equals(Object _pair) {
if (_pair instanceof Pair) {
Pair<?> pair = (Pair<?>) _pair;
if (this.first().equals(pair.first()) == false) return false;
if (this.second() == null & pair.second() == null) {
return true;
}
if (this.second() == null ^ pair.second() == null) {
return false;
}
if (this.second().equals(pair.second()) == false) return false;
}
return true;
}
protected void mergeHandler(
Type type, T store, List<Pair<Type, String>> handlers, Map<String, T> stores) {
for (Pair<Type, String> p : handlers) {
Type handler = p.first();
if (Type.isSubtype(handler, type)) {
T nstore = propagate(handler, store);
merge(p.second(), nstore, stores);
return; // completely subsumed
} else if (Type.isSubtype(type, handler)) {
T nstore = propagate(handler, store);
merge(p.second(), nstore, stores);
// not completely subsumed
type = Type.intersect(type, Type.Negation(handler));
}
}
}
public static ArrayList<Integer> getSubConcept(int now, ArrayList<Pair<Integer, Integer>> edge) {
ArrayList<Integer> temp = new ArrayList<Integer>();
for (Pair<Integer, Integer> e : edge) if (e.second() == now) temp.add(e.first());
return temp;
}
@SuppressWarnings("unchecked")
@Override
public int run(String[] args) throws Exception {
MyOptions options = new MyOptions(args, getConf());
traceWriter = options.clazzTraceOutputter.newInstance();
traceWriter.init(options.traceOutput, getConf());
topologyWriter = new DefaultOutputter<LoggedNetworkTopology>();
topologyWriter.init(options.topologyOutput, getConf());
try {
JobBuilder jobBuilder = null;
for (Path p : options.inputs) {
InputDemuxer inputDemuxer = options.inputDemuxerClass.newInstance();
try {
inputDemuxer.bindTo(p, getConf());
} catch (IOException e) {
LOG.warn("Unable to bind Path " + p + " . Skipping...", e);
continue;
}
Pair<String, InputStream> filePair = null;
try {
while ((filePair = inputDemuxer.getNext()) != null) {
RewindableInputStream ris = new RewindableInputStream(filePair.second());
JobHistoryParser parser = null;
try {
String jobID = JobHistoryUtils.extractJobID(filePair.first());
if (jobID == null) {
LOG.warn("File skipped: Invalid file name: " + filePair.first());
continue;
}
if ((jobBuilder == null) || (!jobBuilder.getJobID().equals(jobID))) {
if (jobBuilder != null) {
traceWriter.output(jobBuilder.build());
}
jobBuilder = new JobBuilder(jobID);
}
if (JobHistoryUtils.isJobConfXml(filePair.first())) {
processJobConf(JobConfigurationParser.parse(ris.rewind()), jobBuilder);
} else {
parser = JobHistoryParserFactory.getParser(ris);
if (parser == null) {
LOG.warn("File skipped: Cannot find suitable parser: " + filePair.first());
} else {
processJobHistory(parser, jobBuilder);
}
}
} finally {
if (parser == null) {
ris.close();
} else {
parser.close();
parser = null;
}
}
}
} catch (Throwable t) {
if (filePair != null) {
LOG.warn(
"TraceBuilder got an error while processing the [possibly virtual] file "
+ filePair.first()
+ " within Path "
+ p,
t);
}
} finally {
inputDemuxer.close();
}
}
if (jobBuilder != null) {
traceWriter.output(jobBuilder.build());
jobBuilder = null;
} else {
LOG.warn("No job found in traces: ");
}
topologyWriter.output(topologyBuilder.build());
} finally {
traceWriter.close();
topologyWriter.close();
}
return 0;
}
/** Test if {@link CurrentJHParser} can read events from current JH files. */
@Test
public void testCurrentJHParser() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser");
lfs.delete(tempDir, true);
String queueName = "testQueue";
// Run a MR job
// create a MR cluster
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
conf.set("mapred.queue.names", queueName);
MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf));
// run a job
Path inDir = new Path(tempDir, "input");
Path outDir = new Path(tempDir, "output");
JobHistoryParser parser = null;
RewindableInputStream ris = null;
ArrayList<String> seenEvents = new ArrayList<String>(10);
RunningJob rJob = null;
try {
JobConf jobConf = mrCluster.createJobConf();
jobConf.setQueueName(queueName);
// construct a job with 1 map and 1 reduce task.
rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1);
rJob.waitForCompletion();
assertTrue("Job failed", rJob.isSuccessful());
JobID id = rJob.getID();
// get the jobhistory filepath
Path inputPath =
new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id)));
// wait for 10 secs for the jobhistory file to move into the done folder
for (int i = 0; i < 100; ++i) {
if (lfs.exists(inputPath)) {
break;
}
TimeUnit.MILLISECONDS.wait(100);
}
assertTrue("Missing job history file", lfs.exists(inputPath));
InputDemuxer inputDemuxer = new DefaultInputDemuxer();
inputDemuxer.bindTo(inputPath, conf);
Pair<String, InputStream> filePair = inputDemuxer.getNext();
assertNotNull(filePair);
ris = new RewindableInputStream(filePair.second());
// Test if the JobHistoryParserFactory can detect the parser correctly
parser = JobHistoryParserFactory.getParser(ris);
// Get ParsedJob
String jobId = TraceBuilder.extractJobID(filePair.first());
JobBuilder builder = new JobBuilder(jobId);
HistoryEvent e;
while ((e = parser.nextEvent()) != null) {
String eventString = e.getEventType().toString();
System.out.println(eventString);
seenEvents.add(eventString);
if (builder != null) {
builder.process(e);
}
}
ParsedJob parsedJob = builder.build();
// validate the obtainXXX api of ParsedJob, ParsedTask and
// ParsedTaskAttempt.
validateParsedJob(parsedJob, 1, 1, queueName);
} finally {
// stop the MR cluster
mrCluster.shutdown();
if (ris != null) {
ris.close();
}
if (parser != null) {
parser.close();
}
// cleanup the filesystem
lfs.delete(tempDir, true);
}
// Check against the gold standard
System.out.println("testCurrentJHParser validating using gold std ");
String[] goldLines =
new String[] {
"JOB_SUBMITTED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
"JOB_INITED",
"JOB_INFO_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"REDUCE_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"JOB_FINISHED"
};
// Check the output with gold std
assertEquals("Size mismatch", goldLines.length, seenEvents.size());
int index = 0;
for (String goldLine : goldLines) {
assertEquals("Content mismatch", goldLine, seenEvents.get(index++));
}
}
protected T propagate(int start, int end, T store, List<Pair<Type, String>> handlers) {
for (int i = start; i < end; ++i) {
Entry entry = block.get(i);
try {
Code code = entry.code;
// First, check for a label which may have incoming information.
if (code instanceof Code.Label) {
Code.Label l = (Code.Label) code;
T tmp = stores.get(l.label);
if (tmp != null && store != null) {
store = join(store, tmp);
} else if (tmp != null) {
store = tmp;
}
}
T oldStore = store;
if (store == null) {
// this indicates dead-code has been reached.
continue;
} else if (code instanceof Code.Loop) {
Code.Loop loop = (Code.Loop) code;
Block.Entry nEntry = entry;
int s = i;
// Note, I could make this more efficient!
while (++i < block.size()) {
nEntry = block.get(i);
if (nEntry.code instanceof Code.Label) {
Code.Label l = (Code.Label) nEntry.code;
if (l.label.equals(loop.target)) {
// end of loop body found
break;
}
}
}
store = propagate(s, i, loop, entry, store, handlers);
continue;
} else if (code instanceof Code.IfGoto) {
Code.IfGoto ifgoto = (Code.IfGoto) code;
Pair<T, T> r = propagate(i, ifgoto, entry, store);
store = r.second();
merge(ifgoto.target, r.first(), stores);
} else if (code instanceof Code.IfType) {
Code.IfType ifgoto = (Code.IfType) code;
Pair<T, T> r = propagate(i, ifgoto, entry, store);
store = r.second();
merge(ifgoto.target, r.first(), stores);
} else if (code instanceof Code.Switch) {
Code.Switch sw = (Code.Switch) code;
List<T> r = propagate(i, sw, entry, store);
// assert r.second().size() == nsw.branches.size()
Code.Switch nsw = (Code.Switch) entry.code;
for (int j = 0; j != nsw.branches.size(); ++j) {
String target = nsw.branches.get(j).second();
T nstore = r.get(j);
merge(target, nstore, stores);
}
merge(sw.defaultTarget, store, stores);
store = null;
} else if (code instanceof Code.TryCatch) {
Code.TryCatch sw = (Code.TryCatch) code;
int s = i;
// Note, I could make this more efficient!
while (++i < block.size()) {
entry = block.get(i);
if (entry.code instanceof Code.Label) {
Code.Label l = (Code.Label) entry.code;
if (l.label.equals(sw.target)) {
// end of loop body found
break;
}
}
}
ArrayList<Pair<Type, String>> nhandlers = new ArrayList<Pair<Type, String>>(handlers);
nhandlers.addAll(0, sw.catches);
store = propagate(s + 1, i, store, nhandlers);
i = i - 1; // this is necessary since last label of
// try-catch is first label of catch handler
} else if (code instanceof Code.Goto) {
Code.Goto gto = (Code.Goto) entry.code;
merge(gto.target, store, stores);
store = null;
} else {
// This indicates a sequential statement was encountered.
store = propagate(i, entry, store);
if (entry.code instanceof Code.Return || entry.code instanceof Code.Throw) {
store = null;
}
}
mergeHandlers(i, code, oldStore, handlers, stores);
} catch (SyntaxError se) {
throw se;
} catch (Throwable ex) {
internalFailure("internal failure", filename, entry, ex);
}
}
return store;
}
