@Test
public void totalOrder() throws RuleBaseException {
SortBlock block = define("<sort by='ascending' priority='5'>@foo</sort>");
final OrderGraph graph = mockery.mock(OrderGraph.class);
mockery.checking(
new Expectations() {
{
one(graph).order("x2", "x1", 5);
one(graph).order("x2", "x3", 5);
one(graph).order("x2", "x4", 5);
one(graph).order("x1", "x4", 5);
one(graph).order("x3", "x4", 5);
}
});
List<Pair<String, Integer>> list = new ArrayList<Pair<String, Integer>>();
list.add(Pair.of("x1", 23));
list.add(Pair.of("x2", 18));
list.add(Pair.of("x3", 23));
list.add(Pair.of("x4", 35));
Comparator<Pair<String, Integer>> comparator =
new Comparator<Pair<String, Integer>>() {
public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
return o1.second.compareTo(o2.second);
}
};
block.totalOrder(list, comparator, graph);
}
@Test
public void restore() throws RuleBaseException, TransformException {
Node comp1 = content.query().single("/id('comp1')").node();
Node comp2 = content.query().single("/id('comp2')").node();
Node cname = content.query().single("/id('cname')").node();
setModNearestAncestorImplementing(
NodeTarget.class,
new NodeTarget() {
public ItemList targets() throws TransformException {
return content.query().all("/id('comp1 comp2')");
}
});
setModData("(<sort-siblings run-length='2'/>, <sort-siblings run-length='1'/>)");
setModReferences(cname, comp2, cname);
SortBySiblingBlock block = define("<sort before='sibling'>uml:name</sort>");
SortBySiblingBlock.SortBySiblingSeg seg =
(SortBySiblingBlock.SortBySiblingSeg) block.createSeg(mod);
seg.restore();
assertEquals(
Arrays.asList(
new Pair[] {
Pair.of(comp1, Arrays.asList(cname, comp2)), Pair.of(comp2, Arrays.asList(cname))
}),
seg.siblingsByTarget);
}
@SuppressWarnings("unchecked")
@Test
public void sortAfter() throws RuleBaseException {
final Node comp1 = content.query().single("/id('comp1')").node();
final Node comp2 = content.query().single("/id('comp2')").node();
final Node cname = content.query().single("/id('cname')").node();
SortBySiblingBlock block = define("<sort after='sibling'>uml:name</sort>");
SortBySiblingBlock.SortBySiblingSeg seg1 =
(SortBySiblingBlock.SortBySiblingSeg) block.createSeg(mod);
SortBySiblingBlock.SortBySiblingSeg seg2 =
(SortBySiblingBlock.SortBySiblingSeg) block.createSeg(mod);
seg1.siblingsByTarget =
Arrays.<Pair<Node, List<Node>>>asList(
new Pair[] {
Pair.of(comp1, Arrays.asList(cname, comp2)), Pair.of(comp2, Arrays.asList(cname))
});
seg2.siblingsByTarget =
Arrays.<Pair<Node, List<Node>>>asList(new Pair[] {Pair.of(cname, Arrays.asList(comp2))});
final OrderGraph graph = mockery.mock(OrderGraph.class);
mockery.checking(
new Expectations() {
{
one(graph).order(cname, comp1, 0);
one(graph).order(comp2, comp1, 0);
one(graph).order(cname, comp2, 0);
one(graph).order(comp2, cname, 0);
}
});
block.sort(Arrays.asList(seg1, seg2), graph);
}
@Test(expected = TransformException.class)
public void verifyBad() throws RuleBaseException, TransformException {
SortByValueBlock block = define("<sort by='ascending'>@name</sort>");
SortByValueBlock.SortByValueSeg seg = (SortByValueBlock.SortByValueSeg) block.createSeg(mod);
setModData(
"(<sort-value refid='m1'>start</sort-value>, <sort-value refid='m2'>foo</sort-value>)");
seg.values = new ArrayList<Pair<String, Item>>();
seg.values.add(Pair.of("m1", content.query().single("/id('m1')/@name").toAtomicItem()));
seg.values.add(Pair.of("m2", content.query().single("/id('m2')/@name").toAtomicItem()));
seg.verify();
}
@Test(expected = IllegalArgumentException.class)
public void test_flatMap_null() {
// prepare
final String left = "hoge";
final String right = "foo";
Pair<String> pair = new Pair<>(left, right);
// action
pair.flatMap(null);
// check
// - nothing
}
@Test
public void test() {
test = new PairTest<Object, Object>();
Pair<String, String> twoStrings = new Pair<>("Ha", "Te");
Pair<String, Double> stringDouble = new Pair<>("Ha", 3.4);
Pair<Double, Integer> doubleInt = new Pair<>(3.0, 5);
Pair<Integer, Integer> intInt = new Pair<>(6, 66);
String result = twoStrings.toString();
assertEquals("1:Ha\n2:Te", result); // tests 2 strings
String resultTwo = stringDouble.toString();
assertEquals("1:Ha\n2:3.4", resultTwo); // test string and a double
String resultThree = doubleInt.toString();
assertEquals("1:3.0\n2:5", resultThree);
String resultFour = intInt.toString();
assertEquals("1:6\n2:66", resultFour);
/*sting string
string double
double int
int string*/
}
@SuppressWarnings("unchecked")
@Test(expected = TransformException.class)
public void verifyMismatch() throws RuleBaseException, TransformException {
Node uc1 = content.query().single("/id('uc1')").node();
Node comp1 = content.query().single("/id('comp1')").node();
Node comp2 = content.query().single("/id('comp2')").node();
Node cname = content.query().single("/id('cname')").node();
setModScope(uc1.query());
SortBySiblingBlock block = define("<sort before='sibling'>uml:name</sort>");
SortBySiblingBlock.SortBySiblingSeg seg =
(SortBySiblingBlock.SortBySiblingSeg) block.createSeg(mod);
seg.siblingsByTarget =
Arrays.<Pair<Node, List<Node>>>asList(
new Pair[] {
Pair.of(comp1, Arrays.asList(comp2)), Pair.of(comp2, Arrays.asList(cname))
});
seg.verify();
}
@Test
public void sortDescending() throws RuleBaseException {
SortByValueBlock block = define("<sort by='descending'>@name</sort>");
SortByValueBlock.SortByValueSeg seg1 = (SortByValueBlock.SortByValueSeg) block.createSeg(mod);
SortByValueBlock.SortByValueSeg seg2 = (SortByValueBlock.SortByValueSeg) block.createSeg(mod);
seg1.values = new ArrayList<Pair<String, Item>>();
seg1.values.add(Pair.of("m1", content.query().single("/id('m1')/@name").toAtomicItem()));
seg2.values = new ArrayList<Pair<String, Item>>();
seg2.values.add(Pair.of("m2", content.query().single("/id('m2')/@name").toAtomicItem()));
final OrderGraph graph = mockery.mock(OrderGraph.class);
mockery.checking(
new Expectations() {
{
one(graph).order("m1", "m2", 0);
}
});
block.sort(Arrays.asList(seg1, seg2), graph);
}
@Override
public void restore() throws TransformException {
List<Node> references = mod.references();
ItemList targets = mod.nearest(NodeTarget.class).targets();
proxies = new ArrayList<Pair<String, Node>>(references.size());
for (int i = 0; i < references.size(); i++) {
proxies.add(Pair.of(targets.get(i).query().single("@xml:id").value(), references.get(i)));
}
}
@Test
public void sort() throws RuleBaseException {
SortByProxyBlock block = define("<sort as='corresponding'>$source</sort>");
SortByProxyBlock.SortByProxySeg seg1 = (SortByProxyBlock.SortByProxySeg) block.createSeg(mod);
SortByProxyBlock.SortByProxySeg seg2 = (SortByProxyBlock.SortByProxySeg) block.createSeg(mod);
seg1.proxies = new ArrayList<Pair<String, Node>>();
seg2.proxies = new ArrayList<Pair<String, Node>>();
seg1.proxies.add(Pair.of("um1", content.query().single("/id('m1')").node()));
seg2.proxies.add(Pair.of("uf1", content.query().single("/id('f1')").node()));
seg1.proxies.add(Pair.of("um2", content.query().single("/id('m2')").node()));
seg2.proxies.add(Pair.of("uf2", content.query().single("/id('f1')").node()));
final OrderGraph graph = mockery.mock(OrderGraph.class);
mockery.checking(
new Expectations() {
{
one(graph).order("um1", "um2", 0);
}
});
block.sort(Arrays.asList(seg1, seg2), graph);
}
@Test
public void test_flatMap_null_right() {
// prepare
final String left = "foo";
Pair<String> pair = new Pair<>(left, null);
// action
Pair<Integer> result =
pair.flatMap(
(l, r) -> {
final int lengthL = (l == null ? 0 : l.length());
final int lengthR = (r == null ? 0 : r.length());
Pair<Integer> tmp = new Pair<>(lengthL, lengthR);
reverse(tmp);
return tmp;
});
// check
assertEquals(left.length(), result.getRight().intValue());
assertEquals(0, result.getLeft().intValue());
}
@Override
public void restore() throws TransformException {
ItemList runLengths = mod.supplementQuery().all("sort-siblings/@run-length");
ItemList targets = mod.nearest(NodeTarget.class).targets();
assert runLengths.size() == targets.size();
for (int i = 0, j = 0; i < targets.size(); i++) {
int runLength = runLengths.get(i).intValue();
siblingsByTarget.add(
Pair.of(targets.get(i).node(), mod.references().subList(j, j + runLength)));
j += runLength;
}
}
@Override
public void restore() throws TransformException {
ItemList targets = mod.nearest(NodeTarget.class).targets();
values = new ArrayList<Pair<String, Item>>(targets.size());
for (Node node : targets.nodes()) {
ItemList items = query.runOn(mod.scope(node.query()));
if (items.size() != 1)
throw new TransformException(
"sort by value query did not select exactly one item: " + items);
values.add(Pair.of(node.query().single("@xml:id").value(), items.get(0).toAtomicItem()));
}
}
@Test
public void restore() throws RuleBaseException, TransformException {
setModNearestAncestorImplementing(
NodeTarget.class,
new NodeTarget() {
public ItemList targets() throws TransformException {
return content.query().all("/id('m1 m2')");
}
});
setModScope(
content.query().single("/id('m1')").node().query(),
content.query().single("/id('m2')").node().query());
SortByValueBlock block = define("<sort by='ascending'>@name</sort>");
SortByValueBlock.SortByValueSeg seg = (SortByValueBlock.SortByValueSeg) block.createSeg(mod);
seg.restore();
assertEquals(
Arrays.asList(
new Pair[] {
Pair.of("m1", content.query().single("/id('m1')/@name").toAtomicItem()),
Pair.of("m2", content.query().single("/id('m2')/@name").toAtomicItem())
}),
seg.values);
}
@Test
public void restore() throws RuleBaseException, TransformException {
setModNearestAncestorImplementing(
NodeTarget.class,
new NodeTarget() {
public ItemList targets() throws TransformException {
return content.query().all("/id('um1 um2')");
}
});
setModData("(<sort-proxy position='1'/>, <sort-proxy position='2'/>)");
setModReferences(
content.query().single("/id('m1')").node(), content.query().single("/id('m2')").node());
SortByProxyBlock block = define("<sort as='corresponding'>$source</sort>");
SortByProxyBlock.SortByProxySeg seg = (SortByProxyBlock.SortByProxySeg) block.createSeg(mod);
seg.restore();
assertEquals(
Arrays.asList(
new Pair[] {
Pair.of("um1", content.query().single("/id('m1')").node()),
Pair.of("um2", content.query().single("/id('m2')").node())
}),
seg.proxies);
}
private void runVerifyScenario(String proxyid, String modData)
throws TransformException, RuleBaseException {
Node um1 = content.query().single("/id('um1')").node();
Node m1 = content.query().single("/id('m1')").node();
setModNearestAncestorImplementing(
NodeTarget.class,
new NodeTarget() {
public ItemList targets() throws TransformException {
return content.query().all("/id('um1')");
}
});
setModData(modData);
setModReferences(content.query().single("/id($_1)", proxyid).node());
setModScope(um1.query().let("$source", m1));
SortByProxyBlock block = define("<sort as='corresponding'>$source</sort>");
SortByProxyBlock.SortByProxySeg seg = (SortByProxyBlock.SortByProxySeg) block.createSeg(mod);
seg.proxies = new ArrayList<Pair<String, Node>>();
seg.proxies.add(Pair.of("um1", mod.references().get(0)));
seg.verify();
}
@Test
public void testTwo() {
testTwo = new PairTest<Object, Object>();
// make two arraylists
ArrayList<String> first = new ArrayList<String>();
ArrayList<String> second = new ArrayList<String>();
ArrayList<Integer> third = new ArrayList<Integer>();
// initialize both with my values
first.add("Hi");
first.add("My name");
second.add("is");
second.add("Neil");
third.add(66);
third.add(6);
// "Hi,My name,is"
String al1 = first.toString();
String al2 = second.toString();
String al3 = third.toString();
// pair them
Pair<ArrayList, ArrayList> twoStrings = new Pair<>(first, second);
Pair<ArrayList, ArrayList> stringAndInt = new Pair<>(second, third);
Pair<ArrayList, ArrayList> intAndInt = new Pair<>(third, third);
Pair<ArrayList, Double> arrayListAndDouble = new Pair<>(first, 7.9);
// to string the
String resultFirst = twoStrings.toString();
String resultSecond = stringAndInt.toString();
String resultThird = intAndInt.toString();
String resultFourth = arrayListAndDouble.toString();
assertEquals("1:" + al1 + "\n2:" + al2, resultFirst);
assertEquals("1:" + al2 + "\n2:" + al3, resultSecond);
assertEquals("1:" + al3 + "\n2:" + al3, resultThird);
assertEquals("1:" + al1 + "\n2:" + 7.9, resultFourth);
}
private <T> void reverse(Pair<T> pair) {
final T left = pair.getLeft();
pair.setLeft(pair.getRight());
pair.setRight(left);
}
/** Test if {@link CurrentJHParser} can read events from current JH files. */
@Test
public void testCurrentJHParser() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser");
lfs.delete(tempDir, true);
String queueName = "testQueue";
// Run a MR job
// create a MR cluster
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
conf.set("mapred.queue.names", queueName);
MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf));
// run a job
Path inDir = new Path(tempDir, "input");
Path outDir = new Path(tempDir, "output");
JobHistoryParser parser = null;
RewindableInputStream ris = null;
ArrayList<String> seenEvents = new ArrayList<String>(10);
RunningJob rJob = null;
try {
JobConf jobConf = mrCluster.createJobConf();
jobConf.setQueueName(queueName);
// construct a job with 1 map and 1 reduce task.
rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1);
rJob.waitForCompletion();
assertTrue("Job failed", rJob.isSuccessful());
JobID id = rJob.getID();
// get the jobhistory filepath
Path inputPath =
new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id)));
// wait for 10 secs for the jobhistory file to move into the done folder
for (int i = 0; i < 100; ++i) {
if (lfs.exists(inputPath)) {
break;
}
TimeUnit.MILLISECONDS.wait(100);
}
assertTrue("Missing job history file", lfs.exists(inputPath));
InputDemuxer inputDemuxer = new DefaultInputDemuxer();
inputDemuxer.bindTo(inputPath, conf);
Pair<String, InputStream> filePair = inputDemuxer.getNext();
assertNotNull(filePair);
ris = new RewindableInputStream(filePair.second());
// Test if the JobHistoryParserFactory can detect the parser correctly
parser = JobHistoryParserFactory.getParser(ris);
// Get ParsedJob
String jobId = TraceBuilder.extractJobID(filePair.first());
JobBuilder builder = new JobBuilder(jobId);
HistoryEvent e;
while ((e = parser.nextEvent()) != null) {
String eventString = e.getEventType().toString();
System.out.println(eventString);
seenEvents.add(eventString);
if (builder != null) {
builder.process(e);
}
}
ParsedJob parsedJob = builder.build();
// validate the obtainXXX api of ParsedJob, ParsedTask and
// ParsedTaskAttempt.
validateParsedJob(parsedJob, 1, 1, queueName);
} finally {
// stop the MR cluster
mrCluster.shutdown();
if (ris != null) {
ris.close();
}
if (parser != null) {
parser.close();
}
// cleanup the filesystem
lfs.delete(tempDir, true);
}
// Check against the gold standard
System.out.println("testCurrentJHParser validating using gold std ");
String[] goldLines =
new String[] {
"JOB_SUBMITTED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
"JOB_INITED",
"JOB_INFO_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"REDUCE_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"JOB_FINISHED"
};
// Check the output with gold std
assertEquals("Size mismatch", goldLines.length, seenEvents.size());
int index = 0;
for (String goldLine : goldLines) {
assertEquals("Content mismatch", goldLine, seenEvents.get(index++));
}
}
