private static void assertFloat(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
slice.setFloat(index, intBitsToFloat(0xAAAA_5555));
assertEquals(floatToIntBits(slice.getFloat(index)), 0xAAAA_5555);
try {
slice.getFloat(-1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getFloat((slice.length() - SIZE_OF_FLOAT) + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getFloat(slice.length());
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getFloat(slice.length() + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
}
public boolean intersects(
List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues) {
for (Slice slice : this) {
if (slice.intersects(comparator, minClusteringValues, maxClusteringValues)) return true;
}
return false;
}
private static void assertShort(Slice slice, short index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
slice.setShort(index, 0xAA55);
assertEquals(slice.getShort(index), (short) 0xAA55);
try {
slice.getShort(-1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getShort((slice.length() - SIZE_OF_SHORT) + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getShort(slice.length());
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getShort(slice.length() + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
}
private static void assertDouble(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
slice.setDouble(index, longBitsToDouble(0xAAAA_AAAA_5555_5555L));
assertEquals(doubleToLongBits(slice.getDouble(index)), 0xAAAA_AAAA_5555_5555L);
try {
slice.getDouble(-1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getDouble((slice.length() - SIZE_OF_DOUBLE) + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getDouble(slice.length());
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getDouble(slice.length() + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
}
public void run() {
try {
for (Slice.FileData data = slice.readNextFile(); data != null; data = slice.readNextFile()) {
FileList.FileLocation file = data.file;
ByteArrayInputStream in = new ByteArrayInputStream(data.data);
decoder.SetDecoderProperties(data.props);
String fileName = destpath + "/" + file.fileName;
File f = new File(fileName);
f.getParentFile().mkdirs();
// check if we need to undo the tranformation on x86 executables
if ((file.fileName.contains(".exe")
|| file.fileName.contains(".dll"))) { // TODO there is an attribute for this
ByteArrayOutputStream out = new ByteArrayOutputStream();
decoder.Code(in, out, file.originalSize);
byte[] decompressedData = out.toByteArray();
Util.transformCallInstructions(decompressedData);
BufferedOutputStream outfile = new BufferedOutputStream(new FileOutputStream(f));
outfile.write(decompressedData);
outfile.close();
} else {
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(f));
decoder.Code(in, out, file.originalSize);
out.close();
}
f.setLastModified(file.mtime.getTime());
ui.increaseProgress();
}
} catch (Exception e) {
ui.showError(e.getLocalizedMessage());
e.printStackTrace();
}
}
private static void assertLexicographicallySortableLong(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
slice.setLexicographicallySortableLong(index, 0xAAAA_AAAA_5555_5555L);
assertEquals(slice.getLexicographicallySortableLong(index), 0xAAAA_AAAA_5555_5555L);
try {
slice.getLexicographicallySortableLong(-1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getLexicographicallySortableLong((slice.length() - SIZE_OF_LONG) + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getLexicographicallySortableLong(slice.length());
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getLexicographicallySortableLong(slice.length() + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
}
public Builder add(Slice slice) {
assert comparator.compare(slice.start(), slice.end()) <= 0;
if (slices.size() > 0
&& comparator.compare(slices.get(slices.size() - 1).end(), slice.start()) > 0)
needsNormalizing = true;
slices.add(slice);
return this;
}
public boolean selects(Clustering clustering) {
for (int i = 0; i < slices.length; i++) {
Slice slice = slices[i];
if (comparator.compare(clustering, slice.start()) < 0) return false;
if (comparator.compare(clustering, slice.end()) <= 0) return true;
}
return false;
}
@Test
public void testUtf8Conversion() {
String s = "apple \u2603 snowman";
Slice slice = Slices.copiedBuffer(s, UTF_8);
assertEquals(Slices.utf8Slice(s), slice);
assertEquals(slice.toStringUtf8(), s);
assertEquals(Slices.utf8Slice(s).toStringUtf8(), s);
}
private void calcLevels() {
LinkedList<LinkedList<Slice>> levelsList = new LinkedList<LinkedList<Slice>>();
HashSet<Slice> visited = new HashSet<Slice>();
LinkedList<Slice> queue = new LinkedList<Slice>();
// add the top slices and set their level
for (int i = 0; i < topSlices.length; i++) {
queue.add(topSlices[i]);
levelMap.put(topSlices[i], 0);
}
while (!queue.isEmpty()) {
Slice slice = queue.removeFirst();
if (!visited.contains(slice)) {
visited.add(slice);
for (Edge destEdge : slice.getTail().getDestSet(SchedulingPhase.STEADY)) {
Slice current = destEdge.getDest().getParent();
if (!visited.contains(current)) {
// only add if all sources has been visited
boolean addMe = true;
int maxParentLevel = 0;
for (Edge sourceEdge : current.getHead().getSourceSet(SchedulingPhase.STEADY)) {
if (!visited.contains(sourceEdge.getSrc().getParent())) {
addMe = false;
break;
}
// remember the max parent level
if (levelMap.get(sourceEdge.getSrc().getParent()).intValue() > maxParentLevel)
maxParentLevel = levelMap.get(sourceEdge.getSrc().getParent()).intValue();
}
if (addMe) {
levelMap.put(current, maxParentLevel + 1);
queue.add(current);
}
}
}
// add the slice to the appropriate level
int sliceLevel = levelMap.get(slice);
if (levelsList.size() <= sliceLevel) {
int levelsToAdd = sliceLevel - levelsList.size() + 1;
for (int i = 0; i < levelsToAdd; i++) ;
levelsList.add(new LinkedList<Slice>());
}
levelsList.get(sliceLevel).add(slice);
}
}
// set the multi-dim array for the levels from the linkedlist of lists
levels = new Slice[levelsList.size()][];
for (int i = 0; i < levels.length; i++) {
levels[i] = new Slice[levelsList.get(i).size()];
for (int j = 0; j < levels[i].length; j++) levels[i][j] = levelsList.get(i).get(j);
}
}
@Test
public void testReading() {
assertEquals(ByteOrder.nativeOrder(), ByteOrder.LITTLE_ENDIAN);
byte[] bytes = new byte[10];
Slice slice = Slices.wrappedBuffer(bytes);
slice.setInt(0, 0xDEADBEEF);
slice.setInt(4, 0xCAFEBABE);
// little endian memory layout: EF BE AD DE BE BA FE CA 00 00
assertEquals(
slice.getBytes(),
new byte[] {
(byte) 0xEF,
(byte) 0xBE,
(byte) 0xAD,
(byte) 0xDE,
(byte) 0xBE,
(byte) 0xBA,
(byte) 0xFE,
(byte) 0xCA,
0x00,
0x00
});
assertEquals(ByteArrays.getShort(bytes, 0), (short) 0xBEEF);
assertEquals(ByteArrays.getShort(bytes, 1), (short) 0xADBE);
assertEquals(ByteArrays.getShort(bytes, 2), (short) 0xDEAD);
assertEquals(ByteArrays.getShort(bytes, 3), (short) 0xBEDE);
assertEquals(ByteArrays.getInt(bytes, 0), 0xDEADBEEF);
assertEquals(ByteArrays.getInt(bytes, 1), 0xBEDEADBE);
assertEquals(ByteArrays.getInt(bytes, 2), 0xBABEDEAD);
assertEquals(ByteArrays.getInt(bytes, 3), 0xFEBABEDE);
assertEquals(ByteArrays.getInt(bytes, 4), 0xCAFEBABE);
assertEquals(ByteArrays.getLong(bytes, 0), 0xCAFEBABE_DEADBEEFL);
assertEquals(ByteArrays.getLong(bytes, 1), 0x00CAFEBA_BEDEADBEL);
assertEquals(ByteArrays.getLong(bytes, 2), 0x0000CAFE_BABEDEADL);
assertEquals(ByteArrays.getFloat(bytes, 0), intBitsToFloat(0xDEADBEEF));
assertEquals(ByteArrays.getFloat(bytes, 1), intBitsToFloat(0xBEDEADBE));
assertEquals(ByteArrays.getFloat(bytes, 2), intBitsToFloat(0xBABEDEAD));
assertEquals(ByteArrays.getFloat(bytes, 3), intBitsToFloat(0xFEBABEDE));
assertEquals(ByteArrays.getFloat(bytes, 4), intBitsToFloat(0xCAFEBABE));
assertEquals(ByteArrays.getDouble(bytes, 0), longBitsToDouble(0xCAFEBABE_DEADBEEFL));
assertEquals(ByteArrays.getDouble(bytes, 1), longBitsToDouble(0x00CAFEBA_BEDEADBEL));
assertEquals(ByteArrays.getDouble(bytes, 2), longBitsToDouble(0x0000CAFE_BABEDEADL));
}
private Slices forForwardPaging(
ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) {
for (int i = 0; i < slices.length; i++) {
Slice slice = slices[i];
Slice newSlice = slice.forPaging(comparator, lastReturned, inclusive, false);
if (newSlice == null) continue;
if (slice == newSlice && i == 0) return this;
ArrayBackedSlices newSlices =
new ArrayBackedSlices(comparator, Arrays.copyOfRange(slices, i, slices.length));
newSlices.slices[0] = newSlice;
return newSlices;
}
return Slices.NONE;
}
private Slices forReversePaging(
ClusteringComparator comparator, Clustering lastReturned, boolean inclusive) {
for (int i = slices.length - 1; i >= 0; i--) {
Slice slice = slices[i];
Slice newSlice = slice.forPaging(comparator, lastReturned, inclusive, true);
if (newSlice == null) continue;
if (slice == newSlice && i == slices.length - 1) return this;
ArrayBackedSlices newSlices =
new ArrayBackedSlices(comparator, Arrays.copyOfRange(slices, 0, i + 1));
newSlices.slices[i] = newSlice;
return newSlices;
}
return Slices.NONE;
}
public static ComponentOfSlice fromSlice(int component, Slice slice) {
Slice.Bound start = slice.start();
Slice.Bound end = slice.end();
if (component >= start.size() && component >= end.size()) return null;
boolean startInclusive = true, endInclusive = true;
ByteBuffer startValue = null, endValue = null;
if (component < start.size()) {
startInclusive = start.isInclusive();
startValue = start.get(component);
}
if (component < end.size()) {
endInclusive = end.isInclusive();
endValue = end.get(component);
}
return new ComponentOfSlice(startInclusive, startValue, endInclusive, endValue);
}
/**
* See if a structure is "whole", which means that none of its fields is missing from the
* constraint, all of fields use default (non-constrained) dimension), and all of its fields are
* also whole. This must be done recursively.
*
* @param dstruct to test
* @return true if this structure is whole.
*/
protected boolean isWholeCompound(DapStructure dstruct) {
int processed = 0;
List<DapVariable> fields = dstruct.getFields();
for (DapVariable field : fields) {
// not contractable if this field has non-original dimensions
Segment seg = findSegment(field);
if (seg == null) break; // this compound is not whole
List<Slice> slices = seg.slices;
if (slices != null) {
for (Slice slice : slices) {
if (slice.isConstrained()) break;
}
}
if (field.getSort() == DapSort.STRUCTURE || field.getSort() == DapSort.SEQUENCE) {
if (!isWholeCompound((DapStructure) field)) break; // this compound is not whole
}
processed++;
}
return (processed == fields.size());
}
@Test
public void testFillAndClear() {
for (byte size = 0; size < 100; size++) {
Slice slice = allocate(size);
for (int i = 0; i < slice.length(); i++) {
assertEquals(slice.getByte(i), (byte) 0);
}
slice.fill((byte) 0xA5);
for (int i = 0; i < slice.length(); i++) {
assertEquals(slice.getByte(i), (byte) 0xA5);
}
slice.clear();
for (int i = 0; i < slice.length(); i++) {
assertEquals(slice.getByte(i), (byte) 0);
}
}
}
@SuppressWarnings("CharUsedInArithmeticContext")
private static void assertToStrings(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
char[] chars = new char[(slice.length() - index) / 2];
for (int i = 0; i < chars.length; i++) {
chars[i] = (char) ('a' + (i % 26));
}
String string = new String(chars);
Slice value = Slices.copiedBuffer(string, UTF_8);
slice.setBytes(index, value);
assertEquals(slice.toString(index, value.length(), UTF_8), string);
for (int length = 0; length < value.length(); length++) {
slice.fill((byte) 0xFF);
slice.setBytes(index, value, 0, length);
assertEquals(slice.toString(index, length, UTF_8), string.substring(0, length));
}
}
/**
* Given an array of slices (potentially overlapping and in any order) and return an equivalent
* array of non-overlapping slices in clustering order.
*
* @param slices an array of slices. This may be modified by this method.
* @return the smallest possible array of non-overlapping slices in clustering order. If the
* original slices are already non-overlapping and in comparator order, this may or may not
* return the provided slices directly.
*/
private List<Slice> normalize(List<Slice> slices) {
if (slices.size() <= 1) return slices;
Collections.sort(
slices,
new Comparator<Slice>() {
@Override
public int compare(Slice s1, Slice s2) {
int c = comparator.compare(s1.start(), s2.start());
if (c != 0) return c;
return comparator.compare(s1.end(), s2.end());
}
});
List<Slice> slicesCopy = new ArrayList<>(slices.size());
Slice last = slices.get(0);
for (int i = 1; i < slices.size(); i++) {
Slice s2 = slices.get(i);
boolean includesStart = last.includes(comparator, s2.start());
boolean includesFinish = last.includes(comparator, s2.end());
if (includesStart && includesFinish) continue;
if (!includesStart && !includesFinish) {
slicesCopy.add(last);
last = s2;
continue;
}
if (includesStart) {
last = Slice.make(last.start(), s2.end());
continue;
}
assert !includesFinish;
}
slicesCopy.add(last);
return slicesCopy;
}
/**
* Recursive helper for tostring/toConstraintString
*
* @param seg
* @param buf
* @param forconstraint
*/
protected void dumpvar(Segment seg, StringBuilder buf, boolean forconstraint) {
if (seg.var.isTopLevel()) buf.append(seg.var.getFQN());
else buf.append(seg.var.getShortName());
List<DapDimension> dimset = seg.var.getDimensions();
// Add any slices
List<Slice> slices = seg.slices;
if (slices == null) dimset = new ArrayList<DapDimension>();
else assert dimset.size() == slices.size();
for (int i = 0; i < dimset.size(); i++) {
Slice slice = slices.get(i);
DapDimension dim = dimset.get(i);
try {
buf.append(forconstraint ? slice.toConstraintString() : slice.toString());
} catch (DapException de) {
}
}
// if the var is atomic, then we are done
if (seg.var.getSort() == DapSort.ATOMICVARIABLE) return;
// If structure and all fields are in the view, then done
if (seg.var.getSort() == DapSort.STRUCTURE || seg.var.getSort() == DapSort.SEQUENCE) {
if (!isWholeCompound((DapStructure) seg.var)) {
// Need to insert {...} and recurse
buf.append(LBRACE);
DapStructure struct = (DapStructure) seg.var;
boolean first = true;
for (DapVariable field : struct.getFields()) {
if (!first) buf.append(";");
first = false;
Segment fseg = findSegment(field);
dumpvar(fseg, buf, forconstraint);
}
buf.append(RBRACE);
}
if (seg.var.getSort() == DapSort.SEQUENCE && seg.filter != null) {
buf.append("|");
buf.append(seg.filter.toString());
}
}
}
/**
* Compute dimension related information using slicing and redef info. In effect, this is where
* projection constraints are applied
*
* <p>Assume that the constraint compiler has given us the following info:
*
* <ol>
* <li>A list of the variables to include.
* <li>A pair (DapDimension,Slice) for each redef
* <li>For each variable in #1, a list of slices taken from the constraint expression
* </ol>
*
* <p>Two products will be produced.
*
* <ol>
* <li>The variables map will be modified so that the slices properly reflect any original or
* redef dimensions.
* <li>A set, dimrefs, of all referenced original dimensions.
* </ol>
*
* <p>The processing is as follows
*
* <ol>
* <li>For each redef create a new redef dimension
* <li>For each variable:
* <ol>
* <li>if the variable is scalar, do nothing.
* <li>if the variable has no associated slices, then make its new dimensions be the
* original dimensions.
* <li>otherwise, walk the slices and create new dimensions from them; use redefs where
* indicated
* <li>
* </ol>
* </ol>
*/
protected void computedimensions() throws DapException {
// Build the redefmap
for (DapDimension key : redefslice.keySet()) {
Slice slice = redefslice.get(key);
DapDimension newdim = (DapDimension) key.clone();
newdim.setSize(slice.getCount());
redef.put(key, newdim);
}
// Process each variable
for (int i = 0; i < segments.size(); i++) {
Segment seg = segments.get(i);
if (seg.var.getRank() == 0) continue;
List<Slice> slices = seg.slices;
List<DapDimension> orig = seg.var.getDimensions();
List<DapDimension> newdims = new ArrayList<>();
// If the slice list is short then pad it with
// default slices
if (slices == null) slices = new ArrayList<Slice>();
while (slices.size() < orig.size()) // pad
{
slices.add(new Slice().setConstrained(false));
}
assert (slices != null && slices.size() == orig.size());
for (int j = 0; j < slices.size(); j++) {
Slice slice = slices.get(j);
DapDimension dim0 = orig.get(j);
DapDimension newdim = redef.get(dim0);
if (newdim == null) newdim = dim0;
// fill in the undefined last value
slice.setMaxSize(newdim.getSize());
slice.finish();
Slice newslice = null;
if (slice.isConstrained()) {
// Construct an anonymous dimension for this slice
newdim = new DapDimension(slice.getCount());
} else { // replace with a new slice from the dim
newslice = new Slice(newdim);
if (newslice != null) {
// track set of referenced non-anonymous dimensions
if (!dimrefs.contains(dim0)) dimrefs.add(dim0);
slices.set(j, newslice);
}
}
// record the dimension per variable
newdims.add(newdim);
}
seg.setDimset(newdims);
}
}
private static void assertBytesArray(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
byte[] value = new byte[slice.length()];
Arrays.fill(value, (byte) 0xFF);
assertEquals(slice.getBytes(), value);
// set and get the value
value = new byte[(slice.length() - index) / 2];
for (int i = 0; i < value.length; i++) {
value[i] = (byte) i;
}
slice.setBytes(index, value);
assertEquals(slice.getBytes(index, value.length), value);
for (int length = 0; length < value.length; length++) {
slice.fill((byte) 0xFF);
slice.setBytes(index, value, 0, length);
assertEquals(slice.getBytes(index, length), Arrays.copyOf(value, length));
}
}
@Test
public void testMemoryMappedReads() throws IOException {
Path path = Files.createTempFile("longs", null);
ImmutableList<Long> values = createRandomLongs(20000);
Slice output = allocate(values.size() * Longs.BYTES);
for (int i = 0; i < values.size(); i++) {
output.setLong(i * Longs.BYTES, values.get(i));
}
Files.write(path, output.getBytes());
Slice slice = Slices.mapFileReadOnly(path.toFile());
for (int i = 0; i < values.size(); i++) {
long actual = slice.getLong(i * Longs.BYTES);
long expected = values.get(i);
assertEquals(actual, expected);
}
assertEquals(slice.getBytes(), output.getBytes());
}
private void assertBytesSlice(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// compare to self slice
assertEquals(slice.slice(0, slice.length()), slice);
Slice value = allocate(slice.length());
slice.getBytes(0, value, 0, slice.length());
assertEquals(value, slice);
// set and get the value
value = allocate((slice.length() - index) / 2);
for (int i = 0; i < value.length(); i++) {
value.setByte(i, i);
}
// check by slicing out the region
slice.setBytes(index, value);
assertEquals(value, slice.slice(index, value.length()));
// check by getting out the region
Slice tempValue = allocate(value.length());
slice.getBytes(index, tempValue, 0, tempValue.length());
assertEquals(tempValue, slice.slice(index, tempValue.length()));
assertTrue(tempValue.equals(0, tempValue.length(), slice, index, tempValue.length()));
for (int length = 0; length < value.length(); length++) {
slice.fill((byte) 0xFF);
slice.setBytes(index, value, 0, length);
// check by slicing out the region
assertEquals(value.slice(0, length), slice.slice(index, length));
assertTrue(value.equals(0, length, slice, index, length));
// check by getting out the region
tempValue = allocate(length);
slice.getBytes(index, tempValue);
assertEquals(tempValue, slice.slice(index, length));
assertTrue(tempValue.equals(0, length, slice, index, length));
}
}
private static void assertByte(Slice slice, byte index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get unsigned value
slice.setByte(index, 0xA5);
assertEquals(slice.getUnsignedByte(index), 0x0000_00A5);
// set and get the value
slice.setByte(index, 0xA5);
assertEquals(slice.getByte(index), (byte) 0xA5);
try {
slice.getByte(-1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getByte((slice.length() - SIZE_OF_BYTE) + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getByte(slice.length());
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
try {
slice.getByte(slice.length() + 1);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException e) {
}
}
public static void run(
SIRStream str,
JInterfaceDeclaration[] interfaces,
SIRInterfaceTable[] interfaceTables,
SIRStructure[] structs,
SIRHelper[] helpers,
SIRGlobal global) {
System.out.println("Entry to SMP Backend...");
checkArguments();
setScheduler();
if (KjcOptions.smp > 16) {
setupLargeConfig();
}
// create cores in desired amount and order
int[] cores = new int[KjcOptions.smp];
for (int x = 0; x < KjcOptions.smp; x++) cores[x] = coreOrder[x];
chip = new SMPMachine(cores);
// create a new structs.h file for typedefs etc.
structs_h = new Structs_h(structs);
// The usual optimizations and transformation to slice graph
CommonPasses commonPasses = new CommonPasses();
// perform standard optimizations, use the number of cores the user wants to target
commonPasses.run(str, interfaces, interfaceTables, structs, helpers, global, chip.size());
// perform some standard cleanup on the slice graph.
commonPasses.simplifySlices();
// dump slice graph to dot file
commonPasses.getSlicer().dumpGraph("traces.dot", null);
// partition the slice graph based on the scheduling policy
SpaceTimeScheduleAndSlicer graphSchedule =
new SpaceTimeScheduleAndSlicer(commonPasses.getSlicer());
scheduler.setGraphSchedule(graphSchedule);
scheduler.run(chip.size());
FilterInfo.reset();
// generate schedules for initialization, primepump and steady-state
scheduleSlices(graphSchedule);
// generate layout for filters
scheduler.runLayout();
// dump final slice graph to dot file
graphSchedule.getSlicer().dumpGraph("after_slice_partition.dot", scheduler);
graphSchedule.getSlicer().dumpGraph("slice_graph.dot", scheduler, false);
// if load balancing, find candidiate fission groups to load balance
if (KjcOptions.loadbalance) {
LoadBalancer.findCandidates();
LoadBalancer.instrumentMainMethods();
}
// create all buffers and set the rotation lengths
RotatingBuffer.createBuffers(graphSchedule);
// now convert to Kopi code plus communication commands
backEndBits = new SMPBackEndFactory(chip, scheduler);
backEndBits.getBackEndMain().run(graphSchedule, backEndBits);
// generate code for file writer
CoreCodeStore.generatePrintOutputCode();
if (KjcOptions.numbers > 0) chip.getNthComputeNode(0).getComputeCode().generateNumbersCode();
// emit c code for all cores
EmitSMPCode.doit(backEndBits);
// dump structs.h file
structs_h.writeToFile();
// display final assignment of filters to cores
System.out.println("Final filter assignments:");
System.out.println("========================================");
for (int x = 0; x < KjcOptions.smp; x++) {
Core core = chip.getNthComputeNode(x);
Set<FilterSliceNode> filters = core.getComputeCode().getFilters();
long totalWork = 0;
System.out.println("Core " + core.getCoreID() + ": ");
for (FilterSliceNode filter : filters) {
long work = SliceWorkEstimate.getWork(filter.getParent());
System.out.format("%16d | " + filter + "\n", work);
totalWork += work;
}
System.out.format("%16d | Total\n", totalWork);
}
// calculate computation to communication ratio
if (KjcOptions.sharedbufs) {
LinkedList<Slice> slices =
DataFlowOrder.getTraversal(graphSchedule.getSlicer().getTopSlices());
HashSet<Slice> compProcessed = new HashSet<Slice>();
HashSet<Slice> commProcessed = new HashSet<Slice>();
long comp = 0;
long comm = 0;
for (Slice slice : slices) {
if (compProcessed.contains(slice)) continue;
comp += SliceWorkEstimate.getWork(slice);
compProcessed.add(slice);
}
/*
for(Slice slice : slices) {
if(commProcessed.contains(slice))
continue;
FilterInfo info = FilterInfo.getFilterInfo(slice.getFirstFilter());
int totalItemsReceived = info.totalItemsReceived(SchedulingPhase.STEADY);
if(totalItemsReceived == 0)
continue;
InputSliceNode input = slice.getHead();
Set<InterSliceEdge> sources = input.getSourceSet(SchedulingPhase.STEADY);
int numInputRots = totalItemsReceived / input.totalWeights(SchedulingPhase.STEADY);
if(!FissionGroupStore.isFizzed(slice)) {
for(InterSliceEdge source : sources) {
Slice srcSlice = source.getSrc().getParent();
// if(srcSlice.getFirstFilter().isFileInput())
// continue;
if(FissionGroupStore.isFizzed(srcSlice)) {
// Filter is not fizzed, source is fizzed
// Filter must receive (N-1)/N of inputs from different cores
comm += numInputRots *
input.getWeight(source, SchedulingPhase.STEADY) /
KjcOptions.smp * (KjcOptions.smp - 1);
}
else {
// Filter is not fizzed, source is not fizzed
// Check to see if on same core
// If not, must communicate all elements
if(!scheduler.getComputeNode(slice.getFirstFilter()).equals(
scheduler.getComputeNode(srcSlice.getFirstFilter()))) {
comm += numInputRots *
input.getWeight(source, SchedulingPhase.STEADY);
}
}
}
}
else {
for(InterSliceEdge source : sources) {
Slice srcSlice = source.getSrc().getParent();
// if(srcSlice.getFirstFilter().isFileInput())
// continue;
if(FissionGroupStore.isFizzed(srcSlice)) {
// Filter is fizzed, source is also fizzed
int totalItemsReceivedPerFizzed = totalItemsReceived /
FissionGroupStore.getFissionGroup(slice).fizzedSlices.length;
int numInputRotsPerFizzed = numInputRots /
FissionGroupStore.getFissionGroup(slice).fizzedSlices.length;
System.out.println("totalItemsReceivedPerFizzed: " + totalItemsReceivedPerFizzed);
System.out.println("numInputRotsPerFizzed: " + numInputRotsPerFizzed);
int inputWeightBeforeSrc =
input.weightBefore(source, SchedulingPhase.STEADY);
int inputWeightSrc = input.getWeight(source, SchedulingPhase.STEADY);
int inputTotalWeight = input.totalWeights(SchedulingPhase.STEADY);
System.out.println("inputWeightBeforeSrc: " + inputWeightBeforeSrc);
System.out.println("inputWeightSrc: " + inputWeightSrc);
System.out.println("copyDown: " + info.copyDown);
int numXmit = 0;
for(int rot = 0 ; rot < numInputRotsPerFizzed ; rot++) {
numXmit += Math.min(inputWeightSrc,
Math.max(0,
info.copyDown +
rot * inputTotalWeight +
inputWeightBeforeSrc + inputWeightSrc -
totalItemsReceivedPerFizzed));
}
System.out.println("numXmit: " + numXmit);
comm += KjcOptions.smp * numXmit;
}
else {
// Filter is fizzed, source is not fizzed
// Source must send (N-1)/N of outputs to different cores
comm += numInputRots *
input.getWeight(source, SchedulingPhase.STEADY) /
KjcOptions.smp * (KjcOptions.smp - 1);
}
}
}
commProcessed.add(slice);
}
*/
// Simple communication estimation
for (Slice slice : slices) {
if (commProcessed.contains(slice)) continue;
FilterInfo info = FilterInfo.getFilterInfo(slice.getFirstFilter());
int totalItemsReceived = info.totalItemsReceived(SchedulingPhase.STEADY);
if (totalItemsReceived == 0) continue;
comm += totalItemsReceived;
if (FissionGroupStore.isFizzed(slice)) {
assert info.peek >= info.pop;
comm += (info.peek - info.pop) * KjcOptions.smp;
}
commProcessed.add(slice);
}
// Simple communication estimation 2
/*
for(Slice slice : slices) {
if(commProcessed.contains(slice))
continue;
FilterInfo info = FilterInfo.getFilterInfo(slice.getFirstFilter());
int totalItemsReceived = info.totalItemsReceived(SchedulingPhase.STEADY);
int totalItemsSent = info.totalItemsSent(SchedulingPhase.STEADY);
comm += totalItemsReceived;
comm += totalItemsSent;
if(totalItemsReceived == 0)
continue;
if(FissionGroupStore.isFizzed(slice)) {
assert info.peek >= info.pop;
comm += (info.peek - info.pop) * KjcOptions.smp;
}
commProcessed.add(slice);
}
*/
System.out.println("Final Computation: " + comp);
System.out.println("Final Communication: " + comm);
System.out.println("Final Comp/Comm Ratio: " + (float) comp / (float) comm);
}
System.exit(0);
}
void setSlices(List<Slice> slices) throws DapException {
this.slices = slices;
// Make sure they are finished
for (Slice sl : slices) sl.finish();
}
@Test
public void testEqualsHashCodeCompare() {
for (int size = 0; size < 100; size++) {
// self equals
Slice slice = allocate(size);
assertSlicesEquals(slice, slice);
// equals other all zero
Slice other = allocate(size);
assertSlicesEquals(slice, other);
// equals self fill pattern
slice = allocate(size); // create a new slice since slices cache the hash code value
slice.fill((byte) 0xA5);
assertSlicesEquals(slice, slice);
// equals other fill pattern
other = allocate(size); // create a new slice since slices cache the hash code value
other.fill((byte) 0xA5);
assertSlicesEquals(slice, other);
// different types
assertNotEquals(slice, new Object());
//noinspection MisorderedAssertEqualsArgumentsTestNG
assertNotEquals(new Object(), slice);
// different sizes
Slice oneBigger = allocate(size + 1);
oneBigger.fill((byte) 0xA5);
assertNotEquals(slice, oneBigger);
assertNotEquals(oneBigger, slice);
assertTrue(slice.compareTo(oneBigger) < 0);
assertTrue(oneBigger.compareTo(slice) > 0);
assertFalse(slice.equals(0, size, oneBigger, 0, size + 1));
assertFalse(oneBigger.equals(0, size + 1, slice, 0, size));
assertTrue(slice.compareTo(0, size, oneBigger, 0, size + 1) < 0);
assertTrue(oneBigger.compareTo(0, size + 1, slice, 0, size) > 0);
// different in one byte
for (int i = 1; i < slice.length(); i++) {
slice.setByte(i - 1, 0xA5);
assertTrue(slice.equals(i - 1, size - i, other, i - 1, size - i));
slice.setByte(i, 0xFF);
assertNotEquals(slice, other);
assertFalse(slice.equals(i, size - i, other, i, size - i));
assertTrue(slice.compareTo(0, size, oneBigger, 0, size + 1) > 0);
}
// compare with empty slice
if (slice.length() > 0) {
assertNotEquals(slice, EMPTY_SLICE);
//noinspection MisorderedAssertEqualsArgumentsTestNG
assertNotEquals(EMPTY_SLICE, slice);
assertFalse(slice.equals(0, size, EMPTY_SLICE, 0, 0));
assertFalse(EMPTY_SLICE.equals(0, 0, slice, 0, size));
assertTrue(slice.compareTo(0, size, EMPTY_SLICE, 0, 0) > 0);
assertTrue(EMPTY_SLICE.compareTo(0, 0, slice, 0, size) < 0);
try {
//noinspection ResultOfMethodCallIgnored
slice.equals(0, size, EMPTY_SLICE, 0, size);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException expected) {
}
try {
//noinspection ResultOfMethodCallIgnored
EMPTY_SLICE.equals(0, size, slice, 0, size);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException expected) {
}
try {
slice.compareTo(0, size, EMPTY_SLICE, 0, size);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException expected) {
}
try {
EMPTY_SLICE.compareTo(0, size, slice, 0, size);
fail("expected IndexOutOfBoundsException");
} catch (IndexOutOfBoundsException expected) {
}
}
}
}
private static void assertBytesStreams(Slice slice, int index) throws Exception {
// fill slice with FF
slice.fill((byte) 0xFF);
byte[] value = new byte[slice.length()];
Arrays.fill(value, (byte) 0xFF);
assertEquals(slice.getBytes(), value);
// set and get the value
value = new byte[(slice.length() - index) / 2];
for (int i = 0; i < value.length; i++) {
value[i] = (byte) i;
}
slice.setBytes(index, new ByteArrayInputStream(value), value.length);
assertEquals(slice.getBytes(index, value.length), value);
ByteArrayOutputStream out = new ByteArrayOutputStream();
slice.getBytes(index, out, value.length);
assertEquals(slice.getBytes(index, value.length), out.toByteArray());
for (int length = 0; length < value.length; length++) {
slice.fill((byte) 0xFF);
slice.setBytes(index, new ByteArrayInputStream(value), length);
assertEquals(slice.getBytes(index, length), Arrays.copyOf(value, length));
out = new ByteArrayOutputStream();
slice.getBytes(index, out, length);
assertEquals(slice.getBytes(index, length), out.toByteArray());
}
}
public long getCardinality(Slice slice, BindingSet bindings) {
long card = getCardinality(slice.getArg(), bindings);
return Math.min(card, slice.getLimit());
}
private static void assertSlicesEquals(Slice slice, Slice other) {
int size = slice.length();
assertEquals(slice, other);
assertTrue(slice.equals(0, size, other, 0, size));
assertEquals(slice.hashCode(), other.hashCode());
assertEquals(slice.hashCode(), other.hashCode(0, size));
assertEquals(slice.compareTo(other), 0);
assertEquals(slice.compareTo(0, size, other, 0, size), 0);
for (int i = 0; i < slice.length(); i++) {
assertTrue(slice.equals(i, size - i, other, i, size - i));
assertEquals(slice.hashCode(i, size - i), other.hashCode(i, size - i));
assertEquals(slice.compareTo(i, size - i, other, i, size - i), 0);
}
for (int i = 0; i < slice.length(); i++) {
assertTrue(slice.equals(0, size - i, other, 0, size - i));
assertEquals(slice.hashCode(0, size - i), other.hashCode(0, size - i));
assertEquals(slice.compareTo(0, size - i, other, 0, size - i), 0);
}
}