/**
* Inserts tuples read from child into the tableid specified by the constructor. It returns a one
* field tuple containing the number of inserted records. Inserts should be passed through
* BufferPool. An instances of BufferPool is available via Database.getBufferPool(). Note that
* insert DOES NOT need check to see if a particular tuple is a duplicate before inserting it.
*
* @return A 1-field tuple containing the number of inserted records, or null if called more than
* once.
* @see Database#getBufferPool
* @see BufferPool#insertTuple
*/
protected Tuple fetchNext() throws TransactionAbortedException, DbException {
Tuple result = new Tuple(td); // use to store the insertion result
int count = 0; // use to keep track of numbers of tuple insertion
if (fetchNextNum
> 0) // meaning this is not the first time calling fetchNext(), and should not return any
// tuples
return null;
else {
try {
while (dbIt.hasNext()) {
try {
Database.getBufferPool().insertTuple(tranId, tableId, dbIt.next());
} catch (IOException e) {
e.printStackTrace();
}
count++;
}
result.setField(0, new IntField(count));
fetchNextNum++;
} catch (DbException e) {
e.printStackTrace();
} catch (TransactionAbortedException e) {
e.printStackTrace();
}
}
return result;
}
/**
* Create a DbIterator over group aggregate results.
*
* @return a DbIterator whose tuples are the pair (groupVal, aggregateVal) if using group, or a
* single (aggregateVal) if no grouping. The aggregateVal is determined by the type of
* aggregate specified in the constructor.
*/
public DbIterator iterator() {
/* TODO: write about iterator() call recalculates all the aggregates, which seems necessary given how the test case is formulated
(merge one tuple, get iterator, expect the result to be updated). Note that recalculate is not necessary unless the source tuples
change, but as a simplification here we implement it like this. */
TupleDesc td;
ArrayList<Tuple> tuples = new ArrayList<Tuple>();
if (isGrouping()) {
Type[] typeAr = new Type[2];
typeAr[0] = this.gbFieldType;
typeAr[1] = Type.INT_TYPE;
td = new TupleDesc(typeAr);
for (Map.Entry<Field, ArrayList<IntField>> entry : this.tupleStorage.entrySet()) {
IntField aggregate = this.getAggregate(entry.getValue());
Tuple tup = new Tuple(td);
tup.setField(0, entry.getKey());
tup.setField(1, aggregate);
tuples.add(tup);
}
} else {
Type[] typeAr = new Type[1];
typeAr[0] = Type.INT_TYPE;
td = new TupleDesc(typeAr);
Tuple tup = new Tuple(td);
tup.setField(0, this.getAggregate(this.tupleStorageNoGrouping));
tuples.add(tup);
}
TupleIterator iterator = new TupleIterator(td, tuples);
return iterator;
}
/** Suck up tuples from the source file. */
private Tuple readNextTuple(DataInputStream dis, int slotId) throws NoSuchElementException {
// if associated bit is not set, read forward to the next tuple, and
// return null.
if (!getSlot(slotId)) {
for (int i = 0; i < td.getSize(); i++) {
try {
dis.readByte();
} catch (IOException e) {
throw new NoSuchElementException("error reading empty tuple");
}
}
return null;
}
// read fields in the tuple
Tuple t = new Tuple(td);
RecordID rid = new RecordID(pid, slotId);
t.setRecordID(rid);
try {
for (int j = 0; j < td.numFields(); j++) {
Field f = td.getType(j).parse(dis);
t.setField(j, f);
}
} catch (java.text.ParseException e) {
// e.printStackTrace();
throw new NoSuchElementException("parsing error!");
}
return t;
}
/**
* Merge a new tuple into the aggregate, grouping as indicated in the constructor
*
* @param tup the Tuple containing an aggregate field and a group-by field
*/
public void mergeTupleIntoGroup(Tuple tup) {
// some code goes here
Field field;
if (m_gbfield != Aggregator.NO_GROUPING) field = tup.getField(m_gbfield);
else field = null;
int value = ((IntField) tup.getField(m_afield)).getValue();
if (!m_aggmap.containsKey(field)) {
m_aggmap.put(field, new AggregateCounter(value, 1));
} else {
AggregateCounter currentvalue = m_aggmap.get(field);
currentvalue.count += 1;
switch (m_op) {
case MIN:
if (currentvalue.value > value) currentvalue.value = value;
break;
case MAX:
if (currentvalue.value < value) currentvalue.value = value;
break;
case SUM:
case AVG: // we will calculate average at the end
currentvalue.value += value;
break;
// case AVG:
// currentvalue.value = (value+currentvalue.value)/(currentvalue.count);
default:
break;
}
m_aggmap.put(field, currentvalue);
}
}
/**
* Merge a new tuple into the aggregate, grouping as indicated in the constructor
*
* @param tup the Tuple containing an aggregate field and a group-by field
*/
public void mergeTupleIntoGroup(Tuple tup) {
if (this.groupByFieldName == null && this.groupByField != NO_GROUPING) {
this.groupByFieldName = tup.getTupleDesc().getFieldName(this.groupByField);
}
Field key;
if (this.groupByField != NO_GROUPING) {
key = tup.getField(this.groupByField);
} else {
key = new IntField(-1);
}
// TODO: clean up average logic
int aggregationValue = ((IntField) tup.getField(this.aggregateField)).getValue();
if (this.groups.containsKey(key)) {
this.groups.put(key, this.performOperation(key, this.groups.get(key), aggregationValue));
if (this.op == Op.AVG) {
this.counts.put(key, this.counts.get(key) + 1);
}
} else {
this.groups.put(key, this.initValue(aggregationValue));
if (this.op == Op.AVG) {
this.counts.put(key, 1);
}
}
}
/**
* Create a DbIterator over group aggregate results.
*
* @return a DbIterator whose tuples are the pair (groupVal, aggregateVal) if using group, or a
* single (aggregateVal) if no grouping. The aggregateVal is determined by the type of
* aggregate specified in the constructor.
*/
public DbIterator iterator() {
// some code goes here
// throw new
// UnsupportedOperationException("please implement me for lab2");
ArrayList<Tuple> tuplearray = new ArrayList<Tuple>();
if (m_gbfield == Aggregator.NO_GROUPING) {
Tuple newtuple = new Tuple(m_td);
if (m_op == Op.AVG) {
newtuple.setField(0, new IntField(m_aggmap.get(null).value / m_aggmap.get(null).count));
} else if (m_op == Op.COUNT) {
newtuple.setField(0, new IntField(m_aggmap.get(null).count));
} else {
newtuple.setField(0, new IntField(m_aggmap.get(null).value));
}
tuplearray.add(newtuple);
} else {
for (Field fieldkey : m_aggmap.keySet()) {
Tuple newtuple = new Tuple(m_td);
newtuple.setField(0, fieldkey);
// calculate average now
if (m_op == Op.AVG) {
newtuple.setField(
1, new IntField(m_aggmap.get(fieldkey).value / m_aggmap.get(fieldkey).count));
} else if (m_op == Op.COUNT) {
newtuple.setField(1, new IntField(m_aggmap.get(fieldkey).count));
} else newtuple.setField(1, new IntField(m_aggmap.get(fieldkey).value));
tuplearray.add(newtuple);
}
}
return new TupleIterator(m_td, tuplearray);
}
/** Unit test for Tuple.resetTupleDesc() */
@Test
public void resetTupleDesc() {
// create a tuple with the original tuple desc
TupleDesc origTd = Utility.getTupleDesc(2, "orig");
Tuple tup = new Tuple(origTd);
assertEquals("orig0", tup.getTupleDesc().getFieldName(0));
// rename the fields by changing the tuple desc to a new one
TupleDesc newTd = Utility.getTupleDesc(2, "new");
tup.resetTupleDesc(newTd);
assertEquals("new0", tup.getTupleDesc().getFieldName(0));
}
/**
* Create a new TableStats object, that keeps track of statistics on each column of a table
*
* @param tableid The table over which to compute statistics
* @param ioCostPerPage The cost per page of IO. This doesn't differentiate between
* sequential-scan IO and disk seeks.
*/
public TableStats(int tableid, int ioCostPerPage) {
// For this function, we use the DbFile for the table in question,
// then scan through its tuples and calculate the values that you
// to build the histograms.
// TODO: Fill out the rest of the constructor.
// Feel free to change anything already written, it's only a guideline
this.ioCostPerPage = ioCostPerPage;
DbFile file = Database.getCatalog().getDbFile(tableid);
tupleDesc = file.getTupleDesc();
numPages = ((HeapFile) file).numPages();
numTuples = 0;
int numFields = tupleDesc.numFields();
// TODO: what goes here?
statistics = new ArrayList<Object>();
for (int i = 0; i < numFields; i++) {
if (Type.INT_TYPE.equals(tupleDesc.getFieldType(i))) {
statistics.add(new IntStatistics(NUM_HIST_BINS));
} else {
statistics.add(new StringHistogram(NUM_HIST_BINS));
}
}
final DbFileIterator iter = file.iterator(null);
try {
iter.open();
while (iter.hasNext()) {
Tuple t = iter.next();
numTuples++;
// TODO: and here?
for (int i = 0; i < numFields; i++) {
if (Type.INT_TYPE.equals(tupleDesc.getFieldType(i))) {
((IntStatistics) statistics.get(i)).addValue(((IntField) t.getField(i)).getValue());
} else {
((StringHistogram) statistics.get(i))
.addValue(((StringField) t.getField(i)).getValue());
}
}
}
iter.close();
} catch (DbException e) {
e.printStackTrace();
} catch (TransactionAbortedException e) {
e.printStackTrace();
}
}
/**
* Merge a new tuple into the aggregate, grouping as indicated in the constructor
*
* @param tup the Tuple containing an aggregate field and a group-by field
*/
public void mergeTupleIntoGroup(Tuple tup) {
if (isGrouping()) {
Field gbValue = tup.getField(this.gbField);
if (this.tupleStorage.containsKey(gbValue)) {
this.tupleStorage.get(gbValue).add((IntField) tup.getField(this.aField));
} else {
this.tupleStorage.put(gbValue, new ArrayList<IntField>());
this.tupleStorage.get(gbValue).add((IntField) tup.getField(this.aField));
}
} else {
this.tupleStorageNoGrouping.add((IntField) tup.getField(this.aField));
}
}
/**
* Remove the specified tuple from the buffer pool. Will acquire a write lock on the page the
* tuple is removed from. May block if the lock cannot be acquired.
*
* <p>Marks any pages that were dirtied by the operation as dirty by calling their markDirty bit.
* Does not need to update cached versions of any pages that have been dirtied, as it is not
* possible that a new page was created during the deletion (note difference from addTuple).
*
* @param tid the transaction deleting the tuple.
* @param t the tuple to delete
*/
public void deleteTuple(TransactionId tid, Tuple t)
throws DbException, TransactionAbortedException {
HeapFile heapFile =
(HeapFile) Database.getCatalog().getDatabaseFile(t.getRecordId().getPageId().getTableId());
Page dirtiedPage = heapFile.deleteTuple(tid, t);
dirtiedPage.markDirty(true, tid);
}
@Test
public void new_fieldsTest() {
int length = 10;
String name = "td";
Tuple t = new Tuple(Utility.getTupleDesc(length, name));
Iterator<Field> fs = t.fields();
int i = 0;
while (fs.hasNext()) {
i++;
fs.next();
}
assertEquals(length, i);
}
public Tuple getNext() throws NoSuchElementException, TransactionAbortedException {
try {
Tuple tuple = new Tuple(td);
for (int i = 0; i < td.numFields(); i++) {
IntField intf = IntField.createIntField(in.readInt());
tuple.setField(i, intf);
}
return tuple;
} catch (EOFException eof) {
throw new NoSuchElementException(eof.getMessage());
} catch (Exception e) {
e.printStackTrace();
BufferPool.Instance().abortTransaction(tid);
closeConnection();
throw new TransactionAbortedException(e);
}
}
/**
* Adds the specified tuple to the page; the tuple should be updated to reflect that it is now
* stored on this page.
*
* @throws DbException if the page is full (no empty slots) or tupledesc is mismatch.
* @param t The tuple to add.
*/
public void insertTuple(Tuple t) throws DbException {
// some code goes here
// not necessary for lab1
RecordId targetrid = t.getRecordId();
if (getNumEmptySlots() == 0 || !td.equals(t.getTupleDesc())) {
throw new DbException("Either page is full or tuple desc doesn't match");
}
for (int i = 0; i < getNumTuples(); i++) {
if (!isSlotUsed(i)) {
markSlotUsed(i, true);
RecordId rid = new RecordId(pid, i);
t.setRecordId(rid);
tuples[i] = t;
return;
}
}
}
/** Unit test for Tuple.getRecordId() and Tuple.setRecordId() */
@Test
public void modifyRecordId() {
Tuple tup1 = new Tuple(Utility.getTupleDesc(1));
HeapPageId pid1 = new HeapPageId(0, 0);
RecordId rid1 = new RecordId(pid1, 0);
tup1.setRecordId(rid1);
try {
assertEquals(rid1, tup1.getRecordId());
} catch (java.lang.UnsupportedOperationException e) {
// rethrow the exception with an explanation
throw new UnsupportedOperationException(
"modifyRecordId() test failed due to "
+ "RecordId.equals() not being implemented. This is not required for Lab 1, "
+ "but should pass when you do implement the RecordId class.");
}
}
/** Unit test for fields() iterator */
@Test
public void fields() {
int numfields = 4;
TupleDesc td = Utility.getTupleDesc(numfields);
// set up the tuple
Tuple tup = new Tuple(td);
for (int i = 0; i < numfields; i++) tup.setField(i, new IntField(i));
// use the iterator, make sure get the same number of fields out
Iterator<Field> iter = tup.fields();
int count = 0;
while (iter.hasNext()) {
iter.next();
count++;
}
assertEquals(numfields, count);
}
/**
* Merge a new tuple into the aggregate, grouping as indicated in the constructor
*
* @param tup the Tuple containing an aggregate field and a group-by field
*/
public void merge(Tuple tup) {
if (_gb == NO_GROUPING) {
_count++;
} else {
Field f = tup.getField(_gb);
Integer c = _counts.get(f);
if (c == null) c = 1;
else c++;
_counts.put(f, c);
}
}
/**
* Delete the specified tuple from the page; the tuple should be updated to reflect that it is no
* longer stored on any page.
*
* @throws DbException if this tuple is not on this page, or tuple slot is already empty.
* @param t The tuple to delete
*/
public void deleteTuple(Tuple t) throws DbException {
// some code goes here
// not necessary for lab1
RecordId targetrid = t.getRecordId();
int tslotnum = targetrid.tupleno();
if (!targetrid.getPageId().equals(pid) || !isSlotUsed(tslotnum)) {
throw new DbException("Either wrong page number or requested tuple didn't exist");
}
markSlotUsed(tslotnum, false);
tuples[tslotnum] = null;
}
/** Unit test for Tuple.getField() and Tuple.setField() */
@Test
public void modifyFields() {
TupleDesc td = Utility.getTupleDesc(2);
Tuple tup = new Tuple(td);
tup.setField(0, new IntField(-1));
tup.setField(1, new IntField(0));
assertEquals(new IntField(-1), tup.getField(0));
assertEquals(new IntField(0), tup.getField(1));
tup.setField(0, new IntField(1));
tup.setField(1, new IntField(37));
assertEquals(new IntField(1), tup.getField(0));
assertEquals(new IntField(37), tup.getField(1));
}
// see DbFile.java for javadocs
public Page deleteTuple(TransactionId tid, Tuple t)
throws DbException, TransactionAbortedException {
// some code goes here
BufferPool bp = Database.getBufferPool();
RecordId rid = t.getRecordId();
if (rid == null) {
throw new DbException("Tuple is not a member of this file");
}
HeapPage p = (HeapPage) bp.getPage(tid, rid.getPageId(), Permissions.READ_WRITE);
p.deleteTuple(t);
return p;
}
@Test
public void new_toStringTest() {
int length = 10;
String name = "td";
Tuple t = new Tuple(Utility.getTupleDesc(length, name));
for (int i = 0; i < length; i++) {
t.setField(i, new TestField());
}
String tString = t.toString();
// Last character should be \n.
assertEquals("\n", tString.substring(tString.length() - 1));
// Only last character is \n.
assertFalse(tString.substring(0, tString.length() - 1).contains("\n"));
// Split string on any white character.
String[] tStringAr = tString.substring(0, tString.length() - 1).split("\\s+");
assertEquals(length, tStringAr.length);
}
/**
* Inserts tuples read from child into the relation with the tableid specified by the constructor.
* It returns a one field tuple containing the number of inserted records (even if there are 0!).
* Insertions should be passed through BufferPool.insertTuple() with the TransactionId from the
* constructor. An instance of BufferPool is available via Database.getBufferPool(). Note that
* insert DOES NOT need to check to see if a particular tuple is a duplicate before inserting it.
*
* <p>This operator should keep track if its fetchNext() has already been called, returning null
* if called multiple times.
*
* @return A 1-field tuple containing the number of inserted records, or null if called more than
* once.
* @see Database#getBufferPool
* @see BufferPool#insertTuple
*/
protected Tuple fetchNext() throws TransactionAbortedException, DbException {
if (!hasFetched) {
this.hasFetched = true;
int numInserts = 0;
while (this.child.hasNext()) {
Tuple t = this.child.next();
try {
Database.getBufferPool().insertTuple(this.tid, this.tableid, t);
numInserts += 1;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
Type[] fields = new Type[1];
fields[0] = Type.INT_TYPE;
TupleDesc td = new TupleDesc(fields);
Tuple newTuple = new Tuple(td);
newTuple.setField(0, new IntField(numInserts));
return newTuple;
} else {
return null;
}
}
/**
* Create a DbIterator over group aggregate results.
*
* @return a DbIterator whose tuples are the pair (groupVal, aggregateVal) if using group, or a
* single (aggregateVal) if no grouping. The aggregateVal is determined by the type of
* aggregate specified in the constructor.
*/
public DbIterator iterator() {
// TODO: This current shares a ton of code of with StringAggregator.
// Refactor?
ArrayList<Tuple> aggregateTuples = new ArrayList<Tuple>();
TupleDesc tupleDesc = this.getTupleDesc();
for (Map.Entry<Field, Integer> entry : this.groups.entrySet()) {
Field key = entry.getKey();
Integer value = entry.getValue();
if (this.op == Op.AVG) {
value /= this.counts.get(key);
}
Tuple tuple = new Tuple(tupleDesc);
if (this.groupByField == NO_GROUPING) {
tuple.setField(0, new IntField(value));
} else {
tuple.setField(0, key);
tuple.setField(1, new IntField(value));
}
aggregateTuples.add(tuple);
}
return new TupleIterator(tupleDesc, aggregateTuples);
}
public static void writeHeapFile(
ArrayList<Tuple> tuples, File outFile, int npagebytes, Type[] typeAr) throws IOException {
int nrecbytes = 0;
for (int i = 0; i < typeAr.length; i++) {
nrecbytes += typeAr[i].getLen();
}
int maxNumRecordsInAPage = (npagebytes * 8) / (nrecbytes * 8 + 1); // floor comes
// for free
// per record, we need one bit; there are nrecords per page, so we need
// nrecords bits, i.e., ((nrecords/32)+1) integers.
int nheaderbytes = (maxNumRecordsInAPage / 8);
if (nheaderbytes * 8 < maxNumRecordsInAPage) nheaderbytes++; // ceiling
int nheaderbits = nheaderbytes * 8;
FileOutputStream os = new FileOutputStream(outFile);
int npages = 0;
int numRecordInCurrPage = 0;
int totalRecordCount = 0;
ByteArrayOutputStream headerBAOS = new ByteArrayOutputStream(nheaderbytes);
DataOutputStream headerStream = new DataOutputStream(headerBAOS);
ByteArrayOutputStream pageBAOS = new ByteArrayOutputStream(npagebytes);
DataOutputStream pageStream = new DataOutputStream(pageBAOS);
for (Tuple t : tuples) {
int fieldNo = 0;
numRecordInCurrPage++;
totalRecordCount++;
Iterator<Field> it = t.fields();
while (it.hasNext()) {
Field f = it.next();
if (typeAr[fieldNo] == Type.INT_TYPE) {
IntField i = (IntField) f;
pageStream.writeInt(i.getValue());
} else if (typeAr[fieldNo] == Type.STRING_TYPE) {
StringField sf = (StringField) f;
String s = sf.getValue();
int overflow = Type.STRING_LEN - s.length();
if (overflow < 0) {
String news = s.substring(0, Type.STRING_LEN);
s = news;
}
pageStream.writeInt(s.length());
pageStream.writeBytes(s);
while (overflow-- > 0) pageStream.write((byte) 0);
}
fieldNo++;
}
// if we wrote a full page of records, or if we're done altogether,
// write out the header of the page.
//
// in the header, write a 1 for bits that correspond to records
// we've
// written and 0 for empty slots.
//
// when we're done, also flush the page to disk, but only if it has
// records on it. however, if this file is empty, do flush an empty
// page to disk.
if (numRecordInCurrPage >= maxNumRecordsInAPage
|| totalRecordCount == tuples.size() && numRecordInCurrPage > 0
|| totalRecordCount == tuples.size() && npages == 0) {
int i = 0;
byte headerbyte = 0;
for (i = 0; i < nheaderbits; i++) {
if (i < numRecordInCurrPage) headerbyte |= (1 << (i % 8));
if (((i + 1) % 8) == 0) {
headerStream.writeByte(headerbyte);
headerbyte = 0;
}
}
if (i % 8 > 0) headerStream.writeByte(headerbyte);
// pad the rest of the page with zeroes
for (i = 0; i < (npagebytes - (numRecordInCurrPage * nrecbytes + nheaderbytes)); i++)
pageStream.writeByte(0);
// write header and body to file
headerStream.flush();
headerBAOS.writeTo(os);
pageStream.flush();
pageBAOS.writeTo(os);
// reset header and body for next page
headerBAOS = new ByteArrayOutputStream(nheaderbytes);
headerStream = new DataOutputStream(headerBAOS);
pageBAOS = new ByteArrayOutputStream(npagebytes);
pageStream = new DataOutputStream(pageBAOS);
numRecordInCurrPage = 0;
npages++;
}
}
os.close();
}
/**
* Apply the predicate to the two specified tuples. The comparison can be made through Field's
* compare method.
*
* @return true if the tuples satisfy the predicate.
*/
public boolean filter(Tuple t1, Tuple t2) {
// Done
Field operand = t1.getField(this.field1);
Field otherOp = t2.getField(this.field2);
return operand.compare(this.op, otherOp);
}
/** Unit test for Tuple.getTupleDesc() */
@Test
public void getTupleDesc() {
TupleDesc td = Utility.getTupleDesc(5);
Tuple tup = new Tuple(td);
assertEquals(td, tup.getTupleDesc());
}
/**
* Compares the field number of t specified in the constructor to the operand field specified in
* the constructor using the operator specific in the constructor. The comparison can be made
* through Field's compare method.
*
* @param t The tuple to compare against
* @return true if the comparison is true, false otherwise.
*/
public boolean filter(Tuple t) {
return t.getField(field).compare(op, operand);
}