{
try {
pigContext.connect();
} catch (ExecException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
static {
pc = new PigContext();
try {
pc.connect();
} catch (ExecException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
r = new Random(SEED);
}
public static void oneTimeSetUp() throws Exception {
cluster = MiniGenericCluster.buildCluster();
pc = new PigContext(cluster.getExecType(), cluster.getProperties());
try {
pc.connect();
} catch (ExecException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
GenPhyOp.setPc(pc);
Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
}
public void reduce(
BytesWritable key,
Iterator<Tuple> values,
OutputCollector<BytesWritable, Tuple> output,
Reporter reporter)
throws IOException {
try {
for (; values.hasNext(); ) {
output.collect(key, values.next());
}
} catch (ExecException e) {
e.printStackTrace();
}
}
@Override
public int compare(Tuple t1, Tuple t2) {
// TODO Auto-generated method stub
int result = 0;
try {
int i1 = (Integer) t1.get(1);
int i2 = (Integer) t2.get(1);
result = (i1 - 50) * (i1 - 50) - (i2 - 50) * (i2 - 50);
} catch (ExecException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return result;
}
@BeforeClass
public static void setUp() throws Exception {
init();
pathTable = getTableFullPath("TestCogroup");
removeDir(pathTable);
BasicTable.Writer writer =
new BasicTable.Writer(
pathTable,
"SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
"[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]",
conf);
Schema schema = writer.getSchema();
Tuple tuple = TypesUtils.createTuple(schema);
final int numsBatch = 10;
final int numsInserters = 1;
TableInserter[] inserters = new TableInserter[numsInserters];
for (int i = 0; i < numsInserters; i++) {
inserters[i] = writer.getInserter("ins" + i, false);
}
for (int b = 0; b < numsBatch; b++) {
for (int i = 0; i < numsInserters; i++) {
TypesUtils.resetTuple(tuple);
for (int k = 0; k < tuple.size(); ++k) {
try {
tuple.set(k, (9 - b) + "_" + i + "" + k);
} catch (ExecException e) {
e.printStackTrace();
}
}
inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
}
}
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
writer.close();
}
public static void createFirstTable() throws IOException, ParseException {
BasicTable.Writer writer = new BasicTable.Writer(pathTable1, STR_SCHEMA1, STR_STORAGE1, conf);
Schema schema = writer.getSchema();
// System.out.println("typeName" + schema.getColumn("a").type.pigDataType());
Tuple tuple = TypesUtils.createTuple(schema);
TableInserter[] inserters = new TableInserter[numsInserters];
for (int i = 0; i < numsInserters; i++) {
inserters[i] = writer.getInserter("ins" + i, false);
}
Tuple tupRecord1;
try {
tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1").getSchema());
} catch (ParseException e) {
e.printStackTrace();
throw new IOException(e);
}
Map<String, String> m1 = new HashMap<String, String>();
for (int b = 0; b < numsBatch; b++) {
for (int i = 0; i < numsInserters; i++) {
TypesUtils.resetTuple(tupRecord1);
TypesUtils.resetTuple(tuple);
m1.clear();
try {
// first row of the table , the biggest row
if (i == 0 && b == 0) {
tuple.set(0, 100);
tuple.set(1, 100.1f);
tuple.set(2, 100L);
tuple.set(3, 50e+2);
tuple.set(4, "something");
tuple.set(5, new DataByteArray("something"));
}
// the middle + 1 row of the table, the smallest row
else if (i == 0 && b == (numsBatch / 2)) {
tuple.set(0, -100);
tuple.set(1, -100.1f);
tuple.set(2, -100L);
tuple.set(3, -50e+2);
tuple.set(4, "so");
tuple.set(5, new DataByteArray("so"));
} else {
Float f = 1.1f;
long l = 11;
double d = 1.1;
tuple.set(0, b);
tuple.set(1, f);
tuple.set(2, l);
tuple.set(3, d);
tuple.set(4, "some");
tuple.set(5, new DataByteArray("some"));
}
// insert record
tupRecord1.set(0, "" + b);
tupRecord1.set(1, "" + b);
tuple.set(6, tupRecord1);
// insert map
m1.put("a", "" + b);
m1.put("b", "" + b);
m1.put("c", "" + b);
tuple.set(7, m1);
} catch (ExecException e) {
e.printStackTrace();
}
inserters[i].insert(new BytesWritable(("key_" + b).getBytes()), tuple);
}
}
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
writer.close();
// check table is setup correctly
String projection = new String("a,b,c,d,e,f,r1,m1");
BasicTable.Reader reader = new BasicTable.Reader(pathTable1, conf);
reader.setProjection(projection);
List<RangeSplit> splits = reader.rangeSplit(1);
TableScanner scanner = reader.getScanner(splits.get(0), true);
BytesWritable key = new BytesWritable();
Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
scanner.getValue(RowValue);
System.out.println("rowvalue size:" + RowValue.size());
System.out.println("read a : " + RowValue.get(0).toString());
System.out.println("read string: " + RowValue.get(1).toString());
scanner.advance();
scanner.getValue(RowValue);
System.out.println("read float in 2nd row: " + RowValue.get(1).toString());
System.out.println("done insert table");
reader.close();
}