public static Collection<IJsonNode> readFromFile(
final File file, final SopremoFormat format, final SopremoRecordLayout layout)
throws IOException {
Configuration config = new Configuration();
final EvaluationContext context = new EvaluationContext();
SopremoUtil.setEvaluationContext(config, context);
SopremoUtil.setLayout(config, layout);
SopremoUtil.transferFieldsToConfiguration(
format, SopremoFormat.class, config, format.getInputFormat(), InputFormat.class);
@SuppressWarnings("unchecked")
final SopremoFileInputFormat inputFormat =
FormatUtil.openInput(
(Class<? extends SopremoFileInputFormat>) format.getInputFormat(),
file.toURI().toString(),
config);
List<IJsonNode> values = new ArrayList<IJsonNode>();
while (!inputFormat.reachedEnd()) {
final SopremoRecord record = new SopremoRecord(layout);
if (inputFormat.nextRecord(record)) values.add(record.getNode().clone());
}
inputFormat.close();
return values;
}
/*
* (non-Javadoc)
* @see eu.stratosphere.core.io.IOReadableWritable#read(java.io.DataInput)
*/
@Override
public void read(final DataInput in) throws IOException {
this.mode = ExecutionMode.values()[in.readInt()];
final ArrayList<String> requiredPackages = new ArrayList<String>();
for (int count = in.readInt(); count > 0; count--) requiredPackages.add(in.readUTF());
this.query = null;
final byte[] planBuffer = new byte[in.readInt()];
in.readFully(planBuffer);
final JobID dummId = new JobID();
try {
LibraryCacheManager.register(
dummId, requiredPackages.toArray(new String[requiredPackages.size()]));
SopremoEnvironment.getInstance().setClassLoader(LibraryCacheManager.getClassLoader(dummId));
this.query = SopremoUtil.deserialize(planBuffer, SopremoPlan.class);
} catch (final IOException e) {
e.printStackTrace();
} finally {
try {
LibraryCacheManager.unregister(dummId);
} catch (final IOException e) {
}
}
}
/*
* (non-Javadoc)
* @see eu.stratosphere.pact.common.stubs.Stub#open(eu.stratosphere.nephele.configuration.Configuration)
*/
@Override
public void open(final Configuration parameters) throws Exception {
SopremoEnvironment.getInstance().setConfigurationAndContext(parameters, getRuntimeContext());
this.context = SopremoEnvironment.getInstance().getEvaluationContext();
this.collector = createCollector(SopremoEnvironment.getInstance().getLayout());
this.cachedIterator1 = new RecordToJsonIterator<LeftElem>();
this.cachedIterator2 = new RecordToJsonIterator<RightElem>();
SopremoUtil.configureWithTransferredState(this, GenericSopremoCoGroup.class, parameters);
this.leftArray.setNodeIterator(this.cachedIterator1);
this.rightArray.setNodeIterator(this.cachedIterator2);
}
/*
* (non-Javadoc)
* @see eu.stratosphere.sopremo.serialization.TypeComparator#setReference(java.lang.Object)
*/
@Override
public void setReference(final SopremoRecord reference) {
final IJsonNode node = reference.getNode();
if (node == null)
for (int index = 0; index < this.numKeys; index++)
this.keyHolders1[index] = reference.getKey(this.keyFields1[index], this.nodeCache1[index]);
else
for (int index = 0; index < this.numKeys; index++)
this.keyHolders1[index] =
SopremoUtil.copyInto(
this.keyExpressions1[index].evaluate(node), this.nodeCache1[index]);
}
/*
* (non-Javadoc)
* @see eu.stratosphere.core.io.IOReadableWritable#write(java.io.DataOutput)
*/
@Override
public void write(final DataOutput out) throws IOException {
out.writeInt(this.mode.ordinal());
final List<String> requiredPackages = this.query.getRequiredPackages();
out.writeInt(requiredPackages.size());
for (final String packageName : requiredPackages) out.writeUTF(packageName);
final byte[] planBuffer = SopremoUtil.serializable(this.query);
out.writeInt(planBuffer.length);
out.write(planBuffer);
}
@Test
public void testSimpleMapping() throws IOException {
String query =
"using cleansing;"
+ "$usCongressMembers = read from '"
+ this.usCongressMembers.toURI()
+ "';\n"
+ "$usCongressBiographies = read from '"
+ this.usCongressBiographies.toURI()
+ "';\n"
+ "$person, $legalEntity = transform records $usCongressMembers, $usCongressBiographies\n"
+ "where ($usCongressMembers.biography == $usCongressBiographies.biographyId)\n"
+ "into [\n"
+ " entity $person with {"
+ // identified by $person.pname
" pname: $usCongressMembers.name,\n"
+ " pworksFor: $legalEntity.id"
+ " },"
+ " entity $legalEntity identified by $legalEntity.lname with {"
+ " lname: $usCongressBiographies.worksFor"
+ " }"
+ "];\n"
+ "write $person to '"
+ this.person.toURI()
+ "';\n"
+ "write $legalEntity to '"
+ this.legalEntity.toURI()
+ "';";
final SopremoPlan plan = parseScript(query);
SopremoUtil.trace();
Assert.assertNotNull(this.client.submit(plan, null, true));
this.testServer.checkContentsOf(
"person.json",
createObjectNode("id", "Andrew Adams", "pname", "Andrew Adams", "pworksFor", "CompanyXYZ"),
createObjectNode("id", "John Adams", "pname", "John Adams", "pworksFor", null),
createObjectNode("id", "John Doe", "pname", "John Doe", "pworksFor", "CompanyUVW"));
this.testServer.checkContentsOf(
"legalEntity.json",
createObjectNode("id", "CompanyXYZ", "lname", "CompanyXYZ"),
createObjectNode("id", "CompanyUVW", "lname", "CompanyUVW"),
createObjectNode("id", "CompanyABC", "lname", "CompanyABC"));
}
/**
* Configures a {@link Stub} with the given {@link Configuration}
*
* @param stub the stub that should be configured
* @param parameters the configuration that should be used
*/
static void configureStub(final Stub stub, final Configuration parameters) {
final Class<? extends Stub> stubClass = stub.getClass();
for (final Field stubField : stubClass.getDeclaredFields())
if ((stubField.getModifiers() & (Modifier.TRANSIENT | Modifier.FINAL | Modifier.STATIC)) == 0)
if (parameters.getString(stubField.getName(), null) != null)
try {
stubField.setAccessible(true);
stubField.set(
stub,
SopremoUtil.deserializeCachingAware(
parameters,
stubField.getName(),
stubField.getType(),
stubField.getGenericType(),
stubClass.getClassLoader()));
} catch (final Exception e) {
LOG.error(
String.format(
"Could not set field %s of class %s: %s",
stubField.getName(), stubClass, StringUtils.stringifyException(e)));
}
}
/*
* (non-Javadoc)
* @see eu.stratosphere.api.typeutils.TypeSerializerFactory#writeParametersToConfig(eu.stratosphere.nephele.
* configuration.Configuration)
*/
@Override
public void writeParametersToConfig(final Configuration config) {
SopremoUtil.setObject(config, LAYOUT_KEY, this.layout);
SopremoUtil.setObject(config, TYPES_KEY, this.typeRegistry);
}
/*
* (non-Javadoc)
* @see eu.stratosphere.api.typeutils.TypeSerializerFactory#readParametersFromConfig(eu.stratosphere.nephele.
* configuration.Configuration, java.lang.ClassLoader)
*/
@Override
public void readParametersFromConfig(final Configuration config, final ClassLoader cl)
throws ClassNotFoundException {
this.layout = SopremoUtil.getObject(config, LAYOUT_KEY, null);
this.typeRegistry = SopremoUtil.getObject(config, TYPES_KEY, null);
}