@Override
public EngineException[] bulk(Bulk bulk) throws EngineException {
EngineException[] failures = null;
rwl.readLock().lock();
try {
IndexWriter writer = this.indexWriter;
if (writer == null) {
throw new EngineClosedException(shardId);
}
for (int i = 0; i < bulk.ops().length; i++) {
Operation op = bulk.ops()[i];
if (op == null) {
continue;
}
try {
switch (op.opType()) {
case CREATE:
Create create = (Create) op;
writer.addDocument(create.doc(), create.analyzer());
translog.add(new Translog.Create(create));
break;
case INDEX:
Index index = (Index) op;
writer.updateDocument(index.uid(), index.doc(), index.analyzer());
translog.add(new Translog.Index(index));
break;
case DELETE:
Delete delete = (Delete) op;
writer.deleteDocuments(delete.uid());
translog.add(new Translog.Delete(delete));
break;
}
} catch (Exception e) {
if (failures == null) {
failures = new EngineException[bulk.ops().length];
}
switch (op.opType()) {
case CREATE:
failures[i] = new CreateFailedEngineException(shardId, (Create) op, e);
break;
case INDEX:
failures[i] = new IndexFailedEngineException(shardId, (Index) op, e);
break;
case DELETE:
failures[i] = new DeleteFailedEngineException(shardId, (Delete) op, e);
break;
}
}
}
dirty = true;
} finally {
rwl.readLock().unlock();
}
return failures;
}
@Test
public void shouldCreatePathWithData() throws Exception {
// Given
createCommand.data = "node data";
// When
createCommand.doExecute(curator);
// Then
verify(createBuilder).forPath(createCommand.path, createCommand.data.getBytes());
}
@Test
public void shouldCreateRecursivePathWithoutData() throws Exception {
// Given
createCommand.recursive = true;
// When
createCommand.doExecute(curator);
// Then
verify(createBuilder).creatingParentsIfNeeded();
verify(createBuilder).forPath(createCommand.path);
}
@Override
public void doProcess(final TestContext context, Create t) throws Throwable {
VariableContext.getInstance().pushScope(null == context.getTarget());
VariableContext.getInstance()
.setVariableBindings(t.uriParams(), t.mergeFields(), t.extractors());
try {
InternalHttpUnitRunner runner = new InternalHttpUnitRunner(context, t.action());
runner.run(new RunNotifier());
} finally {
VariableContext.getInstance().clearVariableBindings();
VariableContext.getInstance().popScope();
}
}
/**
* Applies {@code ApproximateUnique(sampleSize)} verifying that the estimation error falls within
* the maximum allowed error of {@code 2/sqrt(sampleSize)}.
*/
private static void runApproximateUniquePipeline(int sampleSize) {
Pipeline p = TestPipeline.create();
PCollection<String> input = p.apply(Create.of(TEST_LINES));
PCollection<Long> approximate = input.apply(ApproximateUnique.<String>globally(sampleSize));
final PCollectionView<Long> exact =
input
.apply(RemoveDuplicates.<String>create())
.apply(Count.<String>globally())
.apply(View.<Long>asSingleton());
PCollection<KV<Long, Long>> approximateAndExact =
approximate.apply(
ParDo.of(
new DoFn<Long, KV<Long, Long>>() {
@Override
public void processElement(ProcessContext c) {
c.output(KV.of(c.element(), c.sideInput(exact)));
}
})
.withSideInputs(exact));
DataflowAssert.that(approximateAndExact).satisfies(new VerifyEstimatePerKeyFn(sampleSize));
p.run();
}
/** Displays Network screen. */
private void displayNetworkMode() {
// true
NetworkMode.setVisible(true);
Online.setVisible(true);
Lan.setVisible(true);
back.setVisible(true);
// false
loginCreate.setVisible(false);
PlayerVsComp.setVisible(false);
PlayerVsPlayer.setVisible(false);
TwoPlayerVsTwoPlayer.setVisible(false);
Credits.setVisible(false);
Login.setVisible(false);
Login2.setVisible(false);
SmallInput.setVisible(false);
SmallInput2.setVisible(false);
SmallInput3.setVisible(false);
SmallInput4.setVisible(false);
SmallInput5.setVisible(false);
CreateNewAccount.setVisible(false);
Create.setVisible(false);
username.setVisible(false);
password.setVisible(false);
newUsername.setVisible(false);
newPassword.setVisible(false);
verifyPassword.setVisible(false);
}
/** Displays LoginScreen */
private void displayLogin() {
// true
back.setVisible(true);
Login.setVisible(true);
Login2.setVisible(true);
SmallInput.setVisible(true);
SmallInput2.setVisible(true);
SmallInput3.setVisible(true);
SmallInput4.setVisible(true);
SmallInput5.setVisible(true);
CreateNewAccount.setVisible(true);
Create.setVisible(true);
username.setVisible(true);
password.setVisible(true);
newUsername.setVisible(true);
newPassword.setVisible(true);
verifyPassword.setVisible(true);
loginCreate.setVisible(true);
loginCreate.setOpaque(false);
// false
serversError.setVisible(false);
Credits.setVisible(false);
NetworkMode.setVisible(false);
Online.setVisible(false);
Lan.setVisible(false);
Servers.setVisible(false);
BigInput2.setVisible(false);
LongInput.setVisible(false);
LongInput2.setVisible(false);
Join.setVisible(false);
Create2.setVisible(false);
returnToMain.setVisible(false);
ip.setVisible(false);
serverName.setVisible(false);
}
@Test
@SuppressWarnings("unchecked")
public void testTop() {
Pipeline p = TestPipeline.create();
PCollection<String> input =
p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));
PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength()));
PCollection<List<String>> top2 = input.apply(Top.<String>largest(2));
PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3));
PCollection<KV<String, Integer>> inputTable = createInputTable(p);
PCollection<KV<String, List<Integer>>> largestPerKey =
inputTable.apply(Top.<String, Integer>largestPerKey(2));
PCollection<KV<String, List<Integer>>> smallestPerKey =
inputTable.apply(Top.<String, Integer>smallestPerKey(2));
DataflowAssert.thatSingletonIterable(top1).containsInAnyOrder(Arrays.asList("bb"));
DataflowAssert.thatSingletonIterable(top2).containsInAnyOrder("z", "c");
DataflowAssert.thatSingletonIterable(top3).containsInAnyOrder("a", "bb", "c");
DataflowAssert.that(largestPerKey)
.containsInAnyOrder(KV.of("a", Arrays.asList(3, 2)), KV.of("b", Arrays.asList(100, 10)));
DataflowAssert.that(smallestPerKey)
.containsInAnyOrder(KV.of("a", Arrays.asList(1, 2)), KV.of("b", Arrays.asList(1, 10)));
p.run();
}
@Test
@SuppressWarnings("unchecked")
public void testTopEmpty() {
Pipeline p = TestPipeline.create();
PCollection<String> input =
p.apply(Create.of(Arrays.asList(EMPTY_COLLECTION)).withCoder(StringUtf8Coder.of()));
PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength()));
PCollection<List<String>> top2 = input.apply(Top.<String>largest(2));
PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3));
PCollection<KV<String, Integer>> inputTable = createEmptyInputTable(p);
PCollection<KV<String, List<Integer>>> largestPerKey =
inputTable.apply(Top.<String, Integer>largestPerKey(2));
PCollection<KV<String, List<Integer>>> smallestPerKey =
inputTable.apply(Top.<String, Integer>smallestPerKey(2));
DataflowAssert.thatSingletonIterable(top1).empty();
DataflowAssert.thatSingletonIterable(top2).empty();
DataflowAssert.thatSingletonIterable(top3).empty();
DataflowAssert.that(largestPerKey).empty();
DataflowAssert.that(smallestPerKey).empty();
p.run();
}
@Test
public void shouldCreatePathWithoutData() throws Exception {
// When
createCommand.doExecute(curator);
// Then
verify(createBuilder).forPath(createCommand.path);
}
@Before
public void setUp() {
createCommand.path = "/foo/bar";
given(createBuilder.withMode(any(CreateMode.class))).willReturn(createBuilder);
given(createBuilder.withACL(anyListOf(ACL.class))).willReturn(createBuilder);
given(curator.create()).willReturn(createBuilder);
}
@Test
public void testCountConstraint() {
Pipeline p = TestPipeline.create();
PCollection<String> input =
p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));
expectedEx.expect(IllegalArgumentException.class);
expectedEx.expectMessage(Matchers.containsString(">= 0"));
input.apply(Top.of(-1, new OrderByLength()));
}
@Override
public void create(Create create) throws EngineException {
rwl.readLock().lock();
try {
IndexWriter writer = this.indexWriter;
if (writer == null) {
throw new EngineClosedException(shardId);
}
writer.addDocument(create.doc(), create.analyzer());
translog.add(new Translog.Create(create));
dirty = true;
if (create.refresh()) {
refresh(new Refresh(false));
}
} catch (IOException e) {
throw new CreateFailedEngineException(shardId, create, e);
} finally {
rwl.readLock().unlock();
}
}
public static HashMap<String, double[]> getTagsHistogramMap(
String[] allTags, double[][] mTagAverageHistogram) {
HashMap<String, double[]> tagsHistogramMap = Create.hashMap();
for (int i = 0; i < allTags.length; i++) {
/** * HashMap for <tag, tag histogram>** */
tagsHistogramMap.put(allTags[i], mTagAverageHistogram[i]);
}
return tagsHistogramMap;
}
// This is a purely compile-time test. If the code compiles, then it worked.
@Test
public void testPerKeySerializabilityRequirement() {
Pipeline p = TestPipeline.create();
p.apply(
"CreateCollection", Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));
PCollection<KV<String, Integer>> inputTable = createInputTable(p);
inputTable.apply(Top.<String, Integer, IntegerComparator>perKey(1, new IntegerComparator()));
inputTable.apply(
"PerKey2", Top.<String, Integer, IntegerComparator2>perKey(1, new IntegerComparator2()));
}
public static HashMap<String, Map<Integer, Double>> getTagsDescriptorMap(
String[] allTags, Vector<Map<Integer, Double>> mTagAverageDescriptor) {
HashMap<String, Map<Integer, Double>> tagsDescriptorMap = Create.hashMap();
for (int i = 0; i < allTags.length; i++) {
/** * HashMap for <tag, tag descriptor map>** */
tagsDescriptorMap.put(allTags[i], mTagAverageDescriptor.elementAt(i));
}
return tagsDescriptorMap;
}
/** Reads a large {@code PCollection<String>}. */
private PCollection<String> readPCollection(Pipeline p) {
// TODO: Read PCollection from a set of text files.
List<String> page = TestUtils.LINES;
final int pages = 1000;
ArrayList<String> file = new ArrayList<>(pages * page.size());
for (int i = 0; i < pages; i++) {
file.addAll(page);
}
assert file.size() == pages * page.size();
PCollection<String> words = p.apply(Create.of(file));
return words;
}
public static void update() {
if (update) {
Sound.random("random", 1f, "OGG", "res/sound/music", 11);
GUI.Background.draw("dirt");
Text.draw("SETTINGS", GUI.Coordinates.x / 2 - 94, GUI.Coordinates.y / 4 * 3, 6, Color.white);
GUI.Button.draw(
"Music",
GUIScale.toUpperCase(),
GUI.Coordinates.x / 2 - 94,
GUI.Coordinates.y / 2,
1,
1,
"planks_oak",
2,
128);
if (Button.isButtonClicked("Music")) {
if (GUIScale.length() == 6) {
GUIScale = "Auto";
Create.createFile("Settings/GUIScale");
Create.write("Auto");
Create.closeFile();
} else {
GUIScale = "Normal";
Create.createFile("Settings/GUIScale");
Create.write("Normal");
Create.closeFile();
game.Main.Sleep(256);
}
}
}
}
/** Example test that tests a PTransform by using an in-memory input and inspecting the output. */
@Test
@Category(RunnableOnService.class)
public void testCountWords() throws Exception {
Pipeline p = TestPipeline.create();
PCollection<String> input = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));
PCollection<String> output = input.apply(new CountWords())
.apply(ParDo.of(new FormatAsTextFn()));
DataflowAssert.that(output).containsInAnyOrder(COUNTS_ARRAY);
p.run();
}
@Test
@Category(RunnableOnService.class)
public void testApproximateUniqueWithSmallInput() {
Pipeline p = TestPipeline.create();
PCollection<Integer> input = p.apply(Create.of(Arrays.asList(1, 2, 3, 3)));
PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(1000));
DataflowAssert.thatSingleton(estimate).isEqualTo(3L);
p.run();
}
@Test
@Category(RunnableOnService.class)
public void withLambdaAndTypeDescriptorShouldSucceed() {
PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12"));
PCollection<KV<Integer, String>> kvs =
values.apply(
WithKeys.of((String s) -> Integer.valueOf(s))
.withKeyType(TypeDescriptor.of(Integer.class)));
PAssert.that(kvs)
.containsInAnyOrder(
KV.of(1234, "1234"), KV.of(0, "0"), KV.of(-12, "-12"), KV.of(3210, "3210"));
p.run();
}
@Test
public void testTopEmptyWithIncompatibleWindows() {
Pipeline p = TestPipeline.create();
Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L)));
PCollection<String> input =
p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList()))
.apply(windowingFn);
expectedEx.expect(IllegalStateException.class);
expectedEx.expectMessage("Top");
expectedEx.expectMessage("GlobalWindows");
expectedEx.expectMessage("withoutDefaults");
expectedEx.expectMessage("asSingletonView");
input.apply(Top.of(1, new OrderByLength()));
}
public static final <K, T, A> Map<K, A> groupAndAggregate(
final Collection<T> objectsToGroup,
final Grouping<T, K> grouping,
final Aggregator<T, A> aggregator) {
final Map<K, Collection<T>> groupMap = group(objectsToGroup, grouping);
final Map<K, A> aggregatedMap = Create.newMap();
Iterate.forEach(
groupMap,
new Closure<Map.Entry<K, Collection<T>>>() {
// @Override
public void execute(Entry<K, Collection<T>> entry) {
aggregatedMap.put(entry.getKey(), aggregator.aggregate(entry.getValue()));
}
});
return aggregatedMap;
}
@Test
public void withLambdaAndNoTypeDescriptorShouldThrow() {
PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12"));
values.apply("ApplyKeysWithWithKeys", WithKeys.of((String s) -> Integer.valueOf(s)));
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Unable to return a default Coder for ApplyKeysWithWithKeys");
thrown.expectMessage("No Coder has been manually specified");
thrown.expectMessage(containsString("Building a Coder using a registered CoderFactory failed"));
thrown.expectMessage(
containsString("Building a Coder from the @DefaultCoder annotation failed"));
thrown.expectMessage(containsString("Building a Coder from the fallback CoderProvider failed"));
p.run();
}
public static final <K, T> Map<K, Collection<T>> group(
final Collection<T> objectsToGroup, final Grouping<T, K> grouping) {
final Map<K, Collection<T>> groupMap = Create.newMap();
Iterate.forEach(
objectsToGroup,
new Closure<T>() {
// @Override
public void execute(T obj) {
K key = grouping.toGroup(obj);
if (!groupMap.containsKey(key)) {
groupMap.put(key, Create.<T>newList());
}
groupMap.get(key).add(obj);
}
});
return groupMap;
}
public static Vector<Map<Integer, Double>> getTagsDescriptor(double[][] mTagAverageHistogram) {
Vector<Map<Integer, Double>> tmpVec = new Vector<Map<Integer, Double>>();
for (int i = 0; i < mTagAverageHistogram.length; i++) {
Map<Integer, Double> des = Create.linkedHashMap();
for (int j = 0; j < BIN_HISTO; j++) {
if (mTagAverageHistogram[i][j] > 0.0) {
// System.out.print(mTagAverageHistogram[i][j] + " ");
des.put(j, mTagAverageHistogram[i][j]);
}
}
// System.out.println();
tmpVec.add(des);
}
return tmpVec;
}
private void runApproximateUniqueWithDuplicates(
int elementCount, int uniqueCount, int sampleSize) {
assert elementCount >= uniqueCount;
List<Double> elements = Lists.newArrayList();
for (int i = 0; i < elementCount; i++) {
elements.add(1.0 / (i % uniqueCount + 1));
}
Collections.shuffle(elements);
Pipeline p = TestPipeline.create();
PCollection<Double> input = p.apply(Create.of(elements));
PCollection<Long> estimate = input.apply(ApproximateUnique.<Double>globally(sampleSize));
DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize));
p.run();
}
@Test
public void testApproximateUniquePerKey() {
List<KV<Long, Long>> elements = Lists.newArrayList();
List<Long> keys = ImmutableList.of(20L, 50L, 100L);
int elementCount = 1000;
int sampleSize = 100;
// Use the key as the number of unique values.
for (long uniqueCount : keys) {
for (long value = 0; value < elementCount; value++) {
elements.add(KV.of(uniqueCount, value % uniqueCount));
}
}
Pipeline p = TestPipeline.create();
PCollection<KV<Long, Long>> input = p.apply(Create.of(elements));
PCollection<KV<Long, Long>> counts =
input.apply(ApproximateUnique.<Long, Long>perKey(sampleSize));
DataflowAssert.that(counts).satisfies(new VerifyEstimatePerKeyFn(sampleSize));
p.run();
}
private void runApproximateUniqueWithSkewedDistributions(
int elementCount, final int uniqueCount, final int sampleSize) {
List<Integer> elements = Lists.newArrayList();
// Zipf distribution with approximately elementCount items.
double s = 1 - 1.0 * uniqueCount / elementCount;
double maxCount = Math.pow(uniqueCount, s);
for (int k = 0; k < uniqueCount; k++) {
int count = Math.max(1, (int) Math.round(maxCount * Math.pow(k, -s)));
// Element k occurs count times.
for (int c = 0; c < count; c++) {
elements.add(k);
}
}
Pipeline p = TestPipeline.create();
PCollection<Integer> input = p.apply(Create.of(elements));
PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(sampleSize));
DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize));
p.run();
}
/** Displays Server Screen. */
private void displayServers() {
// true
back.setVisible(true);
Servers.setVisible(true);
BigInput2.setVisible(true);
LongInput.setVisible(true);
LongInput2.setVisible(true);
Join.setVisible(true);
Create2.setVisible(true);
ip.setVisible(true);
serverName.setVisible(true);
serversError.setVisible(true);
// false
getServerName.setVisible(false);
getIpNumber.setVisible(false);
getPlayerNames.setVisible(false);
loginCreate.setVisible(false);
Credits.setVisible(false);
Login.setVisible(false);
Login2.setVisible(false);
SmallInput.setVisible(false);
SmallInput2.setVisible(false);
SmallInput3.setVisible(false);
SmallInput4.setVisible(false);
SmallInput5.setVisible(false);
CreateNewAccount.setVisible(false);
Create.setVisible(false);
MediumInput.setVisible(false);
Start.setVisible(false);
Cancel.setVisible(false);
username.setVisible(false);
password.setVisible(false);
newUsername.setVisible(false);
newPassword.setVisible(false);
verifyPassword.setVisible(false);
}
