@Override public EngineException[] bulk(Bulk bulk) throws EngineException { EngineException[] failures = null; rwl.readLock().lock(); try { IndexWriter writer = this.indexWriter; if (writer == null) { throw new EngineClosedException(shardId); } for (int i = 0; i < bulk.ops().length; i++) { Operation op = bulk.ops()[i]; if (op == null) { continue; } try { switch (op.opType()) { case CREATE: Create create = (Create) op; writer.addDocument(create.doc(), create.analyzer()); translog.add(new Translog.Create(create)); break; case INDEX: Index index = (Index) op; writer.updateDocument(index.uid(), index.doc(), index.analyzer()); translog.add(new Translog.Index(index)); break; case DELETE: Delete delete = (Delete) op; writer.deleteDocuments(delete.uid()); translog.add(new Translog.Delete(delete)); break; } } catch (Exception e) { if (failures == null) { failures = new EngineException[bulk.ops().length]; } switch (op.opType()) { case CREATE: failures[i] = new CreateFailedEngineException(shardId, (Create) op, e); break; case INDEX: failures[i] = new IndexFailedEngineException(shardId, (Index) op, e); break; case DELETE: failures[i] = new DeleteFailedEngineException(shardId, (Delete) op, e); break; } } } dirty = true; } finally { rwl.readLock().unlock(); } return failures; }
@Test public void shouldCreatePathWithData() throws Exception { // Given createCommand.data = "node data"; // When createCommand.doExecute(curator); // Then verify(createBuilder).forPath(createCommand.path, createCommand.data.getBytes()); }
@Test public void shouldCreateRecursivePathWithoutData() throws Exception { // Given createCommand.recursive = true; // When createCommand.doExecute(curator); // Then verify(createBuilder).creatingParentsIfNeeded(); verify(createBuilder).forPath(createCommand.path); }
@Override public void doProcess(final TestContext context, Create t) throws Throwable { VariableContext.getInstance().pushScope(null == context.getTarget()); VariableContext.getInstance() .setVariableBindings(t.uriParams(), t.mergeFields(), t.extractors()); try { InternalHttpUnitRunner runner = new InternalHttpUnitRunner(context, t.action()); runner.run(new RunNotifier()); } finally { VariableContext.getInstance().clearVariableBindings(); VariableContext.getInstance().popScope(); } }
/** * Applies {@code ApproximateUnique(sampleSize)} verifying that the estimation error falls within * the maximum allowed error of {@code 2/sqrt(sampleSize)}. */ private static void runApproximateUniquePipeline(int sampleSize) { Pipeline p = TestPipeline.create(); PCollection<String> input = p.apply(Create.of(TEST_LINES)); PCollection<Long> approximate = input.apply(ApproximateUnique.<String>globally(sampleSize)); final PCollectionView<Long> exact = input .apply(RemoveDuplicates.<String>create()) .apply(Count.<String>globally()) .apply(View.<Long>asSingleton()); PCollection<KV<Long, Long>> approximateAndExact = approximate.apply( ParDo.of( new DoFn<Long, KV<Long, Long>>() { @Override public void processElement(ProcessContext c) { c.output(KV.of(c.element(), c.sideInput(exact))); } }) .withSideInputs(exact)); DataflowAssert.that(approximateAndExact).satisfies(new VerifyEstimatePerKeyFn(sampleSize)); p.run(); }
/** Displays Network screen. */ private void displayNetworkMode() { // true NetworkMode.setVisible(true); Online.setVisible(true); Lan.setVisible(true); back.setVisible(true); // false loginCreate.setVisible(false); PlayerVsComp.setVisible(false); PlayerVsPlayer.setVisible(false); TwoPlayerVsTwoPlayer.setVisible(false); Credits.setVisible(false); Login.setVisible(false); Login2.setVisible(false); SmallInput.setVisible(false); SmallInput2.setVisible(false); SmallInput3.setVisible(false); SmallInput4.setVisible(false); SmallInput5.setVisible(false); CreateNewAccount.setVisible(false); Create.setVisible(false); username.setVisible(false); password.setVisible(false); newUsername.setVisible(false); newPassword.setVisible(false); verifyPassword.setVisible(false); }
/** Displays LoginScreen */ private void displayLogin() { // true back.setVisible(true); Login.setVisible(true); Login2.setVisible(true); SmallInput.setVisible(true); SmallInput2.setVisible(true); SmallInput3.setVisible(true); SmallInput4.setVisible(true); SmallInput5.setVisible(true); CreateNewAccount.setVisible(true); Create.setVisible(true); username.setVisible(true); password.setVisible(true); newUsername.setVisible(true); newPassword.setVisible(true); verifyPassword.setVisible(true); loginCreate.setVisible(true); loginCreate.setOpaque(false); // false serversError.setVisible(false); Credits.setVisible(false); NetworkMode.setVisible(false); Online.setVisible(false); Lan.setVisible(false); Servers.setVisible(false); BigInput2.setVisible(false); LongInput.setVisible(false); LongInput2.setVisible(false); Join.setVisible(false); Create2.setVisible(false); returnToMain.setVisible(false); ip.setVisible(false); serverName.setVisible(false); }
@Test @SuppressWarnings("unchecked") public void testTop() { Pipeline p = TestPipeline.create(); PCollection<String> input = p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength())); PCollection<List<String>> top2 = input.apply(Top.<String>largest(2)); PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3)); PCollection<KV<String, Integer>> inputTable = createInputTable(p); PCollection<KV<String, List<Integer>>> largestPerKey = inputTable.apply(Top.<String, Integer>largestPerKey(2)); PCollection<KV<String, List<Integer>>> smallestPerKey = inputTable.apply(Top.<String, Integer>smallestPerKey(2)); DataflowAssert.thatSingletonIterable(top1).containsInAnyOrder(Arrays.asList("bb")); DataflowAssert.thatSingletonIterable(top2).containsInAnyOrder("z", "c"); DataflowAssert.thatSingletonIterable(top3).containsInAnyOrder("a", "bb", "c"); DataflowAssert.that(largestPerKey) .containsInAnyOrder(KV.of("a", Arrays.asList(3, 2)), KV.of("b", Arrays.asList(100, 10))); DataflowAssert.that(smallestPerKey) .containsInAnyOrder(KV.of("a", Arrays.asList(1, 2)), KV.of("b", Arrays.asList(1, 10))); p.run(); }
@Test @SuppressWarnings("unchecked") public void testTopEmpty() { Pipeline p = TestPipeline.create(); PCollection<String> input = p.apply(Create.of(Arrays.asList(EMPTY_COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength())); PCollection<List<String>> top2 = input.apply(Top.<String>largest(2)); PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3)); PCollection<KV<String, Integer>> inputTable = createEmptyInputTable(p); PCollection<KV<String, List<Integer>>> largestPerKey = inputTable.apply(Top.<String, Integer>largestPerKey(2)); PCollection<KV<String, List<Integer>>> smallestPerKey = inputTable.apply(Top.<String, Integer>smallestPerKey(2)); DataflowAssert.thatSingletonIterable(top1).empty(); DataflowAssert.thatSingletonIterable(top2).empty(); DataflowAssert.thatSingletonIterable(top3).empty(); DataflowAssert.that(largestPerKey).empty(); DataflowAssert.that(smallestPerKey).empty(); p.run(); }
@Test public void shouldCreatePathWithoutData() throws Exception { // When createCommand.doExecute(curator); // Then verify(createBuilder).forPath(createCommand.path); }
@Before public void setUp() { createCommand.path = "/foo/bar"; given(createBuilder.withMode(any(CreateMode.class))).willReturn(createBuilder); given(createBuilder.withACL(anyListOf(ACL.class))).willReturn(createBuilder); given(curator.create()).willReturn(createBuilder); }
@Test public void testCountConstraint() { Pipeline p = TestPipeline.create(); PCollection<String> input = p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); expectedEx.expect(IllegalArgumentException.class); expectedEx.expectMessage(Matchers.containsString(">= 0")); input.apply(Top.of(-1, new OrderByLength())); }
@Override public void create(Create create) throws EngineException { rwl.readLock().lock(); try { IndexWriter writer = this.indexWriter; if (writer == null) { throw new EngineClosedException(shardId); } writer.addDocument(create.doc(), create.analyzer()); translog.add(new Translog.Create(create)); dirty = true; if (create.refresh()) { refresh(new Refresh(false)); } } catch (IOException e) { throw new CreateFailedEngineException(shardId, create, e); } finally { rwl.readLock().unlock(); } }
public static HashMap<String, double[]> getTagsHistogramMap( String[] allTags, double[][] mTagAverageHistogram) { HashMap<String, double[]> tagsHistogramMap = Create.hashMap(); for (int i = 0; i < allTags.length; i++) { /** * HashMap for <tag, tag histogram>** */ tagsHistogramMap.put(allTags[i], mTagAverageHistogram[i]); } return tagsHistogramMap; }
// This is a purely compile-time test. If the code compiles, then it worked. @Test public void testPerKeySerializabilityRequirement() { Pipeline p = TestPipeline.create(); p.apply( "CreateCollection", Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<KV<String, Integer>> inputTable = createInputTable(p); inputTable.apply(Top.<String, Integer, IntegerComparator>perKey(1, new IntegerComparator())); inputTable.apply( "PerKey2", Top.<String, Integer, IntegerComparator2>perKey(1, new IntegerComparator2())); }
public static HashMap<String, Map<Integer, Double>> getTagsDescriptorMap( String[] allTags, Vector<Map<Integer, Double>> mTagAverageDescriptor) { HashMap<String, Map<Integer, Double>> tagsDescriptorMap = Create.hashMap(); for (int i = 0; i < allTags.length; i++) { /** * HashMap for <tag, tag descriptor map>** */ tagsDescriptorMap.put(allTags[i], mTagAverageDescriptor.elementAt(i)); } return tagsDescriptorMap; }
/** Reads a large {@code PCollection<String>}. */ private PCollection<String> readPCollection(Pipeline p) { // TODO: Read PCollection from a set of text files. List<String> page = TestUtils.LINES; final int pages = 1000; ArrayList<String> file = new ArrayList<>(pages * page.size()); for (int i = 0; i < pages; i++) { file.addAll(page); } assert file.size() == pages * page.size(); PCollection<String> words = p.apply(Create.of(file)); return words; }
public static void update() { if (update) { Sound.random("random", 1f, "OGG", "res/sound/music", 11); GUI.Background.draw("dirt"); Text.draw("SETTINGS", GUI.Coordinates.x / 2 - 94, GUI.Coordinates.y / 4 * 3, 6, Color.white); GUI.Button.draw( "Music", GUIScale.toUpperCase(), GUI.Coordinates.x / 2 - 94, GUI.Coordinates.y / 2, 1, 1, "planks_oak", 2, 128); if (Button.isButtonClicked("Music")) { if (GUIScale.length() == 6) { GUIScale = "Auto"; Create.createFile("Settings/GUIScale"); Create.write("Auto"); Create.closeFile(); } else { GUIScale = "Normal"; Create.createFile("Settings/GUIScale"); Create.write("Normal"); Create.closeFile(); game.Main.Sleep(256); } } } }
/** Example test that tests a PTransform by using an in-memory input and inspecting the output. */ @Test @Category(RunnableOnService.class) public void testCountWords() throws Exception { Pipeline p = TestPipeline.create(); PCollection<String> input = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of())); PCollection<String> output = input.apply(new CountWords()) .apply(ParDo.of(new FormatAsTextFn())); DataflowAssert.that(output).containsInAnyOrder(COUNTS_ARRAY); p.run(); }
@Test @Category(RunnableOnService.class) public void testApproximateUniqueWithSmallInput() { Pipeline p = TestPipeline.create(); PCollection<Integer> input = p.apply(Create.of(Arrays.asList(1, 2, 3, 3))); PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(1000)); DataflowAssert.thatSingleton(estimate).isEqualTo(3L); p.run(); }
@Test @Category(RunnableOnService.class) public void withLambdaAndTypeDescriptorShouldSucceed() { PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12")); PCollection<KV<Integer, String>> kvs = values.apply( WithKeys.of((String s) -> Integer.valueOf(s)) .withKeyType(TypeDescriptor.of(Integer.class))); PAssert.that(kvs) .containsInAnyOrder( KV.of(1234, "1234"), KV.of(0, "0"), KV.of(-12, "-12"), KV.of(3210, "3210")); p.run(); }
@Test public void testTopEmptyWithIncompatibleWindows() { Pipeline p = TestPipeline.create(); Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L))); PCollection<String> input = p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList())) .apply(windowingFn); expectedEx.expect(IllegalStateException.class); expectedEx.expectMessage("Top"); expectedEx.expectMessage("GlobalWindows"); expectedEx.expectMessage("withoutDefaults"); expectedEx.expectMessage("asSingletonView"); input.apply(Top.of(1, new OrderByLength())); }
public static final <K, T, A> Map<K, A> groupAndAggregate( final Collection<T> objectsToGroup, final Grouping<T, K> grouping, final Aggregator<T, A> aggregator) { final Map<K, Collection<T>> groupMap = group(objectsToGroup, grouping); final Map<K, A> aggregatedMap = Create.newMap(); Iterate.forEach( groupMap, new Closure<Map.Entry<K, Collection<T>>>() { // @Override public void execute(Entry<K, Collection<T>> entry) { aggregatedMap.put(entry.getKey(), aggregator.aggregate(entry.getValue())); } }); return aggregatedMap; }
@Test public void withLambdaAndNoTypeDescriptorShouldThrow() { PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12")); values.apply("ApplyKeysWithWithKeys", WithKeys.of((String s) -> Integer.valueOf(s))); thrown.expect(IllegalStateException.class); thrown.expectMessage("Unable to return a default Coder for ApplyKeysWithWithKeys"); thrown.expectMessage("No Coder has been manually specified"); thrown.expectMessage(containsString("Building a Coder using a registered CoderFactory failed")); thrown.expectMessage( containsString("Building a Coder from the @DefaultCoder annotation failed")); thrown.expectMessage(containsString("Building a Coder from the fallback CoderProvider failed")); p.run(); }
public static final <K, T> Map<K, Collection<T>> group( final Collection<T> objectsToGroup, final Grouping<T, K> grouping) { final Map<K, Collection<T>> groupMap = Create.newMap(); Iterate.forEach( objectsToGroup, new Closure<T>() { // @Override public void execute(T obj) { K key = grouping.toGroup(obj); if (!groupMap.containsKey(key)) { groupMap.put(key, Create.<T>newList()); } groupMap.get(key).add(obj); } }); return groupMap; }
public static Vector<Map<Integer, Double>> getTagsDescriptor(double[][] mTagAverageHistogram) { Vector<Map<Integer, Double>> tmpVec = new Vector<Map<Integer, Double>>(); for (int i = 0; i < mTagAverageHistogram.length; i++) { Map<Integer, Double> des = Create.linkedHashMap(); for (int j = 0; j < BIN_HISTO; j++) { if (mTagAverageHistogram[i][j] > 0.0) { // System.out.print(mTagAverageHistogram[i][j] + " "); des.put(j, mTagAverageHistogram[i][j]); } } // System.out.println(); tmpVec.add(des); } return tmpVec; }
private void runApproximateUniqueWithDuplicates( int elementCount, int uniqueCount, int sampleSize) { assert elementCount >= uniqueCount; List<Double> elements = Lists.newArrayList(); for (int i = 0; i < elementCount; i++) { elements.add(1.0 / (i % uniqueCount + 1)); } Collections.shuffle(elements); Pipeline p = TestPipeline.create(); PCollection<Double> input = p.apply(Create.of(elements)); PCollection<Long> estimate = input.apply(ApproximateUnique.<Double>globally(sampleSize)); DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize)); p.run(); }
@Test public void testApproximateUniquePerKey() { List<KV<Long, Long>> elements = Lists.newArrayList(); List<Long> keys = ImmutableList.of(20L, 50L, 100L); int elementCount = 1000; int sampleSize = 100; // Use the key as the number of unique values. for (long uniqueCount : keys) { for (long value = 0; value < elementCount; value++) { elements.add(KV.of(uniqueCount, value % uniqueCount)); } } Pipeline p = TestPipeline.create(); PCollection<KV<Long, Long>> input = p.apply(Create.of(elements)); PCollection<KV<Long, Long>> counts = input.apply(ApproximateUnique.<Long, Long>perKey(sampleSize)); DataflowAssert.that(counts).satisfies(new VerifyEstimatePerKeyFn(sampleSize)); p.run(); }
private void runApproximateUniqueWithSkewedDistributions( int elementCount, final int uniqueCount, final int sampleSize) { List<Integer> elements = Lists.newArrayList(); // Zipf distribution with approximately elementCount items. double s = 1 - 1.0 * uniqueCount / elementCount; double maxCount = Math.pow(uniqueCount, s); for (int k = 0; k < uniqueCount; k++) { int count = Math.max(1, (int) Math.round(maxCount * Math.pow(k, -s))); // Element k occurs count times. for (int c = 0; c < count; c++) { elements.add(k); } } Pipeline p = TestPipeline.create(); PCollection<Integer> input = p.apply(Create.of(elements)); PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(sampleSize)); DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize)); p.run(); }
/** Displays Server Screen. */ private void displayServers() { // true back.setVisible(true); Servers.setVisible(true); BigInput2.setVisible(true); LongInput.setVisible(true); LongInput2.setVisible(true); Join.setVisible(true); Create2.setVisible(true); ip.setVisible(true); serverName.setVisible(true); serversError.setVisible(true); // false getServerName.setVisible(false); getIpNumber.setVisible(false); getPlayerNames.setVisible(false); loginCreate.setVisible(false); Credits.setVisible(false); Login.setVisible(false); Login2.setVisible(false); SmallInput.setVisible(false); SmallInput2.setVisible(false); SmallInput3.setVisible(false); SmallInput4.setVisible(false); SmallInput5.setVisible(false); CreateNewAccount.setVisible(false); Create.setVisible(false); MediumInput.setVisible(false); Start.setVisible(false); Cancel.setVisible(false); username.setVisible(false); password.setVisible(false); newUsername.setVisible(false); newPassword.setVisible(false); verifyPassword.setVisible(false); }