Ejemplo n.º 1
0
 @Override
 public EngineException[] bulk(Bulk bulk) throws EngineException {
   EngineException[] failures = null;
   rwl.readLock().lock();
   try {
     IndexWriter writer = this.indexWriter;
     if (writer == null) {
       throw new EngineClosedException(shardId);
     }
     for (int i = 0; i < bulk.ops().length; i++) {
       Operation op = bulk.ops()[i];
       if (op == null) {
         continue;
       }
       try {
         switch (op.opType()) {
           case CREATE:
             Create create = (Create) op;
             writer.addDocument(create.doc(), create.analyzer());
             translog.add(new Translog.Create(create));
             break;
           case INDEX:
             Index index = (Index) op;
             writer.updateDocument(index.uid(), index.doc(), index.analyzer());
             translog.add(new Translog.Index(index));
             break;
           case DELETE:
             Delete delete = (Delete) op;
             writer.deleteDocuments(delete.uid());
             translog.add(new Translog.Delete(delete));
             break;
         }
       } catch (Exception e) {
         if (failures == null) {
           failures = new EngineException[bulk.ops().length];
         }
         switch (op.opType()) {
           case CREATE:
             failures[i] = new CreateFailedEngineException(shardId, (Create) op, e);
             break;
           case INDEX:
             failures[i] = new IndexFailedEngineException(shardId, (Index) op, e);
             break;
           case DELETE:
             failures[i] = new DeleteFailedEngineException(shardId, (Delete) op, e);
             break;
         }
       }
     }
     dirty = true;
   } finally {
     rwl.readLock().unlock();
   }
   return failures;
 }
Ejemplo n.º 2
0
  @Test
  public void shouldCreatePathWithData() throws Exception {
    // Given
    createCommand.data = "node data";

    // When
    createCommand.doExecute(curator);

    // Then
    verify(createBuilder).forPath(createCommand.path, createCommand.data.getBytes());
  }
Ejemplo n.º 3
0
  @Test
  public void shouldCreateRecursivePathWithoutData() throws Exception {
    // Given
    createCommand.recursive = true;

    // When
    createCommand.doExecute(curator);

    // Then
    verify(createBuilder).creatingParentsIfNeeded();
    verify(createBuilder).forPath(createCommand.path);
  }
Ejemplo n.º 4
0
 @Override
 public void doProcess(final TestContext context, Create t) throws Throwable {
   VariableContext.getInstance().pushScope(null == context.getTarget());
   VariableContext.getInstance()
       .setVariableBindings(t.uriParams(), t.mergeFields(), t.extractors());
   try {
     InternalHttpUnitRunner runner = new InternalHttpUnitRunner(context, t.action());
     runner.run(new RunNotifier());
   } finally {
     VariableContext.getInstance().clearVariableBindings();
     VariableContext.getInstance().popScope();
   }
 }
  /**
   * Applies {@code ApproximateUnique(sampleSize)} verifying that the estimation error falls within
   * the maximum allowed error of {@code 2/sqrt(sampleSize)}.
   */
  private static void runApproximateUniquePipeline(int sampleSize) {
    Pipeline p = TestPipeline.create();

    PCollection<String> input = p.apply(Create.of(TEST_LINES));
    PCollection<Long> approximate = input.apply(ApproximateUnique.<String>globally(sampleSize));
    final PCollectionView<Long> exact =
        input
            .apply(RemoveDuplicates.<String>create())
            .apply(Count.<String>globally())
            .apply(View.<Long>asSingleton());

    PCollection<KV<Long, Long>> approximateAndExact =
        approximate.apply(
            ParDo.of(
                    new DoFn<Long, KV<Long, Long>>() {
                      @Override
                      public void processElement(ProcessContext c) {
                        c.output(KV.of(c.element(), c.sideInput(exact)));
                      }
                    })
                .withSideInputs(exact));

    DataflowAssert.that(approximateAndExact).satisfies(new VerifyEstimatePerKeyFn(sampleSize));

    p.run();
  }
 /** Displays Network screen. */
 private void displayNetworkMode() {
   // true
   NetworkMode.setVisible(true);
   Online.setVisible(true);
   Lan.setVisible(true);
   back.setVisible(true);
   // false
   loginCreate.setVisible(false);
   PlayerVsComp.setVisible(false);
   PlayerVsPlayer.setVisible(false);
   TwoPlayerVsTwoPlayer.setVisible(false);
   Credits.setVisible(false);
   Login.setVisible(false);
   Login2.setVisible(false);
   SmallInput.setVisible(false);
   SmallInput2.setVisible(false);
   SmallInput3.setVisible(false);
   SmallInput4.setVisible(false);
   SmallInput5.setVisible(false);
   CreateNewAccount.setVisible(false);
   Create.setVisible(false);
   username.setVisible(false);
   password.setVisible(false);
   newUsername.setVisible(false);
   newPassword.setVisible(false);
   verifyPassword.setVisible(false);
 }
 /** Displays LoginScreen */
 private void displayLogin() {
   // true
   back.setVisible(true);
   Login.setVisible(true);
   Login2.setVisible(true);
   SmallInput.setVisible(true);
   SmallInput2.setVisible(true);
   SmallInput3.setVisible(true);
   SmallInput4.setVisible(true);
   SmallInput5.setVisible(true);
   CreateNewAccount.setVisible(true);
   Create.setVisible(true);
   username.setVisible(true);
   password.setVisible(true);
   newUsername.setVisible(true);
   newPassword.setVisible(true);
   verifyPassword.setVisible(true);
   loginCreate.setVisible(true);
   loginCreate.setOpaque(false);
   // false
   serversError.setVisible(false);
   Credits.setVisible(false);
   NetworkMode.setVisible(false);
   Online.setVisible(false);
   Lan.setVisible(false);
   Servers.setVisible(false);
   BigInput2.setVisible(false);
   LongInput.setVisible(false);
   LongInput2.setVisible(false);
   Join.setVisible(false);
   Create2.setVisible(false);
   returnToMain.setVisible(false);
   ip.setVisible(false);
   serverName.setVisible(false);
 }
Ejemplo n.º 8
0
  @Test
  @SuppressWarnings("unchecked")
  public void testTop() {
    Pipeline p = TestPipeline.create();
    PCollection<String> input =
        p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));

    PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength()));
    PCollection<List<String>> top2 = input.apply(Top.<String>largest(2));
    PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3));

    PCollection<KV<String, Integer>> inputTable = createInputTable(p);
    PCollection<KV<String, List<Integer>>> largestPerKey =
        inputTable.apply(Top.<String, Integer>largestPerKey(2));
    PCollection<KV<String, List<Integer>>> smallestPerKey =
        inputTable.apply(Top.<String, Integer>smallestPerKey(2));

    DataflowAssert.thatSingletonIterable(top1).containsInAnyOrder(Arrays.asList("bb"));
    DataflowAssert.thatSingletonIterable(top2).containsInAnyOrder("z", "c");
    DataflowAssert.thatSingletonIterable(top3).containsInAnyOrder("a", "bb", "c");
    DataflowAssert.that(largestPerKey)
        .containsInAnyOrder(KV.of("a", Arrays.asList(3, 2)), KV.of("b", Arrays.asList(100, 10)));
    DataflowAssert.that(smallestPerKey)
        .containsInAnyOrder(KV.of("a", Arrays.asList(1, 2)), KV.of("b", Arrays.asList(1, 10)));

    p.run();
  }
Ejemplo n.º 9
0
  @Test
  @SuppressWarnings("unchecked")
  public void testTopEmpty() {
    Pipeline p = TestPipeline.create();
    PCollection<String> input =
        p.apply(Create.of(Arrays.asList(EMPTY_COLLECTION)).withCoder(StringUtf8Coder.of()));

    PCollection<List<String>> top1 = input.apply(Top.of(1, new OrderByLength()));
    PCollection<List<String>> top2 = input.apply(Top.<String>largest(2));
    PCollection<List<String>> top3 = input.apply(Top.<String>smallest(3));

    PCollection<KV<String, Integer>> inputTable = createEmptyInputTable(p);
    PCollection<KV<String, List<Integer>>> largestPerKey =
        inputTable.apply(Top.<String, Integer>largestPerKey(2));
    PCollection<KV<String, List<Integer>>> smallestPerKey =
        inputTable.apply(Top.<String, Integer>smallestPerKey(2));

    DataflowAssert.thatSingletonIterable(top1).empty();
    DataflowAssert.thatSingletonIterable(top2).empty();
    DataflowAssert.thatSingletonIterable(top3).empty();
    DataflowAssert.that(largestPerKey).empty();
    DataflowAssert.that(smallestPerKey).empty();

    p.run();
  }
Ejemplo n.º 10
0
  @Test
  public void shouldCreatePathWithoutData() throws Exception {
    // When
    createCommand.doExecute(curator);

    // Then
    verify(createBuilder).forPath(createCommand.path);
  }
Ejemplo n.º 11
0
  @Before
  public void setUp() {
    createCommand.path = "/foo/bar";

    given(createBuilder.withMode(any(CreateMode.class))).willReturn(createBuilder);
    given(createBuilder.withACL(anyListOf(ACL.class))).willReturn(createBuilder);
    given(curator.create()).willReturn(createBuilder);
  }
Ejemplo n.º 12
0
  @Test
  public void testCountConstraint() {
    Pipeline p = TestPipeline.create();
    PCollection<String> input =
        p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));

    expectedEx.expect(IllegalArgumentException.class);
    expectedEx.expectMessage(Matchers.containsString(">= 0"));

    input.apply(Top.of(-1, new OrderByLength()));
  }
Ejemplo n.º 13
0
 @Override
 public void create(Create create) throws EngineException {
   rwl.readLock().lock();
   try {
     IndexWriter writer = this.indexWriter;
     if (writer == null) {
       throw new EngineClosedException(shardId);
     }
     writer.addDocument(create.doc(), create.analyzer());
     translog.add(new Translog.Create(create));
     dirty = true;
     if (create.refresh()) {
       refresh(new Refresh(false));
     }
   } catch (IOException e) {
     throw new CreateFailedEngineException(shardId, create, e);
   } finally {
     rwl.readLock().unlock();
   }
 }
  public static HashMap<String, double[]> getTagsHistogramMap(
      String[] allTags, double[][] mTagAverageHistogram) {

    HashMap<String, double[]> tagsHistogramMap = Create.hashMap();

    for (int i = 0; i < allTags.length; i++) {
      /** * HashMap for <tag, tag histogram>** */
      tagsHistogramMap.put(allTags[i], mTagAverageHistogram[i]);
    }

    return tagsHistogramMap;
  }
Ejemplo n.º 15
0
  // This is a purely compile-time test.  If the code compiles, then it worked.
  @Test
  public void testPerKeySerializabilityRequirement() {
    Pipeline p = TestPipeline.create();
    p.apply(
        "CreateCollection", Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of()));

    PCollection<KV<String, Integer>> inputTable = createInputTable(p);
    inputTable.apply(Top.<String, Integer, IntegerComparator>perKey(1, new IntegerComparator()));

    inputTable.apply(
        "PerKey2", Top.<String, Integer, IntegerComparator2>perKey(1, new IntegerComparator2()));
  }
  public static HashMap<String, Map<Integer, Double>> getTagsDescriptorMap(
      String[] allTags, Vector<Map<Integer, Double>> mTagAverageDescriptor) {

    HashMap<String, Map<Integer, Double>> tagsDescriptorMap = Create.hashMap();

    for (int i = 0; i < allTags.length; i++) {
      /** * HashMap for <tag, tag descriptor map>** */
      tagsDescriptorMap.put(allTags[i], mTagAverageDescriptor.elementAt(i));
    }

    return tagsDescriptorMap;
  }
 /** Reads a large {@code PCollection<String>}. */
 private PCollection<String> readPCollection(Pipeline p) {
   // TODO: Read PCollection from a set of text files.
   List<String> page = TestUtils.LINES;
   final int pages = 1000;
   ArrayList<String> file = new ArrayList<>(pages * page.size());
   for (int i = 0; i < pages; i++) {
     file.addAll(page);
   }
   assert file.size() == pages * page.size();
   PCollection<String> words = p.apply(Create.of(file));
   return words;
 }
Ejemplo n.º 18
0
 public static void update() {
   if (update) {
     Sound.random("random", 1f, "OGG", "res/sound/music", 11);
     GUI.Background.draw("dirt");
     Text.draw("SETTINGS", GUI.Coordinates.x / 2 - 94, GUI.Coordinates.y / 4 * 3, 6, Color.white);
     GUI.Button.draw(
         "Music",
         GUIScale.toUpperCase(),
         GUI.Coordinates.x / 2 - 94,
         GUI.Coordinates.y / 2,
         1,
         1,
         "planks_oak",
         2,
         128);
     if (Button.isButtonClicked("Music")) {
       if (GUIScale.length() == 6) {
         GUIScale = "Auto";
         Create.createFile("Settings/GUIScale");
         Create.write("Auto");
         Create.closeFile();
       } else {
         GUIScale = "Normal";
         Create.createFile("Settings/GUIScale");
         Create.write("Normal");
         Create.closeFile();
         game.Main.Sleep(256);
       }
     }
   }
 }
Ejemplo n.º 19
0
  /** Example test that tests a PTransform by using an in-memory input and inspecting the output. */
  @Test
  @Category(RunnableOnService.class)
  public void testCountWords() throws Exception {
    Pipeline p = TestPipeline.create();

    PCollection<String> input = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));

    PCollection<String> output = input.apply(new CountWords())
      .apply(ParDo.of(new FormatAsTextFn()));

    DataflowAssert.that(output).containsInAnyOrder(COUNTS_ARRAY);
    p.run();
  }
  @Test
  @Category(RunnableOnService.class)
  public void testApproximateUniqueWithSmallInput() {
    Pipeline p = TestPipeline.create();

    PCollection<Integer> input = p.apply(Create.of(Arrays.asList(1, 2, 3, 3)));

    PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(1000));

    DataflowAssert.thatSingleton(estimate).isEqualTo(3L);

    p.run();
  }
  @Test
  @Category(RunnableOnService.class)
  public void withLambdaAndTypeDescriptorShouldSucceed() {

    PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12"));
    PCollection<KV<Integer, String>> kvs =
        values.apply(
            WithKeys.of((String s) -> Integer.valueOf(s))
                .withKeyType(TypeDescriptor.of(Integer.class)));

    PAssert.that(kvs)
        .containsInAnyOrder(
            KV.of(1234, "1234"), KV.of(0, "0"), KV.of(-12, "-12"), KV.of(3210, "3210"));

    p.run();
  }
Ejemplo n.º 22
0
  @Test
  public void testTopEmptyWithIncompatibleWindows() {
    Pipeline p = TestPipeline.create();
    Bound<String> windowingFn = Window.<String>into(FixedWindows.of(Duration.standardDays(10L)));
    PCollection<String> input =
        p.apply(Create.timestamped(Collections.<String>emptyList(), Collections.<Long>emptyList()))
            .apply(windowingFn);

    expectedEx.expect(IllegalStateException.class);
    expectedEx.expectMessage("Top");
    expectedEx.expectMessage("GlobalWindows");
    expectedEx.expectMessage("withoutDefaults");
    expectedEx.expectMessage("asSingletonView");

    input.apply(Top.of(1, new OrderByLength()));
  }
Ejemplo n.º 23
0
 public static final <K, T, A> Map<K, A> groupAndAggregate(
     final Collection<T> objectsToGroup,
     final Grouping<T, K> grouping,
     final Aggregator<T, A> aggregator) {
   final Map<K, Collection<T>> groupMap = group(objectsToGroup, grouping);
   final Map<K, A> aggregatedMap = Create.newMap();
   Iterate.forEach(
       groupMap,
       new Closure<Map.Entry<K, Collection<T>>>() {
         //			@Override
         public void execute(Entry<K, Collection<T>> entry) {
           aggregatedMap.put(entry.getKey(), aggregator.aggregate(entry.getValue()));
         }
       });
   return aggregatedMap;
 }
  @Test
  public void withLambdaAndNoTypeDescriptorShouldThrow() {

    PCollection<String> values = p.apply(Create.of("1234", "3210", "0", "-12"));

    values.apply("ApplyKeysWithWithKeys", WithKeys.of((String s) -> Integer.valueOf(s)));

    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("Unable to return a default Coder for ApplyKeysWithWithKeys");
    thrown.expectMessage("No Coder has been manually specified");
    thrown.expectMessage(containsString("Building a Coder using a registered CoderFactory failed"));
    thrown.expectMessage(
        containsString("Building a Coder from the @DefaultCoder annotation failed"));
    thrown.expectMessage(containsString("Building a Coder from the fallback CoderProvider failed"));

    p.run();
  }
Ejemplo n.º 25
0
 public static final <K, T> Map<K, Collection<T>> group(
     final Collection<T> objectsToGroup, final Grouping<T, K> grouping) {
   final Map<K, Collection<T>> groupMap = Create.newMap();
   Iterate.forEach(
       objectsToGroup,
       new Closure<T>() {
         //			@Override
         public void execute(T obj) {
           K key = grouping.toGroup(obj);
           if (!groupMap.containsKey(key)) {
             groupMap.put(key, Create.<T>newList());
           }
           groupMap.get(key).add(obj);
         }
       });
   return groupMap;
 }
  public static Vector<Map<Integer, Double>> getTagsDescriptor(double[][] mTagAverageHistogram) {

    Vector<Map<Integer, Double>> tmpVec = new Vector<Map<Integer, Double>>();

    for (int i = 0; i < mTagAverageHistogram.length; i++) {
      Map<Integer, Double> des = Create.linkedHashMap();
      for (int j = 0; j < BIN_HISTO; j++) {
        if (mTagAverageHistogram[i][j] > 0.0) {
          // System.out.print(mTagAverageHistogram[i][j] + " ");
          des.put(j, mTagAverageHistogram[i][j]);
        }
      }
      // System.out.println();
      tmpVec.add(des);
    }
    return tmpVec;
  }
  private void runApproximateUniqueWithDuplicates(
      int elementCount, int uniqueCount, int sampleSize) {

    assert elementCount >= uniqueCount;
    List<Double> elements = Lists.newArrayList();
    for (int i = 0; i < elementCount; i++) {
      elements.add(1.0 / (i % uniqueCount + 1));
    }
    Collections.shuffle(elements);

    Pipeline p = TestPipeline.create();
    PCollection<Double> input = p.apply(Create.of(elements));
    PCollection<Long> estimate = input.apply(ApproximateUnique.<Double>globally(sampleSize));

    DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize));

    p.run();
  }
  @Test
  public void testApproximateUniquePerKey() {
    List<KV<Long, Long>> elements = Lists.newArrayList();
    List<Long> keys = ImmutableList.of(20L, 50L, 100L);
    int elementCount = 1000;
    int sampleSize = 100;
    // Use the key as the number of unique values.
    for (long uniqueCount : keys) {
      for (long value = 0; value < elementCount; value++) {
        elements.add(KV.of(uniqueCount, value % uniqueCount));
      }
    }

    Pipeline p = TestPipeline.create();
    PCollection<KV<Long, Long>> input = p.apply(Create.of(elements));
    PCollection<KV<Long, Long>> counts =
        input.apply(ApproximateUnique.<Long, Long>perKey(sampleSize));

    DataflowAssert.that(counts).satisfies(new VerifyEstimatePerKeyFn(sampleSize));

    p.run();
  }
  private void runApproximateUniqueWithSkewedDistributions(
      int elementCount, final int uniqueCount, final int sampleSize) {
    List<Integer> elements = Lists.newArrayList();
    // Zipf distribution with approximately elementCount items.
    double s = 1 - 1.0 * uniqueCount / elementCount;
    double maxCount = Math.pow(uniqueCount, s);
    for (int k = 0; k < uniqueCount; k++) {
      int count = Math.max(1, (int) Math.round(maxCount * Math.pow(k, -s)));
      // Element k occurs count times.
      for (int c = 0; c < count; c++) {
        elements.add(k);
      }
    }

    Pipeline p = TestPipeline.create();
    PCollection<Integer> input = p.apply(Create.of(elements));
    PCollection<Long> estimate = input.apply(ApproximateUnique.<Integer>globally(sampleSize));

    DataflowAssert.thatSingleton(estimate).satisfies(new VerifyEstimateFn(uniqueCount, sampleSize));

    p.run();
  }
 /** Displays Server Screen. */
 private void displayServers() {
   // true
   back.setVisible(true);
   Servers.setVisible(true);
   BigInput2.setVisible(true);
   LongInput.setVisible(true);
   LongInput2.setVisible(true);
   Join.setVisible(true);
   Create2.setVisible(true);
   ip.setVisible(true);
   serverName.setVisible(true);
   serversError.setVisible(true);
   // false
   getServerName.setVisible(false);
   getIpNumber.setVisible(false);
   getPlayerNames.setVisible(false);
   loginCreate.setVisible(false);
   Credits.setVisible(false);
   Login.setVisible(false);
   Login2.setVisible(false);
   SmallInput.setVisible(false);
   SmallInput2.setVisible(false);
   SmallInput3.setVisible(false);
   SmallInput4.setVisible(false);
   SmallInput5.setVisible(false);
   CreateNewAccount.setVisible(false);
   Create.setVisible(false);
   MediumInput.setVisible(false);
   Start.setVisible(false);
   Cancel.setVisible(false);
   username.setVisible(false);
   password.setVisible(false);
   newUsername.setVisible(false);
   newPassword.setVisible(false);
   verifyPassword.setVisible(false);
 }