@Test
  public void testWaitForNoMoreSplits() throws Exception {
    SourceOperator operator = createExchangeOperator();

    // add a buffer location containing one page and close the buffer
    operator.addSplit(newRemoteSplit(TASK_1_ID));
    // add pages and leave buffers open
    taskBuffers.getUnchecked(TASK_1_ID).addPages(1, true);

    // read page
    waitForPages(operator, 1);

    // verify state
    assertEquals(operator.isFinished(), false);
    assertEquals(operator.needsInput(), false);
    assertEquals(operator.getOutput(), null);

    // add a buffer location
    operator.addSplit(newRemoteSplit(TASK_2_ID));
    // set no more splits (buffer locations)
    operator.noMoreSplits();
    // add two pages and close the last buffer
    taskBuffers.getUnchecked(TASK_2_ID).addPages(2, true);

    // read all pages
    waitForPages(operator, 2);

    // wait for finished
    waitForFinished(operator);
  }
  @Test
  public void testFinish() throws Exception {
    SourceOperator operator = createExchangeOperator();

    operator.addSplit(newRemoteSplit(TASK_1_ID));
    operator.addSplit(newRemoteSplit(TASK_2_ID));
    operator.addSplit(newRemoteSplit(TASK_3_ID));
    operator.noMoreSplits();

    // add pages and leave buffers open
    taskBuffers.getUnchecked(TASK_1_ID).addPages(1, false);
    taskBuffers.getUnchecked(TASK_2_ID).addPages(1, false);
    taskBuffers.getUnchecked(TASK_3_ID).addPages(1, false);

    // read 3 pages
    waitForPages(operator, 3);

    // verify state
    assertEquals(operator.isFinished(), false);
    assertEquals(operator.needsInput(), false);
    assertEquals(operator.getOutput(), null);

    // finish without closing buffers
    operator.finish();

    // wait for finished
    waitForFinished(operator);
  }
  @Override
  public void write(E entity) {
    Preconditions.checkState(
        state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);

    accessor.keyFor(entity, provided, reusedKey);

    DatasetWriter<E> writer = cachedWriters.getIfPresent(reusedKey);
    if (writer == null) {
      // avoid checking in every whether the entity belongs in the view by only
      // checking when a new writer is created
      Preconditions.checkArgument(
          view.includes(entity), "View %s does not include entity %s", view, entity);
      // get a new key because it is stored in the cache
      StorageKey key = StorageKey.copy(reusedKey);
      try {
        writer = cachedWriters.getUnchecked(key);
      } catch (UncheckedExecutionException ex) {
        throw new IllegalArgumentException(
            "Problem creating view for entity: " + entity, ex.getCause());
      }
    }

    writer.write(entity);
  }
 @Override
 public FileObject getFileForOutput(
     Location location, String packageName, String relativeName, FileObject sibling)
     throws IOException {
   URI uri = uriForFileObject(location, packageName, relativeName);
   return inMemoryFileObjects.getUnchecked(uri);
 }
Exemple #5
0
 public void unlock(Object name) {
   Monitor lock;
   synchronized (stringLocks) {
     lock = stringLocks.getUnchecked(name);
   }
   lock.leave();
 }
Exemple #6
0
 public boolean tryLock(Object name) {
   Monitor lock;
   synchronized (stringLocks) {
     lock = stringLocks.getUnchecked(name);
   }
   return lock.tryEnter();
 }
  @VisibleForTesting
  public CxxPreprocessAndCompile createPreprocessBuildRule(String name, CxxSource source) {

    Preconditions.checkArgument(CxxSourceTypes.isPreprocessableType(source.getType()));

    BuildTarget target = createPreprocessBuildTarget(name, source.getType());
    PreprocessorDelegate preprocessorDelegate =
        preprocessorDelegates.getUnchecked(
            PreprocessAndCompilePreprocessorDelegateKey.of(source.getType(), source.getFlags()));

    // Build the CxxCompile rule and add it to our sorted set of build rules.
    CxxPreprocessAndCompile result =
        CxxPreprocessAndCompile.preprocess(
            getParams()
                .copyWithChanges(
                    target,
                    new DepsBuilder()
                        .addPreprocessDeps()
                        .add(preprocessorDelegate.getPreprocessor())
                        .add(source),
                    Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of())),
            getPathResolver(),
            preprocessorDelegate,
            new CompilerDelegate(
                getPathResolver(),
                getCxxPlatform().getDebugPathSanitizer(),
                getCompiler(source.getType()),
                computeCompilerFlags(source.getType(), source.getFlags())),
            getPreprocessOutputPath(target, source.getType(), name),
            source.getPath(),
            source.getType(),
            getCxxPlatform().getDebugPathSanitizer());
    getResolver().addToIndex(result);
    return result;
  }
  @VisibleForTesting
  public Set<String> getSecurityGroupsForTagAndOptions(
      String region, @Nullable String group, TemplateOptions options) {
    Builder<String> groups = ImmutableSet.builder();

    if (group != null) {
      String markerGroup = namingConvention.create().sharedNameForGroup(group);

      groups.add(markerGroup);

      RegionNameAndIngressRules regionNameAndIngressRulesForMarkerGroup;

      if (userSpecifiedTheirOwnGroups(options)) {
        regionNameAndIngressRulesForMarkerGroup =
            new RegionNameAndIngressRules(region, markerGroup, new int[] {}, false);
        groups.addAll(EC2TemplateOptions.class.cast(options).getGroups());
      } else {
        regionNameAndIngressRulesForMarkerGroup =
            new RegionNameAndIngressRules(region, markerGroup, options.getInboundPorts(), true);
      }
      // this will create if not yet exists.
      securityGroupMap.getUnchecked(regionNameAndIngressRulesForMarkerGroup);
    }
    return groups.build();
  }
  @Override
  public Response<List<Address>> ancestorOfAddresses(Integer anyId) {
    Response<List<Address>> result = new Response<List<Address>>();
    if (anyId == null) {
      log.error("id can not be null");
      result.setError("params.not.null");
      return result;
    }
    List<Address> addresses = Lists.newArrayListWithExpectedSize(3);
    try {
      Integer id = anyId;
      while (id > 0) {
        Address address = addressCache.getUnchecked(id);
        addresses.add(address);
        id = address.getParentId();
      }

      result.setResult(addresses);
      return result;
    } catch (Exception e) {
      log.error(
          "failed find ancestors of address(id={}), cause:{}",
          anyId,
          Throwables.getStackTraceAsString(e));
      result.setError("ancestor.query.fail");
      return result;
    }
  }
  private <T> void evaluateBundle(
      final AppliedPTransform<?, ?, ?> transform,
      final CommittedBundle<T> bundle,
      final CompletionCallback onComplete) {
    TransformExecutorService transformExecutor;

    if (isKeyed(bundle.getPCollection())) {
      final StepAndKey stepAndKey = StepAndKey.of(transform, bundle.getKey());
      // This executor will remain reachable until it has executed all scheduled transforms.
      // The TransformExecutors keep a strong reference to the Executor, the ExecutorService keeps
      // a reference to the scheduled TransformExecutor callable. Follow-up TransformExecutors
      // (scheduled due to the completion of another TransformExecutor) are provided to the
      // ExecutorService before the Earlier TransformExecutor callable completes.
      transformExecutor = executorServices.getUnchecked(stepAndKey);
    } else {
      transformExecutor = parallelExecutorService;
    }

    Collection<ModelEnforcementFactory> enforcements =
        MoreObjects.firstNonNull(
            transformEnforcements.get(transform.getTransform().getClass()),
            Collections.<ModelEnforcementFactory>emptyList());

    TransformExecutor<T> callable =
        TransformExecutor.create(
            evaluationContext,
            registry,
            enforcements,
            bundle,
            transform,
            onComplete,
            transformExecutor);
    outstandingWork.incrementAndGet();
    transformExecutor.schedule(callable);
  }
 @Override
 public User find(String username) {
   User user = cache.getUnchecked(username);
   if (user.getUsername() == null) {
     return null;
   }
   return user;
 }
 protected void checkConsistent(
     Map<String, InputStream> map,
     LoadingCache<String, Node> store,
     String key,
     String id,
     String name)
     throws IOException {
   assertEquals(map.size(), 1);
   if (store.size() == 0) store.getUnchecked(key);
   assertEquals(store.size(), 1);
   // checkRepeatedRead
   assertEquals(store.getUnchecked(key), Node.builder().id(id).name(name).build());
   assertEquals(store.getUnchecked(key), Node.builder().id(id).name(name).build());
   // checkRepeatedRead
   checkToYaml(map, key, id, name);
   checkToYaml(map, key, id, name);
 }
 @Override
 public CompletableFuture<Boolean> remove(K key, V value) {
   checkNotNull(key, ERROR_NULL_KEY);
   checkNotNull(value, ERROR_NULL_VALUE);
   checkIfUnmodifiable();
   return database
       .remove(name, keyCache.getUnchecked(key), serializer.encode(value))
       .thenApply(this::unwrapResult);
 }
  private Slice getColumnSlice(
      TpchTableHandle tableHandle, TpchColumnHandle columnHandle, BlocksFileEncoding encoding) {
    checkNotNull(tableHandle, "tableHandle is null");
    checkNotNull(columnHandle, "columnHandle is null");
    checkNotNull(encoding, "encoding is null");

    File columnFile = tpchDataFileLoader.getDataFile(tableHandle, columnHandle, encoding);
    return mappedFileCache.getUnchecked(columnFile.getAbsolutePath());
  }
 @Override
 public ServiceDiscovered discover(final String name) {
   for (DiscoverableProgramType type : DiscoverableProgramType.values()) {
     if (type.isPrefixOf(name)) {
       return clients.getUnchecked(name).discover(name);
     }
   }
   return delegate.discover(name);
 }
 @Override
 public CompletableFuture<Boolean> replace(K key, V oldValue, V newValue) {
   checkNotNull(key, ERROR_NULL_KEY);
   checkNotNull(newValue, ERROR_NULL_VALUE);
   checkIfUnmodifiable();
   byte[] existing = oldValue != null ? serializer.encode(oldValue) : null;
   return database
       .replace(name, keyCache.getUnchecked(key), existing, serializer.encode(newValue))
       .thenApply(this::unwrapResult);
 }
 @Override
 public CompletableFuture<Versioned<V>> get(K key) {
   checkNotNull(key, ERROR_NULL_KEY);
   return database
       .get(name, keyCache.getUnchecked(key))
       .thenApply(
           v ->
               v != null
                   ? new Versioned<>(serializer.decode(v.value()), v.version(), v.creationTime())
                   : null);
 }
 public V get(K key, Errors errors) throws ErrorsException {
   Object resultOrError = delegate.getUnchecked(key);
   if (resultOrError instanceof Errors) {
     errors.merge((Errors) resultOrError);
     throw errors.toException();
   } else {
     @SuppressWarnings("unchecked") // create returned a non-error result, so this is safe
     V result = (V) resultOrError;
     return result;
   }
 }
  @Test
  public void testSimple() throws Exception {
    SourceOperator operator = createExchangeOperator();

    operator.addSplit(newRemoteSplit(TASK_1_ID));
    operator.addSplit(newRemoteSplit(TASK_2_ID));
    operator.addSplit(newRemoteSplit(TASK_3_ID));
    operator.noMoreSplits();

    // add pages and close the buffers
    taskBuffers.getUnchecked(TASK_1_ID).addPages(10, true);
    taskBuffers.getUnchecked(TASK_2_ID).addPages(10, true);
    taskBuffers.getUnchecked(TASK_3_ID).addPages(10, true);

    // read the pages
    waitForPages(operator, 30);

    // wait for finished
    waitForFinished(operator);
  }
 protected void put(
     Map<String, InputStream> map,
     LoadingCache<String, Node> store,
     String key,
     String id,
     String name) {
   assertEquals(store.size(), 0);
   assertEquals(map.size(), 0);
   map.put(
       key, new ByteArrayInputStream(String.format("id: %s\nname: %s\n", id, name).getBytes()));
   store.getUnchecked(key);
 }
Exemple #21
0
 public SourceOperatorFactory compileScanFilterAndProjectOperator(
     int operatorId,
     PlanNodeId sourceId,
     DataStreamProvider dataStreamProvider,
     List<ColumnHandle> columns,
     RowExpression filter,
     List<RowExpression> projections) {
   OperatorCacheKey cacheKey = new OperatorCacheKey(filter, projections, sourceId);
   return sourceOperatorFactories
       .getUnchecked(cacheKey)
       .create(operatorId, dataStreamProvider, columns);
 }
 private TimingResourceFilter(
     AbstractMethod abstractMethod,
     LoadingCache<String, RequestStats> requestStatsLoadingCache) {
   this.abstractMethod = abstractMethod;
   String objectName =
       new ObjectNameBuilder(
               abstractMethod.getResource().getResourceClass().getPackage().getName())
           .withProperty("type", abstractMethod.getResource().getResourceClass().getSimpleName())
           .build();
   RequestStats requestStats = requestStatsLoadingCache.getUnchecked(objectName);
   timingFilter = new TimingFilter(this.abstractMethod, requestStats);
 }
Exemple #23
0
 public void addLogEntry(String id, LogEntry logEntry) {
   try {
     OutputStream outputStream = logStreamCache.getUnchecked(id);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     objectMapper.writer().writeValue(baos, logEntry);
     synchronized (outputStream) {
       outputStream.write(("\n" + baos.size() + "\n").getBytes());
       outputStream.write(baos.toByteArray());
     }
   } catch (Throwable ignored) {
     // do not log errors when logging
   }
 }
  protected void remove(
      Map<String, InputStream> map, LoadingCache<String, Node> store, String key) {
    store.invalidate(key);
    assertEquals(store.size(), 0);
    map.remove(key);
    assertEquals(map.size(), 0);
    try {
      assertEquals(store.getUnchecked(key), null);
      fail("should not work as null is invalid");
    } catch (UncheckedExecutionException e) {

    }
    assertEquals(map.get(key), null);
  }
 @Override
 public CompletableFuture<Versioned<V>> put(K key, V value) {
   checkNotNull(key, ERROR_NULL_KEY);
   checkNotNull(value, ERROR_NULL_VALUE);
   checkIfUnmodifiable();
   return database
       .put(name, keyCache.getUnchecked(key), serializer.encode(value))
       .thenApply(this::unwrapResult)
       .thenApply(
           v ->
               v != null
                   ? new Versioned<>(serializer.decode(v.value()), v.version(), v.creationTime())
                   : null);
 }
 @Override
 public User check(String username, String password) {
   if (username == null || password == null) {
     return null;
   }
   User user = cache.getUnchecked(username);
   if (user == null || user.getUsername() == null || user.getPassword() == null) {
     return null;
   }
   String encryptPassword = generateEncryptPassword(user, password);
   if (user.getPassword().equals(encryptPassword)) {
     return user;
   }
   return null;
 }
 @Override
 public CompletableFuture<Versioned<V>> remove(K key) {
   checkNotNull(key, ERROR_NULL_KEY);
   checkIfUnmodifiable();
   return database
       .remove(name, keyCache.getUnchecked(key))
       .thenApply(this::unwrapResult)
       .thenApply(v -> v != null ? v.<V>map(serializer::decode) : null)
       .whenComplete(
           (r, e) -> {
             if (r != null) {
               notifyListeners(new MapEvent<>(name, MapEvent.Type.REMOVE, key, r));
             }
           });
 }
 @Override
 public synchronized void removeThingLogic(IThingLogic tl) {
   fireThingLogicManagerEvent(EventType.LOGIC_REMOVING, tl);
   long time;
   if (DEBUG) {
     time = System.nanoTime();
   }
   logics.remove(tl);
   tl.setBNAWorld(null);
   typedLogics.remove(tl.getClass());
   if (DEBUG) {
     time = System.nanoTime() - time;
     debugStats.getUnchecked(tl).addAndGet(time);
   }
   fireThingLogicManagerEvent(EventType.LOGIC_REMOVED, tl);
 }
 /**
  * Request the currently configured client
  *
  * @return
  */
 public static ObjectStorageProviderClient getInstance() throws NoSuchElementException {
   if (lastClient.get() == null || UNSET.equals(lastClient.get())) {
     if (!Strings.isNullOrEmpty(providerClient)) {
       if (clients.containsKey(providerClient)) {
         lastClient.set(providerClient);
       }
     } else {
       throw new NoSuchElementException(
           "OSG object storage provider client not configured. Found property 'objectstorage.providerclient' empty or unset manager("
               + lastClient
               + ").  Legal values are: "
               + Joiner.on(",").join(clients.keySet()));
     }
   }
   return clientInstances.getUnchecked(lastClient.get());
 }
 @Override
 public CompletableFuture<Versioned<V>> putAndGet(K key, V value) {
   checkNotNull(key, ERROR_NULL_KEY);
   checkNotNull(value, ERROR_NULL_VALUE);
   checkIfUnmodifiable();
   return database
       .putAndGet(name, keyCache.getUnchecked(key), serializer.encode(value))
       .thenApply(this::unwrapResult)
       .thenApply(
           v -> {
             Versioned<byte[]> rawNewValue = v.newValue();
             return new Versioned<>(
                 serializer.decode(rawNewValue.value()),
                 rawNewValue.version(),
                 rawNewValue.creationTime());
           });
 }