/** Tests if list correctly returns file or folder names for a large directory. */
  @Test
  public void listLargeDirectory() throws IOException {
    LargeDirectoryConfig config = prepareLargeDirectoryTest();
    String[] children = config.getChildren();

    // Retry for some time to allow list operation eventual consistency for S3 and GCS.
    // See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
    // https://cloud.google.com/storage/docs/consistency for more details.
    // Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
    String[] results = new String[] {};
    for (int i = 0; i < 20; i++) {
      results = mUfs.list(config.getTopLevelDirectory());
      if (children.length == results.length) {
        break;
      }
      CommonUtils.sleepMs(500);
    }
    Assert.assertEquals(children.length, results.length);

    Arrays.sort(results);
    for (int i = 0; i < children.length; ++i) {
      Assert.assertTrue(
          results[i].equals(
              CommonUtils.stripPrefixIfPresent(
                  children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
    }
  }
Пример #2
0
  @Override
  public String[] list(String path) throws IOException {
    String prefix = addFolderSuffixIfNotPresent(stripContainerPrefixIfPresent(path));
    prefix = prefix.equals(PATH_SEPARATOR) ? "" : prefix;

    Collection<DirectoryOrObject> objects = listInternal(prefix);
    Set<String> children = new HashSet<>();
    final String self = stripFolderSuffixIfPresent(prefix);
    boolean foundSelf = false;
    for (DirectoryOrObject object : objects) {
      String child = stripFolderSuffixIfPresent(object.getName());
      String noPrefix = CommonUtils.stripPrefixIfPresent(child, prefix);
      if (!noPrefix.equals(self)) {
        children.add(noPrefix);
      } else {
        foundSelf = true;
      }
    }

    if (!foundSelf) {
      // Path does not exist
      return null;
    }

    return children.toArray(new String[children.size()]);
  }
  /**
   * @summary get the worker metrics
   * @return the response object
   */
  @GET
  @Path(GET_METRICS)
  @ReturnType("java.util.SortedMap<String, Long>")
  public Response getMetrics() {
    MetricRegistry metricRegistry = mWorker.getWorkerMetricsSystem().getMetricRegistry();

    // Get all counters.
    Map<String, Counter> counters = metricRegistry.getCounters();

    // Only the gauge for cached blocks is retrieved here, other gauges are statistics of free/used
    // spaces, those statistics can be gotten via other REST apis.
    String blocksCachedProperty =
        CommonUtils.argsToString(
            ".", WorkerContext.getWorkerSource().getName(), WorkerSource.BLOCKS_CACHED);
    @SuppressWarnings("unchecked")
    Gauge<Integer> blocksCached =
        (Gauge<Integer>) metricRegistry.getGauges().get(blocksCachedProperty);

    // Get values of the counters and gauges and put them into a metrics map.
    SortedMap<String, Long> metrics = new TreeMap<>();
    for (Map.Entry<String, Counter> counter : counters.entrySet()) {
      metrics.put(counter.getKey(), counter.getValue().getCount());
    }
    metrics.put(blocksCachedProperty, blocksCached.getValue().longValue());

    return RestUtils.createResponse(metrics);
  }
Пример #4
0
  @Override
  public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
      throws IOException {
    if (file == null) {
      return null;
    }
    if (mStatistics != null) {
      mStatistics.incrementReadOps(1);
    }

    AlluxioURI path = new AlluxioURI(HadoopUtils.getPathWithoutScheme(file.getPath()));
    List<FileBlockInfo> blocks = getFileBlocks(path);
    List<BlockLocation> blockLocations = new ArrayList<>();
    for (FileBlockInfo fileBlockInfo : blocks) {
      long offset = fileBlockInfo.getOffset();
      long end = offset + fileBlockInfo.getBlockInfo().getLength();
      // Check if there is any overlapping between [start, start+len] and [offset, end]
      if (end >= start && offset <= start + len) {
        ArrayList<String> names = new ArrayList<>();
        ArrayList<String> hosts = new ArrayList<>();
        // add the existing in-memory block locations
        for (alluxio.wire.BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
          HostAndPort address =
              HostAndPort.fromParts(
                  location.getWorkerAddress().getHost(), location.getWorkerAddress().getDataPort());
          names.add(address.toString());
          hosts.add(address.getHostText());
        }
        // add under file system locations
        for (String location : fileBlockInfo.getUfsLocations()) {
          names.add(location);
          hosts.add(HostAndPort.fromString(location).getHostText());
        }
        blockLocations.add(
            new BlockLocation(
                CommonUtils.toStringArray(names),
                CommonUtils.toStringArray(hosts),
                offset,
                fileBlockInfo.getBlockInfo().getLength()));
      }
    }

    BlockLocation[] ret = new BlockLocation[blockLocations.size()];
    blockLocations.toArray(ret);
    return ret;
  }
Пример #5
0
 /**
  * Factory for {@link Allocator}.
  *
  * @param view {@link BlockMetadataManagerView} to pass to {@link Allocator}
  * @return the generated {@link Allocator}, it will be a {@link MaxFreeAllocator} by default
  */
 public static Allocator create(BlockMetadataManagerView view) {
   BlockMetadataManagerView managerView = Preconditions.checkNotNull(view);
   try {
     return CommonUtils.createNewClassInstance(
         Configuration.<Allocator>getClass(PropertyKey.WORKER_ALLOCATOR_CLASS),
         new Class[] {BlockMetadataManagerView.class},
         new Object[] {managerView});
   } catch (Exception e) {
     throw Throwables.propagate(e);
   }
 }
 /** Tests that deleting a file from a folder updates the folder's last modification time. */
 @Test
 public void lastModificationTimeDeleteTest() throws Exception {
   mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
   mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults());
   long folderId = mFsMaster.getFileId(new AlluxioURI("/testFolder"));
   long modificationTimeBeforeDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
   CommonUtils.sleepMs(2);
   mFsMaster.delete(new AlluxioURI("/testFolder/testFile"), true);
   long modificationTimeAfterDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
   Assert.assertTrue(modificationTimeBeforeDelete < modificationTimeAfterDelete);
 }
Пример #7
0
  public static CommandLineJobInfo createRandom() {
    CommandLineJobInfo result = new CommandLineJobInfo();
    Random random = new Random();

    String command = CommonUtils.randomString(random.nextInt(10));
    JobConfInfo jobConfInfo = JobConfInfoTest.createRandom();

    result.setCommand(command);
    result.setConf(jobConfInfo);

    return result;
  }
Пример #8
0
  @Override
  public OutputStream create(String path, CreateOptions options) throws IOException {
    LOG.debug("Create method: {}", path);

    // create will attempt to create the parent directory if it does not already exist
    if (!mkdirs(getParentPath(path), true)) {
      // fail if the parent directory does not exist and creation was unsuccessful
      LOG.error("Parent directory creation unsuccessful for {}", path);
      return null;
    }

    final String strippedPath = CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT);
    // TODO(adit): remove special handling of */_SUCCESS objects
    if (strippedPath.endsWith("_SUCCESS")) {
      // when path/_SUCCESS is created, there is need to create path as
      // an empty object. This is required by Spark in case Spark
      // accesses path directly, bypassing Alluxio
      String plainName = CommonUtils.stripSuffixIfPresent(strippedPath, "_SUCCESS");
      SwiftDirectClient.put(mAccess, plainName).close();
    }
    // We do not check if a folder with the same name exists
    return SwiftDirectClient.put(mAccess, strippedPath);
  }
Пример #9
0
 /**
  * Creates a directory flagged file with the folder suffix.
  *
  * @param path the path to create a folder
  * @return true if the operation was successful, false otherwise
  */
 private boolean mkdirsInternal(String path) {
   try {
     String keyAsFolder =
         addFolderSuffixIfNotPresent(
             CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT));
     // We do not check if a file with same name exists, i.e. a file with name
     // 'swift://swift-container/path' and a folder with name 'swift://swift-container/path/'
     // may both exist simultaneously
     SwiftDirectClient.put(mAccess, keyAsFolder).close();
     return true;
   } catch (IOException e) {
     LOG.error("Failed to create directory: {}", path, e);
     return false;
   }
 }
Пример #10
0
 @Override
 public void heartbeat() {
   int masterWorkerTimeoutMs = Configuration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
   for (MasterWorkerInfo worker : mWorkers) {
     synchronized (worker) {
       final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs();
       if (lastUpdate > masterWorkerTimeoutMs) {
         LOG.error(
             "The worker {} timed out after {}ms without a heartbeat!", worker, lastUpdate);
         mLostWorkers.add(worker);
         mWorkers.remove(worker);
         processWorkerRemovedBlocks(worker, worker.getBlocks());
       }
     }
   }
 }
 @Test
 public void ttlExpiredCreateFileTest() throws Exception {
   mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
   long ttl = 1;
   CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl);
   long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options);
   FileInfo folderInfo =
       mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile1")));
   Assert.assertEquals(fileId, folderInfo.getFileId());
   Assert.assertEquals(ttl, folderInfo.getTtl());
   // Sleep for the ttl expiration.
   CommonUtils.sleepMs(2 * TTL_CHECKER_INTERVAL_MS);
   Assert.assertTrue(
       HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
   HeartbeatScheduler.schedule(HeartbeatContext.MASTER_TTL_CHECK);
   Assert.assertTrue(
       HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
   mThrown.expect(FileDoesNotExistException.class);
   mFsMaster.getFileInfo(fileId);
 }
  /** Tests if delete deletes all files or folders for a large directory. */
  @Test
  public void deleteLargeDirectory() throws IOException {
    LargeDirectoryConfig config = prepareLargeDirectoryTest();
    mUfs.delete(config.getTopLevelDirectory(), true);

    String[] children = config.getChildren();
    for (String child : children) {
      // Retry for some time to allow list operation eventual consistency for S3 and GCS.
      // See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
      // https://cloud.google.com/storage/docs/consistency for more details.
      // Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
      boolean childDeleted = false;
      for (int i = 0; i < 20; i++) {
        childDeleted = !mUfs.exists(child);
        if (childDeleted) {
          break;
        }
        CommonUtils.sleepMs(500);
      }
      Assert.assertTrue(childDeleted);
    }
  }
  @Test
  public void readBlockTest() throws Exception {
    Map<String, String> params = new HashMap<>();
    params.put("blockId", "1");
    params.put("sessionId", "1");
    params.put("lockId", "1");
    params.put("offset", "0");
    params.put("length", "-1");

    Random random = new Random();
    byte[] bytes = CommonUtils.randomBytes(random.nextInt(64));
    ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);

    BlockReader blockReader = PowerMockito.mock(BlockReader.class);
    Mockito.doReturn(byteBuffer).when(blockReader).read(Mockito.anyLong(), Mockito.anyLong());
    Mockito.doReturn((long) bytes.length).when(blockReader).getLength();
    Mockito.doReturn(blockReader)
        .when(mBlockWorker)
        .readBlockRemote(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong());

    TestCase testCase =
        TestCaseFactory.newWorkerTestCase(
            getEndpoint(BlockWorkerClientRestServiceHandler.READ_BLOCK),
            params,
            "GET",
            byteBuffer,
            mResource);

    HttpURLConnection connection = (HttpURLConnection) testCase.createURL().openConnection();
    connection.setRequestMethod(testCase.getMethod());
    connection.connect();
    Assert.assertEquals(
        testCase.getEndpoint(), connection.getResponseCode(), Response.Status.OK.getStatusCode());
    Assert.assertEquals(new String(byteBuffer.array()), testCase.getResponse(connection));

    Mockito.verify(mBlockWorker)
        .readBlockRemote(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong());
  }
  @Test
  public void requestBlockLocationTest() throws Exception {
    Map<String, String> params = new HashMap<>();
    params.put("blockId", "1");
    params.put("sessionId", "1");
    params.put("initialBytes", "1");

    String blockLocation = CommonUtils.randomString(10);
    Mockito.doReturn(blockLocation)
        .when(mBlockWorker)
        .createBlock(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyLong());

    TestCaseFactory.newWorkerTestCase(
            getEndpoint(BlockWorkerClientRestServiceHandler.REQUEST_BLOCK_LOCATION),
            params,
            "POST",
            blockLocation,
            mResource)
        .run();

    Mockito.verify(mBlockWorker)
        .createBlock(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyLong());
  }
  @Test
  public void writeBlockTest() throws Exception {
    Map<String, String> params = new HashMap<>();
    params.put("blockId", "1");
    params.put("sessionId", "1");
    params.put("offset", "0");
    params.put("length", "-1");

    Random random = new Random();
    byte[] bytes = CommonUtils.randomBytes(random.nextInt(64));

    BlockWriter blockWriter = PowerMockito.mock(BlockWriter.class);
    Mockito.doReturn(blockWriter)
        .when(mBlockWorker)
        .getTempBlockWriterRemote(Mockito.anyLong(), Mockito.anyLong());

    TestCase testCase =
        TestCaseFactory.newWorkerTestCase(
            getEndpoint(BlockWorkerClientRestServiceHandler.WRITE_BLOCK),
            params,
            "POST",
            null,
            mResource);

    HttpURLConnection connection = (HttpURLConnection) testCase.createURL().openConnection();
    connection.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
    connection.setRequestMethod(testCase.getMethod());
    connection.setDoOutput(true);
    connection.connect();
    connection.getOutputStream().write(bytes);
    Assert.assertEquals(
        testCase.getEndpoint(), Response.Status.OK.getStatusCode(), connection.getResponseCode());
    Assert.assertEquals("", testCase.getResponse(connection));

    Mockito.verify(mBlockWorker).getTempBlockWriterRemote(Mockito.anyLong(), Mockito.anyLong());
    Mockito.verify(blockWriter).append(ByteBuffer.wrap(bytes));
  }
 private void setupShellMocks(String username, List<String> groups) throws IOException {
   PowerMockito.mockStatic(CommonUtils.class);
   PowerMockito.when(CommonUtils.getUnixGroups(Mockito.eq(username))).thenReturn(groups);
 }
Пример #17
0
 /**
  * Strips the folder suffix if it exists. This is a string manipulation utility and does not
  * guarantee the existence of the folder. This method will leave paths without a suffix unaltered.
  *
  * @param path the path to strip the suffix from
  * @return the path with the suffix removed, or the path unaltered if the suffix is not present
  */
 private String stripFolderSuffixIfPresent(final String path) {
   return CommonUtils.stripSuffixIfPresent(path, FOLDER_SUFFIX);
 }
 /**
  * Returns list of groups for a user.
  *
  * @param user get groups for this user
  * @return list of groups for a given user
  * @throws IOException when getting the UNIX groups
  */
 @Override
 public List<String> getGroups(String user) throws IOException {
   List<String> groups = CommonUtils.getUnixGroups(user);
   // remove duplicated primary group
   return new ArrayList<String>(new LinkedHashSet<String>(groups));
 }
Пример #19
0
 /**
  * Strips the Swift container prefix from the path if it is present. For example, for input path
  * swift://my-container-name/my-path/file, the output would be my-path/file. This method will
  * leave paths without a prefix unaltered, ie. my-path/file returns my-path/file.
  *
  * @param path the path to strip
  * @return the path without the Swift container prefix
  */
 private String stripContainerPrefixIfPresent(final String path) {
   return CommonUtils.stripPrefixIfPresent(path, mContainerPrefix);
 }