/** @throws Exception If failed. */
public void testCreateFileColocated() throws Exception {
GridGgfsPath path = new GridGgfsPath("/colocated");
UUID uuid = UUID.randomUUID();
GridUuid affKey;
long idx = 0;
while (true) {
affKey = new GridUuid(uuid, idx);
if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id()))
break;
idx++;
}
try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
// Write 5M, should be enough to test distribution.
for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
}
GridGgfsFile info = fs.info(path);
Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
assertEquals(1, affNodes.size());
Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
assertEquals(1, nodeIds.size());
assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}
/**
* Test file creation.
*
* @param path Path to file to store.
* @param size Size of file to store.
* @param salt Salt for file content generation.
* @throws Exception In case of any exception.
*/
private void testCreateFile(final GridGgfsPath path, final long size, final int salt)
throws Exception {
info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']');
final AtomicInteger cnt = new AtomicInteger(0);
final Collection<GridGgfsPath> cleanUp = new ConcurrentLinkedQueue<>();
long time =
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
int id = cnt.incrementAndGet();
GridGgfsPath f = new GridGgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : ""));
try (GridGgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) {
assertNotNull(out);
cleanUp.add(f); // Add all created into cleanup list.
U.copy(new GridGgfsTestInputStream(size, salt), out);
}
return null;
}
},
WRITING_THREADS_CNT,
"perform-multi-thread-writing");
if (time > 0) {
double rate = size * 1000. / time / 1024 / 1024;
info(
String.format(
"Write file [path=%s, size=%d kB, rate=%2.1f MB/s]",
path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate));
}
info("Read and validate saved file: " + path);
final InputStream expIn = new GridGgfsTestInputStream(size, salt);
final GridGgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10);
// Validate continuous reading of whole file.
assertEqualStreams(expIn, actIn, size, null);
// Validate random seek and reading.
final Random rnd = new Random();
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
long skip = Math.abs(rnd.nextLong() % (size + 1));
long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400));
assertEqualStreams(new GridGgfsTestInputStream(size, salt), actIn, range, skip);
return null;
}
},
READING_THREADS_CNT,
"validate-multi-thread-reading");
expIn.close();
actIn.close();
info("Get stored file info: " + path);
GridGgfsFile desc = fs.info(path);
info("Validate stored file info: " + desc);
assertNotNull(desc);
if (log.isDebugEnabled()) log.debug("File descriptor: " + desc);
Collection<GridGgfsBlockLocation> aff = fs.affinity(path, 0, desc.length());
assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty());
int blockSize = desc.blockSize();
assertEquals("File size", size, desc.length());
assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize);
// assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString());
// assertEquals("Permission sticky bit marks this is file", false,
// desc.getPermission().getStickyBit());
assertEquals("Type", true, desc.isFile());
assertEquals("Type", false, desc.isDirectory());
info("Cleanup files: " + cleanUp);
for (GridGgfsPath f : cleanUp) {
fs.delete(f, true);
assertNull(fs.info(f));
}
}
/** {@inheritDoc} */
@Nullable
@Override
public final Map<? extends GridComputeJob, GridNode> map(
List<GridNode> subgrid, @Nullable GridGgfsTaskArgs<T> args) throws GridException {
assert grid != null;
assert args != null;
GridGgfs ggfs = grid.ggfs(args.ggfsName());
GridGgfsProcessorAdapter ggfsProc = ((GridKernal) grid).context().ggfs();
Map<GridComputeJob, GridNode> splitMap = new HashMap<>();
Map<UUID, GridNode> nodes = mapSubgrid(subgrid);
for (GridGgfsPath path : args.paths()) {
GridGgfsFile file = ggfs.info(path);
if (file == null) {
if (args.skipNonExistentFiles()) continue;
else
throw new GridException("Failed to process GGFS file because it doesn't exist: " + path);
}
Collection<GridGgfsBlockLocation> aff =
ggfs.affinity(path, 0, file.length(), args.maxRangeLength());
long totalLen = 0;
for (GridGgfsBlockLocation loc : aff) {
GridNode node = null;
for (UUID nodeId : loc.nodeIds()) {
node = nodes.get(nodeId);
if (node != null) break;
}
if (node == null)
throw new GridException(
"Failed to find any of block affinity nodes in subgrid [loc="
+ loc
+ ", subgrid="
+ subgrid
+ ']');
GridGgfsJob job =
createJob(path, new GridGgfsFileRange(file.path(), loc.start(), loc.length()), args);
if (job != null) {
GridComputeJob jobImpl =
ggfsProc.createJob(
job, ggfs.name(), file.path(), loc.start(), loc.length(), args.recordResolver());
splitMap.put(jobImpl, node);
}
totalLen += loc.length();
}
assert totalLen == file.length();
}
return splitMap;
}