/*
* Assume that the window is properly positioned. Try to fill the read
* buffer with data from this file handle, starting at the location
* indicated by the starting offset field. If this file contains more
* data, return true. If this file doesn't contain more data, return
* false.
*
* In all cases, leave the the read buffer pointing at the target
* offset and in a state that's ready to support reads, even if there
* is nothing in the buffer. Note that the target offset, which may not
* be the same as starting offset.
* @return true if more data was read, false if not.
*/
protected boolean fillFromFile(FileHandle fileHandle, long targetOffset)
throws DatabaseException {
boolean foundData = false;
readBuffer.clear();
if (fileManager.readFromFile(
fileHandle.getFile(),
readBuffer,
startOffset,
fileHandle.getFileNum(),
false /* dataKnownToBeInFile */)) {
foundData = true;
nReadOperations += 1;
/*
* Ensure that fileNum and logVersion are in sync. setFileNum
* handles changes in the file number. But we must also update
* the logVersion here to handle the first read after we
* initialize fileNum and logVersion is unknown.
*/
logVersion = fileHandle.getLogVersion();
}
/*
* In all cases, setup read buffer for valid reading. If the buffer
* has no data, it will be positioned at the beginning, and will be
* able to correctly return the fact that there is no data present.
*/
endOffset = startOffset + threadSafeBufferPosition(readBuffer);
threadSafeBufferFlip(readBuffer);
threadSafeBufferPosition(readBuffer, (int) (targetOffset - startOffset));
return foundData;
}
@Test
public void testRenameFile() throws Exception {
VOLUME_NAME = "testRenameFile";
String fileName = "testfile";
String renamedFileName = "renamed";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials,
fileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber(),
0777);
fh.close();
// nothing should happen
volume.rename(userCredentials, fileName, fileName);
DirectoryEntries dir = volume.readDir(userCredentials, "/", 0, 100, true);
assertEquals(fileName, dir.getEntries(2).getName());
assertEquals(3, dir.getEntriesCount());
volume.rename(userCredentials, fileName, renamedFileName);
dir = volume.readDir(userCredentials, "/", 0, 100, true);
assertEquals(renamedFileName, dir.getEntries(2).getName());
assertEquals(3, dir.getEntriesCount());
}
/*
* Reposition to the specified file, and fill starting at
* startOffset. Position the window's buffer to point at the log entry
* indicated by targetOffset
*/
public void slideAndFill(
long windowfileNum, long windowStartOffset, long targetOffset, boolean forward)
throws ChecksumException, FileNotFoundException, DatabaseException {
FileHandle fileHandle = fileManager.getFileHandle(windowfileNum);
try {
startOffset = windowStartOffset;
setFileNum(windowfileNum, fileHandle.getLogVersion());
boolean foundData = fillFromFile(fileHandle, targetOffset);
/*
* When reading backwards, we need to guarantee there is no log
* gap, throws out an EnvironmentFailreException if it exists.
*/
if (!foundData && !forward) {
throw EnvironmentFailureException.unexpectedState(
"Detected a log file gap when reading backwards. "
+ "Target position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, targetOffset))
+ " starting position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, windowStartOffset))
+ " end position = "
+ DbLsn.getNoFormatString(DbLsn.makeLsn(windowfileNum, endOffset)));
}
} finally {
fileHandle.release();
}
}
@Test
public void testReplicaAddListRemove() throws Exception {
VOLUME_NAME = "testReplicaAddListRemove";
final String fileName = "testfile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
// set replication update policy of the file
int flags = ReplicationFlags.setSequentialStrategy(0);
flags = ReplicationFlags.setFullReplica(flags);
volume.setDefaultReplicationPolicy(
userCredentials, "/", ReplicaUpdatePolicies.REPL_UPDATE_PC_WARONE, 2, flags);
FileHandle fileHandle =
volume.openFile(
userCredentials, fileName, SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber(), 0777);
fileHandle.close();
assertEquals(2, volume.listReplicas(userCredentials, fileName).getReplicasCount());
String osdUUID = volume.getSuitableOSDs(userCredentials, fileName, 1).get(0);
Replica replica =
Replica.newBuilder()
.addOsdUuids(osdUUID)
.setStripingPolicy(defaultStripingPolicy)
.setReplicationFlags(flags)
.build();
volume.addReplica(userCredentials, fileName, replica);
assertEquals(3, volume.listReplicas(userCredentials, fileName).getReplicasCount());
volume.removeReplica(userCredentials, fileName, replica.getOsdUuids(0));
assertEquals(2, volume.listReplicas(userCredentials, fileName).getReplicasCount());
}
@Test
public void testGetXattrSize() throws Exception {
VOLUME_NAME = "testGetXattrSize";
String fileName = "testfile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials,
fileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_SYNC.getNumber(),
0777);
fh.close();
int size = volume.getXAttrSize(userCredentials, fileName, "xtreemfs.set_repl_update_policy");
assertEquals(0, size);
size = volume.getXAttrSize(userCredentials, fileName, "xtreemfs.owner");
assertEquals(userCredentials.getUsername().length(), size);
size = volume.getXAttrSize(userCredentials, fileName, "doesnt-exist");
assertEquals(-1, size);
volume.setXAttr(userCredentials, fileName, "foo", "bar", XATTR_FLAGS.XATTR_FLAGS_CREATE);
size = volume.getXAttrSize(userCredentials, fileName, "foo");
assertEquals(3, size);
size = volume.getXAttrSize(userCredentials, fileName, "doesnt-exist-in-cache");
assertEquals(-1, size);
}
public ArrayList<String> available() {
ArrayList<String> scenes = new ArrayList<String>();
FileHandle[] files = Gdx.files.internal("bdx/scenes/").list("bdx");
for (FileHandle file : files) {
scenes.add(file.name().replace(".bdx", ""));
}
return scenes;
}
@Test
public void testVolumeQuota() throws Exception {
final String VOLUME_NAME = "testVolumeQuota";
client.createVolume(
mrcAddress,
auth,
userCredentials,
VOLUME_NAME,
0,
userCredentials.getUsername(),
userCredentials.getGroups(0),
AccessControlPolicyType.ACCESS_CONTROL_POLICY_NULL,
StripingPolicyType.STRIPING_POLICY_RAID0,
defaultStripingPolicy.getStripeSize(),
defaultStripingPolicy.getWidth(),
new ArrayList<KeyValuePair>());
Volume volume = client.openVolume(VOLUME_NAME, null, options);
volume.setXAttr(userCredentials, "/", "xtreemfs.quota", "8", XATTR_FLAGS.XATTR_FLAGS_CREATE);
assertEquals("8", volume.getXAttr(userCredentials, "/", "xtreemfs.quota"));
int flags =
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber();
byte[] content = "foo foo foo".getBytes(); // 11 bytes will exceed quota
FileHandle file = volume.openFile(userCredentials, "/test1.txt", flags, 0777);
boolean osdWriteException = false;
try {
file.write(userCredentials, content, content.length, 0);
} catch (Exception ex) {
osdWriteException = true;
}
if (!osdWriteException) {
fail("OSD performed write operation although it exceeds the quota.");
}
content = "foo bar ".getBytes(); // 8 bytes to fit quota perfectly
file.write(userCredentials, content, content.length, 0);
file.close();
try {
file = volume.openFile(userCredentials, "/test2.txt", flags, 0777);
assertTrue(false);
} catch (PosixErrorException exc) {
// check if right exception was thrown
if (!exc.getMessage().contains("POSIX_ERROR_ENOSPC")) {
assertTrue(false);
}
}
}
@Test
public void testSetGetListXattr() throws Exception {
VOLUME_NAME = "testSetGetListXattr";
final String TESTFILE = "testfile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials, TESTFILE, SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fh.close();
int initialNumberOfXattr = volume.listXAttrs(userCredentials, TESTFILE).getXattrsCount();
// This xattr should not exist
String xattr = volume.getXAttr(userCredentials, TESTFILE, "foobarasdf");
assertNull(xattr);
// and therefore should have no size
assertEquals(-1, volume.getXAttrSize(userCredentials, TESTFILE, "foobarasdf"));
// creating a new Xattr should increase the number of xattrs
volume.setXAttr(
userCredentials, TESTFILE, "foobarasdf", "nyancat", XATTR_FLAGS.XATTR_FLAGS_CREATE);
assertEquals(
initialNumberOfXattr + 1, volume.listXAttrs(userCredentials, TESTFILE).getXattrsCount());
xattr = volume.getXAttr(userCredentials, TESTFILE, "foobarasdf");
assertEquals("nyancat", xattr);
// delete the created Xattr
volume.removeXAttr(userCredentials, TESTFILE, "foobarasdf");
assertEquals(
initialNumberOfXattr, volume.listXAttrs(userCredentials, TESTFILE).getXattrsCount());
xattr = volume.getXAttr(userCredentials, TESTFILE, "foobarasdf");
assertNull(xattr);
// same with xtreemfs. attributes.
try {
xattr = volume.getXAttr(userCredentials, TESTFILE, "xtreemfs.nyancat");
fail("nyancat is not a valid system attribute");
} catch (Exception e) {
}
xattr =
volume.getXAttr(
userCredentials, TESTFILE, "xtreemfs." + SysAttrs.set_repl_update_policy.toString());
assertEquals(ReplicaUpdatePolicies.REPL_UPDATE_PC_NONE, xattr);
// read existing systemflag
xattr =
volume.getXAttr(userCredentials, TESTFILE, "xtreemfs." + SysAttrs.object_type.toString());
assertEquals("1", xattr);
}
/** Override so that we always start at the last file. */
protected void initStartingPosition(long endOfFileLsn, Long singleFileNum)
throws IOException, DatabaseException {
eof = false;
/*
* Start at what seems like the last file. If it doesn't exist, we're
* done.
*/
Long lastNum =
((singleFileNum != null) && (singleFileNum.longValue() >= 0))
? singleFileNum
: fileManager.getLastFileNum();
FileHandle fileHandle = null;
readBufferFileEnd = 0;
long fileLen = 0;
while ((fileHandle == null) && !eof) {
if (lastNum == null) {
eof = true;
} else {
try {
readBufferFileNum = lastNum.longValue();
fileHandle = fileManager.getFileHandle(readBufferFileNum);
/*
* Check the size of this file. If it opened successfully
* but only held a header or is 0 length, backup to the
* next "last" file unless this is the only file in the
* log. Note that an incomplete header will end up throwing
* a checksum exception, but a 0 length file will open
* successfully in read only mode.
*/
fileLen = fileHandle.getFile().length();
if (fileLen <= FileManager.firstLogEntryOffset()) {
lastNum = fileManager.getFollowingFileNum(lastNum.longValue(), false);
if (lastNum != null) {
fileHandle.release();
fileHandle = null;
}
}
} catch (DatabaseException e) {
lastNum = attemptToMoveBadFile(e);
fileHandle = null;
} finally {
if (fileHandle != null) {
fileHandle.release();
}
}
}
}
nextEntryOffset = 0;
}
public static void test(String[] args) {
FileHandle filehandle = new FileHandle();
Map<String, ErrorFile> mapfiles = filehandle.copyFile("d:\\home", "d:\\bak");
java.util.Iterator<ErrorFile> files = mapfiles.values().iterator();
while (files.hasNext()) {
ErrorFile file = files.next();
System.out.println(
file.getDestfile().getName() + ",,," + file.getSgsnid() + ",,," + file.getSrcfilename());
}
}
@Test(expected = PosixErrorException.class)
public void testReadLinkWithoutLink() throws Exception {
VOLUME_NAME = "testReadLinkWithoutLink";
String fileName = "testfile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials, fileName, SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fh.close();
volume.readLink(userCredentials, fileName);
}
@Test
public void testAccessSuccess() throws Exception {
VOLUME_NAME = "testAccessSuccess";
String fileName = "testfile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials,
fileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber(),
0753);
fh.close();
volume.access(userCredentials, fileName, 0753);
}
/**
* Fill up the read buffer with more data, moving along to the following file (next largest
* number) if needed.
*
* @return true if the fill moved us to a new file.
*/
protected boolean fillNext(boolean singleFile, int bytesNeeded)
throws ChecksumException, FileNotFoundException, EOFException, DatabaseException {
adjustReadBufferSize(bytesNeeded);
FileHandle fileHandle = null;
try {
/* Get a file handle to read in more log. */
fileHandle = fileManager.getFileHandle(fileNum);
/*
* Check to see if we've come to the end of the file. If so,
* get the next file.
*/
startOffset = endOffset;
if (fillFromFile(fileHandle, startOffset)) {
/*
* Successfully filled the read buffer, but didn't move to
* a new file.
*/
return false;
}
/* This file is done -- can we read in the next file? */
if (singleFile) {
throw new EOFException("Single file only");
}
Long nextFile = fileManager.getFollowingFileNum(fileNum, true /* forward */);
if (nextFile == null) {
throw new EOFException();
}
fileHandle.release();
fileHandle = null;
fileHandle = fileManager.getFileHandle(nextFile);
setFileNum(nextFile, fileHandle.getLogVersion());
startOffset = 0;
fillFromFile(fileHandle, 0);
return true;
} finally {
if (fileHandle != null) {
fileHandle.release();
}
}
}
@Test
public void testReadDirMultipleChunks() throws Exception {
options.setReaddirChunkSize(2);
VOLUME_NAME = "testReadDirMultipleChunks";
final String TESTFILE = "test";
final int fileCount = 10;
// create volume
client.createVolume(
mrcAddress,
auth,
userCredentials,
VOLUME_NAME,
0,
userCredentials.getUsername(),
userCredentials.getGroups(0),
AccessControlPolicyType.ACCESS_CONTROL_POLICY_NULL,
StripingPolicyType.STRIPING_POLICY_RAID0,
defaultStripingPolicy.getStripeSize(),
defaultStripingPolicy.getWidth(),
new ArrayList<KeyValuePair>());
Volume volume = client.openVolume(VOLUME_NAME, null, options);
// create some files
for (int i = 0; i < fileCount; i++) {
FileHandle fh =
volume.openFile(
userCredentials,
"/" + TESTFILE + i,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fh.close();
}
// test 'readDir' across multiple readDir chunks.
DirectoryEntries entrySet = null;
entrySet = volume.readDir(userCredentials, "/", 0, 1000, false);
assertEquals(2 + fileCount, entrySet.getEntriesCount());
assertEquals("..", entrySet.getEntries(0).getName());
assertEquals(".", entrySet.getEntries(1).getName());
for (int i = 0; i < fileCount; i++) {
assertEquals(TESTFILE + i, entrySet.getEntries(2 + i).getName());
}
}
public void run() {
try {
handle = fileManager.getFileHandle(fileNum);
handle.release();
} catch (Exception e) {
fail(e.getMessage());
}
}
@Test
public void testReadLinkWithLink() throws Exception {
VOLUME_NAME = "testReadLinkWithLink";
String fileName = "testfile";
String linkName = "linkToFile";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fh =
volume.openFile(
userCredentials,
fileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber(),
0777);
fh.close();
volume.symlink(userCredentials, fileName, linkName);
assertEquals(fileName, volume.readLink(userCredentials, linkName));
}
/**
* Test if files inherit the group from its parent if the setgid bit is set, and if subdirectories
* inherit the group and the setgid bit.
*/
@Test
public void testSetgid() throws Exception {
VOLUME_NAME = "testSetgid";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
volume.createDirectory(userCredentials, "/DIR1", 0777);
Stat stat;
stat = volume.getAttr(userCredentials, "/DIR1");
int mode = stat.getMode() | 02000;
stat = stat.toBuilder().setGroupId("foobar").setMode(mode).build();
UserCredentials rootCreds =
userCredentials.toBuilder().setUsername("root").setGroups(0, "root").build();
volume.setAttr(
rootCreds,
"/DIR1",
stat,
Setattrs.SETATTR_MODE.getNumber() | Setattrs.SETATTR_GID.getNumber());
// Test if the setgid bit and the group is set
stat = volume.getAttr(userCredentials, "/DIR1");
assertEquals(02000, stat.getMode() & 02000);
assertEquals("foobar", stat.getGroupId());
// Test if new subdirectories inherit the setgid bit and the group
volume.createDirectory(userCredentials, "/DIR1/DIR2", 0777);
stat = volume.getAttr(userCredentials, "/DIR1/DIR2");
assertEquals(02000, stat.getMode() & 02000);
assertEquals("foobar", stat.getGroupId());
// Test if new files inherit the group
FileHandle fh =
volume.openFile(
userCredentials, "/DIR1/FILE1", SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fh.close();
stat = volume.getAttr(userCredentials, "/DIR1/FILE1");
assertEquals("foobar", stat.getGroupId());
volume.close();
}
public Data getData(int place) {
Data ret = null;
synchronized (m_sync) {
try {
ret = m_DataArray[place];
} catch (ArrayIndexOutOfBoundsException e) {
FileHandle.getInstance()
.printError("DataStorage: Exception while getting data in place - " + place);
}
}
return ret;
}
/** Test log file creation. */
public void testFileCreation() throws IOException, DatabaseException {
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
String[] jeFiles = fileManager.listFiles(FileManager.JE_SUFFIXES);
assertEquals("Should have two files", 2, jeFiles.length);
/* Make a fake files with confusing names. */
File fakeFile1 = new File(envHome, "00000abx.jdb");
File fakeFile2 = new File(envHome, "10.10.jdb");
fakeFile1.createNewFile();
fakeFile2.createNewFile();
jeFiles = fileManager.listFiles(FileManager.JE_SUFFIXES);
assertEquals("Should have two files", 2, jeFiles.length);
/* Open the two existing log files. */
FileHandle file0Handle = fileManager.getFileHandle(0L);
FileHandle file1Handle = fileManager.getFileHandle(1L);
jeFiles = fileManager.listFiles(FileManager.JE_SUFFIXES);
assertEquals("Should have two files", 2, jeFiles.length);
file0Handle.release();
file1Handle.release();
/* Empty the cache and get them again. */
fileManager.clear();
file0Handle = fileManager.getFileHandle(0L);
file1Handle = fileManager.getFileHandle(1L);
jeFiles = fileManager.listFiles(FileManager.JE_SUFFIXES);
assertEquals("Should have two files", 2, jeFiles.length);
file0Handle.close();
file1Handle.close();
file0Handle.release();
file1Handle.release();
fakeFile1.delete();
fakeFile2.delete();
}
@Test
public void testTruncate() throws Exception {
VOLUME_NAME = "testTruncate";
String fileName = "testfile";
String emptyFileName = "emptyFileName";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
FileHandle fileHandle =
volume.openFile(
userCredentials,
fileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber(),
0777);
String data = "1234567890";
fileHandle.write(userCredentials, data.getBytes(), data.length(), 0);
fileHandle.flush();
assertEquals(data.length(), volume.getAttr(userCredentials, fileName).getSize());
volume.truncate(userCredentials, fileName, 5);
assertEquals(5, volume.getAttr(userCredentials, fileName).getSize());
fileHandle.close();
fileHandle =
volume.openFile(
userCredentials,
emptyFileName,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_RDWR.getNumber(),
0777);
assertEquals(0, fileHandle.getAttr(userCredentials).getSize());
volume.truncate(userCredentials, emptyFileName, 1000);
assertEquals(1000, fileHandle.getAttr(userCredentials).getSize());
fileHandle.close();
}
@Test
public void testCreateDelete() throws Exception {
VOLUME_NAME = "testCreateDelete";
// Both directories should be created under "/"
final String DIR1 = "/testdir1";
final String DIR2 = "testdir2";
final String TESTFILE = "testfile";
// create volume
client.createVolume(
mrcAddress,
auth,
userCredentials,
VOLUME_NAME,
0,
userCredentials.getUsername(),
userCredentials.getGroups(0),
AccessControlPolicyType.ACCESS_CONTROL_POLICY_NULL,
StripingPolicyType.STRIPING_POLICY_RAID0,
defaultStripingPolicy.getStripeSize(),
defaultStripingPolicy.getWidth(),
new ArrayList<KeyValuePair>());
Volume volume = client.openVolume(VOLUME_NAME, null, options);
// create some files and directories
try {
volume.createDirectory(userCredentials, DIR1, 0755);
volume.createDirectory(userCredentials, DIR2, 0755);
} catch (IOException ioe) {
fail("failed to create testdirs");
}
for (int i = 0; i < 10; i++) {
FileHandle fh =
volume.openFile(
userCredentials,
DIR1 + "/" + TESTFILE + i,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fh.close();
}
// // try to create a file w/o a name
try {
volume.openFile(userCredentials, "", SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fail("missing filename");
} catch (Exception e) {
}
// try to create an already existing file
try {
volume.openFile(
userCredentials,
DIR1 + "/" + TESTFILE + "1",
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_EXCL.getNumber()
| SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fail("file already exists");
} catch (Exception e) {
}
// file in file creation should fail
try {
volume.openFile(
userCredentials,
DIR1 + "/" + TESTFILE + "1/foo.txt",
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber());
fail("file in file creation");
} catch (Exception e) {
}
// should fail
try {
volume.createDirectory(userCredentials, "/", 0);
fail("directory already exists");
} catch (PosixErrorException exc) {
}
// test 'readDir' and 'stat'
DirectoryEntries entrySet = null;
entrySet = volume.readDir(userCredentials, "", 0, 1000, false);
assertEquals(4, entrySet.getEntriesCount());
assertEquals("..", entrySet.getEntries(0).getName());
assertEquals(".", entrySet.getEntries(1).getName());
assertEquals(DIR1, "/" + entrySet.getEntries(2).getName());
assertEquals("/" + DIR2, "/" + entrySet.getEntries(3).getName());
entrySet = volume.readDir(userCredentials, DIR1, 0, 1000, false);
assertEquals(12, entrySet.getEntriesCount());
// test 'delete'
volume.unlink(userCredentials, DIR1 + "/" + TESTFILE + "4");
entrySet = volume.readDir(userCredentials, DIR1, 0, 1000, false);
assertEquals(11, entrySet.getEntriesCount());
volume.removeDirectory(userCredentials, DIR2);
entrySet = volume.readDir(userCredentials, "", 0, 1000, false);
assertEquals(3, entrySet.getEntriesCount());
}
/** Initialize starting position to the last file with a complete header with a valid checksum. */
private void startAtLastGoodFile(Long singleFileNum) throws ChecksumException {
eof = false;
window.initAtFileStart(DbLsn.makeLsn(0, 0));
/*
* Start at what seems like the last file. If it doesn't exist, we're
* done.
*/
Long lastNum =
((singleFileNum != null) && (singleFileNum.longValue() >= 0))
? singleFileNum
: fileManager.getLastFileNum();
FileHandle fileHandle = null;
long fileLen = 0;
while ((fileHandle == null) && !eof) {
if (lastNum == null) {
eof = true;
} else {
try {
try {
window.initAtFileStart(DbLsn.makeLsn(lastNum, 0));
fileHandle = fileManager.getFileHandle(lastNum);
/*
* Check the size of this file. If it opened
* successfully but only held a header or is 0 length,
* backup to the next "last" file unless this is the
* only file in the log. Note that an incomplete header
* will end up throwing a checksum exception, but a 0
* length file will open successfully in read only
* mode.
*/
fileLen = fileHandle.getFile().length();
if (fileLen <= FileManager.firstLogEntryOffset()) {
lastNum = fileManager.getFollowingFileNum(lastNum, false);
if (lastNum != null) {
fileHandle.release();
fileHandle = null;
}
}
} catch (DatabaseException e) {
lastNum = attemptToMoveBadFile(e);
fileHandle = null;
} catch (ChecksumException e) {
lastNum = attemptToMoveBadFile(e);
fileHandle = null;
} finally {
if (fileHandle != null) {
fileHandle.release();
}
}
} catch (IOException e) {
throw new EnvironmentFailureException(envImpl, EnvironmentFailureReason.LOG_READ, e);
}
}
}
nextEntryOffset = 0;
}
/** Test the file cache. */
public void testCache() throws Throwable {
try {
/*
* Make five log files. The file descriptor cache should be empty.
*/
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
FileManagerTestUtils.createLogFile(fileManager, envImpl, FILE_SIZE);
Long f0 = new Long(0L);
Long f1 = new Long(1L);
Long f2 = new Long(2L);
Long f3 = new Long(3L);
Long f4 = new Long(4L);
Set keySet = fileManager.getCacheKeys();
assertEquals("should have 0 keys", 0, keySet.size());
/*
* Get file descriptors for three files, expect 3 handles in the
* cache.
*/
FileHandle f0Handle = fileManager.getFileHandle(0);
FileHandle f1Handle = fileManager.getFileHandle(1);
FileHandle f2Handle = fileManager.getFileHandle(2);
keySet = fileManager.getCacheKeys();
assertEquals("should have 3 keys", 3, keySet.size());
assertTrue(keySet.contains(f0));
assertTrue(keySet.contains(f1));
assertTrue(keySet.contains(f2));
/*
* Ask for a fourth handle, the cache should grow even though it
* was set to 3 as a starting size, because all handles are
* locked. Do it within another thread, otherwise we'll get a
* latch-already-held exception when we test the other handles in
* the cache. The other thread will get the handle and then release
* it.
*/
CachingThread otherThread = new CachingThread(fileManager, 3);
otherThread.start();
otherThread.join();
keySet = fileManager.getCacheKeys();
assertEquals("should have 4 keys", 4, keySet.size());
assertTrue(keySet.contains(f0));
assertTrue(keySet.contains(f1));
assertTrue(keySet.contains(f2));
assertTrue(keySet.contains(f3));
/*
* Now ask for another file. The cache should not grow, because no
* handles are locked and there's room to evict one.
*/
f0Handle.release();
f1Handle.release();
f2Handle.release();
FileHandle f4Handle = fileManager.getFileHandle(4);
keySet = fileManager.getCacheKeys();
assertEquals("should have 4 keys", 4, keySet.size());
assertTrue(keySet.contains(f4));
f4Handle.release();
/* Clearing should release all file descriptors. */
fileManager.clear();
assertEquals("should have 0 keys", 0, fileManager.getCacheKeys().size());
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
@Test
public void testGetStripeLocations() throws Exception {
VOLUME_NAME = "testGetStripeLocatations";
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
// set replication update policy of the file
int flags = ReplicationFlags.setSequentialStrategy(0);
flags = ReplicationFlags.setFullReplica(flags);
volume.setDefaultReplicationPolicy(
userCredentials, "/", ReplicaUpdatePolicies.REPL_UPDATE_PC_RONLY, 2, flags);
final String FILENAME = "/foobar.tzt";
FileHandle fileHandle =
volume.openFile(
userCredentials, FILENAME, SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber(), 0777);
byte[] data = new byte[6];
for (int i = 0; i < data.length; i++) {
data[i] = (byte) "FOOBAR".charAt(i);
}
fileHandle.write(userCredentials, data, data.length, 0);
fileHandle.close();
List<StripeLocation> stripeLocations =
volume.getStripeLocations(userCredentials, FILENAME, 0, 100);
assertEquals(1, stripeLocations.size());
assertEquals(2, stripeLocations.get(0).getUuids().length);
assertEquals(2, stripeLocations.get(0).getHostnames().length);
assertEquals(0, stripeLocations.get(0).getStartSize());
assertEquals(100, stripeLocations.get(0).getLength());
stripeLocations = volume.getStripeLocations(userCredentials, FILENAME, 200, 123);
assertEquals(1, stripeLocations.size());
assertEquals(2, stripeLocations.get(0).getUuids().length);
assertEquals(2, stripeLocations.get(0).getHostnames().length);
assertEquals(200, stripeLocations.get(0).getStartSize());
assertEquals(123, stripeLocations.get(0).getLength());
List<String> suitableOsds = volume.getSuitableOSDs(userCredentials, FILENAME, 1);
Replica replica =
Replica.newBuilder()
.setStripingPolicy(defaultStripingPolicy)
.setReplicationFlags(flags)
.addOsdUuids(suitableOsds.get(0))
.build();
volume.addReplica(userCredentials, FILENAME, replica);
stripeLocations = volume.getStripeLocations(userCredentials, FILENAME, 0, 100);
assertEquals(1, stripeLocations.size());
assertEquals(3, stripeLocations.get(0).getUuids().length);
assertEquals(3, stripeLocations.get(0).getHostnames().length);
assertEquals(0, stripeLocations.get(0).getStartSize());
assertEquals(100, stripeLocations.get(0).getLength());
stripeLocations = volume.getStripeLocations(userCredentials, FILENAME, 200, 123);
assertEquals(1, stripeLocations.size());
assertEquals(3, stripeLocations.get(0).getUuids().length);
assertEquals(3, stripeLocations.get(0).getHostnames().length);
assertEquals(200, stripeLocations.get(0).getStartSize());
assertEquals(123, stripeLocations.get(0).getLength());
volume.close();
}
@Test
public void testGetStripeLocationsWithMultipleStripes() throws Exception {
VOLUME_NAME = "testGetStripeLocatationsWithMultipleStripes";
client.createVolume(
mrcAddress,
auth,
userCredentials,
VOLUME_NAME,
0777,
userCredentials.getUsername(),
userCredentials.getGroups(0),
AccessControlPolicyType.ACCESS_CONTROL_POLICY_POSIX,
StripingPolicyType.STRIPING_POLICY_RAID0,
2,
1,
new ArrayList<GlobalTypes.KeyValuePair>());
Volume volume = client.openVolume(VOLUME_NAME, options.generateSSLOptions(), options);
// set replication update policy of the file
int replicationFlags = ReplicationFlags.setSequentialStrategy(0);
replicationFlags = ReplicationFlags.setFullReplica(replicationFlags);
volume.setDefaultReplicationPolicy(
userCredentials, "/", ReplicaUpdatePolicies.REPL_UPDATE_PC_RONLY, 2, replicationFlags);
// create new striping policy with width 2
StripingPolicy stripingPolicy =
StripingPolicy.newBuilder()
.setStripeSize(2)
.setWidth(2)
.setType(StripingPolicyType.STRIPING_POLICY_RAID0)
.build();
// create file
final String FILENAME = "/foobar.tzt";
int flags = SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber();
FileHandle fileHandle = volume.openFile(userCredentials, FILENAME, flags, 0777);
// write more than 2kb to the byte to ensure the second stripe is used
byte[] data = new byte[3000];
for (int i = 0; i < data.length; i++) {
data[i] = 'F';
}
fileHandle.write(userCredentials, data, data.length, 0);
fileHandle.close();
// create replica and add it
List<String> suitableOsds = volume.getSuitableOSDs(userCredentials, FILENAME, 2);
Replica replica =
Replica.newBuilder()
.setReplicationFlags(replicationFlags)
.setStripingPolicy(stripingPolicy)
.addAllOsdUuids(suitableOsds)
.build();
volume.addReplica(userCredentials, FILENAME, replica);
List<StripeLocation> stripeLocations =
volume.getStripeLocations(userCredentials, FILENAME, 0, 4000);
assertEquals(2, stripeLocations.size());
assertEquals(3, stripeLocations.get(0).getUuids().length);
assertEquals(3, stripeLocations.get(0).getHostnames().length);
assertEquals(0, stripeLocations.get(0).getStartSize());
assertEquals(2048, stripeLocations.get(0).getLength());
assertEquals(2048, stripeLocations.get(1).getStartSize());
assertEquals(4000 - 2048, stripeLocations.get(1).getLength());
assertEquals(stripeLocations.get(0).getUuids()[0], stripeLocations.get(1).getUuids()[0]);
assertEquals(stripeLocations.get(0).getUuids()[1], stripeLocations.get(1).getUuids()[1]);
assertNotSame(stripeLocations.get(0).getUuids()[2], stripeLocations.get(1).getUuids()[2]);
}
@Test
public void testGetSetAttr() throws Exception {
VOLUME_NAME = "testGetSetAttr";
final String TESTFILE = "testfile";
final String TESTDIR = "testdir";
final int TESTMODE = 0731;
client.createVolume(mrcAddress, auth, userCredentials, VOLUME_NAME);
Volume volume = client.openVolume(VOLUME_NAME, null, options);
volume.createDirectory(userCredentials, TESTDIR, TESTMODE);
FileHandle fh =
volume.openFile(
userCredentials,
TESTDIR + "/" + TESTFILE,
SYSTEM_V_FCNTL.SYSTEM_V_FCNTL_H_O_CREAT.getNumber(),
TESTMODE);
fh.close();
Stat stat = volume.getAttr(userCredentials, TESTDIR);
assertEquals(userCredentials.getUsername(), stat.getUserId());
assertEquals(userCredentials.getGroups(0), stat.getGroupId());
assertEquals(0, stat.getSize());
assertTrue(stat.getAtimeNs() > 0);
assertTrue(stat.getCtimeNs() > 0);
assertTrue(stat.getMtimeNs() > 0);
assertTrue((stat.getMode() & TESTMODE) > 0);
assertEquals(1, stat.getNlink());
stat = volume.getAttr(userCredentials, TESTDIR + "/" + TESTFILE);
assertEquals(userCredentials.getUsername(), stat.getUserId());
assertEquals(userCredentials.getGroups(0), stat.getGroupId());
assertEquals(0, stat.getSize());
assertTrue(stat.getAtimeNs() > 0);
assertTrue(stat.getCtimeNs() > 0);
assertTrue(stat.getMtimeNs() > 0);
assertTrue((stat.getMode() & TESTMODE) == TESTMODE);
assertEquals(1, stat.getNlink());
stat = stat.toBuilder().setGroupId("foobar").setUserId("FredFoobar").build();
try {
volume.setAttr(
userCredentials,
TESTDIR,
stat,
Setattrs.SETATTR_UID.getNumber() | Setattrs.SETATTR_GID.getNumber());
fail("changing username and groups should be restricted to superuser");
} catch (PosixErrorException e) {
}
UserCredentials rootCreds =
userCredentials.toBuilder().setUsername("root").setGroups(0, "root").build();
volume.setAttr(
rootCreds,
TESTDIR,
stat,
Setattrs.SETATTR_UID.getNumber() | Setattrs.SETATTR_GID.getNumber());
stat = volume.getAttr(userCredentials, TESTDIR);
assertEquals("FredFoobar", stat.getUserId());
assertEquals("foobar", stat.getGroupId());
assertTrue((stat.getMode() & TESTMODE) > 0);
stat = stat.toBuilder().setMode(0777).build();
volume.setAttr(
userCredentials, TESTDIR + "/" + TESTFILE, stat, Setattrs.SETATTR_MODE.getNumber());
stat = volume.getAttr(userCredentials, TESTDIR + "/" + TESTFILE);
assertTrue((stat.getMode() & 0777) == 0777);
}