public static long parseMemorySize(String memorySize) {
double alpha = 0.0001;
String ori = memorySize;
String end = "";
int tIndex = memorySize.length() - 1;
while (tIndex >= 0) {
if (memorySize.charAt(tIndex) > '9' || memorySize.charAt(tIndex) < '0') {
end = memorySize.charAt(tIndex) + end;
} else {
break;
}
tIndex--;
}
memorySize = memorySize.substring(0, tIndex + 1);
double ret = Double.parseDouble(memorySize);
end = end.toLowerCase();
switch (end) {
case "":
case "b":
return (long) (ret + alpha);
case "kb":
return (long) (ret * Constants.KB + alpha);
case "mb":
return (long) (ret * Constants.MB + alpha);
case "gb":
return (long) (ret * Constants.GB + alpha);
case "tb":
return (long) (ret * Constants.TB + alpha);
}
runtimeException("Fail to parse " + ori + " as memory size");
return -1;
}
public static void writeParition()
throws IOException, TableDoesNotExistException, InvalidPathException,
FileAlreadyExistException, TException {
RawTable rawTable = sTachyonClient.getRawTable(sTablePath);
LOG.info("Writing data...");
for (int column = 0; column < COLS; column++) {
RawColumn rawColumn = rawTable.getRawColumn(column);
if (!rawColumn.createPartition(0)) {
CommonUtils.runtimeException(
"Failed to create partition in table " + sTablePath + " under column " + column);
}
ByteBuffer buf = ByteBuffer.allocate(80);
buf.order(ByteOrder.nativeOrder());
for (int k = 0; k < 20; k++) {
buf.putInt(k);
}
buf.flip();
CommonUtils.printByteBuffer(LOG, buf);
buf.flip();
TachyonFile tFile = rawColumn.getPartition(0);
OutStream os = tFile.createOutStream(sWriteType);
os.write(buf);
os.close();
}
}
public HdfsClient() {
try {
mFs = FileSystem.get(new Configuration());
} catch (IOException e) {
CommonUtils.runtimeException(e);
}
}
@Override
public void run() {
try {
readPartition();
} catch (Exception e) {
CommonUtils.runtimeException(e);
}
LOG.info("ReadWorker " + mWorkerId + " just finished.");
}
@Override
public void run() {
try {
memoryCopyParition();
} catch (IOException e) {
CommonUtils.runtimeException(e);
}
LOG.info(mMsg + mWorkerId + " just finished.");
}
public RawColumn getRawColumn(int columnIndex) {
if (columnIndex < 0 || columnIndex >= CLIENT_RAW_TABLE_INFO.getColumns()) {
CommonUtils.runtimeException(
CLIENT_RAW_TABLE_INFO.getPath()
+ " does not have column "
+ columnIndex
+ ". It has "
+ CLIENT_RAW_TABLE_INFO.getColumns()
+ " columns.");
}
return new RawColumn(TACHYON_CLIENT, this, columnIndex);
}
private static void memoryCopyTest(boolean write, boolean memoryOnly) {
ByteBuffer[] bufs = new ByteBuffer[THREADS];
for (int thread = 0; thread < THREADS; thread++) {
ByteBuffer sRawData = ByteBuffer.allocate(BLOCK_SIZE_BYTES);
sRawData.order(ByteOrder.nativeOrder());
for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
sRawData.putInt(k);
}
bufs[thread] = sRawData;
}
String msg = (write ? "Write" : "Read") + (memoryOnly ? "_Memory " : "_RamFile ");
GeneralWorker[] WWs = new GeneralWorker[THREADS];
int t = FILES / THREADS;
for (int thread = 0; thread < THREADS; thread++) {
WWs[thread] =
new GeneralWorker(
thread, t * thread, t * (thread + 1), bufs[thread], write, memoryOnly, msg);
}
long startTimeMs = System.currentTimeMillis();
for (int thread = 0; thread < THREADS; thread++) {
WWs[thread].start();
}
for (int thread = 0; thread < THREADS; thread++) {
try {
WWs[thread].join();
} catch (InterruptedException e) {
CommonUtils.runtimeException(e);
}
}
long takenTimeMs = System.currentTimeMillis() - startTimeMs;
double result = 1000L * FILES_BYTES / takenTimeMs / 1024 / 1024;
LOG.info(
result
+ " Mb/sec. "
+ RESULT_PREFIX
+ "Entire "
+ msg
+ " Test : "
+ " Took "
+ takenTimeMs
+ " ms. Current System Time: "
+ System.currentTimeMillis());
}
public void delete(Path f, boolean recursive) {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
mFs.delete(f, recursive);
} catch (IOException e) {
cnt++;
LOG.error(cnt + " : " + e.getMessage(), e);
te = e;
continue;
}
return;
}
CommonUtils.runtimeException(te);
}
public void copyToLocalFile(boolean delSrc, Path src, Path dst) {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
mFs.copyToLocalFile(delSrc, src, dst);
} catch (IOException e) {
cnt++;
LOG.error(cnt + " : " + e.getMessage(), e);
te = e;
continue;
}
return;
}
CommonUtils.runtimeException(te);
}
private static void TachyonTest(boolean write) {
ByteBuffer[] bufs = new ByteBuffer[THREADS];
for (int thread = 0; thread < THREADS; thread++) {
ByteBuffer sRawData = ByteBuffer.allocate(BLOCK_SIZE_BYTES);
sRawData.order(ByteOrder.nativeOrder());
for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
sRawData.putInt(k);
}
bufs[thread] = sRawData;
}
Worker[] WWs = new Worker[THREADS];
int t = FILES / THREADS;
for (int thread = 0; thread < THREADS; thread++) {
if (write) {
WWs[thread] = new TachyonWriterWorker(thread, t * thread, t * (thread + 1), bufs[thread]);
} else {
WWs[thread] = new TachyonReadWorker(thread, t * thread, t * (thread + 1), bufs[thread]);
}
}
long startTimeMs = System.currentTimeMillis();
for (int thread = 0; thread < THREADS; thread++) {
WWs[thread].start();
}
for (int thread = 0; thread < THREADS; thread++) {
try {
WWs[thread].join();
} catch (InterruptedException e) {
CommonUtils.runtimeException(e);
}
}
long takenTimeMs = System.currentTimeMillis() - startTimeMs;
double result = FILES_BYTES * 1000L / takenTimeMs / 1024 / 1024;
LOG.info(
result
+ " Mb/sec. "
+ RESULT_PREFIX
+ "Entire "
+ (write ? "Write " : "Read ")
+ " Took "
+ takenTimeMs
+ " ms. Current System Time: "
+ System.currentTimeMillis());
}
public void copyFromLocalFile(boolean delSrc, boolean overwrite, String src, String dst) {
IOException te = null;
LOG.info("Trying to copy from " + src + " to " + dst);
int cnt = 0;
while (cnt < MAX_TRY) {
try {
mFs.copyFromLocalFile(delSrc, overwrite, new Path(src), new Path(dst));
} catch (IOException e) {
cnt++;
LOG.error(cnt + " : " + e.getMessage(), e);
te = e;
continue;
}
LOG.info("Finished the copy from " + src + " to " + dst);
return;
}
CommonUtils.runtimeException(te);
}
public void readPartition()
throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
ByteBuffer buf;
if (DEBUG_MODE) {
LOG.info("Verifying the reading data...");
for (int pId = mLeft; pId < mRight; pId++) {
TachyonFile file = mTC.getFile(FILE_NAME + mWorkerId);
buf = file.readByteBuffer();
IntBuffer intBuf;
intBuf = buf.asIntBuffer();
int tmp;
for (int i = 0; i < BLOCKS_PER_FILE; i++) {
for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
tmp = intBuf.get();
if ((k == 0 && tmp == (i + mWorkerId)) || (k != 0 && tmp == k)) {
} else {
CommonUtils.runtimeException("WHAT? " + tmp + " " + k);
}
}
}
file.readByteBuffer();
}
}
long sum = 0;
for (int pId = mLeft; pId < mRight; pId++) {
long startTimeMs = System.currentTimeMillis();
TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
buf = file.readByteBuffer();
for (int i = 0; i < BLOCKS_PER_FILE; i++) {
buf.get(mBuf.array());
}
sum += mBuf.get(pId % 16);
if (DEBUG_MODE) {
buf.flip();
CommonUtils.printByteBuffer(LOG, buf);
}
buf.clear();
logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
}
Results[mWorkerId] = sum;
}
public static void main(String[] args)
throws IOException, InvalidPathException, FileAlreadyExistException {
if (args.length != 9) {
System.out.println(
"java -cp target/tachyon-"
+ Version.VERSION
+ "-jar-with-dependencies.jar tachyon.examples.Performance "
+ "<MasterIp> <FileName> <WriteBlockSizeInBytes> <BlocksPerFile> "
+ "<DebugMode:true/false> <Threads> <FilesPerThread> <TestCaseNumber> <BaseFileNumber>\n"
+ "1: Files Write Test\n"
+ "2: Files Read Test\n"
+ "3: RamFile Write Test \n"
+ "4: RamFile Read Test \n"
+ "5: ByteBuffer Write Test \n"
+ "6: ByteBuffer Read Test \n");
System.exit(-1);
}
MASTER_ADDRESS = args[0];
FILE_NAME = args[1];
BLOCK_SIZE_BYTES = Integer.parseInt(args[2]);
BLOCKS_PER_FILE = Integer.parseInt(args[3]);
DEBUG_MODE = ("true".equals(args[4]));
THREADS = Integer.parseInt(args[5]);
FILES = Integer.parseInt(args[6]) * THREADS;
int testCase = Integer.parseInt(args[7]);
BASE_FILE_NUMBER = Integer.parseInt(args[8]);
FILE_BYTES = BLOCKS_PER_FILE * BLOCK_SIZE_BYTES;
FILES_BYTES = 1L * FILE_BYTES * FILES;
RESULT_PREFIX =
String.format(
"Threads %d FilesPerThread %d TotalFiles %d "
+ "BLOCK_SIZE_KB %d BLOCKS_PER_FILE %d FILE_SIZE_MB %d "
+ "Tachyon_WRITE_BUFFER_SIZE_KB %d BaseFileNumber %d : ",
THREADS,
FILES / THREADS,
FILES,
BLOCK_SIZE_BYTES / 1024,
BLOCKS_PER_FILE,
CommonUtils.getMB(FILE_BYTES),
UserConf.get().FILE_BUFFER_BYTES / 1024,
BASE_FILE_NUMBER);
if (testCase == 1) {
RESULT_PREFIX = "TachyonFilesWriteTest " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
MTC = TachyonFS.get(MASTER_ADDRESS);
createFiles();
TachyonTest(true);
} else if (testCase == 2) {
RESULT_PREFIX = "TachyonFilesReadTest " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
MTC = TachyonFS.get(MASTER_ADDRESS);
TachyonTest(false);
} else if (testCase == 3) {
RESULT_PREFIX = "RamFile Write " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
memoryCopyTest(true, false);
} else if (testCase == 4) {
RESULT_PREFIX = "RamFile Read " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
memoryCopyTest(false, false);
} else if (testCase == 5) {
RESULT_PREFIX = "ByteBuffer Write Test " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
memoryCopyTest(true, true);
} else if (testCase == 6) {
RESULT_PREFIX = "ByteBuffer Read Test " + RESULT_PREFIX;
LOG.info(RESULT_PREFIX);
memoryCopyTest(false, true);
} else {
CommonUtils.runtimeException("No Test Case " + testCase);
}
for (int k = 0; k < RESULT_ARRAY_SIZE; k++) {
System.out.print(Results[k] + " ");
}
System.out.println();
System.exit(0);
}