/** {@inheritDoc} */
public void updateBlock(Block oldblock, Block newblock) throws IOException {
if (oldblock.getBlockId() != newblock.getBlockId()) {
throw new IOException(
"Cannot update oldblock (=" + oldblock + ") to newblock (=" + newblock + ").");
}
for (; ; ) {
final List<Thread> threads = tryUpdateBlock(oldblock, newblock);
if (threads == null) {
return;
}
// interrupt and wait for all ongoing create threads
for (Thread t : threads) {
t.interrupt();
}
for (Thread t : threads) {
try {
t.join();
} catch (InterruptedException e) {
DataNode.LOG.warn("interruptOngoingCreates: t=" + t, e);
}
}
}
}
ActiveFile(File f, List<Thread> list) {
file = f;
if (list != null) {
threads.addAll(list);
}
threads.add(Thread.currentThread());
}
public static LinkedHashSet<String> findJars(LogicalPlan dag, Class<?>[] defaultClasses) {
List<Class<?>> jarClasses = new ArrayList<Class<?>>();
for (String className : dag.getClassNames()) {
try {
Class<?> clazz = Thread.currentThread().getContextClassLoader().loadClass(className);
jarClasses.add(clazz);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Failed to load class " + className, e);
}
}
for (Class<?> clazz : Lists.newArrayList(jarClasses)) {
// process class and super classes (super does not require deploy annotation)
for (Class<?> c = clazz; c != Object.class && c != null; c = c.getSuperclass()) {
// LOG.debug("checking " + c);
jarClasses.add(c);
jarClasses.addAll(Arrays.asList(c.getInterfaces()));
}
}
jarClasses.addAll(Arrays.asList(defaultClasses));
if (dag.isDebug()) {
LOG.debug("Deploy dependencies: {}", jarClasses);
}
LinkedHashSet<String> localJarFiles = new LinkedHashSet<String>(); // avoid duplicates
HashMap<String, String> sourceToJar = new HashMap<String, String>();
for (Class<?> jarClass : jarClasses) {
if (jarClass.getProtectionDomain().getCodeSource() == null) {
// system class
continue;
}
String sourceLocation =
jarClass.getProtectionDomain().getCodeSource().getLocation().toString();
String jar = sourceToJar.get(sourceLocation);
if (jar == null) {
// don't create jar file from folders multiple times
jar = JarFinder.getJar(jarClass);
sourceToJar.put(sourceLocation, jar);
LOG.debug("added sourceLocation {} as {}", sourceLocation, jar);
}
if (jar == null) {
throw new AssertionError("Cannot resolve jar file for " + jarClass);
}
localJarFiles.add(jar);
}
String libJarsPath = dag.getValue(LogicalPlan.LIBRARY_JARS);
if (!StringUtils.isEmpty(libJarsPath)) {
String[] libJars = StringUtils.splitByWholeSeparator(libJarsPath, LIB_JARS_SEP);
localJarFiles.addAll(Arrays.asList(libJars));
}
LOG.info("Local jar file dependencies: " + localJarFiles);
return localJarFiles;
}
private static void ignoreAndWait(final Exception e, boolean printException) {
H2O.ignore(e, "Hit HDFS reset problem, retrying...", printException);
try {
Thread.sleep(500);
} catch (InterruptedException ie) {
}
}
/**
* The run method lives for the life of the JobTracker, and removes Jobs that are not still
* running, but which finished a long time ago.
*/
public void run() {
while (shouldRun) {
try {
Thread.sleep(RETIRE_JOB_CHECK_INTERVAL);
} catch (InterruptedException ie) {
}
synchronized (jobs) {
synchronized (jobInitQueue) {
synchronized (jobsByArrival) {
for (Iterator it = jobs.keySet().iterator(); it.hasNext(); ) {
String jobid = (String) it.next();
JobInProgress job = (JobInProgress) jobs.get(jobid);
if (job.getStatus().getRunState() != JobStatus.RUNNING
&& job.getStatus().getRunState() != JobStatus.PREP
&& (job.getFinishTime() + RETIRE_JOB_INTERVAL < System.currentTimeMillis())) {
it.remove();
jobInitQueue.remove(job);
jobsByArrival.remove(job);
}
}
}
}
}
}
}
/*
* Fetch a file that is in a Hadoop file system. Return a local File.
* Interruptible.
*/
private File hdfsFetch(Path fromPath, Reporter reporter)
throws IOException, InterruptedException {
UUID uniqueId = UUID.randomUUID();
File toFile = new File(tempDir, uniqueId.toString() + "/" + fromPath.getName());
File toDir = new File(toFile.getParent());
if (toDir.exists()) {
FileUtils.deleteDirectory(toDir);
}
toDir.mkdirs();
Path toPath = new Path(toFile.getCanonicalPath());
FileSystem fS = fromPath.getFileSystem(hadoopConf);
FileSystem tofS = FileSystem.getLocal(hadoopConf);
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
try {
for (FileStatus fStatus : fS.globStatus(fromPath)) {
log.info("Copying " + fStatus.getPath() + " to " + toPath);
long bytesSoFar = 0;
FSDataInputStream iS = fS.open(fStatus.getPath());
FSDataOutputStream oS = tofS.create(toPath);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
// Needed to being able to be interrupted at any moment.
if (Thread.interrupted()) {
iS.close();
oS.close();
cleanDirNoExceptions(toDir);
throw new InterruptedException();
}
bytesSoFar += nRead;
oS.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
oS.close();
iS.close();
}
return toDir;
} catch (ClosedByInterruptException e) {
// This can be thrown by the method read.
cleanDirNoExceptions(toDir);
throw new InterruptedIOException();
}
}
private static void deleteCache(Configuration conf, MRAsyncDiskService asyncDiskService)
throws IOException {
List<CacheStatus> deleteSet = new LinkedList<CacheStatus>();
// try deleting cache Status with refcount of zero
synchronized (cachedArchives) {
for (Iterator<String> it = cachedArchives.keySet().iterator(); it.hasNext(); ) {
String cacheId = (String) it.next();
CacheStatus lcacheStatus = cachedArchives.get(cacheId);
if (lcacheStatus.refcount == 0) {
// delete this cache entry from the global list
// and mark the localized file for deletion
deleteSet.add(lcacheStatus);
it.remove();
}
}
}
// do the deletion asynchronously, after releasing the global lock
Thread cacheFileCleaner =
new Thread(new CacheFileCleanTask(asyncDiskService, FileSystem.getLocal(conf), deleteSet));
cacheFileCleaner.start();
}
public static InputStream openStream(Key k, ProgressMonitor pmon) throws IOException {
H2OHdfsInputStream res = null;
Path p = new Path(k.toString());
try {
res = new H2OHdfsInputStream(p, 0, pmon);
} catch (IOException e) {
try {
Thread.sleep(1000);
} catch (Exception ex) {
}
Log.warn("Error while opening HDFS key " + k.toString() + ", will wait and retry.");
res = new H2OHdfsInputStream(p, 0, pmon);
}
return res;
}
/**
* The run method lives for the life of the JobTracker, and removes TaskTrackers that have not
* checked in for some time.
*/
public void run() {
while (shouldRun) {
//
// Thread runs periodically to check whether trackers should be expired.
// The sleep interval must be no more than half the maximum expiry time
// for a task tracker.
//
try {
Thread.sleep(TASKTRACKER_EXPIRY_INTERVAL / 3);
} catch (InterruptedException ie) {
}
//
// Loop through all expired items in the queue
//
synchronized (taskTrackers) {
synchronized (trackerExpiryQueue) {
long now = System.currentTimeMillis();
TaskTrackerStatus leastRecent = null;
while ((trackerExpiryQueue.size() > 0)
&& ((leastRecent = (TaskTrackerStatus) trackerExpiryQueue.first()) != null)
&& (now - leastRecent.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL)) {
// Remove profile from head of queue
trackerExpiryQueue.remove(leastRecent);
String trackerName = leastRecent.getTrackerName();
// Figure out if last-seen time should be updated, or if tracker is dead
TaskTrackerStatus newProfile =
(TaskTrackerStatus) taskTrackers.get(leastRecent.getTrackerName());
// Items might leave the taskTracker set through other means; the
// status stored in 'taskTrackers' might be null, which means the
// tracker has already been destroyed.
if (newProfile != null) {
if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
// Remove completely
updateTaskTrackerStatus(trackerName, null);
lostTaskTracker(leastRecent.getTrackerName());
} else {
// Update time by inserting latest profile
trackerExpiryQueue.add(newProfile);
}
}
}
}
}
}
}
public static void startTracker(Configuration conf) throws IOException {
if (tracker != null) throw new IOException("JobTracker already running.");
while (true) {
try {
tracker = new JobTracker(conf);
break;
} catch (IOException e) {
LOG.log(Level.WARNING, "Starting tracker", e);
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
tracker.offerService();
}
private static String lock(String lock) {
String realPath = "";
String parent = "/lock";
String lockName = parent + "/" + lock;
logger.debug("Getting lock " + lockName);
try {
if (zkInstance.exists(parent, false) == null)
zkInstance.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.fromFlag(0));
} catch (Exception E) {
logger.error("Error creating lock node: " + E.toString());
return null;
}
List<String> children = new LinkedList<String>();
try {
// List <ACL> ACLList = zkInstance.getACL(lockName, zkInstance.exists(lock, false));
realPath =
zkInstance.create(
lockName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
// children = zkInstance.getChildren(realPath, false);
checkLock:
while (true) {
children = zkInstance.getChildren(parent, false);
for (String curChild : children) {
String child = parent + "/" + curChild;
// System.out.println(child + " " + realPath + " " +
// Integer.toString(child.compareTo(realPath)));
if (child.compareTo(realPath) < 0
&& child.length() == realPath.length()
&& curChild.startsWith(lock)) {
// System.out.println(child + " cmp to " + realPath);
Thread.sleep(300);
continue checkLock;
}
}
logger.info("Got lock " + lockName);
return realPath;
}
} catch (Exception E) {
logger.error("Exception while trying to get lock " + lockName + " :" + E.toString());
E.printStackTrace();
return null;
}
}
/**
* Map method. Copies one file from source file system to destination.
*
* @param key src len
* @param value FilePair (FileStatus src, Path dst)
* @param out Log of failed copies
* @param reporter
*/
public void map(
LongWritable key,
FilePair value,
OutputCollector<WritableComparable<?>, Text> out,
Reporter reporter)
throws IOException {
final FileStatus srcstat = value.input;
final Path relativedst = new Path(value.output);
try {
copy(srcstat, relativedst, out, reporter);
} catch (IOException e) {
++failcount;
reporter.incrCounter(Counter.FAIL, 1);
updateStatus(reporter);
final String sfailure = "FAIL " + relativedst + " : " + StringUtils.stringifyException(e);
out.collect(null, new Text(sfailure));
LOG.info(sfailure);
try {
for (int i = 0; i < 3; ++i) {
try {
final Path tmp = new Path(job.get(TMP_DIR_LABEL), relativedst);
if (destFileSys.delete(tmp, true)) break;
} catch (Throwable ex) {
// ignore, we are just cleaning up
LOG.debug("Ignoring cleanup exception", ex);
}
// update status, so we don't get timed out
updateStatus(reporter);
Thread.sleep(3 * 1000);
}
} catch (InterruptedException inte) {
throw (IOException) new IOException().initCause(inte);
}
} finally {
updateStatus(reporter);
}
}
public void incrementAndThrottle(int bytes) {
if (bytesPerSec < 1) { // no throttle at all
return;
}
long currentTime = System.currentTimeMillis();
long timeDiff = currentTime - lastTime;
if (timeDiff == 0) {
timeDiff = 1;
}
double bytesPerSec = (bytes / (double) timeDiff) * 1000;
if (bytesPerSec > this.bytesPerSec) {
// Throttle
double exceededByFactorOf = bytesPerSec / this.bytesPerSec;
try {
long mustSleep = (long) ((exceededByFactorOf - 1) * timeDiff);
Thread.sleep(mustSleep);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
lastTime = System.currentTimeMillis();
}
/**
* Start writing to a block file If isRecovery is true and the block pre-exists, then we kill all
* volumeMap.put(b, v); volumeMap.put(b, v); other threads that might be writing to this block,
* and then reopen the file.
*/
public BlockWriteStreams writeToBlock(Block b, boolean isRecovery) throws IOException {
//
// Make sure the block isn't a valid one - we're still creating it!
//
if (isValidBlock(b)) {
if (!isRecovery) {
throw new BlockAlreadyExistsException(
"Block " + b + " is valid, and cannot be written to.");
}
// If the block was successfully finalized because all packets
// were successfully processed at the Datanode but the ack for
// some of the packets were not received by the client. The client
// re-opens the connection and retries sending those packets.
// The other reason is that an "append" is occurring to this block.
detachBlock(b, 1);
}
long blockSize = b.getNumBytes();
//
// Serialize access to /tmp, and check if file already there.
//
File f = null;
List<Thread> threads = null;
synchronized (this) {
//
// Is it already in the create process?
//
ActiveFile activeFile = ongoingCreates.get(b);
if (activeFile != null) {
f = activeFile.file;
threads = activeFile.threads;
if (!isRecovery) {
throw new BlockAlreadyExistsException(
"Block "
+ b
+ " has already been started (though not completed), and thus cannot be created.");
} else {
for (Thread thread : threads) {
thread.interrupt();
}
}
ongoingCreates.remove(b);
}
FSVolume v = null;
if (!isRecovery) {
v = volumes.getNextVolume(blockSize);
// create temporary file to hold block in the designated volume
f = createTmpFile(v, b);
volumeMap.put(b, new DatanodeBlockInfo(v));
} else if (f != null) {
DataNode.LOG.info("Reopen already-open Block for append " + b);
// create or reuse temporary file to hold block in the
// designated volume
v = volumeMap.get(b).getVolume();
volumeMap.put(b, new DatanodeBlockInfo(v));
} else {
// reopening block for appending to it.
DataNode.LOG.info("Reopen Block for append " + b);
v = volumeMap.get(b).getVolume();
f = createTmpFile(v, b);
File blkfile = getBlockFile(b);
File oldmeta = getMetaFile(b);
File newmeta = getMetaFile(f, b);
// rename meta file to tmp directory
DataNode.LOG.debug("Renaming " + oldmeta + " to " + newmeta);
if (!oldmeta.renameTo(newmeta)) {
throw new IOException(
"Block "
+ b
+ " reopen failed. "
+ " Unable to move meta file "
+ oldmeta
+ " to tmp dir "
+ newmeta);
}
// rename block file to tmp directory
DataNode.LOG.debug("Renaming " + blkfile + " to " + f);
if (!blkfile.renameTo(f)) {
if (!f.delete()) {
throw new IOException(
"Block " + b + " reopen failed. " + " Unable to remove file " + f);
}
if (!blkfile.renameTo(f)) {
throw new IOException(
"Block "
+ b
+ " reopen failed. "
+ " Unable to move block file "
+ blkfile
+ " to tmp dir "
+ f);
}
}
volumeMap.put(b, new DatanodeBlockInfo(v));
}
if (f == null) {
DataNode.LOG.warn("Block " + b + " reopen failed " + " Unable to locate tmp file.");
throw new IOException("Block " + b + " reopen failed " + " Unable to locate tmp file.");
}
ongoingCreates.put(b, new ActiveFile(f, threads));
}
try {
if (threads != null) {
for (Thread thread : threads) {
thread.join();
}
}
} catch (InterruptedException e) {
throw new IOException("Recovery waiting for thread interrupted.");
}
//
// Finally, allow a writer to the block file
// REMIND - mjc - make this a filter stream that enforces a max
// block size, so clients can't go crazy
//
File metafile = getMetaFile(f, b);
DataNode.LOG.debug("writeTo blockfile is " + f + " of size " + f.length());
DataNode.LOG.debug("writeTo metafile is " + metafile + " of size " + metafile.length());
return createBlockWriteStreams(f, metafile);
}
/**
* Try to update an old block to a new block. If there are ongoing create threads running for the
* old block, the threads will be returned without updating the block.
*
* @return ongoing create threads if there is any. Otherwise, return null.
*/
private synchronized List<Thread> tryUpdateBlock(Block oldblock, Block newblock)
throws IOException {
// check ongoing create threads
final ActiveFile activefile = ongoingCreates.get(oldblock);
if (activefile != null && !activefile.threads.isEmpty()) {
// remove dead threads
for (Iterator<Thread> i = activefile.threads.iterator(); i.hasNext(); ) {
final Thread t = i.next();
if (!t.isAlive()) {
i.remove();
}
}
// return living threads
if (!activefile.threads.isEmpty()) {
return new ArrayList<Thread>(activefile.threads);
}
}
// No ongoing create threads is alive. Update block.
File blockFile = findBlockFile(oldblock.getBlockId());
if (blockFile == null) {
throw new IOException("Block " + oldblock + " does not exist.");
}
File oldMetaFile = findMetaFile(blockFile);
long oldgs = parseGenerationStamp(blockFile, oldMetaFile);
// rename meta file to a tmp file
File tmpMetaFile =
new File(
oldMetaFile.getParent(),
oldMetaFile.getName() + "_tmp" + newblock.getGenerationStamp());
if (!oldMetaFile.renameTo(tmpMetaFile)) {
throw new IOException("Cannot rename block meta file to " + tmpMetaFile);
}
// update generation stamp
if (oldgs > newblock.getGenerationStamp()) {
throw new IOException(
"Cannot update block (id="
+ newblock.getBlockId()
+ ") generation stamp from "
+ oldgs
+ " to "
+ newblock.getGenerationStamp());
}
// update length
if (newblock.getNumBytes() > oldblock.getNumBytes()) {
throw new IOException(
"Cannot update block file (="
+ blockFile
+ ") length from "
+ oldblock.getNumBytes()
+ " to "
+ newblock.getNumBytes());
}
if (newblock.getNumBytes() < oldblock.getNumBytes()) {
truncateBlock(blockFile, tmpMetaFile, oldblock.getNumBytes(), newblock.getNumBytes());
}
// rename the tmp file to the new meta file (with new generation stamp)
File newMetaFile = getMetaFile(blockFile, newblock);
if (!tmpMetaFile.renameTo(newMetaFile)) {
throw new IOException("Cannot rename tmp meta file to " + newMetaFile);
}
updateBlockMap(ongoingCreates, oldblock, newblock);
updateBlockMap(volumeMap, oldblock, newblock);
// paranoia! verify that the contents of the stored block
// matches the block file on disk.
validateBlockMetadata(newblock);
return null;
}
/**
* Method to move files from HDFS to local filesystem
*
* <p>localPath: Path on the machines filesystem fs:FileSystem object from HDFS pathList:List of
* paths for files that might need to be backed up size:max size in bytes to be backed up
*
* <p>ReturnsDate of the last files backed up if reached size limit, else, zero
*/
public long backupFiles(
String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) {
Path fsPath;
long tmpSize = 0;
long tmpDate = 0;
// Start iterating over all paths
for (Path hdfsPath : pathList) {
try {
long nFileSize = fs.getContentSummary(hdfsPath).getLength();
tmpSize = tmpSize + nFileSize;
if ((tmpSize <= size) || (size == 0)) {
FileStatus stat = fs.getFileStatus(hdfsPath);
System.err.println(
"File "
+ hdfsPath.toUri().getPath()
+ " "
+ nFileSize
+ " bytes, "
+ "perms: "
+ stat.getOwner()
+ "/"
+ stat.getGroup()
+ ", "
+ stat.getPermission().toString());
tmpDate = stat.getModificationTime() / 1000;
String sFsPath = localPath + hdfsPath.toUri().getPath();
fsPath = new Path(sFsPath);
File f = new File(sFsPath);
// COMMENTED OUT: until a few backup cycles run
// and the mtime gets in fact set on all copied
// files.
//
// ignore it if the file exists and has the same mtime
// if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime())
// {
// System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs");
// continue;
// }
if (false == m_bDryRun) {
// check if we need to back up the local file
// (not directory), if it already exists.
if (f.exists() && f.isFile()) {
// ignore files with substrings in the
// no-preserve file
if (true == doPreserveFile(sFsPath)) {
// move it to the backup path
String sNewPath = preservePath + hdfsPath.toUri().getPath();
File newFile = new File(sNewPath);
// create directory structure for new file?
if (false == newFile.getParentFile().exists()) {
if (false == newFile.getParentFile().mkdirs()) {
System.err.println("Failed to mkdirs " + newFile.getParentFile().toString());
System.exit(1);
}
}
// rename existing file to new location
if (false == f.renameTo(newFile)) {
System.err.println(
"Failed to renameTo " + f.toString() + " to " + newFile.toString());
System.exit(1);
}
System.out.println("preserved " + f.toString() + " into " + newFile.toString());
} else {
System.out.println("skipped preservation of " + f.toString());
}
}
// copy from hdfs to local filesystem
fs.copyToLocalFile(hdfsPath, fsPath);
// set the mtime to match hdfs file
f.setLastModified(stat.getModificationTime());
// compare checksums on both files
compareChecksums(fs, hdfsPath, sFsPath);
}
// don't print the progress after every file -- go
// by at least 1% increments
long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes);
if (nPercentDone > m_nLastPercentBytesDone) {
System.out.println(
"progress: copied "
+ prettyPrintBytes(tmpSize)
+ ", "
+ nPercentDone
+ "% done"
+ ", tstamp="
+ tmpDate);
m_nLastPercentBytesDone = nPercentDone;
}
if (m_nSleepSeconds > 0) {
try {
Thread.sleep(1000 * m_nSleepSeconds);
} catch (Exception e2) {
// ignore
}
}
} else {
return tmpDate;
}
} catch (IOException e) {
System.err.println("FATAL ERROR: Something wrong with the file");
System.err.println(e);
System.out.println(tmpDate);
System.exit(1);
return 0;
}
}
return 0;
}
/** In case of interrupted, written file is not deleted. */
private void copyFile(File sourceFile, File destFile, Reporter reporter)
throws IOException, InterruptedException {
if (!destFile.exists()) {
destFile.createNewFile();
}
FileChannel source = null;
FileChannel destination = null;
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
FileInputStream iS = null;
FileOutputStream oS = null;
try {
iS = new FileInputStream(sourceFile);
oS = new FileOutputStream(destFile);
source = iS.getChannel();
destination = oS.getChannel();
long bytesSoFar = 0;
long reportingBytesSoFar = 0;
long size = source.size();
int transferred = 0;
while (bytesSoFar < size) {
// Needed to being able to be interrupted at any moment.
if (Thread.interrupted()) {
throw new InterruptedException();
}
// Casting to int here is safe since we will transfer at most "downloadBufferSize" bytes.
// This is done on purpose for being able to implement Throttling.
transferred = (int) destination.transferFrom(source, bytesSoFar, downloadBufferSize);
bytesSoFar += transferred;
reportingBytesSoFar += transferred;
throttler.incrementAndThrottle(transferred);
if (reportingBytesSoFar >= bytesToReportProgress) {
reporter.progress(reportingBytesSoFar);
reportingBytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(reportingBytesSoFar);
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
if (iS != null) {
iS.close();
}
if (oS != null) {
oS.close();
}
if (source != null) {
source.close();
}
if (destination != null) {
destination.close();
}
}
}
/*
* Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
* Interruptible.
*/
private File s3Fetch(URI uri, Reporter reporter) throws IOException, InterruptedException {
String bucketName = uri.getHost();
String path = uri.getPath();
UUID uniqueId = UUID.randomUUID();
File destFolder = new File(tempDir, uniqueId.toString() + "/" + path);
if (destFolder.exists()) {
FileUtils.deleteDirectory(destFolder);
}
destFolder.mkdirs();
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
boolean done = false;
try {
s3Service = new RestS3Service(getCredentials());
if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
}
if (path.startsWith("/")) {
path = path.substring(1, path.length());
}
for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
long bytesSoFar = 0;
String fileName = path;
if (path.contains("/")) {
fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
}
File fileDest = new File(destFolder, fileName);
log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");
if (fileDest.exists()) {
fileDest.delete();
}
object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
InputStream iS = object.getDataInputStream();
FileOutputStream writer = new FileOutputStream(fileDest);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
// Needed to being able to be interrupted at any moment.
if (Thread.interrupted()) {
iS.close();
writer.close();
cleanDirNoExceptions(destFolder);
throw new InterruptedException();
}
bytesSoFar += nRead;
writer.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
writer.close();
iS.close();
done = true;
}
if (!done) {
throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
}
} catch (S3ServiceException e) {
throw new IOException(e);
}
return destFolder;
}