public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
throws IOException {
super(file, fs.getFileStatus(file).getReplication());
this.datas = fs.getRawFileSystem().open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.getChecksumFile(file);
try {
int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize);
sums = fs.getRawFileSystem().open(sumFile, sumBufferSize);
byte[] version = new byte[CHECKSUM_VERSION.length];
sums.readFully(version);
if (!Arrays.equals(version, CHECKSUM_VERSION))
throw new IOException("Not a checksum file: " + sumFile);
this.bytesPerSum = sums.readInt();
set(fs.verifyChecksum, new CRC32(), bytesPerSum, 4);
} catch (FileNotFoundException e) { // quietly ignore
set(fs.verifyChecksum, null, 1, 0);
} catch (IOException e) { // loudly ignore
LOG.warn(
"Problem opening checksum file: "
+ file
+ ". Ignoring exception: "
+ StringUtils.stringifyException(e));
set(fs.verifyChecksum, null, 1, 0);
}
}
private InputStream OpenMultiplePartsWithOffset(FileSystem fs, Path pt, long offset)
throws IOException {
RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
while (rit.hasNext()) {
Path path = rit.next().getPath();
String filename =
path.toString().substring(path.getParent().toString().length(), path.toString().length());
if (filename.startsWith("/part-")) {
long filesize = fs.getFileStatus(path).getLen();
if (offset < filesize) {
FSDataInputStream handle = fs.open(path);
if (offset > 0) {
handle.seek(offset);
}
fileHandleList.add(handle);
}
offset -= filesize;
}
}
if (fileHandleList.size() == 1) return fileHandleList.get(0);
else if (fileHandleList.size() > 1) {
Enumeration<FSDataInputStream> enu = fileHandleList.elements();
return new SequenceInputStream(enu);
} else {
System.err.println("Error, no source file loaded. run genSeedDataset.sh fisrt!");
return null;
}
}
public void close() throws IOException {
datas.close();
if (sums != null) {
sums.close();
}
set(fs.verifyChecksum, null, 1, 0);
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
long sumsPos = getChecksumFilePos(targetPos);
fs.reportChecksumFailure(file, datas, targetPos, sums, sumsPos);
boolean newDataSource = datas.seekToNewSource(targetPos);
return sums.seekToNewSource(sumsPos) || newDataSource;
}
@Override
protected int readChunk(long pos, byte[] buf, int offset, int len, byte[] checksum)
throws IOException {
boolean eof = false;
if (needChecksum()) {
try {
long checksumPos = getChecksumFilePos(pos);
if (checksumPos != sums.getPos()) {
sums.seek(checksumPos);
}
sums.readFully(checksum);
} catch (EOFException e) {
eof = true;
}
len = bytesPerSum;
}
if (pos != datas.getPos()) {
datas.seek(pos);
}
int nread = readFully(datas, buf, offset, len);
if (eof && nread > 0) {
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return nread;
}
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
m_Sb.setLength(0);
m_Start = split.getStart();
m_End = m_Start + split.getLength();
final Path file = split.getPath();
compressionCodecs = new CompressionCodecFactory(job);
final CompressionCodec codec = compressionCodecs.getCodec(file);
// open the file and seek to the m_Start of the split
FileSystem fs = file.getFileSystem(job);
// getFileStatus fileStatus = fs.getFileStatus(split.getPath());
//noinspection deprecation
@SuppressWarnings(value = "deprecated")
long length = fs.getLength(file);
FSDataInputStream fileIn = fs.open(split.getPath());
if (m_Start > 0) fileIn.seek(m_Start);
if (codec != null) {
CompressionInputStream inputStream = codec.createInputStream(fileIn);
m_Input = new BufferedReader(new InputStreamReader(inputStream));
m_End = length;
} else {
m_Input = new BufferedReader(new InputStreamReader(fileIn));
}
m_Current = m_Start;
m_Key = split.getPath().getName();
}
/*
* Fetch a file that is in a Hadoop file system. Return a local File.
* Interruptible.
*/
private File hdfsFetch(Path fromPath, Reporter reporter)
throws IOException, InterruptedException {
UUID uniqueId = UUID.randomUUID();
File toFile = new File(tempDir, uniqueId.toString() + "/" + fromPath.getName());
File toDir = new File(toFile.getParent());
if (toDir.exists()) {
FileUtils.deleteDirectory(toDir);
}
toDir.mkdirs();
Path toPath = new Path(toFile.getCanonicalPath());
FileSystem fS = fromPath.getFileSystem(hadoopConf);
FileSystem tofS = FileSystem.getLocal(hadoopConf);
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
try {
for (FileStatus fStatus : fS.globStatus(fromPath)) {
log.info("Copying " + fStatus.getPath() + " to " + toPath);
long bytesSoFar = 0;
FSDataInputStream iS = fS.open(fStatus.getPath());
FSDataOutputStream oS = tofS.create(toPath);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
// Needed to being able to be interrupted at any moment.
if (Thread.interrupted()) {
iS.close();
oS.close();
cleanDirNoExceptions(toDir);
throw new InterruptedException();
}
bytesSoFar += nRead;
oS.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
oS.close();
iS.close();
}
return toDir;
} catch (ClosedByInterruptException e) {
// This can be thrown by the method read.
cleanDirNoExceptions(toDir);
throw new InterruptedIOException();
}
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
public static int readInt(Path path, Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), conf);
//FileSystem fs = path.getFileSystem(conf);
FSDataInputStream in = fs.open(path);
try {
return in.readInt();
} finally {
in.close();
}
}
@Override
protected int readChunk(long pos, byte[] buf, int offset, int len, byte[] checksum)
throws IOException {
boolean eof = false;
if (needChecksum()) {
assert checksum != null; // we have a checksum buffer
assert checksum.length % CHECKSUM_SIZE == 0; // it is sane length
assert len >= bytesPerSum; // we must read at least one chunk
final int checksumsToRead =
Math.min(
len / bytesPerSum, // number of checksums based on len to read
checksum.length / CHECKSUM_SIZE); // size of checksum buffer
long checksumPos = getChecksumFilePos(pos);
if (checksumPos != sums.getPos()) {
sums.seek(checksumPos);
}
int sumLenRead = sums.read(checksum, 0, CHECKSUM_SIZE * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % CHECKSUM_SIZE != 0) {
throw new ChecksumException(
"Checksum file not a length multiple of checksum size "
+ "in "
+ file
+ " at "
+ pos
+ " checksumpos: "
+ checksumPos
+ " sumLenread: "
+ sumLenRead,
pos);
}
if (sumLenRead <= 0) { // we're at the end of the file
eof = true;
} else {
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.min(len, bytesPerSum * (sumLenRead / CHECKSUM_SIZE));
}
}
if (pos != datas.getPos()) {
datas.seek(pos);
}
int nread = readFully(datas, buf, offset, len);
if (eof && nread > 0) {
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return nread;
}
/** position readable again. */
@Override
public void readFully(long pos, byte[] b, int offset, int length) throws IOException {
if (start + length + pos > end) {
throw new IOException("Not enough bytes to read.");
}
underLyingStream.readFully(pos + start, b, offset, length);
}
/**
* Seek to the given position in the stream. The next read() will be from that position.
*
* <p>This method does not allow seek past the end of the file. This produces IOException.
*
* @param pos the postion to seek to.
* @exception IOException if an I/O error occurs or seeks after EOF ChecksumException if the
* chunk to seek to is corrupted
*/
@Override
public synchronized void seek(long pos) throws IOException {
if (pos > getFileLength()) {
throw new IOException("Cannot seek after EOF");
}
super.seek(pos);
}
@Override
public synchronized int available() throws IOException {
long remaining = end - underLyingStream.getPos();
if (remaining > Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
return (int) remaining;
}
public void map(
Text key, LongWritable value, OutputCollector<K, LongWritable> collector, Reporter reporter)
throws IOException {
String name = key.toString();
long size = value.get();
long seed = Long.parseLong(name);
if (size == 0) return;
reporter.setStatus("opening " + name);
FSDataInputStream in = fs.open(new Path(DATA_DIR, name));
try {
for (int i = 0; i < SEEKS_PER_FILE; i++) {
// generate a random position
long position = Math.abs(random.nextLong()) % size;
// seek file to that position
reporter.setStatus("seeking " + name);
in.seek(position);
byte b = in.readByte();
// check that byte matches
byte checkByte = 0;
// advance random state to that position
random.setSeed(seed);
for (int p = 0; p <= position; p += check.length) {
reporter.setStatus("generating data for " + name);
if (fastCheck) {
checkByte = (byte) random.nextInt(Byte.MAX_VALUE);
} else {
random.nextBytes(check);
checkByte = check[(int) (position % check.length)];
}
}
assertEquals(b, checkByte);
}
} finally {
in.close();
}
}
@Override
public synchronized int read(byte[] b, int offset, int len) throws IOException {
int newlen = len;
int ret = -1;
if (position + len > end) {
newlen = (int) (end - position);
}
// end case
if (newlen == 0) return ret;
ret = underLyingStream.read(b, offset, newlen);
position += ret;
return ret;
}
/** implementing position readable. */
@Override
public int read(long pos, byte[] b, int offset, int length) throws IOException {
int nlength = length;
if (start + nlength + pos > end) {
// length corrected to the real remaining length:
nlength = (int) (end - start - pos);
}
if (nlength <= 0) {
// EOS:
return -1;
}
return underLyingStream.read(pos + start, b, offset, nlength);
}
HarFsInputStream(FileSystem fs, Path path, long start, long length, int bufferSize)
throws IOException {
if (length < 0) {
throw new IllegalArgumentException("Negative length [" + length + "]");
}
underLyingStream = fs.open(path, bufferSize);
underLyingStream.seek(start);
// the start of this file in the part file
this.start = start;
// the position pointer in the part file
this.position = start;
// the end pointer in the part file
this.end = start + length;
}
public BufferedReader loadDataFromFile(String filepath, long offset) {
try {
Path pt = new Path(filepath);
FileSystem fs = FileSystem.get(fsConf);
InputStreamReader isr;
if (fs.isDirectory(pt)) { // multiple parts
isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
} else { // single file
FSDataInputStream fileHandler = fs.open(pt);
if (offset > 0) fileHandler.seek(offset);
isr = new InputStreamReader(fileHandler);
}
BufferedReader reader = new BufferedReader(isr);
if (offset > 0) reader.readLine(); // skip first line in case of seek
return reader;
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
assert false : "Should not reach here!";
return null;
}
@Override
public synchronized long skip(long n) throws IOException {
long tmpN = n;
if (tmpN > 0) {
final long actualRemaining = end - position;
if (tmpN > actualRemaining) {
tmpN = actualRemaining;
}
underLyingStream.seek(tmpN + position);
position += tmpN;
return tmpN;
}
// NB: the contract is described in java.io.InputStream.skip(long):
// this method returns the number of bytes actually skipped, so,
// the return value should never be negative.
return 0;
}
@Override
public void seek(long p) throws IOException {
stream.seek(p);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
return stream.read(b, off, len);
}
private void parseMetaData() throws IOException {
Text line = new Text();
long read;
FSDataInputStream in = null;
LineReader lin = null;
try {
in = fs.open(masterIndexPath);
FileStatus masterStat = fs.getFileStatus(masterIndexPath);
masterIndexTimestamp = masterStat.getModificationTime();
lin = new LineReader(in, getConf());
read = lin.readLine(line);
// the first line contains the version of the index file
String versionLine = line.toString();
String[] arr = versionLine.split(" ");
version = Integer.parseInt(arr[0]);
// make it always backwards-compatible
if (this.version > HarFileSystem.VERSION) {
throw new IOException(
"Invalid version " + this.version + " expected " + HarFileSystem.VERSION);
}
// each line contains a hashcode range and the index file name
String[] readStr;
while (read < masterStat.getLen()) {
int b = lin.readLine(line);
read += b;
readStr = line.toString().split(" ");
int startHash = Integer.parseInt(readStr[0]);
int endHash = Integer.parseInt(readStr[1]);
stores.add(
new Store(
Long.parseLong(readStr[2]), Long.parseLong(readStr[3]), startHash, endHash));
line.clear();
}
} catch (IOException ioe) {
LOG.warn("Encountered exception ", ioe);
throw ioe;
} finally {
IOUtils.cleanup(LOG, lin, in);
}
FSDataInputStream aIn = fs.open(archiveIndexPath);
try {
FileStatus archiveStat = fs.getFileStatus(archiveIndexPath);
archiveIndexTimestamp = archiveStat.getModificationTime();
LineReader aLin;
// now start reading the real index file
for (Store s : stores) {
read = 0;
aIn.seek(s.begin);
aLin = new LineReader(aIn, getConf());
while (read + s.begin < s.end) {
int tmp = aLin.readLine(line);
read += tmp;
String lineFeed = line.toString();
String[] parsed = lineFeed.split(" ");
parsed[0] = decodeFileName(parsed[0]);
archive.put(new Path(parsed[0]), new HarStatus(lineFeed));
line.clear();
}
}
} finally {
IOUtils.cleanup(LOG, aIn);
}
}
public void map(Text key, Text value, Context context)
throws InterruptedException, IOException {
String filename = key.toString();
String json = value.toString();
// Make sure the input is valid
if (!(filename.isEmpty() || json.isEmpty())) {
// Change the json-type feature to Mat-type feature
Mat descriptor = json2mat(json);
if (descriptor != null) {
// Read the query feature from the cache in Hadoop
Mat query_features;
String pathStr = context.getConfiguration().get("featureFilePath");
FileSystem fs = FileSystem.get(context.getConfiguration());
FSDataInputStream fsDataInputStream = fs.open(new Path(pathStr));
StringBuilder sb = new StringBuilder();
// Use a buffer to read the query_feature
int remain = fsDataInputStream.available();
while (remain > 0) {
int read;
byte[] buf = new byte[BUF_SIZE];
read = fsDataInputStream.read(buf, fsDataInputStream.available() - remain, BUF_SIZE);
sb.append(new String(buf, 0, read, StandardCharsets.UTF_8));
remain = remain - read;
System.out.println("remain:" + remain + "\tread:" + read + "\tsb.size:" + sb.length());
}
// Read the query_feature line by line
// Scanner sc = new Scanner(fsDataInputStream, "UTF-8");
// StringBuilder sb = new StringBuilder();
// while (sc.hasNextLine()) {
// sb.append(sc.nextLine());
// }
// String query_json = sb.toString();
// String query_json = new String(buf, StandardCharsets.UTF_8);
String query_json = sb.toString();
fsDataInputStream.close();
query_features = json2mat(query_json);
// Get the similarity of the current database image against the query image
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);
MatOfDMatch matches = new MatOfDMatch();
// Ensure the two features have same length of cols (the feature extracted are all 128
// cols(at least in this case))
if (query_features.cols() == descriptor.cols()) {
matcher.match(query_features, descriptor, matches);
DMatch[] dMatches = matches.toArray();
// Calculate the max/min distances
// double max_dist = Double.MAX_VALUE;
// double min_dist = Double.MIN_VALUE;
double max_dist = 0;
double min_dist = 100;
for (int i = 0; i < dMatches.length; i++) {
double dist = dMatches[i].distance;
if (min_dist > dist) min_dist = dist;
if (max_dist < dist) max_dist = dist;
}
// Only distances ≤ threshold are good matches
double threshold = max_dist * THRESHOLD_FACTOR;
// double threshold = min_dist * 2;
LinkedList<DMatch> goodMatches = new LinkedList<DMatch>();
for (int i = 0; i < dMatches.length; i++) {
if (dMatches[i].distance <= threshold) {
goodMatches.addLast(dMatches[i]);
}
}
// Get the ratio of good_matches to all_matches
double ratio = (double) goodMatches.size() / (double) dMatches.length;
System.out.println("*** current_record_filename:" + filename + " ***");
System.out.println("feature:" + descriptor + "\nquery_feature:" + query_features);
System.out.println(
"min_dist of keypoints:" + min_dist + " max_dist of keypoints:" + max_dist);
System.out.println(
"total_matches:" + dMatches.length + "\tgood_matches:" + goodMatches.size());
// System.out.println("type:" + descriptor.type() + " channels:" +
// descriptor.channels() + " rows:" + descriptor.rows() + " cols:" + descriptor.cols());
// System.out.println("qtype:" + query_features.type() + "
// qchannels:" + query_features.channels() + " qrows:" + query_features.rows() + "
// qcols:" + query_features.cols());
System.out.println();
if (ratio > PERCENTAGE_THRESHOLD) {
// Key:1 Value:filename|ratio
context.write(ONE, new Text(filename + "|" + ratio));
// context.write(ONE, new Text(filename + "|" +
// String.valueOf(goodMatches.size())));
}
} else {
System.out.println("The size of the features are not equal");
}
} else {
// a null pointer, do nothing
System.out.println("A broken/null feature:" + filename);
System.out.println();
}
}
}
@Override
public synchronized void close() throws IOException {
underLyingStream.close();
super.close();
}
public int available() throws IOException {
return datas.available() + super.available();
}
@Override
public synchronized void seek(final long pos) throws IOException {
validatePosition(pos);
position = start + pos;
underLyingStream.seek(position);
}
/**
* Copy a file to a destination.
*
* @param srcstat src path and metadata
* @param dstpath dst path
* @param reporter
*/
private void copy(
FileStatus srcstat,
Path relativedst,
OutputCollector<WritableComparable<?>, Text> outc,
Reporter reporter)
throws IOException {
Path absdst = new Path(destPath, relativedst);
int totfiles = job.getInt(SRC_COUNT_LABEL, -1);
assert totfiles >= 0 : "Invalid file count " + totfiles;
// if a directory, ensure created even if empty
if (srcstat.isDir()) {
if (destFileSys.exists(absdst)) {
if (!destFileSys.getFileStatus(absdst).isDir()) {
throw new IOException("Failed to mkdirs: " + absdst + " is a file.");
}
} else if (!destFileSys.mkdirs(absdst)) {
throw new IOException("Failed to mkdirs " + absdst);
}
// TODO: when modification times can be set, directories should be
// emitted to reducers so they might be preserved. Also, mkdirs does
// not currently return an error when the directory already exists;
// if this changes, all directory work might as well be done in reduce
return;
}
if (destFileSys.exists(absdst) && !overwrite && !needsUpdate(srcstat, destFileSys, absdst)) {
outc.collect(null, new Text("SKIP: " + srcstat.getPath()));
++skipcount;
reporter.incrCounter(Counter.SKIP, 1);
updateStatus(reporter);
return;
}
Path tmpfile = new Path(job.get(TMP_DIR_LABEL), relativedst);
long cbcopied = 0L;
FSDataInputStream in = null;
FSDataOutputStream out = null;
try {
// open src file
try {
in = srcstat.getPath().getFileSystem(job).open(srcstat.getPath());
} catch (IOException e) {
LOG.error("Failed to open src file " + srcstat.getPath() + ", ignore and return");
in = null;
return;
}
reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
// open tmp file
out = create(tmpfile, reporter, srcstat);
// copy file
for (int cbread; (cbread = in.read(buffer)) >= 0; ) {
out.write(buffer, 0, cbread);
cbcopied += cbread;
reporter.setStatus(
String.format("%.2f ", cbcopied * 100.0 / srcstat.getLen())
+ absdst
+ " [ "
+ StringUtils.humanReadableInt(cbcopied)
+ " / "
+ StringUtils.humanReadableInt(srcstat.getLen())
+ " ]");
}
} finally {
checkAndClose(in);
checkAndClose(out);
}
if (cbcopied != srcstat.getLen()) {
if (srcstat.getLen() == 0 && cbcopied > 0) {
LOG.info("most likely see a WAL file corruption: " + srcstat.getPath());
} else {
throw new IOException(
"File size not matched: copied "
+ bytesString(cbcopied)
+ " to tmpfile (="
+ tmpfile
+ ") but expected "
+ bytesString(srcstat.getLen())
+ " from "
+ srcstat.getPath());
}
} else {
if (totfiles == 1) {
// Copying a single file; use dst path provided by user as destination
// rather than destination directory, if a file
Path dstparent = absdst.getParent();
if (!(destFileSys.exists(dstparent) && destFileSys.getFileStatus(dstparent).isDir())) {
absdst = dstparent;
}
}
if (destFileSys.exists(absdst) && destFileSys.getFileStatus(absdst).isDir()) {
throw new IOException(absdst + " is a directory");
}
if (!destFileSys.mkdirs(absdst.getParent())) {
throw new IOException("Failed to craete parent dir: " + absdst.getParent());
}
rename(tmpfile, absdst);
FileStatus dststat = destFileSys.getFileStatus(absdst);
if (dststat.getLen() != srcstat.getLen()) {
destFileSys.delete(absdst, false);
throw new IOException(
"File size not matched: copied "
+ bytesString(dststat.getLen())
+ " to dst (="
+ absdst
+ ") but expected "
+ bytesString(srcstat.getLen())
+ " from "
+ srcstat.getPath());
}
updatePermissions(srcstat, dststat);
}
// report at least once for each file
++copycount;
reporter.incrCounter(Counter.BYTESCOPIED, cbcopied);
reporter.incrCounter(Counter.COPY, 1);
updateStatus(reporter);
}
@Override
public long tell() throws IOException {
return stream.getPos();
}
@Override
public void close() throws IOException {
stream.close();
}
@Override
protected InputStream open(long offset) throws IOException {
FSDataInputStream is = _fs.open(_path);
is.seek(offset);
return is;
}