/**
* @param schema a MIME compliant non-localizable identifier, that matches file categories (and
* XSD schema names).
* @param descriptionKey a media identifier that can be used to retreive a localizable descriptive
* text.
* @param extensions a list of all file extensions of this type. Must be all lowercase. If null,
* this matches any file.
*/
public MediaType(String schema, String descriptionKey, String[] extensions) {
if (schema == null) {
throw new NullPointerException("schema must not be null");
}
this.schema = schema;
this.descriptionKey = descriptionKey;
if (extensions == null) {
this.exts = Collections.emptySet();
this.extsArray = new String[0];
} else {
Set<String> set = new TreeSet<String>(Comparators.caseInsensitiveStringComparator());
set.addAll(Arrays.asList(extensions));
this.exts = set;
this.extsArray = exts.toArray(new String[0]);
}
}
/**
* A repository of temporary filenames. Gives out file names for temporary files, ensuring that two
* duplicate files always get the same name. This enables smart resumes across hosts. Also keeps
* track of the blocks downloaded, for smart downloading purposes. <b>Thread safe.</b>
*
* <p>
*/
@Singleton
public class IncompleteFileManager {
/**
* The delimiter to use between the size and a real name of a temporary file. To make it easier to
* break the temporary name into its constituent parts, this should not contain a number.
*/
static final String SEPARATOR = "-";
/** The prefix added to preview copies of incomplete files. */
public static final String PREVIEW_PREFIX = "Preview-";
/**
* A mapping from incomplete files (File) to the blocks of the file stored on disk
* (VerifyingFile). Needed for resumptive smart downloads. INVARIANT: all blocks disjoint, no two
* intervals can be coalesced into one interval. Note that blocks are not sorted; there are
* typically few blocks so performance isn't an issue.
*/
private final Map<File, VerifyingFile> blocks =
new TreeMap<File, VerifyingFile>(Comparators.fileComparator());
/**
* Bijection between sha1 hashes (URN) and incomplete files (File). This is used to ensure that
* any two RemoteFileDesc with the same hash get the same incomplete file, regardless of name. The
* inverse of this map is used to get the hash of an incomplete file for query-by-hash and
* resuming. Note that the hash is that of the desired completed file, not that of the incomplete
* file.
*
* <p>Entries are added to hashes before the temp file is actually created on disk. For this
* reason, there can be files in the value set of hashes that are not in the key set of blocks.
* These entries are not serialized to disk in the downloads.dat file. Similarly there may be
* files in the key set of blocks that are not in the value set of hashes. This happens if we
* received RemoteFileDesc's without hashes, or when loading old downloads.dat files without hash
* info.
*
* <p>INVARIANT: the range (value set) of hashes contains no duplicates. INVARIANT: for all keys k
* in hashes, k.isSHA1()
*/
private final Map<URN, File> hashes = new HashMap<URN, File>();
private final Provider<FileManager> fileManager;
private final Provider<HashTreeCache> tigerTreeCache;
private final VerifyingFileFactory verifyingFileFactory;
@Inject
public IncompleteFileManager(
Provider<FileManager> fileManager,
Provider<HashTreeCache> tigerTreeCache,
VerifyingFileFactory verifyingFileFactory) {
this.fileManager = fileManager;
this.tigerTreeCache = tigerTreeCache;
this.verifyingFileFactory = verifyingFileFactory;
}
/**
* Removes entries in this for which there is no file on disk.
*
* @return true iff any entries were purged
*/
public synchronized boolean purge() {
boolean ret = false;
// Remove any blocks for which the file doesn't exist.
for (Iterator<File> iter = blocks.keySet().iterator(); iter.hasNext(); ) {
File file = iter.next();
if (!file.exists()) {
ret = true;
fileManager.get().getManagedFileList().remove(file);
file.delete(); // always safe to call; return value ignored
iter.remove();
}
}
return ret;
}
/**
* Deletes incomplete files more than INCOMPLETE_PURGE_TIME days old from disk Then removes
* entries in this for which there is no file on disk.
*
* @param activeFiles which files are currently being downloaded.
* @return true iff any entries were purged
*/
public synchronized boolean initialPurge(Collection<File> activeFiles) {
// Remove any files that are old.
boolean ret = false;
for (Iterator<File> iter = blocks.keySet().iterator(); iter.hasNext(); ) {
File file = iter.next();
try {
file = FileUtils.getCanonicalFile(file);
} catch (IOException iox) {
file = file.getAbsoluteFile();
}
if (!file.exists() || (isOld(file) && !activeFiles.contains(file))) {
ret = true;
fileManager.get().getManagedFileList().remove(file);
file.delete();
iter.remove();
}
}
for (Iterator<File> iter = hashes.values().iterator(); iter.hasNext(); ) {
File file = iter.next();
if (!file.exists()) {
iter.remove();
ret = true;
}
}
return ret;
}
/** Returns true iff file is "too old". */
private static final boolean isOld(File file) {
// Inlining this method allows some optimizations--not that they matter.
long days = SharingSettings.INCOMPLETE_PURGE_TIME.getValue();
// Back up a couple days.
// 24 hour/day * 60 min/hour * 60 sec/min * 1000 msec/sec
long purgeTime = System.currentTimeMillis() - days * 24l * 60l * 60l * 1000l;
return file.lastModified() < purgeTime;
}
/*
* Returns true if both rfd "have the same content". Currently
* rfd1~=rfd2 iff either of the following conditions hold:
*
* <ul>
* <li>Both files have the same hash, i.e.,
* rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn(). Note that this (almost)
* always means that rfd1.getSize()==rfd2.getSize(), though rfd1 and
* rfd2 may have different names.
* <li>Both files have the same name and size and don't have conflicting
* hashes, i.e., rfd1.getName().equals(rfd2.getName()) &&
* rfd1.getSize()==rfd2.getSize() && (rfd1.getSHA1Urn()==null ||
* rfd2.getSHA1Urn()==null ||
* rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn())).
* </ul>
* Note that the second condition allows risky resumes, i.e., resumes when
* one (or both) of the files doesn't have a hash.
*
* @see getFile
*/
static boolean same(RemoteFileDesc rfd1, RemoteFileDesc rfd2) {
return same(
rfd1.getFileName(),
rfd1.getSize(),
rfd1.getSHA1Urn(),
rfd2.getFileName(),
rfd2.getSize(),
rfd2.getSHA1Urn());
}
/** @see similar(RemoteFileDesc, RemoteFileDesc) */
static boolean same(String name1, long size1, URN hash1, String name2, long size2, URN hash2) {
// Either they have the same hashes...
if (hash1 != null && hash2 != null) return hash1.equals(hash2);
// ..or same name and size and no conflicting hashes.
else return size1 == size2 && name1.equals(name2);
}
/**
* Canonicalization is not as important on windows, and is causing problems. Therefore, don't do
* it.
*/
private static File canonicalize(File f) throws IOException {
f = f.getAbsoluteFile();
if (OSUtils.isWindows()) return f;
else return f.getCanonicalFile();
}
/**
* Same as getFile(String, urn, int), except taking the values from the RFD. getFile(rfd) ==
* getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize());
*/
public synchronized File getFile(RemoteFileDesc rfd) throws IOException {
return getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize());
}
/**
* Stub for calling getFile(String, URN, int, SharingSettings.INCOMPLETE_DIRECTORY.getValue());
*/
public synchronized File getFile(String name, URN sha1, long size) throws IOException {
return getFile(name, sha1, size, SharingSettings.INCOMPLETE_DIRECTORY.get());
}
/**
* Returns the fully-qualified temporary download file for the given file/location pair. If an
* incomplete file already exists for this URN, that file is returned. Otherwise, the location of
* the file is determined by the "incDir" variable. For example, getFile("test.txt", 1999) may
* return "C:\Program Files\LimeWire\Incomplete\T-1999-Test.txt" if "incDir" is "C:\Program
* Files\LimeWire\Incomplete". The disk is not modified, except for the file possibly being
* created.
*
* <p>This method gives duplicate files the same temporary file, which is critical for resume and
* swarmed downloads. That is, for all rfd_i and rfd_j
*
* <pre>
* similar(rfd_i, rfd_j) <==> getFile(rfd_i).equals(getFile(rfd_j))<p>
* </pre>
*
* It is imperative that the files are compared as in their canonical formats to preserve the
* integrity of the filesystem. Otherwise, multiple downloads could be downloading to "FILE A",
* and "file a", although only "file a" exists on disk and is being written to by both.
*
* @throws IOException if there was an IOError while determining the file's name.
*/
public synchronized File getFile(String name, URN sha1, long size, File incDir)
throws IOException {
boolean dirsMade = false;
File baseFile = null;
File canonFile = null;
// make sure its created.. (the user might have deleted it)
dirsMade = incDir.mkdirs();
String convertedName = CommonUtils.convertFileName(name);
try {
if (sha1 != null) {
File file = hashes.get(sha1);
if (file != null) {
// File already allocated for hash
return file;
} else {
// Allocate unique file for hash. By "unique" we mean not in
// the value set of HASHES. Because we allow risky resumes,
// there's no need to look at BLOCKS as well...
for (int i = 1; ; i++) {
file = new File(incDir, tempName(convertedName, size, i));
baseFile = file;
file = canonicalize(file);
canonFile = file;
if (!hashes.values().contains(file)) break;
}
// ...and record the hash for later.
hashes.put(sha1, file);
// ...and make sure the file exists on disk, so that
// future File.getCanonicalFile calls will match this
// file. This was a problem on OSX, where
// File("myfile") and File("MYFILE") aren't equal,
// but File("myfile").getCanonicalFile() will only return
// a File("MYFILE") if that already existed on disk.
// This means that in order for the canonical-checking
// within this class to work, the file must exist on disk.
FileUtils.touch(file);
return file;
}
} else {
// No hash.
File f = new File(incDir, tempName(convertedName, size, 0));
baseFile = f;
f = canonicalize(f);
canonFile = f;
return f;
}
} catch (IOException ioe) {
IOException ioe2 =
new IOException(
"dirsMade: "
+ dirsMade
+ "\ndirExist: "
+ incDir.exists()
+ "\nbaseFile: "
+ baseFile
+ "\ncannFile: "
+ canonFile);
ioe2.initCause(ioe);
throw ioe2;
}
}
/**
* Returns the file associated with the specified URN. If no file matches, returns null.
*
* @return the file associated with the URN, or null if none.
*/
public synchronized File getFileForUrn(URN urn) {
if (urn == null) throw new NullPointerException("null urn");
return hashes.get(urn);
}
/**
* Returns the unqualified file name for a file with the given name and size, with an optional
* suffix to make it unique.
*
* @param count a suffix to attach before the file extension in parens before the file extension,
* or 1 for none.
*/
private static String tempName(String filename, long size, int suffix) {
if (suffix <= 1) {
// a) No suffix
return "T-" + size + "-" + filename;
}
int i = filename.lastIndexOf('.');
if (i < 0) {
// b) Suffix, no extension
return "T-" + size + "-" + filename + " (" + suffix + ")";
} else {
// c) Suffix, file extension
String noExtension = filename.substring(0, i);
String extension = filename.substring(i); // e.g., ".txt"
return "T-" + size + "-" + noExtension + " (" + suffix + ")" + extension;
}
}
/**
* Removes the block and hash information for the given incomplete file. Typically this is called
* after incompleteFile has been deleted.
*
* @param incompleteFile a temporary file returned by getFile
*/
public synchronized void removeEntry(File incompleteFile) {
// Remove downloaded blocks.
blocks.remove(incompleteFile);
// Remove any key k from hashes for which hashes[k]=incompleteFile.
// There should be at most one value of k.
for (Iterator<Map.Entry<URN, File>> iter = hashes.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<URN, File> entry = iter.next();
if (incompleteFile.equals(entry.getValue())) iter.remove();
}
// Remove the entry from FileManager
fileManager.get().getManagedFileList().remove(incompleteFile);
}
/** Initializes entries with URNs, Files & Ranges. */
public synchronized void initEntry(
File incompleteFile, List<Range> ranges, URN sha1, boolean publish)
throws InvalidDataException {
try {
incompleteFile = canonicalize(incompleteFile);
} catch (IOException iox) {
throw new InvalidDataException(iox);
}
VerifyingFile verifyingFile;
try {
verifyingFile = verifyingFileFactory.createVerifyingFile(getCompletedSize(incompleteFile));
} catch (IllegalArgumentException iae) {
throw new InvalidDataException(iae);
}
if (ranges != null) {
for (Range range : ranges) {
verifyingFile.addInterval(range);
}
}
if (ranges == null || ranges.isEmpty()) {
try {
verifyingFile.setScanForExistingBlocks(true, incompleteFile.length());
} catch (IOException iox) {
throw new InvalidDataException(iox);
}
}
blocks.put(incompleteFile, verifyingFile);
if (sha1 != null) hashes.put(sha1, incompleteFile);
if (publish) registerIncompleteFile(incompleteFile);
}
/**
* Associates the incompleteFile with the VerifyingFile vf. Notifies FileManager about a new
* Incomplete File.
*/
public synchronized void addEntry(File incompleteFile, VerifyingFile vf, boolean publish) {
// We must canonicalize the file.
try {
incompleteFile = canonicalize(incompleteFile);
} catch (IOException ignored) {
}
blocks.put(incompleteFile, vf);
if (publish) registerIncompleteFile(incompleteFile);
}
public synchronized void addTorrentEntry(URN urn) {
String torrentDirPath =
SharingSettings.INCOMPLETE_DIRECTORY.get().getAbsolutePath()
+ File.separator
+ Base32.encode(urn.getBytes());
File torrentDir = new File(torrentDirPath);
hashes.put(urn, torrentDir);
}
public synchronized void removeTorrentEntry(URN urn) {
hashes.remove(urn);
}
public synchronized VerifyingFile getEntry(File incompleteFile) {
return blocks.get(incompleteFile);
}
public synchronized long getBlockSize(File incompleteFile) {
VerifyingFile vf = blocks.get(incompleteFile);
if (vf == null) return 0;
else return vf.getBlockSize();
}
/** Notifies file manager about all incomplete files. */
public synchronized void registerAllIncompleteFiles() {
for (File file : blocks.keySet()) {
if (file.exists() && !isOld(file)) registerIncompleteFile(file);
}
}
/** Notifies file manager about a single incomplete file. */
private synchronized void registerIncompleteFile(File incompleteFile) {
// Only register if it has a SHA1 -- otherwise we can't share.
Set<URN> completeHashes = getAllCompletedHashes(incompleteFile);
if (completeHashes.size() == 0) return;
fileManager
.get()
.getIncompleteFileList()
.addIncompleteFile(
incompleteFile,
completeHashes,
getCompletedName(incompleteFile),
getCompletedSize(incompleteFile),
getEntry(incompleteFile));
}
/**
* Returns the name of the complete file associated with the given incomplete file, i.e., what
* incompleteFile will be renamed to when the download completes (without path information). Slow;
* runs in linear time with respect to the number of hashes in this.
*
* @param incompleteFile a file returned by getFile
* @return the complete file name, without path
* @exception IllegalArgumentException incompleteFile was not the return value from getFile
*/
public static String getCompletedName(File incompleteFile) throws IllegalArgumentException {
String torrent = getCompletedTorrentName(incompleteFile);
if (torrent != null) return torrent;
// Given T-<size>-<name> return <name>.
// i j
// This is not as strict as it could be. TODO: what about (x) suffix?
String name = incompleteFile.getName();
int i = name.indexOf(SEPARATOR);
if (i < 0) throw new IllegalArgumentException("Missing separator: " + name);
int j = name.indexOf(SEPARATOR, i + 1);
if (j < 0) throw new IllegalArgumentException("Missing separator: " + name);
if (j == name.length() - 1) throw new IllegalArgumentException("No name after last separator");
return name.substring(j + 1);
}
private static String getCompletedTorrentName(File incompleteDir) {
if (!isTorrentFolder(incompleteDir)) return null;
File[] list = incompleteDir.listFiles();
if (list[0].getName().startsWith(".dat")) return list[1].getName();
else return list[0].getName();
}
public static boolean isTorrentFolder(File file) {
if (!file.isDirectory() || file.getName().length() != 32) return false;
File[] files = file.listFiles();
if (files.length != 2) return false;
File datFile = files[0];
File otherFile = files[1];
if (!datFile.getName().startsWith(".dat")) {
datFile = files[1];
otherFile = files[0];
}
if (!datFile.getName().startsWith(".dat")) return false;
return datFile.getName().equals(".dat" + otherFile.getName());
}
/**
* Returns the size of the complete file associated with the given incomplete file, i.e., the
* number of bytes in the file when the download completes.
*
* @param incompleteFile a file returned by getFile
* @return the complete file size
* @exception IllegalArgumentException incompleteFile was not returned by getFile
*/
public static long getCompletedSize(File incompleteFile) throws IllegalArgumentException {
// Given T-<size>-<name>, return <size>.
// i j
String name = incompleteFile.getName();
int i = name.indexOf(SEPARATOR);
if (i < 0) throw new IllegalArgumentException("Missing separator: " + name);
int j = name.indexOf(SEPARATOR, i + 1);
if (j < 0) throw new IllegalArgumentException("Missing separator: " + name);
try {
return Long.parseLong(name.substring(i + 1, j));
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Bad number format: " + name);
}
}
/**
* Returns the hash of the complete file associated with the given incomplete file, i.e., the hash
* of incompleteFile when the download is complete.
*
* @param incompleteFile a file returned by getFile
* @return a SHA1 hash, or null if unknown
*/
public synchronized URN getCompletedHash(File incompleteFile) {
// Return a key k s.t., hashes.get(k)==incompleteFile...
for (Map.Entry<URN, File> entry : hashes.entrySet()) {
if (incompleteFile.equals(entry.getValue()) && entry.getKey().isSHA1()) return entry.getKey();
}
return null; // ...or null if no such k.
}
/**
* Returns any known hashes of the complete file associated with the given incomplete file, i.e.,
* the hashes of incompleteFile when the download is complete.
*
* @param incompleteFile a file returned by getFile
* @return a set of known hashes
*/
public synchronized Set<URN> getAllCompletedHashes(File incompleteFile) {
Set<URN> urns = new UrnSet();
// Return a set S s.t. for each K in S, hashes.get(k)==incpleteFile
for (Map.Entry<URN, File> entry : hashes.entrySet()) {
if (incompleteFile.equals(entry.getValue())) {
urns.add(entry.getKey());
URN ttroot = tigerTreeCache.get().getHashTreeRootForSha1(entry.getKey());
if (ttroot != null) urns.add(ttroot);
}
}
return urns;
}
@Override
public synchronized String toString() {
StringBuilder buf = new StringBuilder();
buf.append("{");
boolean first = true;
for (File file : blocks.keySet()) {
if (!first) buf.append(", ");
List<Range> intervals = blocks.get(file).getVerifiedBlocksAsList();
buf.append(file);
buf.append(":");
buf.append(intervals.toString());
first = false;
}
buf.append("}");
return buf.toString();
}
public synchronized String dumpHashes() {
return hashes.toString();
}
public Collection<File> getUnregisteredIncompleteFilesInDirectory(File value) {
if (value == null) {
return Collections.emptyList();
}
File[] files =
value.listFiles(
new FileFilter() {
@Override
public boolean accept(File incompleteFile) {
if (!incompleteFile.isFile()) {
return false;
}
String name = incompleteFile.getName();
int i = name.indexOf(SEPARATOR);
if (i < 0 || i == name.length() - 1) {
return false;
}
int j = name.indexOf(SEPARATOR, i + 1);
if (j < 0 || j == name.length() - 1) {
return false;
}
try {
Long.parseLong(name.substring(i + 1, j));
} catch (NumberFormatException e) {
return false;
}
synchronized (IncompleteFileManager.this) {
return !blocks.containsKey(FileUtils.canonicalize(incompleteFile));
}
}
});
if (files == null) {
return Collections.emptyList();
} else {
return Arrays.asList(files);
}
}
}
