private static void invalidateIndex() {
LOG.info("Marking VFS as corrupted");
final File indexRoot = PathManager.getIndexRoot();
if (indexRoot.exists()) {
final String[] children = indexRoot.list();
if (children != null && children.length > 0) {
// create index corruption marker only if index directory exists and is non-empty
// It is incorrect to consider non-existing indices "corrupted"
FileUtil.createIfDoesntExist(new File(PathManager.getIndexRoot(), "corruption.marker"));
}
}
}
private void reportIOErrorWithJars(File original, File target, IOException e) {
LOG.warn(e);
String path = original.getPath();
myFileSystem.setNoCopyJarForPath(path);
String message =
VfsBundle.message("jar.copy.error.message", path, target.getPath(), e.getMessage());
ERROR_COPY_NOTIFICATION
.getValue()
.createNotification(message, NotificationType.ERROR)
.notify(null);
}
private synchronized void save() {
myDir.mkdirs();
Properties props = new Properties();
props.setProperty(KIND_KEY, myKind.toString());
props.setProperty(ID_KEY, myRepositoryId);
props.setProperty(PATH_OR_URL_KEY, myRepositoryPathOrUrl);
props.setProperty(INDEX_VERSION_KEY, CURRENT_VERSION);
if (myUpdateTimestamp != null)
props.setProperty(TIMESTAMP_KEY, String.valueOf(myUpdateTimestamp));
if (myDataDirName != null) props.setProperty(DATA_DIR_NAME_KEY, myDataDirName);
if (myFailureMessage != null) props.setProperty(FAILURE_MESSAGE_KEY, myFailureMessage);
try {
FileOutputStream s = new FileOutputStream(new File(myDir, INDEX_INFO_FILE));
try {
props.store(s, null);
} finally {
s.close();
}
} catch (IOException e) {
MavenLog.LOG.warn(e);
}
}
private void updateData(MavenProgressIndicator progress) throws MavenIndexException {
String newDataDirName;
IndexData newData;
newDataDirName = findAvailableDataDirName();
try {
FileUtil.copyDir(getUpdateDir(), getDataContextDir(getDataDir(newDataDirName)));
} catch (IOException e) {
throw new MavenIndexException(e);
}
newData = openData(newDataDirName);
try {
doUpdateIndexData(newData, progress);
newData.flush();
} catch (Throwable e) {
newData.close(true);
FileUtil.delete(getDataDir(newDataDirName));
if (e instanceof MavenServerIndexerException) throw new MavenIndexException(e);
if (e instanceof IOException) throw new MavenIndexException(e);
throw new RuntimeException(e);
}
synchronized (this) {
IndexData oldData = myData;
myData = newData;
myDataDirName = newDataDirName;
myUpdateTimestamp = System.currentTimeMillis();
oldData.close(true);
File[] children = myDir.listFiles();
if (children != null) {
for (File each : children) {
if (each.getName().startsWith(DATA_DIR_PREFIX)
&& !each.getName().equals(newDataDirName)) {
FileUtil.delete(each);
}
}
}
}
}
private static boolean deleteWithSubordinates(File file) {
final String baseName = file.getName();
final File[] files =
file.getParentFile()
.listFiles(
new FileFilter() {
@Override
public boolean accept(final File pathname) {
return pathname.getName().startsWith(baseName);
}
});
boolean ok = true;
if (files != null) {
for (File f : files) {
ok &= FileUtil.delete(f);
}
}
return ok;
}
public IndexData(File dir) throws MavenIndexException {
try {
groupToArtifactMap = createPersistentMap(new File(dir, ARTIFACT_IDS_MAP_FILE));
groupWithArtifactToVersionMap = createPersistentMap(new File(dir, VERSIONS_MAP_FILE));
indexId = createContext(getDataContextDir(dir), dir.getName());
} catch (IOException e) {
close(true);
throw new MavenIndexException(e);
} catch (MavenServerIndexerException e) {
close(true);
throw new MavenIndexException(e);
}
}
private File getMirrorFile(@NotNull File originalFile) {
if (!myFileSystem.isMakeCopyOfJar(originalFile)) return originalFile;
final FileAttributes originalAttributes = FileSystemUtil.getAttributes(originalFile);
if (originalAttributes == null) return originalFile;
final String folderPath = getJarsDir();
if (!new File(folderPath).exists() && !new File(folderPath).mkdirs()) {
return originalFile;
}
if (FSRecords.weHaveContentHashes) {
return getMirrorWithContentHash(originalFile, originalAttributes);
}
final String mirrorName =
originalFile.getName() + "." + Integer.toHexString(originalFile.getPath().hashCode());
final File mirrorFile = new File(folderPath, mirrorName);
final FileAttributes mirrorAttributes = FileSystemUtil.getAttributes(mirrorFile);
return mirrorDiffers(originalAttributes, mirrorAttributes, false)
? copyToMirror(originalFile, mirrorFile)
: mirrorFile;
}
private void open() throws MavenIndexException {
try {
try {
doOpen();
} catch (Exception e1) {
MavenLog.LOG.warn(e1);
try {
doOpen();
} catch (Exception e2) {
throw new MavenIndexException("Cannot open index " + myDir.getPath(), e2);
}
markAsBroken();
}
} finally {
save();
}
}
@NotNull
private File copyToMirror(@NotNull File original, @NotNull File mirror) {
ProgressIndicator progress = ProgressManager.getInstance().getProgressIndicator();
if (progress != null) {
progress.pushState();
progress.setText(VfsBundle.message("jar.copy.progress", original.getPath()));
progress.setFraction(0);
}
try {
FileUtil.copy(original, mirror);
} catch (final IOException e) {
reportIOErrorWithJars(original, mirror, e);
return original;
}
if (progress != null) {
progress.popState();
}
return mirror;
}
private static void init() {
final File basePath = basePath();
basePath.mkdirs();
final File namesFile = new File(basePath, "names.dat");
final File attributesFile = new File(basePath, "attrib.dat");
final File contentsFile = new File(basePath, "content.dat");
final File recordsFile = new File(basePath, "records.dat");
if (!namesFile.exists()) {
invalidateIndex();
}
try {
if (getCorruptionMarkerFile().exists()) {
invalidateIndex();
throw new IOException("Corruption marker file found");
}
PagedFileStorage.StorageLockContext storageLockContext =
new PagedFileStorage.StorageLock(false).myDefaultStorageLockContext;
myNames = new PersistentStringEnumerator(namesFile, storageLockContext);
myAttributes = new Storage(attributesFile.getCanonicalPath(), REASONABLY_SMALL);
myContents =
new RefCountingStorage(
contentsFile.getCanonicalPath(),
CapacityAllocationPolicy
.FIVE_PERCENT_FOR_GROWTH); // sources usually zipped with 4x ratio
boolean aligned = PagedFileStorage.BUFFER_SIZE % RECORD_SIZE == 0;
assert aligned; // for performance
myRecords =
new ResizeableMappedFile(
recordsFile, 20 * 1024, storageLockContext, PagedFileStorage.BUFFER_SIZE, aligned);
if (myRecords.length() == 0) {
cleanRecord(0); // Clean header
cleanRecord(1); // Create root record
setCurrentVersion();
}
if (getVersion() != VERSION) {
throw new IOException("FS repository version mismatch");
}
if (myRecords.getInt(HEADER_CONNECTION_STATUS_OFFSET) != SAFELY_CLOSED_MAGIC) {
throw new IOException("FS repository wasn't safely shut down");
}
markDirty();
scanFreeRecords();
} catch (Exception e) { // IOException, IllegalArgumentException
LOG.info(
"Filesystem storage is corrupted or does not exist. [Re]Building. Reason: "
+ e.getMessage());
try {
closeFiles();
boolean deleted = FileUtil.delete(getCorruptionMarkerFile());
deleted &= deleteWithSubordinates(namesFile);
deleted &= AbstractStorage.deleteFiles(attributesFile.getCanonicalPath());
deleted &= AbstractStorage.deleteFiles(contentsFile.getCanonicalPath());
deleted &= deleteWithSubordinates(recordsFile);
if (!deleted) {
throw new IOException("Cannot delete filesystem storage files");
}
} catch (final IOException e1) {
final Runnable warnAndShutdown =
new Runnable() {
@Override
public void run() {
if (ApplicationManager.getApplication().isUnitTestMode()) {
//noinspection CallToPrintStackTrace
e1.printStackTrace();
} else {
final String message =
"Files in "
+ basePath.getPath()
+ " are locked.\n"
+ ApplicationNamesInfo.getInstance().getProductName()
+ " will not be able to start up.";
if (!ApplicationManager.getApplication().isHeadlessEnvironment()) {
JOptionPane.showMessageDialog(
JOptionPane.getRootFrame(),
message,
"Fatal Error",
JOptionPane.ERROR_MESSAGE);
} else {
//noinspection UseOfSystemOutOrSystemErr
System.err.println(message);
}
}
Runtime.getRuntime().halt(1);
}
};
if (EventQueue.isDispatchThread()) {
warnAndShutdown.run();
} else {
//noinspection SSBasedInspection
SwingUtilities.invokeLater(warnAndShutdown);
}
throw new RuntimeException("Can't rebuild filesystem storage ", e1);
}
init();
}
}
private IndexData openData(String dataDir) throws MavenIndexException {
File dir = getDataDir(dataDir);
dir.mkdirs();
return new IndexData(dir);
}
private int createContext(File contextDir, String suffix) throws MavenServerIndexerException {
String indexId = myDir.getName() + "-" + suffix;
return myIndexer.createIndex(
indexId, myRepositoryId, getRepositoryFile(), getRepositoryUrl(), contextDir);
}
private static void removeStaleJarFilesIfNeeded(
File snapshotInfoFile, PersistentHashMap<String, CacheLibraryInfo> info)
throws IOException {
File versionFile = getVersionFile(snapshotInfoFile);
long lastModified = versionFile.lastModified();
if ((System.currentTimeMillis() - lastModified) < 30 * 24 * 60 * 60 * 1000L) {
return;
}
// snapshotInfo is persistent mapping of project library path -> jar snapshot path
// Stale jars are the jars that do not exist with registered paths, to remove them:
// - Take all snapshot library files in jar directory
// - Collect librarySnapshot -> projectLibraryPaths and existing projectLibraryPath ->
// librarySnapshot
// - Remove all projectLibraryPaths that doesn't exist from persistent mapping
// - Remove jar library snapshots that have no projectLibraryPath
Set<String> availableLibrarySnapshots =
new THashSet<String>(
Arrays.asList(
snapshotInfoFile
.getParentFile()
.list(
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
int lastDotPosition = name.lastIndexOf('.');
if (lastDotPosition == -1) return false;
String extension = name.substring(lastDotPosition + 1);
if (extension.length() != 40 || !consistsOfHexLetters(extension))
return false;
return true;
}
private boolean consistsOfHexLetters(String extension) {
for (int i = 0; i < extension.length(); ++i) {
if (Character.digit(extension.charAt(i), 16) == -1) return false;
}
return true;
}
})));
final List<String> invalidLibraryFilePaths = ContainerUtil.newArrayList();
final List<String> allLibraryFilePaths = ContainerUtil.newArrayList();
MultiMap<String, String> jarSnapshotFileToLibraryFilePaths = new MultiMap<String, String>();
Set<String> validLibraryFilePathToJarSnapshotFilePaths = ContainerUtil.newTroveSet();
info.processKeys(new CommonProcessors.CollectProcessor<String>(allLibraryFilePaths));
for (String filePath : allLibraryFilePaths) {
CacheLibraryInfo libraryInfo = info.get(filePath);
if (libraryInfo == null) continue;
jarSnapshotFileToLibraryFilePaths.putValue(libraryInfo.mySnapshotPath, filePath);
if (new File(filePath).exists()) {
validLibraryFilePathToJarSnapshotFilePaths.add(filePath);
} else {
invalidLibraryFilePaths.add(filePath);
}
}
for (String invalidLibraryFilePath : invalidLibraryFilePaths) {
LOG.info("removing stale library reference:" + invalidLibraryFilePath);
info.remove(invalidLibraryFilePath);
}
for (Map.Entry<String, Collection<String>> e : jarSnapshotFileToLibraryFilePaths.entrySet()) {
for (String libraryFilePath : e.getValue()) {
if (validLibraryFilePathToJarSnapshotFilePaths.contains(libraryFilePath)) {
availableLibrarySnapshots.remove(e.getKey());
break;
}
}
}
for (String availableLibrarySnapshot : availableLibrarySnapshots) {
File librarySnapshotFileToDelete =
new File(snapshotInfoFile.getParentFile(), availableLibrarySnapshot);
LOG.info("removing stale library snapshot:" + librarySnapshotFileToDelete);
FileUtil.delete(librarySnapshotFileToDelete);
}
saveVersion(
versionFile); // time stamp will change to start another time interval when stale jar
// files are tracked
}
@NotNull
private static File getVersionFile(File file) {
return new File(file.getParentFile(), file.getName() + ".version");
}
static {
File snapshotInfoFile = new File(new File(getJarsDir()), "snapshots_info");
int currentVersion = -1;
File versionFile = getVersionFile(snapshotInfoFile);
if (versionFile.exists()) {
try {
DataInputStream versionStream =
new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)));
try {
currentVersion = DataInputOutputUtil.readINT(versionStream);
} finally {
versionStream.close();
}
} catch (IOException ignore) {
}
}
if (currentVersion != VERSION) {
PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
saveVersion(versionFile);
}
PersistentHashMap<String, CacheLibraryInfo> info = null;
for (int i = 0; i < 2; ++i) {
try {
info =
new PersistentHashMap<String, CacheLibraryInfo>(
snapshotInfoFile,
new EnumeratorStringDescriptor(),
new DataExternalizer<CacheLibraryInfo>() {
@Override
public void save(@NotNull DataOutput out, CacheLibraryInfo value)
throws IOException {
IOUtil.writeUTF(out, value.mySnapshotPath);
out.writeLong(value.myModificationTime);
out.writeLong(value.myFileLength);
}
@Override
public CacheLibraryInfo read(@NotNull DataInput in) throws IOException {
return new CacheLibraryInfo(IOUtil.readUTF(in), in.readLong(), in.readLong());
}
});
if (i == 0) removeStaleJarFilesIfNeeded(snapshotInfoFile, info);
break;
} catch (IOException ex) {
PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
saveVersion(versionFile);
}
}
assert info != null;
ourCachedLibraryInfo = info;
FlushingDaemon.everyFiveSeconds(
new Runnable() {
@Override
public void run() {
flushCachedLibraryInfos();
}
});
ShutDownTracker.getInstance()
.registerShutdownTask(
new Runnable() {
@Override
public void run() {
flushCachedLibraryInfos();
}
});
}
private File getMirrorWithContentHash(File originalFile, FileAttributes originalAttributes) {
File mirrorFile = null;
String jarDir = getJarsDir();
try {
String path = originalFile.getPath();
CacheLibraryInfo info = CacheLibraryInfo.ourCachedLibraryInfo.get(path);
if (info != null
&& originalAttributes.length == info.myFileLength
&& Math.abs(originalAttributes.lastModified - info.myModificationTime)
<= FS_TIME_RESOLUTION) {
mirrorFile = new File(jarDir, info.mySnapshotPath);
if (!mirrorDiffers(originalAttributes, FileSystemUtil.getAttributes(mirrorFile), true)) {
return mirrorFile;
}
}
MessageDigest sha1 = null;
File tempJarFile = null;
try {
tempJarFile =
FileUtil.createTempFile(new File(jarDir), originalFile.getName(), "", true, false);
DataOutputStream os = new DataOutputStream(new FileOutputStream(tempJarFile));
try {
FileInputStream is = new FileInputStream(originalFile);
try {
sha1 = MessageDigest.getInstance("SHA1");
sha1.update(
String.valueOf(originalAttributes.length).getBytes(Charset.defaultCharset()));
sha1.update((byte) 0);
byte[] buffer = new byte[Math.min(1024 * 1024, (int) originalAttributes.length)];
long totalBytes = 0;
while (true) {
int read = is.read(buffer);
if (read < 0) break;
totalBytes += read;
sha1.update(buffer, 0, read);
os.write(buffer, 0, read);
if (totalBytes == originalAttributes.length) break;
}
} finally {
is.close();
}
} finally {
os.close();
}
} catch (IOException ex) {
File target =
mirrorFile != null ? mirrorFile : tempJarFile != null ? tempJarFile : new File(jarDir);
reportIOErrorWithJars(originalFile, target, ex);
return originalFile;
} catch (NoSuchAlgorithmException ex) {
LOG.error(ex);
return originalFile; // should never happen for sha1
}
String mirrorName = getSnapshotName(originalFile.getName(), sha1.digest());
mirrorFile = new File(jarDir, mirrorName);
if (mirrorDiffers(originalAttributes, FileSystemUtil.getAttributes(mirrorFile), true)) {
try {
FileUtil.delete(mirrorFile);
FileUtil.rename(tempJarFile, mirrorFile);
FileUtil.setLastModified(mirrorFile, originalAttributes.lastModified);
} catch (IOException ex) {
reportIOErrorWithJars(originalFile, mirrorFile, ex);
return originalFile;
}
} else {
FileUtil.delete(tempJarFile);
}
info =
new CacheLibraryInfo(
mirrorFile.getName(), originalAttributes.lastModified, originalAttributes.length);
CacheLibraryInfo.ourCachedLibraryInfo.put(path, info);
return mirrorFile;
} catch (IOException ex) {
CacheLibraryInfo.ourCachedLibraryInfo.markCorrupted();
reportIOErrorWithJars(
originalFile,
mirrorFile != null ? mirrorFile : new File(jarDir, originalFile.getName()),
ex);
return originalFile;
}
}
