private static void invalidateIndex() {
   LOG.info("Marking VFS as corrupted");
   final File indexRoot = PathManager.getIndexRoot();
   if (indexRoot.exists()) {
     final String[] children = indexRoot.list();
     if (children != null && children.length > 0) {
       // create index corruption marker only if index directory exists and is non-empty
       // It is incorrect to consider non-existing indices "corrupted"
       FileUtil.createIfDoesntExist(new File(PathManager.getIndexRoot(), "corruption.marker"));
     }
   }
 }
Esempio n. 2
0
  private void reportIOErrorWithJars(File original, File target, IOException e) {
    LOG.warn(e);

    String path = original.getPath();
    myFileSystem.setNoCopyJarForPath(path);

    String message =
        VfsBundle.message("jar.copy.error.message", path, target.getPath(), e.getMessage());
    ERROR_COPY_NOTIFICATION
        .getValue()
        .createNotification(message, NotificationType.ERROR)
        .notify(null);
  }
Esempio n. 3
0
  private synchronized void save() {
    myDir.mkdirs();

    Properties props = new Properties();

    props.setProperty(KIND_KEY, myKind.toString());
    props.setProperty(ID_KEY, myRepositoryId);
    props.setProperty(PATH_OR_URL_KEY, myRepositoryPathOrUrl);
    props.setProperty(INDEX_VERSION_KEY, CURRENT_VERSION);
    if (myUpdateTimestamp != null)
      props.setProperty(TIMESTAMP_KEY, String.valueOf(myUpdateTimestamp));
    if (myDataDirName != null) props.setProperty(DATA_DIR_NAME_KEY, myDataDirName);
    if (myFailureMessage != null) props.setProperty(FAILURE_MESSAGE_KEY, myFailureMessage);

    try {
      FileOutputStream s = new FileOutputStream(new File(myDir, INDEX_INFO_FILE));
      try {
        props.store(s, null);
      } finally {
        s.close();
      }
    } catch (IOException e) {
      MavenLog.LOG.warn(e);
    }
  }
Esempio n. 4
0
  private void updateData(MavenProgressIndicator progress) throws MavenIndexException {
    String newDataDirName;
    IndexData newData;

    newDataDirName = findAvailableDataDirName();
    try {
      FileUtil.copyDir(getUpdateDir(), getDataContextDir(getDataDir(newDataDirName)));
    } catch (IOException e) {
      throw new MavenIndexException(e);
    }
    newData = openData(newDataDirName);

    try {
      doUpdateIndexData(newData, progress);
      newData.flush();
    } catch (Throwable e) {
      newData.close(true);
      FileUtil.delete(getDataDir(newDataDirName));

      if (e instanceof MavenServerIndexerException) throw new MavenIndexException(e);
      if (e instanceof IOException) throw new MavenIndexException(e);
      throw new RuntimeException(e);
    }

    synchronized (this) {
      IndexData oldData = myData;

      myData = newData;
      myDataDirName = newDataDirName;

      myUpdateTimestamp = System.currentTimeMillis();

      oldData.close(true);

      File[] children = myDir.listFiles();
      if (children != null) {
        for (File each : children) {
          if (each.getName().startsWith(DATA_DIR_PREFIX)
              && !each.getName().equals(newDataDirName)) {
            FileUtil.delete(each);
          }
        }
      }
    }
  }
    private static boolean deleteWithSubordinates(File file) {
      final String baseName = file.getName();
      final File[] files =
          file.getParentFile()
              .listFiles(
                  new FileFilter() {
                    @Override
                    public boolean accept(final File pathname) {
                      return pathname.getName().startsWith(baseName);
                    }
                  });

      boolean ok = true;
      if (files != null) {
        for (File f : files) {
          ok &= FileUtil.delete(f);
        }
      }

      return ok;
    }
Esempio n. 6
0
    public IndexData(File dir) throws MavenIndexException {
      try {
        groupToArtifactMap = createPersistentMap(new File(dir, ARTIFACT_IDS_MAP_FILE));
        groupWithArtifactToVersionMap = createPersistentMap(new File(dir, VERSIONS_MAP_FILE));

        indexId = createContext(getDataContextDir(dir), dir.getName());
      } catch (IOException e) {
        close(true);
        throw new MavenIndexException(e);
      } catch (MavenServerIndexerException e) {
        close(true);
        throw new MavenIndexException(e);
      }
    }
Esempio n. 7
0
  private File getMirrorFile(@NotNull File originalFile) {
    if (!myFileSystem.isMakeCopyOfJar(originalFile)) return originalFile;

    final FileAttributes originalAttributes = FileSystemUtil.getAttributes(originalFile);
    if (originalAttributes == null) return originalFile;

    final String folderPath = getJarsDir();
    if (!new File(folderPath).exists() && !new File(folderPath).mkdirs()) {
      return originalFile;
    }

    if (FSRecords.weHaveContentHashes) {
      return getMirrorWithContentHash(originalFile, originalAttributes);
    }

    final String mirrorName =
        originalFile.getName() + "." + Integer.toHexString(originalFile.getPath().hashCode());
    final File mirrorFile = new File(folderPath, mirrorName);
    final FileAttributes mirrorAttributes = FileSystemUtil.getAttributes(mirrorFile);
    return mirrorDiffers(originalAttributes, mirrorAttributes, false)
        ? copyToMirror(originalFile, mirrorFile)
        : mirrorFile;
  }
Esempio n. 8
0
 private void open() throws MavenIndexException {
   try {
     try {
       doOpen();
     } catch (Exception e1) {
       MavenLog.LOG.warn(e1);
       try {
         doOpen();
       } catch (Exception e2) {
         throw new MavenIndexException("Cannot open index " + myDir.getPath(), e2);
       }
       markAsBroken();
     }
   } finally {
     save();
   }
 }
Esempio n. 9
0
  @NotNull
  private File copyToMirror(@NotNull File original, @NotNull File mirror) {
    ProgressIndicator progress = ProgressManager.getInstance().getProgressIndicator();
    if (progress != null) {
      progress.pushState();
      progress.setText(VfsBundle.message("jar.copy.progress", original.getPath()));
      progress.setFraction(0);
    }

    try {
      FileUtil.copy(original, mirror);
    } catch (final IOException e) {
      reportIOErrorWithJars(original, mirror, e);
      return original;
    }

    if (progress != null) {
      progress.popState();
    }

    return mirror;
  }
    private static void init() {
      final File basePath = basePath();
      basePath.mkdirs();

      final File namesFile = new File(basePath, "names.dat");
      final File attributesFile = new File(basePath, "attrib.dat");
      final File contentsFile = new File(basePath, "content.dat");
      final File recordsFile = new File(basePath, "records.dat");

      if (!namesFile.exists()) {
        invalidateIndex();
      }

      try {
        if (getCorruptionMarkerFile().exists()) {
          invalidateIndex();
          throw new IOException("Corruption marker file found");
        }

        PagedFileStorage.StorageLockContext storageLockContext =
            new PagedFileStorage.StorageLock(false).myDefaultStorageLockContext;
        myNames = new PersistentStringEnumerator(namesFile, storageLockContext);
        myAttributes = new Storage(attributesFile.getCanonicalPath(), REASONABLY_SMALL);
        myContents =
            new RefCountingStorage(
                contentsFile.getCanonicalPath(),
                CapacityAllocationPolicy
                    .FIVE_PERCENT_FOR_GROWTH); // sources usually zipped with 4x ratio
        boolean aligned = PagedFileStorage.BUFFER_SIZE % RECORD_SIZE == 0;
        assert aligned; // for performance
        myRecords =
            new ResizeableMappedFile(
                recordsFile, 20 * 1024, storageLockContext, PagedFileStorage.BUFFER_SIZE, aligned);

        if (myRecords.length() == 0) {
          cleanRecord(0); // Clean header
          cleanRecord(1); // Create root record
          setCurrentVersion();
        }

        if (getVersion() != VERSION) {
          throw new IOException("FS repository version mismatch");
        }

        if (myRecords.getInt(HEADER_CONNECTION_STATUS_OFFSET) != SAFELY_CLOSED_MAGIC) {
          throw new IOException("FS repository wasn't safely shut down");
        }
        markDirty();
        scanFreeRecords();
      } catch (Exception e) { // IOException, IllegalArgumentException
        LOG.info(
            "Filesystem storage is corrupted or does not exist. [Re]Building. Reason: "
                + e.getMessage());
        try {
          closeFiles();

          boolean deleted = FileUtil.delete(getCorruptionMarkerFile());
          deleted &= deleteWithSubordinates(namesFile);
          deleted &= AbstractStorage.deleteFiles(attributesFile.getCanonicalPath());
          deleted &= AbstractStorage.deleteFiles(contentsFile.getCanonicalPath());
          deleted &= deleteWithSubordinates(recordsFile);

          if (!deleted) {
            throw new IOException("Cannot delete filesystem storage files");
          }
        } catch (final IOException e1) {
          final Runnable warnAndShutdown =
              new Runnable() {
                @Override
                public void run() {
                  if (ApplicationManager.getApplication().isUnitTestMode()) {
                    //noinspection CallToPrintStackTrace
                    e1.printStackTrace();
                  } else {
                    final String message =
                        "Files in "
                            + basePath.getPath()
                            + " are locked.\n"
                            + ApplicationNamesInfo.getInstance().getProductName()
                            + " will not be able to start up.";
                    if (!ApplicationManager.getApplication().isHeadlessEnvironment()) {
                      JOptionPane.showMessageDialog(
                          JOptionPane.getRootFrame(),
                          message,
                          "Fatal Error",
                          JOptionPane.ERROR_MESSAGE);
                    } else {
                      //noinspection UseOfSystemOutOrSystemErr
                      System.err.println(message);
                    }
                  }
                  Runtime.getRuntime().halt(1);
                }
              };

          if (EventQueue.isDispatchThread()) {
            warnAndShutdown.run();
          } else {
            //noinspection SSBasedInspection
            SwingUtilities.invokeLater(warnAndShutdown);
          }

          throw new RuntimeException("Can't rebuild filesystem storage ", e1);
        }

        init();
      }
    }
Esempio n. 11
0
 private IndexData openData(String dataDir) throws MavenIndexException {
   File dir = getDataDir(dataDir);
   dir.mkdirs();
   return new IndexData(dir);
 }
Esempio n. 12
0
 private int createContext(File contextDir, String suffix) throws MavenServerIndexerException {
   String indexId = myDir.getName() + "-" + suffix;
   return myIndexer.createIndex(
       indexId, myRepositoryId, getRepositoryFile(), getRepositoryUrl(), contextDir);
 }
Esempio n. 13
0
    private static void removeStaleJarFilesIfNeeded(
        File snapshotInfoFile, PersistentHashMap<String, CacheLibraryInfo> info)
        throws IOException {
      File versionFile = getVersionFile(snapshotInfoFile);
      long lastModified = versionFile.lastModified();
      if ((System.currentTimeMillis() - lastModified) < 30 * 24 * 60 * 60 * 1000L) {
        return;
      }

      // snapshotInfo is persistent mapping of project library path -> jar snapshot path
      // Stale jars are the jars that do not exist with registered paths, to remove them:
      // - Take all snapshot library files in jar directory
      // - Collect librarySnapshot -> projectLibraryPaths and existing projectLibraryPath ->
      // librarySnapshot
      // - Remove all projectLibraryPaths that doesn't exist from persistent mapping
      // - Remove jar library snapshots that have no projectLibraryPath
      Set<String> availableLibrarySnapshots =
          new THashSet<String>(
              Arrays.asList(
                  snapshotInfoFile
                      .getParentFile()
                      .list(
                          new FilenameFilter() {
                            @Override
                            public boolean accept(File dir, String name) {
                              int lastDotPosition = name.lastIndexOf('.');
                              if (lastDotPosition == -1) return false;
                              String extension = name.substring(lastDotPosition + 1);
                              if (extension.length() != 40 || !consistsOfHexLetters(extension))
                                return false;
                              return true;
                            }

                            private boolean consistsOfHexLetters(String extension) {
                              for (int i = 0; i < extension.length(); ++i) {
                                if (Character.digit(extension.charAt(i), 16) == -1) return false;
                              }
                              return true;
                            }
                          })));

      final List<String> invalidLibraryFilePaths = ContainerUtil.newArrayList();
      final List<String> allLibraryFilePaths = ContainerUtil.newArrayList();
      MultiMap<String, String> jarSnapshotFileToLibraryFilePaths = new MultiMap<String, String>();
      Set<String> validLibraryFilePathToJarSnapshotFilePaths = ContainerUtil.newTroveSet();

      info.processKeys(new CommonProcessors.CollectProcessor<String>(allLibraryFilePaths));
      for (String filePath : allLibraryFilePaths) {
        CacheLibraryInfo libraryInfo = info.get(filePath);
        if (libraryInfo == null) continue;

        jarSnapshotFileToLibraryFilePaths.putValue(libraryInfo.mySnapshotPath, filePath);
        if (new File(filePath).exists()) {
          validLibraryFilePathToJarSnapshotFilePaths.add(filePath);
        } else {
          invalidLibraryFilePaths.add(filePath);
        }
      }

      for (String invalidLibraryFilePath : invalidLibraryFilePaths) {
        LOG.info("removing stale library reference:" + invalidLibraryFilePath);
        info.remove(invalidLibraryFilePath);
      }
      for (Map.Entry<String, Collection<String>> e : jarSnapshotFileToLibraryFilePaths.entrySet()) {
        for (String libraryFilePath : e.getValue()) {
          if (validLibraryFilePathToJarSnapshotFilePaths.contains(libraryFilePath)) {
            availableLibrarySnapshots.remove(e.getKey());
            break;
          }
        }
      }
      for (String availableLibrarySnapshot : availableLibrarySnapshots) {
        File librarySnapshotFileToDelete =
            new File(snapshotInfoFile.getParentFile(), availableLibrarySnapshot);
        LOG.info("removing stale library snapshot:" + librarySnapshotFileToDelete);
        FileUtil.delete(librarySnapshotFileToDelete);
      }

      saveVersion(
          versionFile); // time stamp will change to start another time interval when stale jar
      // files are tracked
    }
Esempio n. 14
0
 @NotNull
 private static File getVersionFile(File file) {
   return new File(file.getParentFile(), file.getName() + ".version");
 }
Esempio n. 15
0
    static {
      File snapshotInfoFile = new File(new File(getJarsDir()), "snapshots_info");

      int currentVersion = -1;
      File versionFile = getVersionFile(snapshotInfoFile);
      if (versionFile.exists()) {
        try {
          DataInputStream versionStream =
              new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)));
          try {
            currentVersion = DataInputOutputUtil.readINT(versionStream);
          } finally {
            versionStream.close();
          }
        } catch (IOException ignore) {
        }
      }

      if (currentVersion != VERSION) {
        PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
        saveVersion(versionFile);
      }

      PersistentHashMap<String, CacheLibraryInfo> info = null;
      for (int i = 0; i < 2; ++i) {
        try {
          info =
              new PersistentHashMap<String, CacheLibraryInfo>(
                  snapshotInfoFile,
                  new EnumeratorStringDescriptor(),
                  new DataExternalizer<CacheLibraryInfo>() {

                    @Override
                    public void save(@NotNull DataOutput out, CacheLibraryInfo value)
                        throws IOException {
                      IOUtil.writeUTF(out, value.mySnapshotPath);
                      out.writeLong(value.myModificationTime);
                      out.writeLong(value.myFileLength);
                    }

                    @Override
                    public CacheLibraryInfo read(@NotNull DataInput in) throws IOException {
                      return new CacheLibraryInfo(IOUtil.readUTF(in), in.readLong(), in.readLong());
                    }
                  });

          if (i == 0) removeStaleJarFilesIfNeeded(snapshotInfoFile, info);
          break;
        } catch (IOException ex) {
          PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
          saveVersion(versionFile);
        }
      }

      assert info != null;
      ourCachedLibraryInfo = info;
      FlushingDaemon.everyFiveSeconds(
          new Runnable() {
            @Override
            public void run() {
              flushCachedLibraryInfos();
            }
          });

      ShutDownTracker.getInstance()
          .registerShutdownTask(
              new Runnable() {
                @Override
                public void run() {
                  flushCachedLibraryInfos();
                }
              });
    }
Esempio n. 16
0
  private File getMirrorWithContentHash(File originalFile, FileAttributes originalAttributes) {
    File mirrorFile = null;
    String jarDir = getJarsDir();

    try {
      String path = originalFile.getPath();
      CacheLibraryInfo info = CacheLibraryInfo.ourCachedLibraryInfo.get(path);

      if (info != null
          && originalAttributes.length == info.myFileLength
          && Math.abs(originalAttributes.lastModified - info.myModificationTime)
              <= FS_TIME_RESOLUTION) {
        mirrorFile = new File(jarDir, info.mySnapshotPath);
        if (!mirrorDiffers(originalAttributes, FileSystemUtil.getAttributes(mirrorFile), true)) {
          return mirrorFile;
        }
      }

      MessageDigest sha1 = null;
      File tempJarFile = null;

      try {
        tempJarFile =
            FileUtil.createTempFile(new File(jarDir), originalFile.getName(), "", true, false);

        DataOutputStream os = new DataOutputStream(new FileOutputStream(tempJarFile));
        try {
          FileInputStream is = new FileInputStream(originalFile);
          try {
            sha1 = MessageDigest.getInstance("SHA1");
            sha1.update(
                String.valueOf(originalAttributes.length).getBytes(Charset.defaultCharset()));
            sha1.update((byte) 0);

            byte[] buffer = new byte[Math.min(1024 * 1024, (int) originalAttributes.length)];
            long totalBytes = 0;
            while (true) {
              int read = is.read(buffer);
              if (read < 0) break;
              totalBytes += read;
              sha1.update(buffer, 0, read);
              os.write(buffer, 0, read);
              if (totalBytes == originalAttributes.length) break;
            }
          } finally {
            is.close();
          }
        } finally {
          os.close();
        }
      } catch (IOException ex) {
        File target =
            mirrorFile != null ? mirrorFile : tempJarFile != null ? tempJarFile : new File(jarDir);
        reportIOErrorWithJars(originalFile, target, ex);
        return originalFile;
      } catch (NoSuchAlgorithmException ex) {
        LOG.error(ex);
        return originalFile; // should never happen for sha1
      }

      String mirrorName = getSnapshotName(originalFile.getName(), sha1.digest());
      mirrorFile = new File(jarDir, mirrorName);

      if (mirrorDiffers(originalAttributes, FileSystemUtil.getAttributes(mirrorFile), true)) {
        try {
          FileUtil.delete(mirrorFile);
          FileUtil.rename(tempJarFile, mirrorFile);
          FileUtil.setLastModified(mirrorFile, originalAttributes.lastModified);
        } catch (IOException ex) {
          reportIOErrorWithJars(originalFile, mirrorFile, ex);
          return originalFile;
        }
      } else {
        FileUtil.delete(tempJarFile);
      }

      info =
          new CacheLibraryInfo(
              mirrorFile.getName(), originalAttributes.lastModified, originalAttributes.length);
      CacheLibraryInfo.ourCachedLibraryInfo.put(path, info);
      return mirrorFile;
    } catch (IOException ex) {
      CacheLibraryInfo.ourCachedLibraryInfo.markCorrupted();
      reportIOErrorWithJars(
          originalFile,
          mirrorFile != null ? mirrorFile : new File(jarDir, originalFile.getName()),
          ex);
      return originalFile;
    }
  }