private void removeFilesInIndex(
     List<Object> filesToRemove,
     List<IIndexFragmentFile> indexFilesToRemove,
     IProgressMonitor monitor)
     throws InterruptedException, CoreException {
   if (!filesToRemove.isEmpty() || !indexFilesToRemove.isEmpty()) {
     fIndex.acquireWriteLock();
     try {
       for (Object tu : filesToRemove) {
         if (monitor.isCanceled()) {
           return;
         }
         IIndexFileLocation ifl = fResolver.resolveFile(tu);
         if (ifl == null) continue;
         IIndexFragmentFile[] ifiles = fIndex.getWritableFiles(ifl);
         for (IIndexFragmentFile ifile : ifiles) {
           fIndex.clearFile(ifile);
         }
         incrementRequestedFilesCount(-1);
       }
       for (IIndexFragmentFile ifile : indexFilesToRemove) {
         if (monitor.isCanceled()) {
           return;
         }
         fIndex.clearFile(ifile);
         incrementRequestedFilesCount(-1);
       }
     } finally {
       fIndex.releaseWriteLock();
     }
   }
   filesToRemove.clear();
 }
 private void setResume(boolean value) throws InterruptedException, CoreException {
   fIndex.acquireWriteLock();
   try {
     fIndex
         .getWritableFragment()
         .setProperty(IIndexFragment.PROPERTY_RESUME_INDEXER, String.valueOf(value));
   } finally {
     fIndex.releaseWriteLock();
   }
 }
Ejemplo n.º 3
0
  public static void buildIndex(List<String> files) throws Exception {
    File writableIndex = new File("/tmp/standaloneIndexFile.pdom");
    IIndexLocationConverter converter = new MyIndexLocationConverter();
    // See PDOMCPPLinkageFactory if needed ..
    Map<String, IPDOMLinkageFactory> linkageFactoryMappings =
        new HashMap<String, IPDOMLinkageFactory>();
    linkageFactoryMappings.put(ILinkage.C_LINKAGE_NAME, new PDOMCLinkageFactory());
    linkageFactoryMappings.put(ILinkage.CPP_LINKAGE_NAME, new PDOMCPPLinkageFactory());
    IStandaloneScannerInfoProvider scannerProvider = new MyStandaloneScannerInfoProvider();
    ILanguageMapper mapper = new MyLanguageMapper();
    IParserLogService log = new MyParserLogService();
    IProgressMonitor monitor = new MyProgressMonitor();
    StandaloneIndexer indexer =
        new StandaloneFastIndexer(
            writableIndex, converter, linkageFactoryMappings, scannerProvider, mapper, log);
    indexer.setTraceStatistics(true);

    indexer.rebuild(files, monitor);
    IWritableIndex i = indexer.getIndex();
    GlobalResultCollector.bindingindex = i;
    i.acquireReadLock();
    try {
      System.out.println("internals");
      for (IIndexFile file : i.getAllFiles()) {
        System.out.printf("Index has file: %s\n", file.toString());
      }
      char[] prefix = {};
      for (IIndexBinding binding :
          i.findBindingsForPrefix(prefix, false, IndexFilter.ALL, monitor)) {
        System.out.println(binding);
      }
      System.out.println(i);
    } finally {
      i.releaseReadLock();
    }
  }
 public IIndexFragmentFile[] getAvailableIndexFiles(int linkageID, IIndexFileLocation ifl)
     throws CoreException {
   IIndexFragmentFile[] files = fIndexFilesCache.get(ifl);
   if (files == null) {
     IIndexFragmentFile[] fragFiles = fIndex.getWritableFiles(linkageID, ifl);
     int j = 0;
     for (int i = 0; i < fragFiles.length; i++) {
       if (fragFiles[i].hasContent()) {
         if (j != i) fragFiles[j] = fragFiles[i];
         j++;
       }
     }
     if (j == fragFiles.length) {
       files = fragFiles;
     } else {
       files = new IIndexFragmentFile[j];
       System.arraycopy(fragFiles, 0, files, 0, j);
     }
     fIndexFilesCache.put(ifl, files);
   }
   return files;
 }
  private void parseLinkage(int linkageID, List<IIndexFileLocation> files, IProgressMonitor monitor)
      throws CoreException, InterruptedException {
    LinkageTask map = findRequestMap(linkageID);
    if (map == null || files == null || files.isEmpty()) return;

    // First parse the required sources
    for (Iterator<IIndexFileLocation> it = files.iterator(); it.hasNext(); ) {
      IIndexFileLocation ifl = it.next();
      LocationTask locTask = map.find(ifl);
      if (locTask == null || locTask.isCompleted()) {
        it.remove();
      } else if (locTask.fKind == UpdateKind.REQUIRED_SOURCE) {
        if (monitor.isCanceled() || hasUrgentTasks()) return;
        final Object tu = locTask.fTu;
        final IScannerInfo scannerInfo = getScannerInfo(linkageID, tu);
        parseFile(tu, getLanguage(tu, linkageID), ifl, scannerInfo, null, monitor);
      }
    }

    // Files with context
    for (Iterator<IIndexFileLocation> it = files.iterator(); it.hasNext(); ) {
      IIndexFileLocation ifl = it.next();
      LocationTask locTask = map.find(ifl);
      if (locTask == null || locTask.isCompleted()) {
        it.remove();
      } else {
        for (FileVersionTask versionTask : locTask.fVersionTasks) {
          if (versionTask.fOutdated) {
            if (monitor.isCanceled() || hasUrgentTasks()) return;
            parseVersionInContext(
                linkageID,
                map,
                ifl,
                versionTask,
                locTask.fTu,
                new LinkedHashSet<IIndexFile>(),
                monitor);
          }
        }
      }
    }

    // Files without context
    for (Iterator<IIndexFileLocation> it = files.iterator(); it.hasNext(); ) {
      IIndexFileLocation ifl = it.next();
      LocationTask locTask = map.find(ifl);
      if (locTask == null || locTask.isCompleted()) {
        it.remove();
      } else {
        if (locTask.needsVersion()) {
          if (monitor.isCanceled() || hasUrgentTasks()) return;
          final Object tu = locTask.fTu;
          final IScannerInfo scannerInfo = getScannerInfo(linkageID, tu);
          parseFile(tu, getLanguage(tu, linkageID), ifl, scannerInfo, null, monitor);
          if (locTask.isCompleted()) it.remove();
        }
      }
    }

    // Delete remaining files.
    fIndex.acquireWriteLock();
    try {
      for (IIndexFileLocation ifl : files) {
        LocationTask locTask = map.find(ifl);
        if (locTask != null && !locTask.isCompleted()) {
          if (!locTask.needsVersion()) {
            if (monitor.isCanceled() || hasUrgentTasks()) return;
            Iterator<FileVersionTask> it = locTask.fVersionTasks.iterator();
            while (it.hasNext()) {
              FileVersionTask v = it.next();
              if (v.fOutdated) {
                fIndex.clearFile(v.fIndexFile);
                reportFile(true, locTask.fKind);
                locTask.removeVersionTask(it);
                fIndexContentCache.remove(v.fIndexFile);
                fIndexFilesCache.remove(ifl);
              }
            }
          }
        }
      }
    } finally {
      fIndex.releaseWriteLock();
    }
  }
  private void extractFiles(
      HashMap<Integer, List<IIndexFileLocation>> files,
      List<IIndexFragmentFile> iFilesToRemove,
      IProgressMonitor monitor)
      throws CoreException {
    final boolean forceAll = (fUpdateFlags & IIndexManager.UPDATE_ALL) != 0;
    final boolean checkTimestamps = (fUpdateFlags & IIndexManager.UPDATE_CHECK_TIMESTAMPS) != 0;
    final boolean checkFileContentsHash =
        (fUpdateFlags & IIndexManager.UPDATE_CHECK_CONTENTS_HASH) != 0;
    final boolean forceUnresolvedIncludes =
        (fUpdateFlags & IIndexManager.UPDATE_UNRESOLVED_INCLUDES) != 0;
    final boolean both = fIndexHeadersWithoutContext == UnusedHeaderStrategy.useBoth;
    int count = 0;
    int forceFirst = fForceNumberFiles;
    BitSet linkages = new BitSet();
    for (final Object tu : fFilesToUpdate) {
      if (monitor.isCanceled()) return;

      final boolean force = forceAll || --forceFirst >= 0;
      final IIndexFileLocation ifl = fResolver.resolveFile(tu);
      if (ifl == null) continue;

      final IIndexFragmentFile[] indexFiles = fIndex.getWritableFiles(ifl);
      final boolean isSourceUnit = fResolver.isSourceUnit(tu);
      linkages.clear();
      final boolean regularContent = isRequiredInIndex(tu, ifl, isSourceUnit);
      final boolean indexedUnconditionally = fResolver.isIndexedUnconditionally(ifl);
      if (regularContent || indexedUnconditionally) {
        // Headers or sources required with a specific linkage
        final UpdateKind updateKind =
            isSourceUnit
                ? UpdateKind.REQUIRED_SOURCE
                : regularContent && both
                    ? UpdateKind.REQUIRED_HEADER
                    : UpdateKind.ONE_LINKAGE_HEADER;
        if (regularContent || indexFiles.length == 0) {
          AbstractLanguage[] langs = fResolver.getLanguages(tu, fIndexHeadersWithoutContext);
          for (AbstractLanguage lang : langs) {
            int linkageID = lang.getLinkageID();
            boolean foundInLinkage = false;
            for (int i = 0; i < indexFiles.length; i++) {
              IIndexFragmentFile ifile = indexFiles[i];
              if (ifile != null && ifile.getLinkageID() == linkageID && ifile.hasContent()) {
                foundInLinkage = true;
                indexFiles[i] = null; // Take the file.
                boolean update =
                    force
                        || (forceUnresolvedIncludes && ifile.hasUnresolvedInclude())
                        || isModified(checkTimestamps, checkFileContentsHash, ifl, tu, ifile);
                if (update && requestUpdate(linkageID, ifl, ifile, tu, updateKind)) {
                  count++;
                  linkages.set(linkageID);
                }
              }
            }
            if (!foundInLinkage && requestUpdate(linkageID, ifl, null, tu, updateKind)) {
              linkages.set(linkageID);
              count++;
            }
          }
        }
      }

      // Handle other files present in index.
      for (IIndexFragmentFile ifile : indexFiles) {
        if (ifile != null) {
          IIndexInclude ctx = ifile.getParsedInContext();
          if (ctx == null && !indexedUnconditionally && ifile.hasContent()) {
            iFilesToRemove.add(ifile);
            count++;
          } else {
            boolean update =
                force
                    || (forceUnresolvedIncludes && ifile.hasUnresolvedInclude())
                    || isModified(checkTimestamps, checkFileContentsHash, ifl, tu, ifile);
            final int linkageID = ifile.getLinkageID();
            if (update && requestUpdate(linkageID, ifl, ifile, tu, UpdateKind.OTHER_HEADER)) {
              count++;
              linkages.set(linkageID);
            }
          }
        }
      }
      for (int lid = linkages.nextSetBit(0); lid >= 0; lid = linkages.nextSetBit(lid + 1)) {
        addPerLinkage(lid, ifl, files);
      }
    }
    synchronized (this) {
      incrementRequestedFilesCount(count - fFilesToUpdate.length);
      fFilesToUpdate = null;
    }
  }
  public final void runTask(IProgressMonitor monitor) throws InterruptedException {
    try {
      if (!fIndexFilesWithoutConfiguration) {
        fIndexHeadersWithoutContext = UnusedHeaderStrategy.skip;
      }

      fIndex = createIndex();
      if (fIndex == null) {
        return;
      }
      fTodoTaskUpdater = createTodoTaskUpdater();

      fASTOptions =
          ILanguage.OPTION_NO_IMAGE_LOCATIONS
              | ILanguage.OPTION_SKIP_TRIVIAL_EXPRESSIONS_IN_AGGREGATE_INITIALIZERS;

      if (getSkipReferences() == SKIP_ALL_REFERENCES) {
        fASTOptions |= ILanguage.OPTION_SKIP_FUNCTION_BODIES;
      }

      fIndex.resetCacheCounters();
      fIndex.acquireReadLock();

      try {
        try {
          // Split into sources and headers, remove excluded sources.
          HashMap<Integer, List<IIndexFileLocation>> files =
              new HashMap<Integer, List<IIndexFileLocation>>();
          final ArrayList<IIndexFragmentFile> indexFilesToRemove =
              new ArrayList<IIndexFragmentFile>();
          extractFiles(files, indexFilesToRemove, monitor);

          setResume(true);

          // Remove files from index
          removeFilesInIndex(fFilesToRemove, indexFilesToRemove, monitor);

          HashMap<Integer, List<IIndexFileLocation>> moreFiles = null;
          while (true) {
            for (int linkageID : getLinkagesToParse()) {
              final List<IIndexFileLocation> filesForLinkage = files.get(linkageID);
              if (filesForLinkage != null) {
                parseLinkage(linkageID, filesForLinkage, monitor);
                for (Iterator<LocationTask> it = fOneLinkageTasks.values().iterator();
                    it.hasNext(); ) {
                  LocationTask task = it.next();
                  if (task.isCompleted()) it.remove();
                }
                fIndexContentCache.clear();
                fIndexFilesCache.clear();
              }
              if (hasUrgentTasks()) break;
            }
            synchronized (this) {
              if (fUrgentTasks.isEmpty()) {
                if (moreFiles == null) {
                  // No urgent tasks and no more files to parse. We are done.
                  fTaskCompleted = true;
                  break;
                } else {
                  files = moreFiles;
                  moreFiles = null;
                }
              }
            }
            AbstractIndexerTask urgentTask;
            while ((urgentTask = getUrgentTask()) != null) {
              // Move the lists of not yet parsed files from 'files' to 'moreFiles'.
              if (moreFiles == null) {
                moreFiles = files;
              } else {
                for (Map.Entry<Integer, List<IIndexFileLocation>> entry : files.entrySet()) {
                  List<IIndexFileLocation> list = moreFiles.get(entry.getKey());
                  if (list == null) {
                    moreFiles.put(entry.getKey(), entry.getValue());
                  } else {
                    list.addAll(0, entry.getValue());
                  }
                }
              }
              // Extract files from the urgent task.
              files = new HashMap<Integer, List<IIndexFileLocation>>();
              fFilesToUpdate = urgentTask.fFilesToUpdate;
              fForceNumberFiles = urgentTask.fForceNumberFiles;
              fFilesToRemove = urgentTask.fFilesToRemove;
              incrementRequestedFilesCount(fFilesToUpdate.length + fFilesToRemove.size());
              extractFiles(files, indexFilesToRemove, monitor);
              removeFilesInIndex(fFilesToRemove, indexFilesToRemove, monitor);
            }
          }
          if (!monitor.isCanceled()) {
            setResume(false);
          }
        } finally {
          fIndex.flush();
        }
      } catch (CoreException e) {
        logException(e);
      } finally {
        fIndex.releaseReadLock();
      }
    } finally {
      synchronized (this) {
        fTaskCompleted = true;
      }
    }
  }