private FlatSearchResponse searchFlatPaged(
FlatSearchRequest request,
Collection<IndexingContext> indexingContexts,
boolean ignoreContext)
throws IOException {
TreeSet<ArtifactInfo> result = new TreeSet<ArtifactInfo>(request.getArtifactInfoComparator());
int totalHits = 0;
for (IndexingContext ctx : indexingContexts) {
if (ignoreContext || ctx.isSearchable()) {
int hitCount =
searchFlat(
request, result, ctx, request.getQuery(), request.getStart(), request.getCount());
if (hitCount == AbstractSearchResponse.LIMIT_EXCEEDED) {
totalHits = hitCount;
} else {
totalHits += hitCount;
}
}
if (request.isHitLimited() && (totalHits > request.getResultHitLimit())
|| totalHits == AbstractSearchResponse.LIMIT_EXCEEDED) {
totalHits = AbstractSearchResponse.LIMIT_EXCEEDED;
result = new TreeSet<ArtifactInfo>(request.getArtifactInfoComparator());
break;
}
}
return new FlatSearchResponse(request.getQuery(), totalHits, result);
}
private GroupedSearchResponse searchGrouped(
GroupedSearchRequest request,
Collection<IndexingContext> indexingContexts,
boolean ignoreContext)
throws IOException {
TreeMap<String, ArtifactInfoGroup> result =
new TreeMap<String, ArtifactInfoGroup>(request.getGroupKeyComparator());
int totalHits = 0;
for (IndexingContext ctx : indexingContexts) {
if (ignoreContext || ctx.isSearchable()) {
int hitCount =
searchGrouped(request, result, request.getGrouping(), ctx, request.getQuery());
if (hitCount == AbstractSearchResponse.LIMIT_EXCEEDED) {
totalHits = hitCount;
} else {
totalHits += hitCount;
}
}
if (request.isHitLimited() && (totalHits > request.getResultHitLimit())
|| totalHits == AbstractSearchResponse.LIMIT_EXCEEDED) {
totalHits = AbstractSearchResponse.LIMIT_EXCEEDED;
result = new TreeMap<String, ArtifactInfoGroup>(request.getGroupKeyComparator());
break;
}
}
return new GroupedSearchResponse(request.getQuery(), totalHits, result);
}
public void updateIndex(
int id, MavenServerSettings settings, MavenServerProgressIndicator indicator)
throws MavenServerIndexerException, MavenServerProcessCanceledException, RemoteException {
IndexingContext index = getIndex(id);
try {
if (isLocal(index)) {
File repository = index.getRepository();
if (repository != null && repository.exists()) {
indicator.setIndeterminate(true);
try {
myIndexer.scan(index, new MyScanningListener(indicator), false);
} finally {
indicator.setIndeterminate(false);
}
}
} else {
IndexUpdateRequest request = new IndexUpdateRequest(index);
Maven2ServerEmbedderImpl embedder = Maven2ServerEmbedderImpl.create(settings);
try {
request.setResourceFetcher(
new Maven2ServerIndexFetcher(
index.getRepositoryId(),
index.getRepositoryUrl(),
embedder.getComponent(WagonManager.class),
new TransferListenerAdapter(indicator) {
@Override
protected void downloadProgress(long downloaded, long total) {
super.downloadProgress(downloaded, total);
try {
myIndicator.setFraction(((double) downloaded) / total);
} catch (RemoteException e) {
throw new RuntimeRemoteException(e);
}
}
@Override
public void transferCompleted(TransferEvent event) {
super.transferCompleted(event);
try {
myIndicator.setText2("Processing indices...");
} catch (RemoteException e) {
throw new RuntimeRemoteException(e);
}
}
}));
myUpdater.fetchAndUpdateIndex(request);
} finally {
embedder.release();
}
}
} catch (RuntimeRemoteException e) {
throw e.getCause();
} catch (ProcessCanceledException e) {
throw new MavenServerProcessCanceledException();
} catch (Exception e) {
throw new MavenServerIndexerException(wrapException(e));
}
}
public Set<MavenArtifactInfo> search(int indexId, Query query, int maxResult)
throws MavenServerIndexerException {
try {
IndexingContext index = getIndex(indexId);
TopDocs docs = null;
try {
BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
docs = index.getIndexSearcher().search(query, null, maxResult);
} catch (BooleanQuery.TooManyClauses ignore) {
// this exception occurs when too wide wildcard is used on too big data.
}
if (docs == null || docs.scoreDocs.length == 0) return Collections.emptySet();
Set<MavenArtifactInfo> result = new THashSet<MavenArtifactInfo>();
for (int i = 0; i < docs.scoreDocs.length; i++) {
int docIndex = docs.scoreDocs[i].doc;
Document doc = index.getIndexReader().document(docIndex);
ArtifactInfo a = IndexUtils.constructArtifactInfo(doc, index);
if (a == null) continue;
a.repository = getRepositoryPathOrUrl(index);
result.add(Maven2ModelConverter.convertArtifactInfo(a));
}
return result;
} catch (Exception e) {
throw new MavenServerIndexerException(wrapException(e));
}
}
private IteratorSearchResponse searchIteratorPaged(
IteratorSearchRequest request,
Collection<IndexingContext> indexingContexts,
boolean ignoreContext)
throws IOException {
// manage defaults!
if (request.getStart() < 0) {
request.setStart(IteratorSearchRequest.UNDEFINED);
}
if (request.getCount() < 0) {
request.setCount(IteratorSearchRequest.UNDEFINED);
}
// to not change the API all away, but we need stable ordering here
// filter for those 1st, that take part in here
ArrayList<IndexingContext> contexts = new ArrayList<IndexingContext>(indexingContexts.size());
for (IndexingContext ctx : indexingContexts) {
if (ignoreContext || ctx.isSearchable()) {
contexts.add(ctx);
}
}
ArrayList<IndexReader> contextsToSearch = new ArrayList<IndexReader>(contexts.size());
for (IndexingContext ctx : contexts) {
contextsToSearch.add(ctx.getIndexReader());
}
MultiReader multiReader =
new MultiReader(contextsToSearch.toArray(new IndexReader[contextsToSearch.size()]));
IndexSearcher indexSearcher = new IndexSearcher(multiReader);
// NEXUS-3482 made us to NOT use reverse ordering (it is a fix I wanted to implement, but user
// contributed patch
// did come in faster! -- Thanks)
Hits hits =
indexSearcher.search(
request.getQuery(),
new Sort(
new SortField[] {
SortField.FIELD_SCORE, new SortField(null, SortField.DOC, false)
}));
return new IteratorSearchResponse(
request.getQuery(),
hits.length(),
new DefaultIteratorResultSet(request, indexSearcher, contexts, hits));
}
protected int searchFlat(
AbstractSearchRequest req,
Collection<ArtifactInfo> result,
IndexingContext context,
Query query,
int from,
int aiCount)
throws IOException {
Hits hits =
context
.getIndexSearcher()
.search(query, new Sort(new SortField(ArtifactInfo.UINFO, SortField.STRING)));
if (hits == null || hits.length() == 0) {
return 0;
}
if (req.isHitLimited() && hits.length() > req.getResultHitLimit()) {
return AbstractSearchResponse.LIMIT_EXCEEDED;
}
int hitCount = hits.length();
int start = 0; // from == FlatSearchRequest.UNDEFINED ? 0 : from;
// we have to pack the results as long: a) we have found aiCount ones b) we depleted hits
for (int i = start; i < hits.length(); i++) {
Document doc = hits.doc(i);
ArtifactInfo artifactInfo = IndexUtils.constructArtifactInfo(doc, context);
if (artifactInfo != null) {
artifactInfo.repository = context.getRepositoryId();
artifactInfo.context = context.getId();
result.add(artifactInfo);
if (req.isHitLimited() && result.size() > req.getResultHitLimit()) {
// we hit limit, back out now !!
return AbstractSearchResponse.LIMIT_EXCEEDED;
}
}
}
return hitCount;
}
public static void addArtifact(
NexusIndexer indexer, IndexingContext index, ArtifactContext artifactContext)
throws IOException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
indexer.addArtifactToIndex(artifactContext, index);
// this hack is necessary to invalidate searcher's and reader's cache (may not be required then
// lucene or nexus library change
Method m = index.getClass().getDeclaredMethod("closeReaders");
m.setAccessible(true);
m.invoke(index);
}
protected int searchGrouped(
AbstractSearchRequest req,
Map<String, ArtifactInfoGroup> result,
Grouping grouping,
IndexingContext context,
Query query)
throws IOException {
Hits hits =
context
.getIndexSearcher()
.search(query, new Sort(new SortField(ArtifactInfo.UINFO, SortField.STRING)));
if (hits != null && hits.length() != 0) {
int hitCount = hits.length();
for (int i = 0; i < hits.length(); i++) {
ArtifactInfo artifactInfo = IndexUtils.constructArtifactInfo(hits.doc(i), context);
if (artifactInfo != null) {
artifactInfo.repository = context.getRepositoryId();
artifactInfo.context = context.getId();
if (!grouping.addArtifactInfo(result, artifactInfo)) {
// fix the hitCount accordingly
hitCount--;
}
}
}
if (req.isHitLimited() && hits.length() > req.getResultHitLimit()) {
return AbstractSearchResponse.LIMIT_EXCEEDED;
}
return hitCount;
} else {
return 0;
}
}
/**
* Will reindex, shift if needed and publish indexes for a "remote" repository (published over
* jetty component).
*
* @param repositoryRoot
* @param repositoryId
* @param deleteIndexFiles
* @param shiftDays
* @throws IOException
*/
protected void reindexRemoteRepositoryAndPublish(
File repositoryRoot, String repositoryId, boolean deleteIndexFiles, int shiftDays)
throws IOException, ComponentLookupException {
File indexDirectory = getIndexFamilyDirectory(repositoryId);
Directory directory = FSDirectory.getDirectory(indexDirectory);
IndexingContext ctx =
nexusIndexer.addIndexingContextForced(
repositoryId + "-temp",
repositoryId,
repositoryRoot,
directory,
null,
null,
new IndexCreatorHelper(getContainer()).getFullCreators());
// shifting if needed (very crude way to do it, but heh)
shiftContextInTime(ctx, shiftDays);
// and scan "today"
nexusIndexer.scan(ctx);
ctx.updateTimestamp(true);
// pack it up
File targetDir = new File(repositoryRoot, ".index");
targetDir.mkdirs();
IndexPackingRequest ipr = new IndexPackingRequest(ctx, targetDir);
ipr.setCreateIncrementalChunks(true);
indexPacker.packIndex(ipr);
nexusIndexer.removeIndexingContext(ctx, deleteIndexFiles);
}
protected void shiftContextInTime(IndexingContext ctx, int shiftDays) throws IOException {
if (shiftDays != 0) {
IndexWriter iw = ctx.getIndexWriter();
for (int docNum = 0; docNum < ctx.getIndexReader().maxDoc(); docNum++) {
if (!ctx.getIndexReader().isDeleted(docNum)) {
Document doc = ctx.getIndexReader().document(docNum);
String lastModified = doc.get(ArtifactInfo.LAST_MODIFIED);
if (lastModified != null) {
long lm = Long.parseLong(lastModified);
lm = lm + (shiftDays * A_DAY_MILLIS);
doc.removeFields(ArtifactInfo.LAST_MODIFIED);
doc.add(
new Field(
ArtifactInfo.LAST_MODIFIED,
Long.toString(lm),
Field.Store.YES,
Field.Index.NO));
iw.updateDocument(new Term(ArtifactInfo.UINFO, doc.get(ArtifactInfo.UINFO)), doc);
}
}
}
iw.optimize();
iw.close();
// shift timestamp too
if (ctx.getTimestamp() != null) {
ctx.updateTimestamp(
true, new Date(ctx.getTimestamp().getTime() + (shiftDays * A_DAY_MILLIS)));
} else {
ctx.updateTimestamp(
true, new Date(System.currentTimeMillis() + (shiftDays * A_DAY_MILLIS)));
}
}
}
private boolean isLocal(IndexingContext index) {
return index.getRepository() != null;
}
private String getRepositoryPathOrUrl(IndexingContext index) {
File file = index.getRepository();
return file == null ? index.getRepositoryUrl() : file.getPath();
}
