protected void addRepositoryDetails( Request request, SearchNGResponse response, Repository repository) { boolean add = true; for (NexusNGRepositoryDetail repoDetail : response.getRepoDetails()) { if (repoDetail.getRepositoryId().equals(repository.getId())) { add = false; break; } } if (add) { NexusNGRepositoryDetail repoDetail = new NexusNGRepositoryDetail(); repoDetail.setRepositoryId(repository.getId()); repoDetail.setRepositoryName(repository.getName()); repoDetail.setRepositoryURL( createRepositoryReference(request, repository.getId()).getTargetRef().toString()); repoDetail.setRepositoryContentClass(repository.getRepositoryContentClass().getId()); repoDetail.setRepositoryKind(extractRepositoryKind(repository)); MavenRepository mavenRepo = repository.adaptToFacet(MavenRepository.class); if (mavenRepo != null) { repoDetail.setRepositoryPolicy(mavenRepo.getRepositoryPolicy().name()); } response.addRepoDetail(repoDetail); } }
protected Model getPom(Variant variant, Request request, Response response) throws ResourceException { Form form = request.getResourceRef().getQueryAsForm(); // TODO: enable only one section retrieval of POM, ie. only mailing lists, or team members String groupId = form.getFirstValue("g"); String artifactId = form.getFirstValue("a"); String version = form.getFirstValue("v"); String repositoryId = form.getFirstValue("r"); if (groupId == null || artifactId == null || version == null || repositoryId == null) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST); } ArtifactStoreRequest gavRequest = getResourceStoreRequest( request, false, repositoryId, groupId, artifactId, version, null, null, "pom"); try { MavenRepository mavenRepository = getMavenRepository(repositoryId); ArtifactStoreHelper helper = mavenRepository.getArtifactStoreHelper(); InputStream pomContent = null; InputStreamReader ir = null; Model pom = null; try { StorageFileItem file = helper.retrieveArtifactPom(gavRequest); pomContent = file.getInputStream(); MavenXpp3Reader reader = new MavenXpp3Reader(); ir = new InputStreamReader(pomContent); pom = reader.read(ir); } finally { IOUtil.close(pomContent); IOUtil.close(ir); } return pom; } catch (Exception e) { handleException(request, response, e); } return null; }
@Test public void shouldUnregisterRepository() throws Exception { MavenRepository repository = createRepository(true); yumRegistry.register(repository); Assert.assertTrue(yumRegistry.isRegistered(repository.getId())); yumRegistry.unregister(repository.getId()); Assert.assertFalse(yumRegistry.isRegistered(repository.getId())); }
@Test public void shouldScanRepository() throws Exception { final MavenRepository repository = mock(MavenRepository.class); when(repository.getId()).thenReturn(REPO_ID); when(repository.getLocalUrl()).thenReturn(rpmsDir().toURI().toASCIIString()); final RepositoryKind repositoryKind = mock(RepositoryKind.class); when(repository.getRepositoryKind()).thenReturn(repositoryKind); when(repositoryKind.isFacetAvailable(HostedRepository.class)).thenReturn(true); yumRegistry.register(repository); waitForAllTasksToBeDone(); Assert.assertNotNull(yumRegistry.get(REPO_ID)); }
private void removeDirectoryIfEmpty(StorageCollectionItem coll) throws StorageException, IllegalOperationException, UnsupportedStorageOperationException { try { if (repository.list(false, coll).size() > 0) { return; } if (getLogger().isDebugEnabled()) { getLogger() .debug( "Removing the empty directory leftover: UID=" + coll.getRepositoryItemUid().toString()); } repository.deleteItem(false, new ResourceStoreRequest(coll)); } catch (ItemNotFoundException e) { // silent, this happens if whole GAV is removed and the dir is removed too } }
@Test public void smoke() throws Exception { final WLManager wm = lookup(WLManager.class); // deploy to hosted something { final MavenRepository mavenRepository = getRepositoryRegistry().getRepositoryWithFacet(HOSTED_REPO_ID, MavenRepository.class); mavenRepository.storeItemWithChecksums( new ResourceStoreRequest("/com/sonatype/test/1.0/test-1.0.txt"), new ByteArrayInputStream("Some fluke content".getBytes()), null); } { try { server.start(); final PrefixSource hostedEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(HOSTED_REPO_ID, MavenRepository.class)); final PrefixSource proxyEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(PROXY_REPO_ID, MavenRepository.class)); final PrefixSource groupEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(GROUP_REPO_ID, MavenRepository.class)); assertThat("Hosted should have ES", hostedEntrySource.exists()); assertThat( "Proxy should not have ES", !proxyEntrySource.exists()); // as we serve 404s for prefix // file assertThat( "Group cannot have ES", !groupEntrySource.exists()); // as proxy member does not have WL } finally { server.stop(); } } { server.stop(); server = Server.withPort(remoteServerPort) .serve("/.meta/prefixes.txt") .withBehaviours(Behaviours.content(prefixFile1(true))); try { server.start(); final MavenProxyRepository mavenProxyRepository = getRepositoryRegistry() .getRepositoryWithFacet(PROXY_REPO_ID, MavenProxyRepository.class); wm.updateWhitelist(mavenProxyRepository); waitForWLBackgroundUpdates(); final PrefixSource hostedEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(HOSTED_REPO_ID, MavenRepository.class)); final PrefixSource proxyEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(PROXY_REPO_ID, MavenRepository.class)); final PrefixSource groupEntrySource = wm.getPrefixSourceFor( getRepositoryRegistry() .getRepositoryWithFacet(GROUP_REPO_ID, MavenRepository.class)); assertThat("Hosted should have ES", hostedEntrySource.exists()); assertThat("Proxy should have ES", proxyEntrySource.exists()); // as we did serve file ok assertThat( "Group should have ES", groupEntrySource.exists()); // as all member should have it // GROUP wl must have 4 entries: 1 from hosted (/com/sonatype) + 3 from proxied prefix file final List<String> groupEntries = groupEntrySource.readEntries(); assertThat(groupEntries.size(), equalTo(4)); assertThat(groupEntries, hasItem("/com/sonatype")); assertThat(groupEntries, hasItem("/org/sonatype")); assertThat(groupEntries, hasItem("/org/apache/maven")); assertThat(groupEntries, hasItem("/eu/flatwhite")); } finally { server.stop(); } } }
public boolean releaseExistsForSnapshot(Gav snapshotGav, Map<String, Object> context) { for (Repository repository : repositoryRegistry.getRepositories()) { // we need to filter for: // repository that is MavenRepository and is hosted or proxy // repository that has release policy if (repository.getRepositoryKind().isFacetAvailable(MavenHostedRepository.class) || repository.getRepositoryKind().isFacetAvailable(MavenProxyRepository.class)) { // actually, we don't care is it proxy or hosted, we only need to filter out groups and // other // "composite" reposes like shadows MavenRepository mrepository = repository.adaptToFacet(MavenRepository.class); // look in release reposes only if (mrepository.isUserManaged() && RepositoryPolicy.RELEASE.equals(mrepository.getRepositoryPolicy())) { try { String releaseVersion = null; // NEXUS-3148 if (snapshotGav.getBaseVersion().endsWith("-SNAPSHOT")) { // "-SNAPSHOT" :== 9 chars releaseVersion = snapshotGav .getBaseVersion() .substring(0, snapshotGav.getBaseVersion().length() - 9); } else { // "SNAPSHOT" :== 8 chars releaseVersion = snapshotGav .getBaseVersion() .substring(0, snapshotGav.getBaseVersion().length() - 8); } Gav releaseGav = new Gav( snapshotGav.getGroupId(), snapshotGav.getArtifactId(), releaseVersion, snapshotGav.getClassifier(), snapshotGav.getExtension(), null, null, null, false, null, false, null); String path = mrepository.getGavCalculator().gavToPath(releaseGav); ResourceStoreRequest req = new ResourceStoreRequest(path, true); req.getRequestContext().putAll(context); mrepository.retrieveItem(false, req); return true; } catch (ItemNotFoundException e) { // nothing } catch (Exception e) { // nothing } } } } return false; }
public void doOnCollectionExit(WalkerContext context, StorageCollectionItem coll) throws Exception { if (getLogger().isDebugEnabled()) { getLogger().debug("onCollectionExit() :: " + coll.getRepositoryItemUid().toString()); } shouldProcessCollection = coll.getPath().endsWith("SNAPSHOT"); if (!shouldProcessCollection) { return; } deletableSnapshotsAndFiles.clear(); remainingSnapshotsAndFiles.clear(); removeWholeGAV = false; Gav gav = null; Collection<StorageItem> items; items = repository.list(false, coll); HashSet<Long> versionsToRemove = new HashSet<Long>(); // gathering the facts for (StorageItem item : items) { if (!item.isVirtual() && !StorageCollectionItem.class.isAssignableFrom(item.getClass())) { gav = ((MavenRepository) coll.getRepositoryItemUid().getRepository()) .getGavCalculator() .pathToGav(item.getPath()); if (gav != null) { // if we find a pom, check for delete on release if (!gav.isHash() && !gav.isSignature() && gav.getExtension().equals("pom")) { if (request.isRemoveIfReleaseExists() && releaseExistsForSnapshot(gav, item.getItemContext())) { getLogger().debug("Found POM and release exists, removing whole gav."); removeWholeGAV = true; // Will break out and junk whole gav break; } } item.getItemContext().put(Gav.class.getName(), gav); if (gav.getSnapshotTimeStamp() != null) { getLogger().debug("Using GAV snapshot timestamp"); long itemTimestamp = gav.getSnapshotTimeStamp().longValue(); getLogger().debug("NOW is " + itemTimestamp); // If this timestamp is already marked to be removed, junk it if (versionsToRemove.contains(new Long(itemTimestamp))) { addStorageFileItemToMap(deletableSnapshotsAndFiles, gav, (StorageFileItem) item); } else { getLogger() .debug("itemTimestamp=" + itemTimestamp + ", dateTreshold=" + dateThreshold); // if dateTreshold is not used (zero days) OR // if itemTimestamp is less then dateTreshold (NB: both are positive!) // below will the retentionCount overrule if needed this if (-1 == dateThreshold || itemTimestamp < dateThreshold) { versionsToRemove.add(new Long(itemTimestamp)); addStorageFileItemToMap(deletableSnapshotsAndFiles, gav, (StorageFileItem) item); } else { addStorageFileItemToMap(remainingSnapshotsAndFiles, gav, (StorageFileItem) item); } } } else { // If no timestamp on gav, then it is a non-unique snapshot // and should _not_ be removed getLogger() .debug("GAV Snapshot timestamp not available, skipping non-unique snapshot"); addStorageFileItemToMap(remainingSnapshotsAndFiles, gav, (StorageFileItem) item); } } } } // and doing the work here if (removeWholeGAV) { try { for (StorageItem item : items) { try { // preserve possible subdirs if (!(item instanceof StorageCollectionItem)) { repository.deleteItem(false, new ResourceStoreRequest(item)); } } catch (ItemNotFoundException e) { if (getLogger().isDebugEnabled()) { getLogger() .debug( "Could not delete whole GAV " + coll.getRepositoryItemUid().toString(), e); } } } } catch (Exception e) { getLogger() .warn("Could not delete whole GAV " + coll.getRepositoryItemUid().toString(), e); } } else { // and now check some things if (remainingSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep()) { // do something if (remainingSnapshotsAndFiles.size() + deletableSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep()) { // delete nothing, since there is less snapshots in total as allowed deletableSnapshotsAndFiles.clear(); } else { TreeSet<Version> keys = new TreeSet<Version>(deletableSnapshotsAndFiles.keySet()); while (!keys.isEmpty() && remainingSnapshotsAndFiles.size() < request.getMinCountOfSnapshotsToKeep()) { Version keyToMove = keys.last(); if (remainingSnapshotsAndFiles.containsKey(keyToMove)) { remainingSnapshotsAndFiles .get(keyToMove) .addAll(deletableSnapshotsAndFiles.get(keyToMove)); } else { remainingSnapshotsAndFiles.put( keyToMove, deletableSnapshotsAndFiles.get(keyToMove)); } deletableSnapshotsAndFiles.remove(keyToMove); keys.remove(keyToMove); } } } // NEXUS-814: is this GAV have remaining artifacts? boolean gavHasMoreTimestampedSnapshots = remainingSnapshotsAndFiles.size() > 0; for (Version key : deletableSnapshotsAndFiles.keySet()) { List<StorageFileItem> files = deletableSnapshotsAndFiles.get(key); deletedSnapshots++; for (StorageFileItem file : files) { try { // NEXUS-814: mark that we are deleting a TS snapshot, but there are still remaining // ones in repository. if (gavHasMoreTimestampedSnapshots) { file.getItemContext().put(MORE_TS_SNAPSHOTS_EXISTS_FOR_GAV, Boolean.TRUE); } gav = (Gav) file.getItemContext().get(Gav.class.getName()); repository.deleteItem(false, new ResourceStoreRequest(file)); deletedFiles++; } catch (ItemNotFoundException e) { if (getLogger().isDebugEnabled()) { getLogger().debug("Could not delete file:", e); } } catch (Exception e) { getLogger().info("Could not delete file:", e); } } } } removeDirectoryIfEmpty(coll); updateMetadataIfNecessary(context, coll); }
/** * Removes the snapshots from maven repository. * * @param repository the repository * @throws Exception the exception */ protected SnapshotRemovalRepositoryResult removeSnapshotsFromMavenRepository( MavenRepository repository, SnapshotRemovalRequest request) { TaskUtil.checkInterruption(); SnapshotRemovalRepositoryResult result = new SnapshotRemovalRepositoryResult(repository.getId(), 0, 0, true); if (!repository.getLocalStatus().shouldServiceRequest()) { return result; } // we are already processed here, so skip repo if (request.isProcessedRepo(repository.getId())) { return new SnapshotRemovalRepositoryResult(repository.getId(), true); } request.addProcessedRepo(repository.getId()); // if this is not snap repo, do nothing if (!RepositoryPolicy.SNAPSHOT.equals(repository.getRepositoryPolicy())) { return result; } if (getLogger().isDebugEnabled()) { getLogger() .debug( "Collecting deletable snapshots on repository " + repository.getId() + " from storage directory " + repository.getLocalUrl()); } request.getMetadataRebuildPaths().clear(); // create a walker to collect deletables and let it loose on collections only SnapshotRemoverWalkerProcessor snapshotRemoveProcessor = new SnapshotRemoverWalkerProcessor(repository, request); DefaultWalkerContext ctxMain = new DefaultWalkerContext( repository, new ResourceStoreRequest("/"), new DottedStoreWalkerFilter()); ctxMain.getProcessors().add(snapshotRemoveProcessor); walker.walk(ctxMain); if (ctxMain.getStopCause() != null) { result.setSuccessful(false); } // and collect results result.setDeletedSnapshots(snapshotRemoveProcessor.getDeletedSnapshots()); result.setDeletedFiles(snapshotRemoveProcessor.getDeletedFiles()); if (getLogger().isDebugEnabled()) { getLogger() .debug( "Collected and deleted " + snapshotRemoveProcessor.getDeletedSnapshots() + " snapshots with alltogether " + snapshotRemoveProcessor.getDeletedFiles() + " files on repository " + repository.getId()); } repository.expireCaches(new ResourceStoreRequest(RepositoryItemUid.PATH_ROOT)); RecreateMavenMetadataWalkerProcessor metadataRebuildProcessor = new RecreateMavenMetadataWalkerProcessor(getLogger()); for (String path : request.getMetadataRebuildPaths()) { DefaultWalkerContext ctxMd = new DefaultWalkerContext( repository, new ResourceStoreRequest(path), new DottedStoreWalkerFilter()); ctxMd.getProcessors().add(metadataRebuildProcessor); try { walker.walk(ctxMd); } catch (WalkerException e) { if (!(e.getCause() instanceof ItemNotFoundException)) { // do not ignore it throw e; } } } return result; }
/** * Centralized way to create ResourceStoreRequests, since we have to fill in various things in * Request context, like authenticated username, etc. * * @param isLocal * @return */ protected ArtifactStoreRequest getResourceStoreRequest( Request request, boolean localOnly, String repositoryId, String g, String a, String v, String p, String c, String e) throws ResourceException { if (StringUtils.isBlank(p) && StringUtils.isBlank(e)) { // if packaging and extension is both blank, it is a bad request throw new ResourceException( Status.CLIENT_ERROR_BAD_REQUEST, "Deployment tried with both 'packaging' and/or 'extension' being empty! One of these values is mandatory!"); } MavenRepository mavenRepository = getMavenRepository(repositoryId); // if extension is not given, fall-back to packaging and apply mapper if (StringUtils.isBlank(e)) { e = mavenRepository.getArtifactPackagingMapper().getExtensionForPackaging(p); } // clean up the classifier if (StringUtils.isBlank(c)) { c = null; } Gav gav = null; try { gav = new Gav( g, a, v, c, e, null, null, null, VersionUtils.isSnapshot(v), false, null, false, null); } catch (IllegalArtifactCoordinateException ex) { throw new ResourceException( Status.CLIENT_ERROR_BAD_REQUEST, "Illegal artifact coordinate.", ex); } ArtifactStoreRequest result = new ArtifactStoreRequest(mavenRepository, gav, localOnly); if (getLogger().isDebugEnabled()) { getLogger().debug("Created ArtifactStoreRequest request for " + result.getRequestPath()); } // stuff in the originating remote address result .getRequestContext() .put(AccessManager.REQUEST_REMOTE_ADDRESS, getValidRemoteIPAddress(request)); // stuff in the user id if we have it in request if (request.getChallengeResponse() != null && request.getChallengeResponse().getIdentifier() != null) { result .getRequestContext() .put(AccessManager.REQUEST_USER, request.getChallengeResponse().getIdentifier()); } // this is HTTPS, get the cert and stuff it too for later if (request.isConfidential()) { result.getRequestContext().put(AccessManager.REQUEST_CONFIDENTIAL, Boolean.TRUE); List<?> certs = (List<?>) request.getAttributes().get("org.restlet.https.clientCertificates"); if (certs != null) { result.getRequestContext().put(AccessManager.REQUEST_CERTIFICATES, certs); } } // put the incoming URLs result.setRequestAppRootUrl(getContextRoot(request).toString()); result.setRequestUrl(request.getOriginalRef().toString()); return result; }
@Override public Object upload(Context context, Request request, Response response, List<FileItem> files) throws ResourceException { // we have "nibbles": (params,fileA,[fileB])+ // the second file is optional // if two files are present, one of them should be POM String repositoryId = null; boolean hasPom = false; boolean isPom = false; InputStream is = null; String groupId = null; String artifactId = null; String version = null; String classifier = null; String packaging = null; String extension = null; ArtifactCoordinate coords = null; PomArtifactManager pomManager = new PomArtifactManager(getNexus().getNexusConfiguration().getTemporaryDirectory()); try { for (FileItem fi : files) { if (fi.isFormField()) { // a parameter if ("r".equals(fi.getFieldName())) { repositoryId = fi.getString(); } else if ("g".equals(fi.getFieldName())) { groupId = fi.getString(); } else if ("a".equals(fi.getFieldName())) { artifactId = fi.getString(); } else if ("v".equals(fi.getFieldName())) { version = fi.getString(); } else if ("p".equals(fi.getFieldName())) { packaging = fi.getString(); } else if ("c".equals(fi.getFieldName())) { classifier = fi.getString(); } else if ("e".equals(fi.getFieldName())) { extension = fi.getString(); } else if ("hasPom".equals(fi.getFieldName())) { hasPom = Boolean.parseBoolean(fi.getString()); } coords = new ArtifactCoordinate(); coords.setGroupId(groupId); coords.setArtifactId(artifactId); coords.setVersion(version); coords.setPackaging(packaging); } else { // a file isPom = fi.getName().endsWith(".pom") || fi.getName().endsWith("pom.xml"); ArtifactStoreRequest gavRequest = null; if (hasPom) { if (isPom) { // let it "thru" the pomManager to be able to get GAV from it on later pass pomManager.storeTempPomFile(fi.getInputStream()); is = pomManager.getTempPomFileInputStream(); } else { is = fi.getInputStream(); } try { coords = pomManager.getArtifactCoordinateFromTempPomFile(); } catch (IOException e) { getLogger().info(e.getMessage()); throw new ResourceException( Status.CLIENT_ERROR_BAD_REQUEST, "Error occurred while reading the POM file. Malformed POM?"); } if (isPom) { gavRequest = getResourceStoreRequest( request, true, repositoryId, coords.getGroupId(), coords.getArtifactId(), coords.getVersion(), coords.getPackaging(), null, null); } else { gavRequest = getResourceStoreRequest( request, true, repositoryId, coords.getGroupId(), coords.getArtifactId(), coords.getVersion(), coords.getPackaging(), classifier, extension); } } else { is = fi.getInputStream(); gavRequest = getResourceStoreRequest( request, true, repositoryId, groupId, artifactId, version, packaging, classifier, extension); } MavenRepository mr = gavRequest.getMavenRepository(); ArtifactStoreHelper helper = mr.getArtifactStoreHelper(); // temporarily we disable SNAPSHOT upload // check is it a Snapshot repo if (RepositoryPolicy.SNAPSHOT.equals(mr.getRepositoryPolicy())) { getLogger() .info("Upload to SNAPSHOT maven repository attempted, returning Bad Request."); throw new ResourceException( Status.CLIENT_ERROR_BAD_REQUEST, "This is a Maven SNAPSHOT repository, and manual upload against it is forbidden!"); } if (!versionMatchesPolicy(gavRequest.getVersion(), mr.getRepositoryPolicy())) { getLogger() .warn("Version (" + gavRequest.getVersion() + ") and Repository Policy mismatch"); throw new ResourceException( Status.CLIENT_ERROR_BAD_REQUEST, "The version " + gavRequest.getVersion() + " does not match the repository policy!"); } if (isPom) { helper.storeArtifactPom(gavRequest, is, null); isPom = false; } else { if (hasPom) { helper.storeArtifact(gavRequest, is, null); } else { helper.storeArtifactWithGeneratedPom(gavRequest, packaging, is, null); } } } } } catch (Throwable t) { return buildUploadFailedHtmlResponse(t, request, response); } finally { if (hasPom) { pomManager.removeTempPomFile(); } } return coords; }
protected Object getContent( Variant variant, boolean redirectTo, Request request, Response response) throws ResourceException { Form form = request.getResourceRef().getQueryAsForm(); String groupId = form.getFirstValue("g"); String artifactId = form.getFirstValue("a"); String version = form.getFirstValue("v"); String packaging = form.getFirstValue("p"); String classifier = form.getFirstValue("c"); String repositoryId = form.getFirstValue("r"); String extension = form.getFirstValue("e"); if (groupId == null || artifactId == null || version == null || repositoryId == null) { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST); } // default the packaging if (StringUtils.isBlank(packaging)) { packaging = "jar"; } ArtifactStoreRequest gavRequest = getResourceStoreRequest( request, false, repositoryId, groupId, artifactId, version, packaging, classifier, extension); try { MavenRepository mavenRepository = getMavenRepository(repositoryId); ArtifactStoreHelper helper = mavenRepository.getArtifactStoreHelper(); StorageFileItem file = helper.retrieveArtifact(gavRequest); if (redirectTo) { Reference fileReference = createRepositoryReference( request, file.getRepositoryItemUid().getRepository().getId(), file.getRepositoryItemUid().getPath()); response.setLocationRef(fileReference); response.setStatus(Status.REDIRECTION_PERMANENT); String redirectMessage = "If you are not automatically redirected use this url: " + fileReference.toString(); return redirectMessage; } else { Representation result = new StorageFileItemRepresentation(file); result.setDownloadable(true); result.setDownloadName(file.getName()); return result; } } catch (Exception e) { handleException(request, response, e); } return null; }
@Test public void shouldFindRepository() throws Exception { final MavenRepository repository = createRepository(true); yumRegistry.register(repository); assertThat(yumRegistry.get(repository.getId()), is(notNullValue())); }