public ArtifactData getCandidateAsync(String arg) throws Exception { reporter.trace("coordinate %s", arg); if (isUrl(arg)) try { ArtifactData data = putAsync(new URI(arg)); data.local = true; return data; } catch (Exception e) { reporter.trace("hmm, not a valid url %s, will try the server", arg); } Coordinate c = new Coordinate(arg); if (c.isSha()) { ArtifactData r = get(c.getSha()); if (r != null) return r; } Revision revision = library.getRevisionByCoordinate(c); if (revision == null) return null; reporter.trace("revision %s", Hex.toHexString(revision._id)); ArtifactData ad = get(revision._id); if (ad != null) { reporter.trace("found in cache"); return ad; } URI url = revision.urls.iterator().next(); ArtifactData artifactData = putAsync(url); artifactData.coordinate = c; return artifactData; }
void download(ResourceDescriptor rds, File path) throws Exception { reporter.trace("starting download %s", path); Exception exception = new Exception(); event(TYPE.START_DOWNLOAD, rds, null); for (int i = 0; i < 3; sleep(3000), i++) try { download0(rds.url, path, rds.id); event(TYPE.END_DOWNLOAD, rds, null); reporter.trace("succesful download %s", path); failures.remove(rds.url); return; } catch (FileNotFoundException e) { reporter.trace("no such file download %s", path); exception = e; break; // no use retrying } catch (Exception e) { reporter.trace("exception download %s", path); exception = e; } failures.put(rds.url, System.currentTimeMillis()); reporter.trace("failed download %s", path); event(TYPE.ERROR, rds, exception); event(TYPE.END_DOWNLOAD, rds, exception); throw exception; }
/** * This is called when JPM runs in the background to start jobs * * @throws Exception */ public void daemon() throws Exception { Runtime.getRuntime() .addShutdownHook( new Thread("Daemon shutdown") { public void run() { for (Service service : startedByDaemon) { try { reporter.error("Stopping " + service); service.stop(); reporter.error("Stopped " + service); } catch (Exception e) { // Ignore } } } }); List<ServiceData> services = getServices(); Map<String, ServiceData> map = new HashMap<String, ServiceData>(); for (ServiceData d : services) { map.put(d.name, d); } List<ServiceData> start = new ArrayList<ServiceData>(); Set<ServiceData> set = new HashSet<ServiceData>(); for (ServiceData sd : services) { checkStartup(map, start, sd, set); } if (start.isEmpty()) reporter.warning("No services to start"); for (ServiceData sd : start) { try { Service service = getService(sd.name); reporter.trace("Starting " + service); String result = service.start(); if (result != null) reporter.error("Started error " + result); else startedByDaemon.add(service); reporter.trace("Started " + service); } catch (Exception e) { reporter.error("Cannot start daemon %s, due to %s", sd.name, e); } } while (true) { for (Service sd : startedByDaemon) { try { if (!sd.isRunning()) { reporter.error("Starting due to failure " + sd); String result = sd.start(); if (result != null) reporter.error("Started error " + result); } } catch (Exception e) { reporter.error("Cannot start daemon %s, due to %s", sd, e); } } Thread.sleep(10000); } }
void download0(URI url, File path, byte[] sha) throws Exception { path.getParentFile().mkdirs(); File tmp = IO.createTempFile(path.getParentFile(), "tmp", ".jar"); URL u = url.toURL(); URLConnection conn = u.openConnection(); InputStream in; if (conn instanceof HttpURLConnection) { HttpURLConnection http = (HttpURLConnection) conn; http.setRequestProperty("Accept-Encoding", "deflate"); http.setInstanceFollowRedirects(true); connector.handle(conn); int result = http.getResponseCode(); if (result / 100 != 2) { String s = ""; InputStream err = http.getErrorStream(); try { if (err != null) s = IO.collect(err); if (result == 404) { reporter.trace("not found "); throw new FileNotFoundException("Cannot find " + url + " : " + s); } throw new IOException( "Failed request " + result + ":" + http.getResponseMessage() + " " + s); } finally { if (err != null) err.close(); } } String deflate = http.getHeaderField("Content-Encoding"); in = http.getInputStream(); if (deflate != null && deflate.toLowerCase().contains("deflate")) { in = new InflaterInputStream(in); reporter.trace("inflate"); } } else { connector.handle(conn); in = conn.getInputStream(); } IO.copy(in, tmp); byte[] digest = SHA1.digest(tmp).digest(); if (Arrays.equals(digest, sha)) { IO.rename(tmp, path); } else { reporter.trace( "sha's did not match %s, expected %s, got %s", tmp, Hex.toHexString(sha), digest); throw new IllegalArgumentException("Invalid sha downloaded"); } }
/** Add a resource descriptor to the index. */ public boolean add(String repoId, ResourceDescriptor rd) throws Exception { ResourceDescriptorImpl rdi = getResourceDescriptor(rd.id); boolean add = false; if (rdi != null) { add = true; reporter.trace("adding repo %s to resource %s to index", repoId, rdi); } else { rdi = new ResourceDescriptorImpl(rd); getIndex().descriptors.add(rdi); reporter.trace("adding resource %s to index", rdi); } rdi.repositories.add(repoId); event(TYPE.ADD, rdi, null); setDirty(); save(); return add; }
/** * Dispatch the events * * @param type * @param rds * @param exception */ private void event(TYPE type, ResourceDescriptor rds, Exception exception) { for (Listener l : listeners) { try { l.events(new ResourceRepositoryEvent(type, rds, exception)); } catch (Exception e) { reporter.trace("listener %s throws exception %s", l, e); } } }
/** Delete a resource from the text file (not from the cache) */ void delete(byte[] id) throws Exception { for (Iterator<ResourceDescriptorImpl> i = getIndex().descriptors.iterator(); i.hasNext(); ) { ResourceDescriptorImpl d = i.next(); if (Arrays.equals(id, d.id)) { i.remove(); reporter.trace("removing resource %s from index", d); event(TYPE.REMOVE, d, null); setDirty(); } } save(); }
void put(final URI uri, ArtifactData data) throws Exception { reporter.trace("put %s %s", uri, data); File tmp = createTempFile(repoDir, "mtp", ".whatever"); tmp.deleteOnExit(); try { copy(uri.toURL(), tmp); byte[] sha = SHA1.digest(tmp).digest(); reporter.trace("SHA %s %s", uri, Hex.toHexString(sha)); ArtifactData existing = get(sha); if (existing != null) { reporter.trace("existing"); xcopy(existing, data); return; } File meta = new File(repoDir, Hex.toHexString(sha) + ".json"); File file = new File(repoDir, Hex.toHexString(sha)); rename(tmp, file); reporter.trace("file %s", file); data.file = file.getAbsolutePath(); data.sha = sha; data.busy = false; CommandData cmddata = parseCommandData(data); if (cmddata.bsn != null) { data.name = cmddata.bsn + "-" + cmddata.version; } else data.name = Strings.display(cmddata.title, cmddata.bsn, cmddata.name, uri); codec.enc().to(meta).put(data); reporter.trace("TD = " + data); } finally { tmp.delete(); reporter.trace("puted %s %s", uri, data); } }
@Override public void log(ServiceReference sr, int level, String message, Throwable exception) { switch (level) { case LogService.LOG_ERROR: reporter.error("%s:%s: %s", sr, exception, message); return; case LogService.LOG_WARNING: reporter.warning("%s:%s: %s", sr, exception, message); return; default: reporter.trace("%s:%s: %s", sr, exception, message); return; } }
/** * @param data * @param target * @throws Exception * @throws IOException */ public String createCommand(CommandData data, boolean force) throws Exception, IOException { // TODO // if (Data.validate(data) != null) // return "Invalid command data: " + Data.validate(data); Map<String, String> map = null; if (data.trace) { map = new HashMap<String, String>(); map.put("java.security.manager", "aQute.jpm.service.TraceSecurityManager"); reporter.trace("tracing"); } String s = platform.createCommand(data, map, force, service.getAbsolutePath()); if (s == null) storeData(new File(commandDir, data.name), data); return s; }
@Override public void log(int level, String message) { if (level > this.level) return; switch (level) { case LogService.LOG_ERROR: reporter.error("%s", message); return; case LogService.LOG_WARNING: reporter.warning("%s", message); return; default: reporter.trace("%s", message); return; } }
public ArtifactData get(byte[] sha) throws Exception { String name = Hex.toHexString(sha); File data = IO.getFile(repoDir, name + ".json"); reporter.trace("artifact data file %s", data); if (data.isFile()) { // Bin + metadata ArtifactData artifact = codec.dec().from(data).get(ArtifactData.class); artifact.file = IO.getFile(repoDir, name).getAbsolutePath(); return artifact; } File bin = IO.getFile(repoDir, name); if (bin.exists()) { // Only bin ArtifactData artifact = new ArtifactData(); artifact.file = bin.getAbsolutePath(); artifact.sha = sha; return artifact; } return null; }
public String what(String key, boolean oneliner) throws Exception { byte[] sha; Matcher m = SHA_P.matcher(key); if (m.matches()) { sha = Hex.toByteArray(key); } else { m = URL_P.matcher(key); if (m.matches()) { URL url = new URL(key); sha = SHA1.digest(url.openStream()).digest(); } else { File jarfile = new File(key); if (!jarfile.exists()) { reporter.error("File does not exist: %s", jarfile.getCanonicalPath()); } sha = SHA1.digest(jarfile).digest(); } } reporter.trace("sha %s", Hex.toHexString(sha)); Revision revision = library.getRevision(sha); if (revision == null) { return null; } StringBuilder sb = new StringBuilder(); Formatter f = new Formatter(sb); Justif justif = new Justif(120, 20, 70, 20, 75); DateFormat dateFormat = DateFormat.getDateInstance(); try { if (oneliner) { f.format("%20s %s%n", Hex.toHexString(revision._id), createCoord(revision)); } else { f.format("Artifact: %s%n", revision.artifactId); if (revision.organization != null && revision.organization.name != null) { f.format(" (%s)", revision.organization.name); } f.format("%n"); f.format("Coordinates\t0: %s%n", createCoord(revision)); f.format("Created\t0: %s%n", dateFormat.format(new Date(revision.created))); f.format("Size\t0: %d%n", revision.size); f.format("Sha\t0: %s%n", Hex.toHexString(revision._id)); f.format("URL\t0: %s%n", createJpmLink(revision)); f.format("%n"); f.format("%s%n", revision.description); f.format("%n"); f.format("Dependencies\t0:%n"); boolean flag = false; Iterable<RevisionRef> closure = library.getClosure(revision._id, true); for (RevisionRef dep : closure) { f.format( " - %s \t2- %s \t3- %s%n", dep.name, createCoord(dep), dateFormat.format(new Date(dep.created))); flag = true; } if (!flag) { f.format(" None%n"); } f.format("%n"); } f.flush(); justif.wrap(sb); return sb.toString(); } finally { f.close(); } }
public CommandData parseCommandData(ArtifactData artifact) throws Exception { File source = new File(artifact.file); if (!source.isFile()) throw new FileNotFoundException(); CommandData data = new CommandData(); data.sha = artifact.sha; data.jpmRepoDir = repoDir.getCanonicalPath(); JarFile jar = new JarFile(source); try { reporter.trace("Parsing %s", source); Manifest m = jar.getManifest(); Attributes main = m.getMainAttributes(); data.name = data.bsn = main.getValue("Bundle-SymbolicName"); String version = main.getValue("Bundle-Version"); if (version == null) data.version = Version.LOWEST; else data.version = new Version(version); data.main = main.getValue("Main-Class"); data.description = main.getValue("Bundle-Description"); data.title = main.getValue("JPM-Name"); reporter.trace("name " + data.name + " " + data.main + " " + data.title); DependencyCollector path = new DependencyCollector(this); path.add(artifact); DependencyCollector bundles = new DependencyCollector(this); if (main.getValue("JPM-Classpath") != null) { Parameters requires = OSGiHeader.parseHeader(main.getValue("JPM-Classpath")); for (Map.Entry<String, Attrs> e : requires.entrySet()) { path.add(e.getKey(), e.getValue().get("name")); // coordinate } } else if (!artifact.local) { // No JPM-Classpath, falling back to // server's revision // Iterable<RevisionRef> closure = // library.getClosure(artifact.sha, // false); // System.out.println("getting closure " + artifact.url + " " + // Strings.join("\n",closure)); // if (closure != null) { // for (RevisionRef ref : closure) { // path.add(Hex.toHexString(ref.revision)); // } // } } if (main.getValue("JPM-Runbundles") != null) { Parameters jpmrunbundles = OSGiHeader.parseHeader(main.getValue("JPM-Runbundles")); for (Map.Entry<String, Attrs> e : jpmrunbundles.entrySet()) { bundles.add(e.getKey(), e.getValue().get("name")); } } reporter.trace("collect digests runpath"); data.dependencies.addAll(path.getDigests()); reporter.trace("collect digests bundles"); data.runbundles.addAll(bundles.getDigests()); Parameters command = OSGiHeader.parseHeader(main.getValue("JPM-Command")); if (command.size() > 1) reporter.error("Only one command can be specified"); for (Map.Entry<String, Attrs> e : command.entrySet()) { data.name = e.getKey(); Attrs attrs = e.getValue(); if (attrs.containsKey("jvmargs")) data.jvmArgs = attrs.get("jvmargs"); if (attrs.containsKey("title")) data.title = attrs.get("title"); if (data.title != null) data.title = data.name; } return data; } finally { jar.close(); } }
/** Get the file belonging to a Resource Descriptor */ public File getResource(byte[] rd, final RepositoryPlugin.DownloadListener... blockers) throws Exception { final ResourceDescriptorImpl rds = getResourceDescriptor(rd); // No such descriptor? if (rds == null) { reporter.trace("no such descriptor %s", Hex.toHexString(rd)); return null; } // // Construct a path // final File path = IO.getFile(cache, Hex.toHexString(rds.id) + "/" + rds.bsn + "-" + rds.version + ".jar"); if (path.isFile()) { // // Ok, it is there, just report // ok(blockers, path); return path; } // // Check if we had an earlier failure // synchronized (failures) { Long l = failures.get(rds.url); if (l != null && (System.currentTimeMillis() - l) < THRESHOLD) { reporter.trace("descriptor %s, had earlier failure not retrying", Hex.toHexString(rd)); return null; } } // // Check if we need to download directly, no blockers // if (blockers == null || blockers.length == 0) { reporter.trace("descriptor %s, not found, immediate download", Hex.toHexString(rd)); download(rds, path); return path; } // // We have blockers so we can download in the background. // reporter.trace("descriptor %s, not found, background download", Hex.toHexString(rd)); // // With download listeners we need to be careful to queue them // appropriately. Don't want to download n times because // requests arrive during downloads. // synchronized (queues) { List<DownloadListener> list = queues.get(path); boolean first = list == null || list.isEmpty(); for (DownloadListener b : blockers) { queues.add(path, b); } if (!first) { // return, file is being downloaded by another and that // other will signal the download listener. reporter.trace("someone else is downloading our file " + queues.get(path)); return path; } } limitDownloads.acquire(); executor.execute( new Runnable() { public void run() { try { download(rds, path); synchronized (queues) { ok(queues.get(path).toArray(EMPTY_LISTENER), path); } } catch (Exception e) { synchronized (queues) { fail(e, queues.get(path).toArray(EMPTY_LISTENER), path); } } finally { synchronized (queues) { queues.remove(path); } limitDownloads.release(); } } }); return path; }