private void _loadConfig() {
try {
_configScope.set("__instance__", this);
_loadConfigFromCloudObject(getSiteObject());
_loadConfigFromCloudObject(getEnvironmentObject());
File f;
if (!_admin) {
f = getFileSafe("_config.js");
if (f == null || !f.exists()) f = getFileSafe("_config");
} else
f =
new File(
Module.getModule("core-modules/admin").getRootFile(getVersionForLibrary("admin")),
"_config.js");
_libraryLogger.info("config file [" + f + "] exists:" + f.exists());
if (f == null || !f.exists()) return;
Convert c = new Convert(f);
JSFunction func = c.get();
func.setUsePassedInScope(true);
func.call(_configScope);
_logger.debug("config things " + _configScope.keySet());
} catch (Exception e) {
throw new RuntimeException("couldn't load config", e);
}
}
private Service getService(File base) throws Exception {
File dataFile = new File(base, "data");
if (!dataFile.isFile()) return null;
ServiceData data = getData(ServiceData.class, dataFile);
return new Service(this, data);
}
public void testSerial() {
assertTrue(_nbhm.isEmpty());
final String k1 = "k1";
final String k2 = "k2";
assertThat(_nbhm.put(k1, "v1"), nullValue());
assertThat(_nbhm.put(k2, "v2"), nullValue());
// Serialize it out
try {
FileOutputStream fos = new FileOutputStream("NBHM_test.txt");
ObjectOutputStream out = new ObjectOutputStream(fos);
out.writeObject(_nbhm);
out.close();
} catch (IOException ex) {
ex.printStackTrace();
}
// Read it back
try {
File f = new File("NBHM_test.txt");
FileInputStream fis = new FileInputStream(f);
ObjectInputStream in = new ObjectInputStream(fis);
NonBlockingIdentityHashMap nbhm = (NonBlockingIdentityHashMap) in.readObject();
in.close();
assertThat(
"serialization works",
nbhm.toString(),
anyOf(is("{k1=v1, k2=v2}"), is("{k2=v2, k1=v1}")));
if (!f.delete()) throw new IOException("delete failed");
} catch (IOException ex) {
ex.printStackTrace();
} catch (ClassNotFoundException ex) {
ex.printStackTrace();
}
}
JxpSource getSource(File f) throws IOException {
if (DEBUG) System.err.println("getSource\n\t " + f);
File temp = _findFile(f);
if (DEBUG) System.err.println("\t " + temp);
if (!temp.exists()) return handleFileNotFound(f);
// if it's a directory (and we know we can't find the index file)
// TODO : at some point, do something where we return an index for the dir?
if (temp.isDirectory()) return null;
// if we are at init time, save it as an initializaiton file
loadedFile(temp);
// Ensure that this is w/in the right tree for the context
if (_localObject != null && _localObject.isIn(temp)) return _localObject.getSource(temp);
// if not, is it core?
if (_core.isIn(temp)) return _core.getSource(temp);
throw new RuntimeException("what? can't find:" + f);
}
void put(final URI uri, ArtifactData data) throws Exception {
reporter.trace("put %s %s", uri, data);
File tmp = createTempFile(repoDir, "mtp", ".whatever");
tmp.deleteOnExit();
try {
copy(uri.toURL(), tmp);
byte[] sha = SHA1.digest(tmp).digest();
reporter.trace("SHA %s %s", uri, Hex.toHexString(sha));
ArtifactData existing = get(sha);
if (existing != null) {
reporter.trace("existing");
xcopy(existing, data);
return;
}
File meta = new File(repoDir, Hex.toHexString(sha) + ".json");
File file = new File(repoDir, Hex.toHexString(sha));
rename(tmp, file);
reporter.trace("file %s", file);
data.file = file.getAbsolutePath();
data.sha = sha;
data.busy = false;
CommandData cmddata = parseCommandData(data);
if (cmddata.bsn != null) {
data.name = cmddata.bsn + "-" + cmddata.version;
} else data.name = Strings.display(cmddata.title, cmddata.bsn, cmddata.name, uri);
codec.enc().to(meta).put(data);
reporter.trace("TD = " + data);
} finally {
tmp.delete();
reporter.trace("puted %s %s", uri, data);
}
}
public Integer call() {
count = 0;
try {
File[] files = directory.listFiles();
ArrayList<Future<Integer>> results = new ArrayList<Future<Integer>>();
for (File file : files)
if (file.isDirectory()) {
MatchCounter counter = new MatchCounter(file, keyword);
FutureTask<Integer> task = new FutureTask<Integer>(counter);
results.add(task);
Thread t = new Thread(task);
t.start();
} else {
if (search(file)) count++;
}
for (Future<Integer> result : results)
try {
count += result.get();
} catch (ExecutionException e) {
e.printStackTrace();
}
} catch (InterruptedException e) {
}
return count;
}
/**
* Maps a servlet-like URI to a jxp file.
*
* @param f File to check
* @return new File with <root>.jxp if exists, orig file if not
* @example /wiki/geir -> maps to wiki.jxp if exists
*/
File tryServlet(File f) {
if (f.exists()) return f;
String uri = f.toString();
if (uri.startsWith(_rootFile.toString())) uri = uri.substring(_rootFile.toString().length());
if (_core != null && uri.startsWith(_core._base.toString()))
uri = "/~~" + uri.substring(_core._base.toString().length());
while (uri.startsWith("/")) uri = uri.substring(1);
int start = 0;
while (true) {
int idx = uri.indexOf("/", start);
if (idx < 0) break;
String foo = uri.substring(0, idx);
File temp = getFileSafe(foo + ".jxp");
if (temp != null && temp.exists()) f = temp;
start = idx + 1;
}
return f;
}
/**
* Tries to find the given file, assuming that it's missing the ".jxp" extension
*
* @param f File to check
* @return same file if not found to be missing the .jxp, or a new File w/ the .jxp appended
*/
File tryNoJXP(File f) {
if (f.exists()) return f;
if (f.getName().indexOf(".") >= 0) return f;
File temp = new File(f.toString() + ".jxp");
return temp.exists() ? temp : f;
}
@Override
protected void setUp() throws Exception {
tmp = IO.getFile("generated/tmp");
tmp.mkdirs();
IO.copy(IO.getFile("testdata/ws"), tmp);
workspace = Workspace.getWorkspace(tmp);
workspace.refresh();
InfoRepository repo = workspace.getPlugin(InfoRepository.class);
t1 = create("bsn-1", new Version(1, 0, 0));
t2 = create("bsn-2", new Version(1, 0, 0));
repo.put(new FileInputStream(t1), null);
repo.put(new FileInputStream(t2), null);
t1 = repo.get("bsn-1", new Version(1, 0, 0), null);
t2 = repo.get("bsn-2", new Version(1, 0, 0), null);
repo.put(new FileInputStream(IO.getFile("generated/biz.aQute.remote.launcher.jar")), null);
workspace.getPlugins().add(repo);
File storage = IO.getFile("generated/storage-1");
storage.mkdirs();
configuration = new HashMap<String, Object>();
configuration.put(
Constants.FRAMEWORK_STORAGE_CLEAN, Constants.FRAMEWORK_STORAGE_CLEAN_ONFIRSTINIT);
configuration.put(Constants.FRAMEWORK_STORAGE, storage.getAbsolutePath());
configuration.put(
Constants.FRAMEWORK_SYSTEMPACKAGES_EXTRA, "org.osgi.framework.launch;version=1.2");
framework = new org.apache.felix.framework.FrameworkFactory().newFramework(configuration);
framework.init();
framework.start();
context = framework.getBundleContext();
location = "reference:" + IO.getFile("generated/biz.aQute.remote.agent.jar").toURI().toString();
agent = context.installBundle(location);
agent.start();
thread =
new Thread() {
@Override
public void run() {
try {
Main.main(
new String[] {
"-s", "generated/storage", "-c", "generated/cache", "-p", "1090", "-et"
});
} catch (Exception e) {
e.printStackTrace();
}
}
};
thread.setDaemon(true);
thread.start();
super.setUp();
}
/**
* Returns the index.jxp for the File argument if it's an existing directory, and the index.jxp
* file exists
*
* @param f directory to check
* @return new File for index.jxp in that directory, or same file object if not
*/
File tryIndex(File f) {
if (!(f.isDirectory() && f.exists())) return f;
for (int i = 0; i < JSFileLibrary._srcExtensions.length; i++) {
File temp = new File(f, "index" + JSFileLibrary._srcExtensions[i]);
if (temp.exists()) return temp;
}
return f;
}
File tryOtherExtensions(File f) {
if (f.exists()) return f;
if (f.getName().indexOf(".") >= 0) return f;
for (int i = 0; i < JSFileLibrary._srcExtensions.length; i++) {
File temp = new File(f.toString() + JSFileLibrary._srcExtensions[i]);
if (temp.exists()) return temp;
}
return f;
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
String filePickerResult = "";
if (data != null && resultCode == RESULT_OK) {
try {
ContentResolver cr = getContentResolver();
Uri uri = data.getData();
Cursor cursor =
GeckoApp.mAppContext
.getContentResolver()
.query(uri, new String[] {OpenableColumns.DISPLAY_NAME}, null, null, null);
String name = null;
if (cursor != null) {
try {
if (cursor.moveToNext()) {
name = cursor.getString(0);
}
} finally {
cursor.close();
}
}
String fileName = "tmp_";
String fileExt = null;
int period;
if (name == null || (period = name.lastIndexOf('.')) == -1) {
String mimeType = cr.getType(uri);
fileExt = "." + GeckoAppShell.getExtensionFromMimeType(mimeType);
} else {
fileExt = name.substring(period);
fileName = name.substring(0, period);
}
File file = File.createTempFile(fileName, fileExt, sGREDir);
FileOutputStream fos = new FileOutputStream(file);
InputStream is = cr.openInputStream(uri);
byte[] buf = new byte[4096];
int len = is.read(buf);
while (len != -1) {
fos.write(buf, 0, len);
len = is.read(buf);
}
fos.close();
filePickerResult = file.getAbsolutePath();
} catch (Exception e) {
Log.e(LOG_FILE_NAME, "showing file picker", e);
}
}
try {
mFilePickerResult.put(filePickerResult);
} catch (InterruptedException e) {
Log.i(LOG_FILE_NAME, "error returning file picker result", e);
}
}
static Folder fromDirectory(File dir) throws IOException {
List<Document> documents = new LinkedList<>();
List<Folder> subFolders = new LinkedList<>();
for (File entry : dir.listFiles()) {
if (entry.isDirectory()) {
subFolders.add(Folder.fromDirectory(entry));
} else {
documents.add(Document.fromFile(entry));
}
}
return new Folder(subFolders, documents);
}
public List<CommandData> getCommands(File commandDir) throws Exception {
List<CommandData> result = new ArrayList<CommandData>();
if (!commandDir.exists()) {
return result;
}
for (File f : commandDir.listFiles()) {
CommandData data = getData(CommandData.class, f);
if (data != null) result.add(data);
}
return result;
}
void removeFiles() throws IOException {
BufferedReader reader = new BufferedReader(new FileReader(new File(sGREDir, "removed-files")));
try {
for (String removedFileName = reader.readLine();
removedFileName != null;
removedFileName = reader.readLine()) {
File removedFile = new File(sGREDir, removedFileName);
if (removedFile.exists()) removedFile.delete();
}
} finally {
reader.close();
}
}
public URL getResource(String path) {
try {
File f = getFile(path);
if (!f.exists()) return null;
return f.toURL();
} catch (FileNotFoundException fnf) {
// the spec says to return null if we can't find it
// even though this is weird...
return null;
} catch (IOException ioe) {
throw new RuntimeException("error opening [" + path + "]", ioe);
}
}
public List<ServiceData> getServices(File serviceDir) throws Exception {
List<ServiceData> result = new ArrayList<ServiceData>();
if (!serviceDir.exists()) {
return result;
}
for (File sdir : serviceDir.listFiles()) {
File dataFile = new File(sdir, "data");
ServiceData data = getData(ServiceData.class, dataFile);
result.add(data);
}
return result;
}
/*
* Launches against the agent
*/
public void testSimpleLauncher() throws Exception {
Project project = workspace.getProject("p1");
Run bndrun = new Run(workspace, project.getBase(), project.getFile("one.bndrun"));
bndrun.setProperty("-runpath", "biz.aQute.remote.launcher");
bndrun.setProperty("-runbundles", "bsn-1,bsn-2");
bndrun.setProperty("-runremote", "test");
final RemoteProjectLauncherPlugin pl =
(RemoteProjectLauncherPlugin) bndrun.getProjectLauncher();
pl.prepare();
final CountDownLatch latch = new CountDownLatch(1);
final AtomicInteger exitCode = new AtomicInteger(-1);
List<? extends RunSession> sessions = pl.getRunSessions();
assertEquals(1, sessions.size());
final RunSession session = sessions.get(0);
Thread t =
new Thread("test-launch") {
public void run() {
try {
exitCode.set(session.launch());
} catch (Exception e) {
e.printStackTrace();
} finally {
latch.countDown();
}
}
};
t.start();
Thread.sleep(500);
for (Bundle b : context.getBundles()) {
System.out.println(b.getLocation());
}
assertEquals(4, context.getBundles().length);
String p1 = t1.getAbsolutePath();
System.out.println(p1);
assertNotNull(context.getBundle(p1));
assertNotNull(context.getBundle(t2.getAbsolutePath()));
pl.cancel();
latch.await();
assertEquals(-3, exitCode.get());
bndrun.close();
}
private boolean unpackFile(ZipFile zip, byte[] buf, ZipEntry fileEntry, String name)
throws IOException, FileNotFoundException {
if (fileEntry == null) fileEntry = zip.getEntry(name);
if (fileEntry == null)
throw new FileNotFoundException("Can't find " + name + " in " + zip.getName());
File outFile = new File(sGREDir, name);
if (outFile.lastModified() == fileEntry.getTime() && outFile.length() == fileEntry.getSize())
return false;
File dir = outFile.getParentFile();
if (!dir.exists()) dir.mkdirs();
InputStream fileStream;
fileStream = zip.getInputStream(fileEntry);
OutputStream outStream = new FileOutputStream(outFile);
while (fileStream.available() > 0) {
int read = fileStream.read(buf, 0, buf.length);
outStream.write(buf, 0, read);
}
fileStream.close();
outStream.close();
outFile.setLastModified(fileEntry.getTime());
return true;
}
private File getAndValidateFile(String pFile, String pWhat) throws IOException {
File ret = new File(pFile);
if (!ret.exists()) {
throw new FileNotFoundException("No such " + pWhat + " " + pFile);
}
if (!ret.canRead()) {
throw new IOException(
pWhat.substring(0, 1).toUpperCase()
+ pWhat.substring(1)
+ " "
+ pFile
+ " is not readable");
}
return ret;
}
private static void build(Iterable<String> in) throws IOException, InterruptedException {
final List<String> args = newArrayList(in);
Collections.sort(args);
final int threads = Runtime.getRuntime().availableProcessors();
final ExecutorService ex = Executors.newFixedThreadPool(threads);
final String base = "base";
new WithVm("base").createIfNotPresent();
for (String pkg : args)
ex.submit(
() -> {
final WithVm newVm = new WithVm("fbuild-" + pkg, TimeUnit.MINUTES.toMillis(30));
final File rbuild = new File("wip-" + pkg + ".rbuild");
try {
newVm.cloneFrom(base);
newVm.start();
newVm.inTee(rbuild, "apt-get", "-oAPT::Get::Only-Source=true", "source", pkg);
newVm.inTee(rbuild, "apt-get", "build-dep", "-y", "--force-yes", pkg);
newVm.inTee(rbuild, "ifdown", "eth0");
final boolean success =
0
== newVm.inTee(
rbuild, "sh", "-c", "cd " + pkg + "-* && dpkg-buildpackage -us -uc");
newVm.stopNow();
if (success) {
rbuild.renameTo(new File("success-" + pkg + ".rbuild"));
newVm.destroy();
System.out.println("success: " + pkg);
} else {
rbuild.renameTo(new File("failure-" + pkg + ".rbuild"));
System.out.println("failure: " + pkg);
}
} catch (Exception e) {
rbuild.renameTo(new File("error-" + pkg + ".rbuild"));
System.err.println("build error: " + pkg);
e.printStackTrace();
newVm.stopNow();
}
return null;
});
ex.shutdown();
ex.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
}
/**
* Loads test configuration.
*
* @throws Exception if configuration is unawailable or broken.
*/
private void loadTestConfiguration() throws Exception {
assert TEST_CONFIGURATION_FILE.isFile();
InputStream in = null;
Properties p = new Properties();
try {
in = new FileInputStream(TEST_CONFIGURATION_FILE);
p.load(in);
} finally {
U.closeQuiet(in);
}
clientNodes = Integer.parseInt(p.getProperty("client.nodes.count"));
srvNodes = Integer.parseInt(p.getProperty("server.nodes.count"));
threadsPerClient = Integer.parseInt(p.getProperty("threads.per.client"));
cancelRate = Integer.parseInt(p.getProperty("cancel.rate"));
submitDelay = Long.parseLong(p.getProperty("submit.delay"));
taskParams =
new GridJobLoadTestParams(
Integer.parseInt(p.getProperty("jobs.count")),
Integer.parseInt(p.getProperty("jobs.test.duration")),
Integer.parseInt(p.getProperty("jobs.test.completion.delay")),
Double.parseDouble(p.getProperty("jobs.failure.probability")));
}
public static void main(final String[] args) throws Exception {
final int length = (1 << 14);
final ICircularLongsBuffer buffer = new UnsafeCircularLongsBuffer(length);
final File file = new File("output.log");
log.info("File " + file);
file.delete();
// file.deleteOnExit();
final FastLoggerImpl logger =
new FastLoggerImpl(
new ThreadFactory() {
@Override
public Thread newThread(final Runnable r) {
return new Thread(r);
}
},
buffer,
WaitingStrategy.SPINNING,
new RawFileWriter(file));
logger.startDraining();
final int workers = Runtime.getRuntime().availableProcessors() - 1;
final ExecutorService workersPool = Executors.newFixedThreadPool(workers);
for (int i = 0; i < workers; i++) {
workersPool.submit(
new Runnable() {
@Override
public void run() {
try {
for (int i = 0; i < 100000000; i++) {
logger.log("Message %f -- %d ").with(25.98 + i).with(100 - i).submit();
}
} catch (Exception e) {
log.error("Error", e);
}
}
});
}
workersPool.shutdown();
workersPool.awaitTermination(1000, TimeUnit.SECONDS);
log.info("Finished");
System.exit(1);
}
/**
* Returns the adapter type for the given file. Will first use the adapter selector function if it
* was specified in init.js, otherwise will use the static type (either set in _init file, as a
* server-wide override in 10gen.properties, or default of DIRECT_10GEN)
*
* @param file to produce type for
* @return adapter type for the specified file
*/
public AdapterType getAdapterType(File file) {
// Q : I think this is the right thing to do
if (inScopeSetup()) {
return AdapterType.DIRECT_10GEN;
}
/*
* cheap hack - prevent any _init.* file from getting run as anythign but DIRECT_10GEN
*/
if (file != null && file.getName().indexOf("_init.") != -1) {
return AdapterType.DIRECT_10GEN;
}
if (_adapterSelector == null) {
return _staticAdapterType;
}
/*
* only let the app select type if file is part of application (i.e.
* don't do it for corejs, core modules, etc...
*/
String fp = file.getAbsolutePath();
String fullRoot = _rootFile.getAbsolutePath(); // there must be a nicer way to do this?
if (!fp.startsWith(fullRoot)) {
return AdapterType.DIRECT_10GEN;
}
Object o = _adapterSelector.call(_initScope, new JSString(fp.substring(fullRoot.length())));
if (o == null) {
return _staticAdapterType;
}
if (!(o instanceof JSString)) {
log("Error : adapter selector not returning string. Ignoring and using static adapter type");
return _staticAdapterType;
}
AdapterType t = getAdapterTypeFromString(o.toString());
return (t == null ? _staticAdapterType : t);
}
public void init() throws IOException {
URL s = getClass().getClassLoader().getResource(SERVICE_JAR_FILE);
if (s == null)
if (underTest) return;
else throw new Error("No " + SERVICE_JAR_FILE + " resource in jar");
service.getParentFile().mkdirs();
IO.copy(s, service);
}
protected void unpackComponents() throws IOException, FileNotFoundException {
File applicationPackage = new File(getApplication().getPackageResourcePath());
File componentsDir = new File(sGREDir, "components");
if (componentsDir.lastModified() == applicationPackage.lastModified()) return;
componentsDir.mkdir();
componentsDir.setLastModified(applicationPackage.lastModified());
GeckoAppShell.killAnyZombies();
ZipFile zip = new ZipFile(applicationPackage);
byte[] buf = new byte[32768];
try {
if (unpackFile(zip, buf, null, "removed-files")) removeFiles();
} catch (Exception ex) {
// This file may not be there, so just log any errors and move on
Log.w(LOG_FILE_NAME, "error removing files", ex);
}
// copy any .xpi file into an extensions/ directory
Enumeration<? extends ZipEntry> zipEntries = zip.entries();
while (zipEntries.hasMoreElements()) {
ZipEntry entry = zipEntries.nextElement();
if (entry.getName().startsWith("extensions/") && entry.getName().endsWith(".xpi")) {
Log.i("GeckoAppJava", "installing extension : " + entry.getName());
unpackFile(zip, buf, entry, entry.getName());
}
}
}
/**
* Read block from file.
*
* @param file - File to read.
* @param off - Marker position in file to start read from if {@code -1} read last blockSz bytes.
* @param blockSz - Maximum number of chars to read.
* @param lastModified - File last modification time.
* @return Read file block.
* @throws IOException In case of error.
*/
public static VisorFileBlock readBlock(File file, long off, int blockSz, long lastModified)
throws IOException {
RandomAccessFile raf = null;
try {
long fSz = file.length();
long fLastModified = file.lastModified();
long pos = off >= 0 ? off : Math.max(fSz - blockSz, 0);
// Try read more that file length.
if (fLastModified == lastModified && fSz != 0 && pos >= fSz)
throw new IOException(
"Trying to read file block with wrong offset: " + pos + " while file size: " + fSz);
if (fSz == 0)
return new VisorFileBlock(file.getPath(), pos, fLastModified, 0, false, EMPTY_FILE_BUF);
else {
int toRead = Math.min(blockSz, (int) (fSz - pos));
byte[] buf = new byte[toRead];
raf = new RandomAccessFile(file, "r");
raf.seek(pos);
int cntRead = raf.read(buf, 0, toRead);
if (cntRead != toRead)
throw new IOException(
"Count of requested and actually read bytes does not match [cntRead="
+ cntRead
+ ", toRead="
+ toRead
+ ']');
boolean zipped = buf.length > 512;
return new VisorFileBlock(
file.getPath(), pos, fSz, fLastModified, zipped, zipped ? zipBytes(buf) : buf);
}
} finally {
U.close(raf, null);
}
}
/**
* Check is text file.
*
* @param f file reference.
* @param emptyOk default value if empty file.
* @return Is text file.
*/
public static boolean textFile(File f, boolean emptyOk) {
if (f.length() == 0) return emptyOk;
String detected = VisorMimeTypes.getContentType(f);
for (String mime : TEXT_MIME_TYPE) if (mime.equals(detected)) return true;
return false;
}
private File create(String bsn, Version v) throws Exception {
String name = bsn + "-" + v;
Builder b = new Builder();
b.setBundleSymbolicName(bsn);
b.setBundleVersion(v);
b.setProperty("Random", random++ + "");
b.setProperty("-resourceonly", true + "");
b.setIncludeResource("foo;literal='foo'");
Jar jar = b.build();
assertTrue(b.check());
File file = IO.getFile(tmp, name + ".jar");
file.getParentFile().mkdirs();
jar.updateModified(System.currentTimeMillis(), "Force it to now");
jar.write(file);
b.close();
return file;
}
JxpSource handleFileNotFound(File f) {
String name = f.getName();
if (name.endsWith(".class")) {
name = name.substring(0, name.length() - 6);
return getJxpServlet(name);
}
return null;
}