@Test
public void testPersistMergeCaseInsensitive() throws Exception {
final long timestamp = System.currentTimeMillis();
IncrementalIndex toPersist1 = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);
IncrementalIndex toPersist2 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("DIm1", "DIM2"),
ImmutableMap.<String, Object>of(
"dim1", "1", "dim2", "2", "DIm1", "10000", "DIM2", "100000000")));
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("dIM1", "dIm2"),
ImmutableMap.<String, Object>of("DIm1", "1", "DIM2", "2", "dim1", "5", "dim2", "6")));
final File tempDir1 = Files.createTempDir();
final File tempDir2 = Files.createTempDir();
final File mergedDir = Files.createTempDir();
try {
QueryableIndex index1 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tempDir1));
Assert.assertEquals(2, index1.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
Assert.assertEquals(2, index1.getColumnNames().size());
QueryableIndex index2 = IndexIO.loadIndex(IndexMerger.persist(toPersist2, tempDir2));
Assert.assertEquals(2, index2.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(index2.getAvailableDimensions()));
Assert.assertEquals(2, index2.getColumnNames().size());
QueryableIndex merged =
IndexIO.loadIndex(
IndexMerger.mergeQueryableIndex(
Arrays.asList(index1, index2), new AggregatorFactory[] {}, mergedDir));
Assert.assertEquals(3, merged.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
Assert.assertEquals(2, merged.getColumnNames().size());
} finally {
FileUtils.deleteQuietly(tempDir1);
FileUtils.deleteQuietly(tempDir2);
FileUtils.deleteQuietly(mergedDir);
}
}
/**
* TextResource should be able to read files as text.
*
* @throws Exception If something goes wrong.
*/
@Test
public void readsFileAsText() throws Exception {
final String text = "<a xmlns='urn:foo'><b>\u0433!</b></a>";
final File file = new File(Files.createTempDir(), "dummy.xml");
FileUtils.writeStringToFile(file, text, CharEncoding.UTF_8);
MatcherAssert.assertThat(new TextResource(file).toString(), Matchers.is(text));
}
public SchemaTupleFrontendGenHelper(PigContext pigContext, Configuration conf) {
codeDir = Files.createTempDir();
codeDir.deleteOnExit();
LOG.debug("Temporary directory for generated code created: " + codeDir.getAbsolutePath());
this.pigContext = pigContext;
this.conf = conf;
}
protected final synchronized File getTestTempDir() {
if (testTempDir == null) {
testTempDir = Files.createTempDir();
testTempDir.deleteOnExit();
}
return testTempDir;
}
/**
* Set embedded cassandra up and spawn it in a new thread.
*
* @throws TTransportException
* @throws IOException
* @throws InterruptedException
*/
public void start()
throws TTransportException, IOException, InterruptedException, ConfigurationException {
File dir = Files.createTempDir();
String dirPath = dir.getAbsolutePath();
System.out.println("Storing Cassandra files in " + dirPath);
URL url = Resources.getResource("cassandra.yaml");
String yaml = Resources.toString(url, Charsets.UTF_8);
yaml = yaml.replaceAll("REPLACEDIR", dirPath);
String yamlPath = dirPath + File.separatorChar + "cassandra.yaml";
org.apache.commons.io.FileUtils.writeStringToFile(new File(yamlPath), yaml);
// make a tmp dir and copy cassandra.yaml and log4j.properties to it
copy("/log4j.properties", dir.getAbsolutePath());
System.setProperty("cassandra.config", "file:" + dirPath + yamlFilePath);
System.setProperty("log4j.configuration", "file:" + dirPath + "/log4j.properties");
System.setProperty("cassandra-foreground", "true");
cleanupAndLeaveDirs();
try {
executor.execute(new CassandraRunner());
} catch (RejectedExecutionException e) {
log.error("RejectError", e);
return;
}
try {
TimeUnit.SECONDS.sleep(WAIT_SECONDS);
} catch (InterruptedException e) {
log.error("InterrputedError", e);
throw new AssertionError(e);
}
}
@BeforeMethod
public void setup() throws Exception {
IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime());
dummyHandle = dbi.open();
dataDir = Files.createTempDir();
shardManager = new DatabaseShardManager(dbi);
}
@Override
protected void setUp() {
javac = ToolProvider.getSystemJavaCompiler();
diagnosticCollector = new DiagnosticCollector<JavaFileObject>();
fileManager = javac.getStandardFileManager(diagnosticCollector, null, null);
tmpDir = Files.createTempDir();
}
private File createSchema(String schemaName)
throws SAXException, IOException, OutputFailedException, InvalidParamsException,
InputFailedException, URISyntaxException {
String rncInputFile =
getClass()
.getClassLoader()
.getResource(schemaName + ".rnc")
.toString()
.replace('\\', '/')
.replaceAll(" ", "%20");
File rngOutputFile = new File(Files.createTempDir(), schemaName + ".rng");
InputFormat inputFormat = Formats.createInputFormat("rnc");
OutputFormat outputFormat = Formats.createOutputFormat("rng");
String[] inputOptions = {};
SchemaCollection schemaCollection =
inputFormat.load(rncInputFile, inputOptions, "rng", new ErrorHandlerImpl(), null);
OutputDirectory outputDirectory =
new LocalOutputDirectory(
schemaCollection.getMainUri(), rngOutputFile, "rng", "UTF-8", 72, 2);
String[] outputOptions = {};
outputFormat.output(
schemaCollection, outputDirectory, outputOptions, "rnc", new ErrorHandlerImpl());
return rngOutputFile;
}
private WorkerTaskMonitor createTaskMonitor() {
return new WorkerTaskMonitor(
jsonMapper,
cf,
workerCuratorCoordinator,
new ThreadPoolTaskRunner(
new TaskToolboxFactory(
new TaskConfig(Files.createTempDir().toString(), null, null, 0, null),
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
new SegmentLoaderFactory(
new SegmentLoaderLocalCacheManager(
null,
new SegmentLoaderConfig() {
@Override
public List<StorageLocationConfig> getLocations() {
return Lists.newArrayList();
}
},
jsonMapper)),
jsonMapper),
null),
new WorkerConfig().setCapacity(1));
}
public static void start() throws Exception {
testStackIndex++;
if (standaloneMain == null) {
LOG.info("Starting standalone instance");
try {
CConfiguration cConf = CConfiguration.create();
tempDirectory = Files.createTempDir();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tempDirectory.getAbsolutePath());
cConf.set(Constants.Router.ADDRESS, HOSTNAME);
cConf.set(Constants.Dangerous.UNRECOVERABLE_RESET, "true");
// Start without UI
standaloneMain = StandaloneMain.create(null, cConf, new Configuration());
standaloneMain.startUp();
waitForStandalone(30000);
} catch (Exception e) {
LOG.error("Failed to start standalone", e);
if (standaloneMain != null) {
standaloneMain.shutDown();
}
try {
DirUtils.deleteDirectoryContents(tempDirectory);
} catch (IOException ex) {
LOG.warn("Failed to delete temp directory: " + tempDirectory.getAbsolutePath(), ex);
}
throw e;
}
}
}
/** Extracts the archive resource and then runs the batch-import process on it. */
protected void importDataArchive(
final Resource resource,
final ArchiveInputStream resourceStream,
BatchImportOptions options) {
final File tempDir = Files.createTempDir();
try {
ArchiveEntry archiveEntry;
while ((archiveEntry = resourceStream.getNextEntry()) != null) {
final File entryFile = new File(tempDir, archiveEntry.getName());
if (archiveEntry.isDirectory()) {
entryFile.mkdirs();
} else {
entryFile.getParentFile().mkdirs();
Files.copy(
new InputSupplier<InputStream>() {
@Override
public InputStream getInput() throws IOException {
return new CloseShieldInputStream(resourceStream);
}
},
entryFile);
}
}
importDataDirectory(tempDir, null, options);
} catch (IOException e) {
throw new RuntimeException(
"Failed to extract data from '" + resource + "' to '" + tempDir + "' for batch import.",
e);
} finally {
FileUtils.deleteQuietly(tempDir);
}
}
@Subscribe
public void onEvent(ModelArchiveDownloadedEvent e) throws IOException {
if (!isRunning()) {
log(INFO_SERVICE_NOT_RUNNING);
return;
}
if (isIndex(e.model)) {
File location = repository.getLocation(e.model, false).orNull();
String remoteUri = e.model.getHint(HINT_REPOSITORY_URL).orNull();
if (remoteUri != null) {
Pair<File, IModelIndex> pair = openDelegates.get(remoteUri);
if (pair == null) {
File folder = createIndexLocation(remoteUri);
folder.mkdir();
Zips.unzip(location, folder);
openDelegate(remoteUri, folder);
} else {
File folder = Files.createTempDir();
Zips.unzip(location, folder);
IModelIndex modelIndex = pair.getSecond();
modelIndex.updateIndex(folder);
bus.post(new ModelIndexOpenedEvent());
FileUtils.deleteDirectory(folder);
}
}
}
}
@BeforeMethod(alwaysRun = true)
@Override
public void setUp() throws Exception {
mementoDir = Files.createTempDir();
mgmt = RebindTestUtils.newPersistingManagementContext(mementoDir, classLoader);
super.setUp();
}
private BerkeleyDBTempTargetRepository createTempTargetRepository() {
final File tmp = Files.createTempDir();
final BerkeleyDBTempTargetRepository targets = new BerkeleyDBTempTargetRepository(tmp);
targets.start();
assertTrue(targets.isRunning());
return targets;
}
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
ObjectMapper jsonMapper = DruidInitialization.getInstance().getObjectMapper();
SegmentLoadSpec spec = readSegmentJobSpec(context.getConfiguration(), jsonMapper);
final List<String> dimensions = spec.getDimensions();
final List<String> metrics = spec.getMetrics();
final DimFilter filter = spec.getFilter();
final Interval interval =
new Interval(context.getConfiguration().get(DruidInputFormat.CONF_DRUID_INTERVAL));
String hdfsPath = ((DruidInputSplit) split).getPath();
logger.info("Reading segment from " + hdfsPath);
segmentDir = Files.createTempDir();
logger.info("segment dir: " + segmentDir);
FileSystem fs = FileSystem.get(context.getConfiguration());
getSegmentFiles(hdfsPath, segmentDir, fs);
logger.info("finished getting segment files");
QueryableIndex index = IndexIO.loadIndex(segmentDir);
StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
List<StorageAdapter> adapters = Lists.newArrayList(adapter);
rowYielder =
new IngestSegmentFirehose(
adapters, dimensions, metrics, filter, interval, QueryGranularity.NONE);
}
/*
* Must be an external file handle referring to a docker image
* must be able to retrieve the docker image
*
* returns the output of 'docker pull'
*/
public String pullDockerImage(FileHandle fileHandle) throws IOException {
String dockerReference = getDockerReferenceFromFileHandle(fileHandle);
// 'shell out' to 'docker pull <reference>'
File tmpDir = Files.createTempDir();
String result = executeDockerCommand(new String[] {"pull", dockerReference}, tmpDir);
return result;
}
@Test
public void test() throws IOException {
URL resource = getClass().getClassLoader().getResource("GeoIpTestData");
File path = new File(resource.getPath());
File tmp = File.createTempFile("compressed", ".db");
File tmpDir = Files.createTempDir();
tmp.deleteOnExit();
tmpDir.deleteOnExit();
GeoIpCompressor.compress(path, tmp);
GeoIpCompressor.decompress(tmp, tmpDir);
List<Location> got = readLocationCSV(new File(tmpDir, "lantern-location.csv"));
List<Location> expected = readLocationCSV(new File(path, "lantern-location-expected.csv"));
assertEquals(got.size(), expected.size());
for (int i = 0; i < got.size(); ++i) {
Location gotLoc = got.get(i);
Location expectedLoc = expected.get(i);
assertEquals(expectedLoc.country, gotLoc.country);
assertTrue(Math.abs(expectedLoc.latitude - gotLoc.latitude) < 1);
assertTrue(Math.abs(expectedLoc.longitude - gotLoc.longitude) < 1);
}
List<Block> gotBlocks = readBlocksCSV(new File(tmpDir, "lantern-blocks.csv"));
List<Block> expectedBlocks = readBlocksCSV(new File(path, "lantern-blocks-expected.csv"));
assertEquals(expectedBlocks, gotBlocks);
}
@Test
public void testPersistEmptyColumn() throws Exception {
final IncrementalIndex toPersist1 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
final IncrementalIndex toPersist2 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
final File tmpDir1 = Files.createTempDir();
final File tmpDir2 = Files.createTempDir();
final File tmpDir3 = Files.createTempDir();
try {
toPersist1.add(
new MapBasedInputRow(
1L,
ImmutableList.of("dim1", "dim2"),
ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "foo")));
toPersist2.add(
new MapBasedInputRow(
1L,
ImmutableList.of("dim1", "dim2"),
ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "bar")));
final QueryableIndex index1 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tmpDir1));
final QueryableIndex index2 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tmpDir2));
final QueryableIndex merged =
IndexIO.loadIndex(
IndexMerger.mergeQueryableIndex(
Arrays.asList(index1, index2), new AggregatorFactory[] {}, tmpDir3));
Assert.assertEquals(1, index1.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions()));
Assert.assertEquals(1, index2.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions()));
Assert.assertEquals(1, merged.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions()));
} finally {
FileUtils.deleteQuietly(tmpDir1);
FileUtils.deleteQuietly(tmpDir2);
FileUtils.deleteQuietly(tmpDir3);
}
}
public static File createAutoDeleteTempDir() {
File tempdir = Files.createTempDir();
tempdir.delete();
tempdir.mkdir();
tempdir.deleteOnExit();
return tempdir;
}
@Before
public void setup() throws Exception {
dataDir = new File(Files.createTempDir(), "sentry_policy_db");
Configuration conf = new Configuration(false);
conf.set(
ServerConfig.SENTRY_STORE_JDBC_URL,
"jdbc:derby:;databaseName=" + dataDir.getPath() + ";create=true");
sentryStore = new SentryStore(conf);
}
@Before
public void setup() throws IOException {
baseDir = Files.createTempDir();
PolicyFiles.copyToDir(baseDir, resourcePath);
backend =
new SimpleFileProviderBackend(
new Configuration(), new File(baseDir, resourcePath).toString());
context = new ProviderBackendContext();
}
@BeforeMethod
public void setUp() throws Exception {
installationHelper = new InstallationHelper();
appleInstallation = installationHelper.getAppleInstallation();
bananaInstallation = installationHelper.getBananaInstallation();
tempDir = Files.createTempDir().getCanonicalFile();
final AgentConfig config = new AgentConfig();
manager = new DirectoryDeploymentManager(tempDir, "/location/test", config.getTarTimeout());
}
private File prepareInstallationDirectory() throws IOException {
File tempDir = Files.createTempDir();
assertTrue(new File(tempDir, "bin").mkdir());
assertTrue(new File(tempDir, "conf").mkdir());
Files.copy(
new File(resourcesDirectory, "conf" + File.separator + "server.xml"),
new File(tempDir, "conf" + File.separator + "server.xml"));
return tempDir;
}
public TestFileSystemStorage() throws IOException {
StorageServiceConfiguration config = new StorageServiceConfiguration();
config.host = "filesystem";
final File tempDir;
tempDir = Files.createTempDir();
config.storagePath = tempDir.getCanonicalPath();
super.initService(new StorageService(config));
}
@Before
public void setup() throws Exception {
tempDir = Files.createTempDir();
databaseFile = new File(tempDir, "GeoLite2-Country.mmdb");
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(databaseFile));
Resources.copy(Resources.getResource("GeoLite2-Country.mmdb"), out);
out.flush();
out.close();
}
/**
* Sets up the necessary work environment prior to generating the mutants.
*
* @throws FatalStrykerStageException
*/
private File createWorkingDirectory() throws IllegalStateException {
File tmpDir = null;
if ((tmpDir = Files.createTempDir()) == null) {
throw new IllegalStateException(
"Couldn't create work environment: failed" + "to create temp directory");
}
log.trace("Using directory " + tmpDir + " as working directory.");
return tmpDir;
}
static {
PLUGIN_DIR = Files.createTempDir();
SETTINGS =
ImmutableSettings.settingsBuilder()
.put("discovery.zen.ping.multicast.enabled", false)
.put("force.http.enabled", true)
.put("plugins.isolation", true)
.put("path.plugins", PLUGIN_DIR.getAbsolutePath())
.build();
}
public void testRedirectFileLinksOutput() throws Exception {
if (!setupSdkHome()) {
System.out.println("Skipping testRedirectFileLinksOutput because sdk-common was not found");
return;
}
// Need file to be named (exactly) values.xml
File tempDir = Files.createTempDir();
File layoutDir = new File(tempDir, "layout-land");
layoutDir.mkdirs();
sourceFile = new File(layoutDir, "main.xml");
sourceFilePath = sourceFile.getAbsolutePath();
writeToFile(
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+ "<LinearLayout xmlns:android=\"http://schemas.android.com/apk/res/android\"\n"
+ " android:orientation=\"vertical\"\n"
+ " android:layout_width=\"fill_parent\"\n"
+ " android:layout_height=\"fill_parent\"\n"
+ " >\n"
+ "<TextView\n"
+ " android:layout_width=\"fill_parent\"\n"
+ " android:layout_height=\"wrap_content\"\n"
+ " android:text=\"Test App - Basic\"\n"
+ " android:id=\"@+id/text\"\n"
+ " />\n"
+ "</LinearLayout>\n"
+ "\n"
+ "<!-- From: file:src/test/resources/testData/resources/incMergeData/filesVsValues/main/layout/main.xml -->");
String messageText = "Random error message here";
String err = sourceFilePath + ":4: error: Error: " + messageText;
Collection<Message> messages = parser.parseToolOutput(err);
assertEquals("[message count]", 1, messages.size());
Message message = messages.iterator().next();
assertNotNull(message);
// NOT sourceFilePath; should be translated back from source comment
String expected =
new File(
"src/test/resources/testData/resources/incMergeData/filesVsValues/main/layout/main.xml")
.getAbsolutePath();
assertEquals("[file path]", expected, getSystemIndependentSourcePath(message));
assertEquals("[message severity]", Message.Kind.ERROR, message.getKind());
assertEquals("[message text]", messageText, message.getText());
assertEquals(
"[position line]",
4,
message.getSourceFilePositions().get(0).getPosition().getStartLine() + 1);
// assertEquals("[position column]", 35, message.getColumn());
// TODO: Test encoding issues (e.g. & in path where the XML source comment would be &
// instead)
}
@Override
protected void runSteps() throws IOException {
String instanceDir = getInstanceDir();
int generationID = getGenerationID();
String generationPrefix = Namespaces.getInstanceGenerationPrefix(instanceDir, generationID);
int lastGenerationID = getLastGenerationID();
File currentInputDir = Files.createTempDir();
currentInputDir.deleteOnExit();
File tempOutDir = Files.createTempDir();
tempOutDir.deleteOnExit();
try {
Store store = Store.get();
store.downloadDirectory(generationPrefix + "inbound/", currentInputDir);
if (lastGenerationID >= 0) {
store.downloadDirectory(
Namespaces.getInstanceGenerationPrefix(instanceDir, lastGenerationID) + "input/",
currentInputDir);
}
List<Example> examples = new ArrayList<>();
Map<Integer, BiMap<String, Integer>> columnToCategoryNameToIDMapping = new HashMap<>();
new ReadInputs(currentInputDir, examples, columnToCategoryNameToIDMapping).call();
DecisionForest forest = DecisionForest.fromExamplesWithDefault(examples);
DecisionForestPMML.write(
new File(tempOutDir, "model.pmml.gz"), forest, columnToCategoryNameToIDMapping);
store.uploadDirectory(generationPrefix + "input/", currentInputDir, false);
store.uploadDirectory(generationPrefix, tempOutDir, false);
} finally {
IOUtils.deleteRecursively(currentInputDir);
IOUtils.deleteRecursively(tempOutDir);
}
}
@Override
protected void run() throws IOException {
Preconditions.checkArgument(arpaFile.exists(), arpaFile + " does not exist. ");
Log.info("Arpa file to convert:" + arpaFile);
if (tmpDir == null) {
tmpDir = com.google.common.io.Files.createTempDir();
tmpDir.deleteOnExit();
Log.info(("Using temporar dir: " + tmpDir));
} else {
Files.createDirectories(tmpDir.toPath());
}
Preconditions.checkArgument(
spaceUsageStr != null && spaceUsageStr.trim().length() > 0,
"Improper -spaceUsageStr value: " + spaceUsageStr + ". Argument seems to be empty.");
List<String> tokens =
Lists.newArrayList(Splitter.on("-").omitEmptyStrings().trimResults().split(spaceUsageStr));
Preconditions.checkArgument(
tokens.size() == 3,
"Improper -spaceUsageStr value: "
+ spaceUsageStr
+ ". Three value is expected in x-y-z format.");
for (int i = 0; i < spaceUsage.length; i++) {
try {
final int val = Integer.parseInt(tokens.get(i));
if (val <= 0 || val > 32 || val % 8 != 0) {
throw new IllegalArgumentException(
"Improper -spaceUsageStr value: " + spaceUsageStr + ". Values can be 8,16,24 or 32");
}
spaceUsage[i] = val;
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"Improper -spaceUsageStr value: " + spaceUsageStr + ". Values must be integers");
}
}
Preconditions.checkArgument(
chunkBits >= 16 && chunkBits <= 31,
"Unexpected chunkBits value. Value must be between 16 to 31. But it is : " + chunkBits);
UncompressedToSmoothLmConverter converter =
new UncompressedToSmoothLmConverter(smoothLmFile, tmpDir);
converter.convertLarge(
MultiFileUncompressedLm.generate(arpaFile, tmpDir, "utf-8").getLmDir(),
new UncompressedToSmoothLmConverter.NgramDataBlock(
spaceUsage[0], spaceUsage[1], spaceUsage[2]),
chunkBits);
}
