@Test
public void testCopyFromFileWithPartition() throws Exception {
execute("create table quotes (id int, " + "quote string) partitioned by (id)");
ensureGreen();
String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");
execute("copy quotes partition (id = 1) from ? with (shared=true)", new Object[] {filePath});
refresh();
execute("select * from quotes");
assertEquals(3L, response.rowCount());
}
@Test
public void testCopyFromFileWithUmlautsWhitespacesAndGlobs() throws Exception {
execute(
"create table t (id int primary key, name string) clustered into 1 shards with (number_of_replicas = 0)");
File tmpFolder = folder.newFolder("äwesöme földer");
File file = new File(tmpFolder, "süpär.json");
List<String> lines = Collections.singletonList("{\"id\": 1, \"name\": \"Arthur\"}");
Files.write(file.toPath(), lines, StandardCharsets.UTF_8);
execute("copy t from ?", new Object[] {tmpFolder.getAbsolutePath() + "/s*.json"});
assertThat(response.rowCount(), is(1L));
}
@Test
public void testCopyFromDirectory() throws Exception {
execute(
"create table quotes (id int primary key, "
+ "quote string index using fulltext) with (number_of_replicas=0)");
ensureYellow();
execute("copy quotes from ? with (shared=true)", new Object[] {copyFilePath + "/*"});
assertEquals(3L, response.rowCount());
refresh();
execute("select * from quotes");
assertEquals(3L, response.rowCount());
}
@Test
public void testCopyFromNestedArrayRow() throws Exception {
// assert that rows with nested arrays aren't imported
execute("create table users (id int, " + "name string) with (number_of_replicas=0)");
ensureYellow();
String uriPath = Joiner.on("/").join(nestedArrayCopyFilePath, "nested_array_copy_from.json");
execute("copy users from ? with (shared=true)", new Object[] {uriPath});
assertEquals(1L, response.rowCount()); // only 1 document got inserted
refresh();
execute("select * from users");
assertThat(response.rowCount(), is(1L));
assertThat(TestingHelpers.printedTable(response.rows()), is("2| Trillian\n"));
}
@Test
public void testCopyFromFilePattern() throws Exception {
execute(
"create table quotes (id int primary key, "
+ "quote string index using fulltext) with (number_of_replicas=0)");
ensureYellow();
String uriPath = Joiner.on("/").join(copyFilePath, "*.json");
execute("copy quotes from ?", new Object[] {uriPath});
assertEquals(3L, response.rowCount());
refresh();
execute("select * from quotes");
assertEquals(3L, response.rowCount());
}
/**
* Show environment variables.
*
* @throws Exception if failed
*/
@Test
public void environment() throws Exception {
String target =
new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath();
File shell = putScript("environment.sh", new File(target));
HadoopScript script =
new HadoopScript(
"testing",
set(),
"com.example.Client",
map(),
map("script", "SCRIPT", "override", "SCRIPT"));
HadoopScriptHandler handler =
handler(
"env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(),
"env.handler", "HANDLER",
"env.override", "HANDLER");
execute(script, handler);
List<String> results = getOutput(shell);
assertThat(results, has(equalToIgnoringWhiteSpace("script=SCRIPT")));
assertThat(results, has(equalToIgnoringWhiteSpace("handler=HANDLER")));
assertThat(results, has(equalToIgnoringWhiteSpace("override=SCRIPT")));
}
/**
* Using complex prefix.
*
* @throws Exception if failed
*/
@Test
public void complex_prefix() throws Exception {
String target =
new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath();
File shell = putScript("arguments.sh", new File(target));
HadoopScript script = new HadoopScript("testing", set(), "com.example.Client", map(), map());
HadoopScriptHandler handler =
handler(
"env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(),
"command.0", "@[0]",
"command.1", "@[2]-@[3]-@[4]");
ExecutionContext context =
new ExecutionContext(
"tbatch", "tflow", "texec", ExecutionPhase.MAIN, map("hello", "world", "key", "value"));
execute(context, script, handler);
List<String> results = getOutput(shell);
assertThat(
results.subList(0, 7),
is(
Arrays.asList(
"tbatch-tflow-texec",
shell.getAbsolutePath(),
"com.example.Client",
"tbatch",
"tflow",
"texec",
context.getArgumentsAsString())));
}
@Test
public void testCopyFromInvalidJson() throws Exception {
execute(
"create table foo (id integer primary key) clustered into 1 shards with (number_of_replicas=0)");
ensureYellow();
File newFile = folder.newFile();
BufferedWriter writer = new BufferedWriter(new FileWriter(newFile));
writer.write("{id:1}\n");
writer.write("{|}");
writer.write("{id:2}\n");
writer.flush();
writer.close();
expectedException.expect(SQLActionException.class);
expectedException.expectMessage("Failed to parse content to map");
execute("copy foo from ?", new Object[] {newFile.getPath()});
}
@Test
public void testCopyFromFileWithoutPK() throws Exception {
execute(
"create table quotes (id int, "
+ "quote string index using fulltext) with (number_of_replicas=0)");
ensureYellow();
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[] {uriPath});
assertEquals(6L, response.rowCount());
assertThat(response.duration(), greaterThanOrEqualTo(0L));
refresh();
execute("select * from quotes");
assertEquals(6L, response.rowCount());
assertThat(response.rows()[0].length, is(2));
}
@Test
public void testCopyToDirectoryPath() throws Exception {
expectedException.expect(SQLActionException.class);
expectedException.expectMessage(
startsWith("Failed to open output: 'Output path is a directory: "));
execute(
"create table characters ("
+ " race string,"
+ " gender string,"
+ " age long,"
+ " birthdate timestamp,"
+ " name string,"
+ " details object"
+ ") with (number_of_replicas=0)");
ensureYellow();
String directoryUri = Paths.get(folder.newFolder().toURI()).toUri().toString();
execute("COPY characters TO ?", new Object[] {directoryUri});
}
@Test
public void testCopyFromFileWithEmptyLine() throws Exception {
execute(
"create table foo (id integer primary key) clustered into 1 shards with (number_of_replicas=0)");
ensureYellow();
File newFile = folder.newFile();
BufferedWriter writer = new BufferedWriter(new FileWriter(newFile));
writer.write("{id:1}\n");
writer.write("\n");
writer.write("{id:2}\n");
writer.flush();
writer.close();
execute("copy foo from ?", new Object[] {newFile.getPath()});
assertEquals(2L, response.rowCount());
refresh();
execute("select * from foo order by id");
assertThat((Integer) response.rows()[0][0], is(1));
assertThat((Integer) response.rows()[1][0], is(2));
}
@Test
public void testCopyToFile() throws Exception {
execute(
"create table singleshard (name string) clustered into 1 shards with (number_of_replicas = 0)");
ensureYellow();
execute("insert into singleshard (name) values ('foo')");
execute("refresh table singleshard");
String uri =
Paths.get(folder.getRoot().toURI()).resolve("testsingleshard.json").toUri().toString();
SQLResponse response = execute("copy singleshard to ?", new Object[] {uri});
assertThat(response.rowCount(), is(1L));
List<String> lines =
Files.readAllLines(
Paths.get(folder.getRoot().toURI().resolve("testsingleshard.json")), UTF8);
assertThat(lines.size(), is(1));
for (String line : lines) {
assertThat(line, startsWith("{"));
assertThat(line, endsWith("}"));
}
}
@Test
public void testCopyFromWithRoutingInPK() throws Exception {
execute(
"create table t (i int primary key, c string primary key, a int)"
+ " clustered by (c) with (number_of_replicas=0)");
ensureGreen();
execute("insert into t (i, c) values (1, 'clusteredbyvalue'), (2, 'clusteredbyvalue')");
refresh();
String uri = Paths.get(folder.getRoot().toURI()).toString();
SQLResponse response = execute("copy t to directory ? with(shared=true)", new Object[] {uri});
assertThat(response.rowCount(), is(2L));
execute("delete from t");
refresh();
execute("copy t from ? with (shared=true)", new Object[] {uri + "/t_*"});
refresh();
// only one shard should have all imported rows, since we have the same routing for both rows
response = execute("select count(*) from sys.shards where num_docs>0 and table_name='t'");
assertThat((long) response.rows()[0][0], is(1L));
}
/**
* Passes runtime context.
*
* @throws Exception if failed
*/
@Test
public void runtime_context() throws Exception {
String target =
new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath();
File shell = putScript("environment.sh", new File(target));
HadoopScript script =
new HadoopScript(
"testing",
set(),
"com.example.Client",
map(),
map("script", "SCRIPT", "override", "SCRIPT"));
HadoopScriptHandler handler =
handler(
"env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(),
"env.handler", "HANDLER",
"env.override", "HANDLER");
RuntimeContext rc =
RuntimeContext.DEFAULT.batchId("b").mode(ExecutionMode.SIMULATION).buildId("OK");
ExecutionContext context =
new ExecutionContext("b", "f", "e", ExecutionPhase.MAIN, map(), rc.unapply());
execute(context, script, handler);
Map<String, String> map = new HashMap<>();
for (String line : getOutput(shell)) {
if (line.trim().isEmpty()) {
continue;
}
String[] kv = line.split("=", 2);
if (kv.length != 2) {
continue;
}
map.put(kv[0], kv[1]);
}
assertThat(RuntimeContext.DEFAULT.apply(map), is(rc));
}
@Test
public void testCopyFromWithOverwriteDuplicates() throws Exception {
execute("create table t (id int primary key) with (number_of_replicas = 0)");
ensureYellow();
execute(
"insert into t (id) values (?)",
new Object[][] {new Object[] {1}, new Object[] {2}, new Object[] {3}, new Object[] {4}});
execute("refresh table t");
File tmpExport = folder.newFolder("tmpExport");
execute("copy t to directory ?", new Object[] {tmpExport.getAbsolutePath()});
assertThat(response.rowCount(), is(4L));
execute("copy t from ?", new Object[] {String.format("%s/*", tmpExport.getAbsolutePath())});
assertThat(response.rowCount(), is(0L));
execute(
"copy t from ? with (overwrite_duplicates = true, shared=true)",
new Object[] {String.format("%s/*", tmpExport.getAbsolutePath())});
assertThat(response.rowCount(), is(4L));
execute("refresh table t");
execute("select count(*) from t");
assertThat(((Long) response.rows()[0][0]), is(4L));
}
/**
* With properties.
*
* @throws Exception if failed
*/
@Test
public void properties() throws Exception {
String target =
new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath();
File shell = putScript("arguments.sh", new File(target));
HadoopScript script =
new HadoopScript(
"testing", set(), "com.example.Client", map("hello", "world", "hoge", "foo"), map());
HadoopScriptHandler handler =
handler(
"env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(),
"prop.hello", "handler",
"prop.bar", "moga");
ExecutionContext context =
new ExecutionContext("tbatch", "tflow", "texec", ExecutionPhase.MAIN, map());
execute(context, script, handler);
List<String> results = getOutput(shell);
assertThat(
results.subList(0, 5),
is(
Arrays.asList(
"com.example.Client", "tbatch", "tflow", "texec", context.getArgumentsAsString())));
List<String> rest = results.subList(5, results.size());
int hello = rest.indexOf("hello=world");
assertThat(hello, greaterThanOrEqualTo(1));
assertThat(rest.get(hello - 1), is("-D"));
int hoge = rest.indexOf("hoge=foo");
assertThat(hoge, greaterThanOrEqualTo(1));
assertThat(rest.get(hoge - 1), is("-D"));
int bar = rest.indexOf("bar=moga");
assertThat(bar, greaterThanOrEqualTo(1));
assertThat(rest.get(bar - 1), is("-D"));
}