@Test public void testCopyFromFileWithPartition() throws Exception { execute("create table quotes (id int, " + "quote string) partitioned by (id)"); ensureGreen(); String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json"); execute("copy quotes partition (id = 1) from ? with (shared=true)", new Object[] {filePath}); refresh(); execute("select * from quotes"); assertEquals(3L, response.rowCount()); }
@Test public void testCopyFromFileWithUmlautsWhitespacesAndGlobs() throws Exception { execute( "create table t (id int primary key, name string) clustered into 1 shards with (number_of_replicas = 0)"); File tmpFolder = folder.newFolder("äwesöme földer"); File file = new File(tmpFolder, "süpär.json"); List<String> lines = Collections.singletonList("{\"id\": 1, \"name\": \"Arthur\"}"); Files.write(file.toPath(), lines, StandardCharsets.UTF_8); execute("copy t from ?", new Object[] {tmpFolder.getAbsolutePath() + "/s*.json"}); assertThat(response.rowCount(), is(1L)); }
@Test public void testCopyFromDirectory() throws Exception { execute( "create table quotes (id int primary key, " + "quote string index using fulltext) with (number_of_replicas=0)"); ensureYellow(); execute("copy quotes from ? with (shared=true)", new Object[] {copyFilePath + "/*"}); assertEquals(3L, response.rowCount()); refresh(); execute("select * from quotes"); assertEquals(3L, response.rowCount()); }
@Test public void testCopyFromNestedArrayRow() throws Exception { // assert that rows with nested arrays aren't imported execute("create table users (id int, " + "name string) with (number_of_replicas=0)"); ensureYellow(); String uriPath = Joiner.on("/").join(nestedArrayCopyFilePath, "nested_array_copy_from.json"); execute("copy users from ? with (shared=true)", new Object[] {uriPath}); assertEquals(1L, response.rowCount()); // only 1 document got inserted refresh(); execute("select * from users"); assertThat(response.rowCount(), is(1L)); assertThat(TestingHelpers.printedTable(response.rows()), is("2| Trillian\n")); }
@Test public void testCopyFromFilePattern() throws Exception { execute( "create table quotes (id int primary key, " + "quote string index using fulltext) with (number_of_replicas=0)"); ensureYellow(); String uriPath = Joiner.on("/").join(copyFilePath, "*.json"); execute("copy quotes from ?", new Object[] {uriPath}); assertEquals(3L, response.rowCount()); refresh(); execute("select * from quotes"); assertEquals(3L, response.rowCount()); }
/** * Show environment variables. * * @throws Exception if failed */ @Test public void environment() throws Exception { String target = new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath(); File shell = putScript("environment.sh", new File(target)); HadoopScript script = new HadoopScript( "testing", set(), "com.example.Client", map(), map("script", "SCRIPT", "override", "SCRIPT")); HadoopScriptHandler handler = handler( "env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(), "env.handler", "HANDLER", "env.override", "HANDLER"); execute(script, handler); List<String> results = getOutput(shell); assertThat(results, has(equalToIgnoringWhiteSpace("script=SCRIPT"))); assertThat(results, has(equalToIgnoringWhiteSpace("handler=HANDLER"))); assertThat(results, has(equalToIgnoringWhiteSpace("override=SCRIPT"))); }
/** * Using complex prefix. * * @throws Exception if failed */ @Test public void complex_prefix() throws Exception { String target = new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath(); File shell = putScript("arguments.sh", new File(target)); HadoopScript script = new HadoopScript("testing", set(), "com.example.Client", map(), map()); HadoopScriptHandler handler = handler( "env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(), "command.0", "@[0]", "command.1", "@[2]-@[3]-@[4]"); ExecutionContext context = new ExecutionContext( "tbatch", "tflow", "texec", ExecutionPhase.MAIN, map("hello", "world", "key", "value")); execute(context, script, handler); List<String> results = getOutput(shell); assertThat( results.subList(0, 7), is( Arrays.asList( "tbatch-tflow-texec", shell.getAbsolutePath(), "com.example.Client", "tbatch", "tflow", "texec", context.getArgumentsAsString()))); }
@Test public void testCopyFromInvalidJson() throws Exception { execute( "create table foo (id integer primary key) clustered into 1 shards with (number_of_replicas=0)"); ensureYellow(); File newFile = folder.newFile(); BufferedWriter writer = new BufferedWriter(new FileWriter(newFile)); writer.write("{id:1}\n"); writer.write("{|}"); writer.write("{id:2}\n"); writer.flush(); writer.close(); expectedException.expect(SQLActionException.class); expectedException.expectMessage("Failed to parse content to map"); execute("copy foo from ?", new Object[] {newFile.getPath()}); }
@Test public void testCopyFromFileWithoutPK() throws Exception { execute( "create table quotes (id int, " + "quote string index using fulltext) with (number_of_replicas=0)"); ensureYellow(); String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json"); execute("copy quotes from ?", new Object[] {uriPath}); assertEquals(6L, response.rowCount()); assertThat(response.duration(), greaterThanOrEqualTo(0L)); refresh(); execute("select * from quotes"); assertEquals(6L, response.rowCount()); assertThat(response.rows()[0].length, is(2)); }
@Test public void testCopyToDirectoryPath() throws Exception { expectedException.expect(SQLActionException.class); expectedException.expectMessage( startsWith("Failed to open output: 'Output path is a directory: ")); execute( "create table characters (" + " race string," + " gender string," + " age long," + " birthdate timestamp," + " name string," + " details object" + ") with (number_of_replicas=0)"); ensureYellow(); String directoryUri = Paths.get(folder.newFolder().toURI()).toUri().toString(); execute("COPY characters TO ?", new Object[] {directoryUri}); }
@Test public void testCopyFromFileWithEmptyLine() throws Exception { execute( "create table foo (id integer primary key) clustered into 1 shards with (number_of_replicas=0)"); ensureYellow(); File newFile = folder.newFile(); BufferedWriter writer = new BufferedWriter(new FileWriter(newFile)); writer.write("{id:1}\n"); writer.write("\n"); writer.write("{id:2}\n"); writer.flush(); writer.close(); execute("copy foo from ?", new Object[] {newFile.getPath()}); assertEquals(2L, response.rowCount()); refresh(); execute("select * from foo order by id"); assertThat((Integer) response.rows()[0][0], is(1)); assertThat((Integer) response.rows()[1][0], is(2)); }
@Test public void testCopyToFile() throws Exception { execute( "create table singleshard (name string) clustered into 1 shards with (number_of_replicas = 0)"); ensureYellow(); execute("insert into singleshard (name) values ('foo')"); execute("refresh table singleshard"); String uri = Paths.get(folder.getRoot().toURI()).resolve("testsingleshard.json").toUri().toString(); SQLResponse response = execute("copy singleshard to ?", new Object[] {uri}); assertThat(response.rowCount(), is(1L)); List<String> lines = Files.readAllLines( Paths.get(folder.getRoot().toURI().resolve("testsingleshard.json")), UTF8); assertThat(lines.size(), is(1)); for (String line : lines) { assertThat(line, startsWith("{")); assertThat(line, endsWith("}")); } }
@Test public void testCopyFromWithRoutingInPK() throws Exception { execute( "create table t (i int primary key, c string primary key, a int)" + " clustered by (c) with (number_of_replicas=0)"); ensureGreen(); execute("insert into t (i, c) values (1, 'clusteredbyvalue'), (2, 'clusteredbyvalue')"); refresh(); String uri = Paths.get(folder.getRoot().toURI()).toString(); SQLResponse response = execute("copy t to directory ? with(shared=true)", new Object[] {uri}); assertThat(response.rowCount(), is(2L)); execute("delete from t"); refresh(); execute("copy t from ? with (shared=true)", new Object[] {uri + "/t_*"}); refresh(); // only one shard should have all imported rows, since we have the same routing for both rows response = execute("select count(*) from sys.shards where num_docs>0 and table_name='t'"); assertThat((long) response.rows()[0][0], is(1L)); }
/** * Passes runtime context. * * @throws Exception if failed */ @Test public void runtime_context() throws Exception { String target = new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath(); File shell = putScript("environment.sh", new File(target)); HadoopScript script = new HadoopScript( "testing", set(), "com.example.Client", map(), map("script", "SCRIPT", "override", "SCRIPT")); HadoopScriptHandler handler = handler( "env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(), "env.handler", "HANDLER", "env.override", "HANDLER"); RuntimeContext rc = RuntimeContext.DEFAULT.batchId("b").mode(ExecutionMode.SIMULATION).buildId("OK"); ExecutionContext context = new ExecutionContext("b", "f", "e", ExecutionPhase.MAIN, map(), rc.unapply()); execute(context, script, handler); Map<String, String> map = new HashMap<>(); for (String line : getOutput(shell)) { if (line.trim().isEmpty()) { continue; } String[] kv = line.split("=", 2); if (kv.length != 2) { continue; } map.put(kv[0], kv[1]); } assertThat(RuntimeContext.DEFAULT.apply(map), is(rc)); }
@Test public void testCopyFromWithOverwriteDuplicates() throws Exception { execute("create table t (id int primary key) with (number_of_replicas = 0)"); ensureYellow(); execute( "insert into t (id) values (?)", new Object[][] {new Object[] {1}, new Object[] {2}, new Object[] {3}, new Object[] {4}}); execute("refresh table t"); File tmpExport = folder.newFolder("tmpExport"); execute("copy t to directory ?", new Object[] {tmpExport.getAbsolutePath()}); assertThat(response.rowCount(), is(4L)); execute("copy t from ?", new Object[] {String.format("%s/*", tmpExport.getAbsolutePath())}); assertThat(response.rowCount(), is(0L)); execute( "copy t from ? with (overwrite_duplicates = true, shared=true)", new Object[] {String.format("%s/*", tmpExport.getAbsolutePath())}); assertThat(response.rowCount(), is(4L)); execute("refresh table t"); execute("select count(*) from t"); assertThat(((Long) response.rows()[0][0]), is(4L)); }
/** * With properties. * * @throws Exception if failed */ @Test public void properties() throws Exception { String target = new File(getAsakusaHome(), ProcessHadoopScriptHandler.PATH_EXECUTE).getAbsolutePath(); File shell = putScript("arguments.sh", new File(target)); HadoopScript script = new HadoopScript( "testing", set(), "com.example.Client", map("hello", "world", "hoge", "foo"), map()); HadoopScriptHandler handler = handler( "env.ASAKUSA_HOME", getAsakusaHome().getAbsolutePath(), "prop.hello", "handler", "prop.bar", "moga"); ExecutionContext context = new ExecutionContext("tbatch", "tflow", "texec", ExecutionPhase.MAIN, map()); execute(context, script, handler); List<String> results = getOutput(shell); assertThat( results.subList(0, 5), is( Arrays.asList( "com.example.Client", "tbatch", "tflow", "texec", context.getArgumentsAsString()))); List<String> rest = results.subList(5, results.size()); int hello = rest.indexOf("hello=world"); assertThat(hello, greaterThanOrEqualTo(1)); assertThat(rest.get(hello - 1), is("-D")); int hoge = rest.indexOf("hoge=foo"); assertThat(hoge, greaterThanOrEqualTo(1)); assertThat(rest.get(hoge - 1), is("-D")); int bar = rest.indexOf("bar=moga"); assertThat(bar, greaterThanOrEqualTo(1)); assertThat(rest.get(bar - 1), is("-D")); }