Пример #1
0
  @Test
  public void testDriverAddSingleRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverAddSingleRecordToExistingRow";
    Iface client = getClient();
    creatTable(tableName, new Path(root, "tables"), true);
    addRow(client, tableName, "row1", "record1", "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr);
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(
        0,
        driver.run(
            new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(2, tableStats.getRecordCount());
  }
Пример #2
0
  @BeforeClass
  public static void setupTest() throws Exception {
    setupJavaHome();
    File file = new File("./target/tmp/BlurInputFormatTest_tmp");
    String pathStr = file.getAbsoluteFile().toURI().toString();
    System.setProperty("test.build.data", pathStr + "/data");
    System.setProperty("hadoop.log.dir", pathStr + "/hadoop_log");
    miniCluster = new MiniCluster();
    miniCluster.startBlurCluster(pathStr + "/blur", 2, 2);
    miniCluster.startMrMiniCluster();
    conf = miniCluster.getMRConfiguration();

    BufferStore.initNewBuffer(128, 128 * 128);
  }
Пример #3
0
 @AfterClass
 public static void teardown() throws IOException {
   if (miniCluster != null) {
     miniCluster.stopMrMiniCluster();
   }
   rm(new File("build"));
 }
Пример #4
0
  @Test
  public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow";
    Iface client = getClient();
    Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName);
    creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString());
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2");

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value1", column.getValue());
    }

    BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand();
    bulkTableUpdateCommand.setAutoLoad(true);
    bulkTableUpdateCommand.setTable(tableName);
    bulkTableUpdateCommand.setWaitForDataBeVisible(true);
    bulkTableUpdateCommand.addExtraConfig(conf);
    assertEquals(0, (int) bulkTableUpdateCommand.run(getClient()));

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value2", column.getValue());
    }
  }
 private static void createTable(String tableName) throws BlurException, TException, IOException {
   Blur.Iface client = client();
   TableDescriptor tableDescriptor = new TableDescriptor();
   tableDescriptor.setName(tableName);
   tableDescriptor.setShardCount(5);
   tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
   client.createTable(tableDescriptor);
   List<String> tableList = client.tableList();
   assertTrue(tableList.contains(tableName));
 }
  private SolrServer createServerAndTableWithSimpleTestDoc(String table)
      throws BlurException, TException, IOException, SolrServerException {
    createTable(table);
    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
    SolrInputDocument doc;

    doc = createSimpleTestDoc();

    server.add(doc);
    return server;
  }
  @BeforeClass
  public static void startCluster() throws IOException {
    GCWatcher.init(0.60);
    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
    testDirectory.mkdirs();

    Path directory = new Path(testDirectory.getPath());
    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
    FsAction userAction = dirPermissions.getUserAction();
    FsAction groupAction = dirPermissions.getGroupAction();
    FsAction otherAction = dirPermissions.getOtherAction();

    StringBuilder builder = new StringBuilder();
    builder.append(userAction.ordinal());
    builder.append(groupAction.ordinal());
    builder.append(otherAction.ordinal());
    String dirPermissionNum = builder.toString();
    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
    testDirectory.delete();
    miniCluster = new MiniCluster();
    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true);
    connectionStr = miniCluster.getControllerConnectionStr();
  }
Пример #8
0
 private void generateData(String mrIncWorkingPathStr, String rowId, String recordId, String value)
     throws IOException {
   Path path = new Path(new Path(mrIncWorkingPathStr), "new");
   Writer writer =
       new SequenceFile.Writer(
           miniCluster.getFileSystem(),
           conf,
           new Path(path, UUID.randomUUID().toString()),
           Text.class,
           BlurRecord.class);
   BlurRecord blurRecord = new BlurRecord();
   blurRecord.setRowId(rowId);
   blurRecord.setRecordId(recordId);
   blurRecord.setFamily("fam0");
   blurRecord.addColumn("col0", value);
   writer.append(new Text(rowId), blurRecord);
   writer.close();
 }
    public SolrServer create() throws Exception {
      createTable(tableName);
      SolrServer server =
          new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), tableName);

      for (int i = 0; i < rows; i++) {
        SolrInputDocument parent = new SolrInputDocument();
        parent.addField(BlurConstants.ROW_ID, i);
        for (int j = 0; j < recordsPerRow; j++) {
          SolrInputDocument child = new SolrInputDocument();
          child.addField(BlurConstants.RECORD_ID, j);

          for (String colName : columns) {
            child.addField(colName, "value" + i + "-" + j);
          }
          parent.addChildDocument(child);
        }
        server.add(parent);
      }
      return server;
    }
  @Test
  public void docShouldBeDiscoverableWithMultiValuedFields()
      throws SolrServerException, IOException, BlurException, TException {
    String table = "docShouldBeDiscoverableWithMultiValuedFields";
    createTable(table);
    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("rowid", "1");

    SolrInputDocument child = new SolrInputDocument();
    child.addField("recordid", "1");
    child.addField("fam.value", "123");
    child.addField("fam.value", "124");

    doc.addChildDocument(child);

    server.add(doc);

    assertTotalResults(table, "fam.value:123", 1l);
    assertTotalResults(table, "fam.value:124", 1l);
    assertTotalResults(table, "fam.value:justincase", 0l);

    removeTable(table);
  }
 @AfterClass
 public static void shutdownCluster() {
   miniCluster.shutdownBlurCluster();
 }
Пример #12
0
 private Iface getClient() {
   return BlurClient.getClientFromZooKeeperConnectionStr(miniCluster.getZkConnectionString());
 }
Пример #13
0
  @Test
  public void testDriverUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverUpdateRecordToExistingRow";
    Iface client = getClient();
    creatTable(tableName, new Path(root, "tables"), true);
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr, rowId, recordId, "value2");
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(
        0,
        driver.run(
            new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value1", column.getValue());
    }

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value2", column.getValue());
    }
  }