Esempio n. 1
0
  @Test
  public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow";
    Iface client = getClient();
    Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName);
    creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString());
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2");

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value1", column.getValue());
    }

    BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand();
    bulkTableUpdateCommand.setAutoLoad(true);
    bulkTableUpdateCommand.setTable(tableName);
    bulkTableUpdateCommand.setWaitForDataBeVisible(true);
    bulkTableUpdateCommand.addExtraConfig(conf);
    assertEquals(0, (int) bulkTableUpdateCommand.run(getClient()));

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value2", column.getValue());
    }
  }
Esempio n. 2
0
  @Test
  public void testDriverAddSingleRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverAddSingleRecordToExistingRow";
    Iface client = getClient();
    creatTable(tableName, new Path(root, "tables"), true);
    addRow(client, tableName, "row1", "record1", "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr);
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(
        0,
        driver.run(
            new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(2, tableStats.getRecordCount());
  }
Esempio n. 3
0
  @Before
  public void ensureCleanTables() throws Exception {
    setupConfigIfNeeded();

    Iface client = Config.getClient();
    List<String> tableList = client.tableList();
    if (!tableList.isEmpty()) {
      for (String table : tableList) {
        client.disableTable(table);
        client.removeTable(table, true);
      }
    }
  }
  private void runSearch(int start, int fetch) throws SQLException {
    currentStart = start;
    currentFetch = fetch;
    BlurQuery blurQuery = new BlurQuery();
    blurQuery.minimumNumberOfResults = Long.MAX_VALUE;
    blurQuery.maxQueryTime = Long.MAX_VALUE;
    blurQuery.uuid = UUID.randomUUID().toString();
    blurQuery.fetch = fetch;
    blurQuery.start = start;
    blurQuery.query = new Query();
    blurQuery.query.query = parser.getWhere();
    blurQuery.query.rowQuery = false;

    try {
      schema = client.schema(tableName);
    } catch (BlurException e) {
      e.printStackTrace();
      throw new SQLException(e);
    } catch (TException e) {
      e.printStackTrace();
      throw new SQLException(e);
    }

    selector = new Selector();
    setupSelector(selector, schema, columnNames);
    selector.recordOnly = !blurQuery.query.rowQuery;
    Map<String, Map<String, ColumnDefinition>> columnFamilies = schema.getFamilies();
    Map<String, ColumnDefinition> cfSet = columnFamilies.get(columnFamily);
    columnFamilies.clear();
    columnFamilies.put(columnFamily, cfSet);
    blurResultSetMetaData = new BlurResultSetMetaData(columnNames, columnFamilies);
    try {
      results = client.query(tableName, blurQuery);
    } catch (BlurException e) {
      e.printStackTrace();
      throw new SQLException(e);
    } catch (TException e) {
      e.printStackTrace();
      throw new SQLException(e);
    }
    if (results.totalResults > 0) {
      size = results.results.size();
    }
    totalResults = results.totalResults;
  }
 private CommandDescriptor getCommandDescriptor(Iface client, String commandName)
     throws BlurException, TException {
   List<CommandDescriptor> listInstalledCommands = client.listInstalledCommands();
   for (CommandDescriptor commandDescriptor : listInstalledCommands) {
     if (commandDescriptor.getCommandName().equals(commandName)) {
       return commandDescriptor;
     }
   }
   return null;
 }
Esempio n. 6
0
 private void waitUntilAllImportsAreCompleted(Iface client, String tableName)
     throws BlurException, TException, InterruptedException {
   while (true) {
     Thread.sleep(1000);
     TableStats tableStats = client.tableStats(tableName);
     if (tableStats.getSegmentImportInProgressCount() == 0
         && tableStats.getSegmentImportPendingCount() == 0) {
       return;
     }
   }
 }
Esempio n. 7
0
 private void addRow(Iface client, String tableName, String rowId, String recordId, String value)
     throws BlurException, TException {
   List<RecordMutation> recordMutations = new ArrayList<RecordMutation>();
   List<Column> columns = new ArrayList<Column>();
   columns.add(new Column("col0", value));
   Record record = new Record(recordId, "fam0", columns);
   recordMutations.add(new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record));
   RowMutation rowMutation =
       new RowMutation(tableName, rowId, RowMutationType.REPLACE_ROW, recordMutations);
   client.mutate(rowMutation);
 }
Esempio n. 8
0
  private void creatTable(String tableName, Path tables, boolean fastDisable, String workingPath)
      throws BlurException, TException {
    Path tablePath = new Path(tables, tableName);
    Iface client = getClient();
    TableDescriptor tableDescriptor = new TableDescriptor();
    tableDescriptor.setTableUri(tablePath.toString());
    tableDescriptor.setName(tableName);
    tableDescriptor.setShardCount(2);
    tableDescriptor.putToTableProperties(
        BlurConstants.BLUR_TABLE_DISABLE_FAST_DIR, Boolean.toString(fastDisable));
    if (workingPath != null) {
      tableDescriptor.putToTableProperties(
          BlurConstants.BLUR_BULK_UPDATE_WORKING_PATH, workingPath);
    }
    client.createTable(tableDescriptor);

    ColumnDefinition colDef = new ColumnDefinition();
    colDef.setFamily("fam0");
    colDef.setColumnName("col0");
    colDef.setFieldType("string");
    client.addColumnDefinition(tableName, colDef);
  }
Esempio n. 9
0
  @SuppressWarnings({"unchecked", "rawtypes"})
  @Test
  public void testGetTableSummaries() throws Exception {
    Iface client = Config.getClient();

    TableDescriptor td = new TableDescriptor();
    td.setShardCount(11);
    td.setTableUri("file://" + TABLE_PATH + "/tableUnitTable");
    td.setCluster("default");
    td.setName("tableUnitTable");
    td.setEnabled(true);
    client.createTable(td);

    Map<String, List> data = TableUtil.getTableSummaries();

    assertEquals(1, data.get("tables").size());
    assertEquals(0l, ((Map<String, Object>) data.get("tables").get(0)).get("rows"));
    assertEquals(0l, ((Map<String, Object>) data.get("tables").get(0)).get("records"));
    assertEquals("default", ((Map<String, Object>) data.get("tables").get(0)).get("cluster"));
    assertEquals("tableUnitTable", ((Map<String, Object>) data.get("tables").get(0)).get("name"));
    assertEquals(
        0,
        ((List<String>) ((Map<String, Object>) data.get("tables").get(0)).get("families")).size());
  }
  @Override
  public boolean next() throws SQLException {
    if (displayRows != null && displayRowsPosition + 1 < displayRows.size()) {
      overallRowPosition++;
      displayRowsPosition++;
      return true;
    } else if (resultPosition + 1 < size) {
      INNER:
      while (resultPosition + 1 < size) {
        displayRows.clear();
        resultPosition++;
        displayRowsPosition = 0;
        overallRowPosition++;

        final BlurResult result = results.results.get(resultPosition);
        try {
          FetchRecordResult recordResult = result.getFetchResult().getRecordResult();
          selector.setRowId(recordResult.getRowid());
          selector.setRecordId(recordResult.getRecord().getRecordId());
          fetchResult = client.fetchRow(tableName, selector);
          Record record = fetchResult.recordResult.record;
          if (!record.family.equals(columnFamily)) {
            continue INNER;
          }
          String rowId = fetchResult.recordResult.rowid;
          displayRows.add(
              addColumns(result.getScore(), result.getLocationId(), rowId, record.family, record));
          return true;

        } catch (Exception e) {
          e.printStackTrace();
          throw new SQLException(e);
        }
      }
      return next();
    } else if (overallRowPosition < totalResults) {
      currentStart += currentFetch;
      runSearch(currentStart, currentFetch);
      displayRowsPosition = 0;
      resultPosition = -1;
      return next();
    }
    return false;
  }
Esempio n. 11
0
  @Test
  public void testDriverUpdateRecordToExistingRow() throws Exception {
    FileSystem fileSystem = miniCluster.getFileSystem();
    Path root = new Path(fileSystem.getUri() + "/");

    String tableName = "testDriverUpdateRecordToExistingRow";
    Iface client = getClient();
    creatTable(tableName, new Path(root, "tables"), true);
    String rowId = "row1";
    String recordId = "record1";
    addRow(client, tableName, rowId, recordId, "value1");

    Driver driver = new Driver();
    driver.setConf(conf);

    String mrIncWorkingPathStr = new Path(root, "working").toString();
    generateData(mrIncWorkingPathStr, rowId, recordId, "value2");
    String outputPathStr = new Path(root, "output").toString();
    String blurZkConnection = miniCluster.getZkConnectionString();

    assertEquals(
        0,
        driver.run(
            new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value1", column.getValue());
    }

    client.loadData(tableName, outputPathStr);

    waitUntilAllImportsAreCompleted(client, tableName);

    TableStats tableStats = client.tableStats(tableName);
    assertEquals(1, tableStats.getRowCount());
    assertEquals(1, tableStats.getRecordCount());

    {
      Selector selector = new Selector();
      selector.setRowId(rowId);
      FetchResult fetchRow = client.fetchRow(tableName, selector);
      Row row = fetchRow.getRowResult().getRow();
      assertEquals(rowId, row.getId());
      List<Record> records = row.getRecords();
      assertEquals(1, records.size());
      Record record = records.get(0);
      assertEquals(recordId, record.getRecordId());
      List<Column> columns = record.getColumns();
      assertEquals(1, columns.size());
      Column column = columns.get(0);
      assertEquals("col0", column.getName());
      assertEquals("value2", column.getValue());
    }
  }