示例#1
0
  /**
   * Regression test for KUDU-1226. Calls to session.flush() concurrent with AUTO_FLUSH_BACKGROUND
   * can end up giving ConvertBatchToListOfResponsesCB a list with nulls if a tablet was already
   * flushed. Only happens with multiple tablets.
   */
  @Test(timeout = 10000)
  public void testConcurrentFlushes() throws Exception {
    String tableName = TABLE_NAME_PREFIX + "-testConcurrentFlushes";
    CreateTableOptions builder = new CreateTableOptions();
    int numTablets = 4;
    int numRowsPerTablet = 100;

    // Create a 4 tablets table split on 1000, 2000, and 3000.
    for (int i = 1; i < numTablets; i++) {
      PartialRow split = basicSchema.newPartialRow();
      split.addInt(0, i * numRowsPerTablet);
      builder.addSplitRow(split);
    }
    table = createTable(tableName, basicSchema, builder);

    // Configure the session to background flush as often as it can (every 1ms).
    KuduSession session = syncClient.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    session.setFlushInterval(1);

    // Fill each tablet in parallel 1 by 1 then flush. Without the fix this would quickly get an
    // NPE.
    for (int i = 0; i < numRowsPerTablet; i++) {
      for (int j = 0; j < numTablets; j++) {
        session.apply(createInsert(i + (numRowsPerTablet * j)));
      }
      session.flush();
    }
  }
示例#2
0
  /**
   * 单纯插入
   *
   * @param tablename
   * @param mid
   * @param fieldsValue
   */
  private void insert(String tablename, String mid, String[] fieldsValue) {

    checkSession();

    KuduTable table;
    if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
      table = table_attr;
    } else {
      table = table_days;
    }
    try {
      Insert insert = table.newInsert();
      PartialRow row = insert.getRow();
      setInsertValue(mid, Constants.DAYS_FIELDS, fieldsValue, row);
      OperationResponse rsInsert = session.apply(insert);
      if (rsInsert.hasRowError()) {
        System.out.println(
            "=======================================INSERT DATA:"
                + mid
                + "---------------"
                + Arrays.toString(fieldsValue)
                + rsInsert.getRowError().getMessage());
      }
    } catch (Exception e) {
      collector.reportError(e);
    }
  }
示例#3
0
  /**
   * 执行插入更新操作
   *
   * @param tablename
   * @param mid
   * @param fieldsValue
   */
  private void insertUpdate(String tablename, String mid, String[] fieldsValue) {

    checkSession();

    KuduTable table;
    if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
      table = table_attr;
    } else {
      table = table_days;
    }
    try {
      Insert insert = table.newInsert();
      PartialRow row = insert.getRow();
      setInsertValue(mid, Constants.ATTR_INSERT_FIELDS, fieldsValue, row);
      OperationResponse rsInsert = session.apply(insert);
      if (rsInsert.hasRowError()) {
        if ("key already present".equals(rsInsert.getRowError().getMessage())) {
          Update update = table.newUpdate();
          PartialRow urow = update.getRow();
          setUpdateValue(mid, Constants.ATTR_UPDATE_FIELDS, fieldsValue, urow);
          OperationResponse rsUpdate = session.apply(update);
          if (rsUpdate.hasRowError()) {
            System.out.println(
                "=======================================ERROR UPDATE :" + rsUpdate.getRowError());
          } else {
            System.out.println(
                "=======================================UPDATE DATA:"
                    + mid
                    + ":"
                    + Arrays.toString(fieldsValue));
          }
        } else {
          System.out.println(
              "=======================================ERROR INSERT :" + rsInsert.getRowError());
        }
      } else {
        System.out.println(
            "=======================================INSERT DATA:"
                + mid
                + ":"
                + Arrays.toString(fieldsValue));
      }
    } catch (Exception e) {
      collector.reportError(e);
    }
  }
示例#4
0
  @Test(timeout = 100000)
  public void testBatchWithSameRow() throws Exception {
    String tableName = TABLE_NAME_PREFIX + "-testBatchWithSameRow";
    table = createTable(tableName, basicSchema, new CreateTableOptions());

    KuduSession session = syncClient.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);

    // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
    // while also clearing the cache between each batch half the time. The delete is added here
    // so that a misplaced update would fail if it happens later than its delete.
    for (int i = 0; i < 25; i++) {
      session.apply(createInsert(i));
      for (int j = 0; j < 50; j++) {
        Update update = table.newUpdate();
        PartialRow row = update.getRow();
        row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
        row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
        session.apply(update);
      }
      Delete del = table.newDelete();
      PartialRow row = del.getRow();
      row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
      session.apply(del);
      session.flush();
      if (i % 2 == 0) {
        client.emptyTabletsCacheForTable(table.getTableId());
      }
    }
    assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
  }
示例#5
0
  @Test(timeout = 10000)
  public void testOverWritingValues() throws Exception {
    String tableName = TABLE_NAME_PREFIX + "-OverridingValues";
    table = createTable(tableName, basicSchema, null);
    KuduSession session = syncClient.newSession();
    Insert insert = createInsert(0);
    PartialRow row = insert.getRow();

    // Overwrite all the normal columns.
    int magicNumber = 9999;
    row.addInt(1, magicNumber);
    row.addInt(2, magicNumber);
    row.addBoolean(4, false);
    // Spam the string column since it's backed by an array.
    for (int i = 0; i <= magicNumber; i++) {
      row.addString(3, i + "");
    }
    // We're supposed to keep a constant size.
    assertEquals(5, row.getVarLengthData().size());
    session.apply(insert);

    KuduScanner scanner = syncClient.newScannerBuilder(table).build();
    RowResult rr = scanner.nextRows().next();
    assertEquals(magicNumber, rr.getInt(1));
    assertEquals(magicNumber, rr.getInt(2));
    assertEquals(magicNumber + "", rr.getString(3));
    assertEquals(false, rr.getBoolean(4));

    // Test setting a value post-apply.
    try {
      row.addInt(1, 0);
      fail("Row should be frozen and throw");
    } catch (IllegalStateException ex) {
      // Ok.
    }
  }
示例#6
0
 /** 检查Session */
 private void checkSession() {
   if (session.isClosed()) {
     System.out.println(
         "================================Session is closed ================================"
             + client);
     if (client == null) {
       client = new KuduClient.KuduClientBuilder(Constants.KUDU_MASTER).build();
     }
     if (StringUtils.isNotEmpty(client.toString())) {
       try {
         table_attr = client.openTable(Constants.UPUSERS_ATTR_TABLE);
         table_days = client.openTable(Constants.UPUSERS_DAYS_TABLE);
         session = client.newSession();
       } catch (Exception e) {
         e.printStackTrace();
       }
     }
     System.out.println(
         "================================Session is created ================================"
             + client);
   }
 }
示例#7
0
  @Test(timeout = 100000)
  public void testBasicOps() throws Exception {
    String tableName = TABLE_NAME_PREFIX + "-testBasicOps";
    table = createTable(tableName, basicSchema, new CreateTableOptions());

    KuduSession session = syncClient.newSession();
    for (int i = 0; i < 10; i++) {
      session.apply(createInsert(i));
    }
    assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build()));

    OperationResponse resp = session.apply(createInsert(0));
    assertTrue(resp.hasRowError());

    session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);

    for (int i = 10; i < 20; i++) {
      session.apply(createInsert(i));
    }
    session.flush();
    assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build()));
  }