示例#1
0
  /**
   * 单纯插入
   *
   * @param tablename
   * @param mid
   * @param fieldsValue
   */
  private void insert(String tablename, String mid, String[] fieldsValue) {

    checkSession();

    KuduTable table;
    if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
      table = table_attr;
    } else {
      table = table_days;
    }
    try {
      Insert insert = table.newInsert();
      PartialRow row = insert.getRow();
      setInsertValue(mid, Constants.DAYS_FIELDS, fieldsValue, row);
      OperationResponse rsInsert = session.apply(insert);
      if (rsInsert.hasRowError()) {
        System.out.println(
            "=======================================INSERT DATA:"
                + mid
                + "---------------"
                + Arrays.toString(fieldsValue)
                + rsInsert.getRowError().getMessage());
      }
    } catch (Exception e) {
      collector.reportError(e);
    }
  }
示例#2
0
  @Test(timeout = 100000)
  public void testBatchWithSameRow() throws Exception {
    String tableName = TABLE_NAME_PREFIX + "-testBatchWithSameRow";
    table = createTable(tableName, basicSchema, new CreateTableOptions());

    KuduSession session = syncClient.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);

    // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
    // while also clearing the cache between each batch half the time. The delete is added here
    // so that a misplaced update would fail if it happens later than its delete.
    for (int i = 0; i < 25; i++) {
      session.apply(createInsert(i));
      for (int j = 0; j < 50; j++) {
        Update update = table.newUpdate();
        PartialRow row = update.getRow();
        row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
        row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
        session.apply(update);
      }
      Delete del = table.newDelete();
      PartialRow row = del.getRow();
      row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
      session.apply(del);
      session.flush();
      if (i % 2 == 0) {
        client.emptyTabletsCacheForTable(table.getTableId());
      }
    }
    assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
  }
示例#3
0
  protected static KuduTable createFourTabletsTableWithNineRows(String tableName) throws Exception {
    CreateTableBuilder builder = new CreateTableBuilder();
    PartialRow splitRow = basicSchema.newPartialRow();
    for (int i : KEYS) {
      splitRow.addInt(0, i);
      builder.addSplitRow(splitRow);
    }
    KuduTable table = createTable(tableName, basicSchema, builder);
    AsyncKuduSession session = client.newSession();

    // create a table with on empty tablet and 3 tablets of 3 rows each
    for (int key1 : KEYS) {
      for (int key2 = 1; key2 <= 3; key2++) {
        Insert insert = table.newInsert();
        PartialRow row = insert.getRow();
        row.addInt(0, key1 + key2);
        row.addInt(1, key1);
        row.addInt(2, key2);
        row.addString(3, "a string");
        row.addBoolean(4, true);
        session.apply(insert).join(DEFAULT_SLEEP);
      }
    }
    session.close().join(DEFAULT_SLEEP);
    return table;
  }
  @Test
  public void test() throws Exception {
    createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());

    KuduTableOutputFormat output = new KuduTableOutputFormat();
    Configuration conf = new Configuration();
    conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
    conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
    output.setConf(conf);

    String multitonKey = conf.get(KuduTableOutputFormat.MULTITON_KEY);
    KuduTable table = KuduTableOutputFormat.getKuduTable(multitonKey);
    assertNotNull(table);

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addInt(0, 1);
    row.addInt(1, 2);
    row.addInt(2, 3);
    row.addString(3, "a string");
    row.addBoolean(4, true);

    RecordWriter<NullWritable, Operation> rw = output.getRecordWriter(null);
    rw.write(NullWritable.get(), insert);
    rw.close(null);
    AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
    assertEquals(1, countRowsInScan(builder.build()));
  }
示例#5
0
  /**
   * 执行插入更新操作
   *
   * @param tablename
   * @param mid
   * @param fieldsValue
   */
  private void insertUpdate(String tablename, String mid, String[] fieldsValue) {

    checkSession();

    KuduTable table;
    if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
      table = table_attr;
    } else {
      table = table_days;
    }
    try {
      Insert insert = table.newInsert();
      PartialRow row = insert.getRow();
      setInsertValue(mid, Constants.ATTR_INSERT_FIELDS, fieldsValue, row);
      OperationResponse rsInsert = session.apply(insert);
      if (rsInsert.hasRowError()) {
        if ("key already present".equals(rsInsert.getRowError().getMessage())) {
          Update update = table.newUpdate();
          PartialRow urow = update.getRow();
          setUpdateValue(mid, Constants.ATTR_UPDATE_FIELDS, fieldsValue, urow);
          OperationResponse rsUpdate = session.apply(update);
          if (rsUpdate.hasRowError()) {
            System.out.println(
                "=======================================ERROR UPDATE :" + rsUpdate.getRowError());
          } else {
            System.out.println(
                "=======================================UPDATE DATA:"
                    + mid
                    + ":"
                    + Arrays.toString(fieldsValue));
          }
        } else {
          System.out.println(
              "=======================================ERROR INSERT :" + rsInsert.getRowError());
        }
      } else {
        System.out.println(
            "=======================================INSERT DATA:"
                + mid
                + ":"
                + Arrays.toString(fieldsValue));
      }
    } catch (Exception e) {
      collector.reportError(e);
    }
  }
示例#6
0
 protected Insert createBasicSchemaInsert(KuduTable table, int key) {
   Insert insert = table.newInsert();
   PartialRow row = insert.getRow();
   row.addInt(0, key);
   row.addInt(1, 2);
   row.addInt(2, 3);
   row.addString(3, "a string");
   row.addBoolean(4, true);
   return insert;
 }
示例#7
0
  /**
   * Helper method to easily kill a tablet server that serves the given table's only tablet's
   * leader. The currently running test case will be failed if there's more than one tablet, if the
   * tablet has no leader after some retries, or if the tablet server was already killed.
   *
   * <p>This method is thread-safe.
   *
   * @param table a KuduTable which will get its single tablet's leader killed.
   * @throws Exception
   */
  protected static void killTabletLeader(KuduTable table) throws Exception {
    LocatedTablet.Replica leader = null;
    DeadlineTracker deadlineTracker = new DeadlineTracker();
    deadlineTracker.setDeadline(DEFAULT_SLEEP);
    while (leader == null) {
      if (deadlineTracker.timedOut()) {
        fail("Timed out while trying to find a leader for this table: " + table.getName());
      }
      List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
      if (tablets.isEmpty() || tablets.size() > 1) {
        fail(
            "Currently only support killing leaders for tables containing 1 tablet, table "
                + table.getName()
                + " has "
                + tablets.size());
      }
      LocatedTablet tablet = tablets.get(0);
      if (tablet.getReplicas().size() == 1) {
        fail("Table " + table.getName() + " only has 1 tablet, please enable replication");
      }
      leader = tablet.getLeaderReplica();
      if (leader == null) {
        LOG.info(
            "Sleeping while waiting for a tablet LEADER to arise, currently slept "
                + deadlineTracker.getElapsedMillis()
                + "ms");
        Thread.sleep(50);
      }
    }

    Integer port = leader.getRpcPort();
    Process ts = TABLET_SERVERS.get(port);
    if (ts == null) {
      // The TS is already dead, good.
      return;
    }
    LOG.info("Killing server at port " + port);
    ts.destroy();
    ts.waitFor();
    TABLET_SERVERS.remove(port);
  }