Esempio n. 1
0
  public static void deleteTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);

      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] startRow = Bytes.toBytes("rowKey_1");
      byte[] stopRow = Bytes.toBytes("rowKey_3");
      byte[] family = f0;

      Scan scan = new Scan();
      scan.addFamily(family);
      scan.setMaxVersions(1);

      //            scan.setStartRow(startRow);
      //            scan.setStopRow(stopRow);

      ResultScanner scanner = table.getScanner(scan);
      Result result = scanner.next();
      List<Delete> delete = new ArrayList<Delete>();
      while (result != null) {
        Delete del = new Delete(result.getRow());
        delete.add(del);
        result = scanner.next();
      }
      table.delete(delete);
      System.out.println("delete done");
      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Esempio n. 2
0
  public static void writeTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);
      byte[] family = f0;

      List<Put> puts = new ArrayList<Put>();
      for (int k = 0; k < 10; k++) // 写10行数据
      {
        byte[] rowkey = Bytes.toBytes("rowKey_" + k);
        Put p = new Put(rowkey);

        byte[] value_id = Bytes.toBytes("123456");
        byte[] value_user = Bytes.toBytes("mengqinghao" + k);

        p.add(family, qualifier_id, value_id);
        p.add(family, qualifier_user, value_user);

        puts.add(p);
      }
      table.put(puts);
      System.out.println("Puts done: " + puts.size());

      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Esempio n. 3
0
  public static void readTest(String tableStr, String row) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] rowkey = Bytes.toBytes(row);
      Get get = new Get(rowkey);
      get.addFamily(f0);

      Result result = table.get(get);
      NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0);

      if (m == null || m.isEmpty()) {
        System.err.println("Empty." + m);
        return;
      }

      for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
        String qualifier = Bytes.toString(entry.getKey());
        String value = Bytes.toString(entry.getValue());
        System.out.println(qualifier + ":" + value);
      }
      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Esempio n. 4
0
  /**
   * Test waiting on meta w/ no timeout specified.
   *
   * @throws Exception
   */
  @Ignore // Can't make it work reliably on all platforms; mockito gets confused
  // Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
  // Result cannot be returned by locateRegion()
  // If you plug locateRegion, it then throws for incCounter, and if you plug
  // that ... and so one.
  @Test
  public void testNoTimeoutWaitForMeta() throws Exception {
    // Mock an HConnection and a HRegionInterface implementation.  Have the
    // HConnection return the HRI.  Have the HRI return a few mocked up responses
    // to make our test work.
    // Mock an HRegionInterface.
    final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    // Now the ct is up... set into the mocks some answers that make it look
    // like things have been getting assigned. Make it so we'll return a
    // location (no matter what the Get is). Same for getHRegionInfo -- always
    // just return the meta region.
    final Result result = getMetaTableRowResult();

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
        .thenReturn(result);
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
        .thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ServerName hsa = ct.getMetaLocation();
    Assert.assertNull(hsa);

    // Now test waiting on meta location getting set.
    Thread t =
        new WaitOnMetaThread(ct) {
          @Override
          void doWaiting() throws InterruptedException {
            this.ct.waitForMeta();
          }
        };
    startWaitAliveThenWaitItLives(t, 1000);

    // This should trigger wake up of meta wait (Its the removal of the meta
    // region unassigned node that triggers catalogtrackers that a meta has
    // been assigned).
    String node = ct.getMetaNodeTracker().getNode();
    ZKUtil.createAndFailSilent(this.watcher, node);
    MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
    ZKUtil.deleteNode(this.watcher, node);
    // Go get the new meta location. waitForMeta gets and verifies meta.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
    // Join the thread... should exit shortly.
    t.join();
    // Now meta is available.
    Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
  }
Esempio n. 5
0
 @Override
 protected void cleanup(Context context) throws IOException, InterruptedException {
   super.cleanup(context);
   table.flushCommits();
   table.close();
   connection.close();
 }
  /**
   * Do the commit. This is the 2nd phase of the 2-phase protocol.
   *
   * @param transactionState
   * @throws CommitUnsuccessfulException
   */
  void doCommit(final TransactionState transactionState) throws CommitUnsuccessfulException {
    try {
      LOG.trace("Commiting [" + transactionState.getTransactionId() + "]");

      transactionLogger.setStatusForTransaction(
          transactionState.getTransactionId(), TransactionLogger.TransactionStatus.COMMITTED);

      for (HRegionLocation location : transactionState.getParticipatingRegions()) {
        if (transactionState.getRegionsToIngore().contains(location)) {
          continue;
        }
        TransactionalRegionInterface transactionalRegionServer =
            (TransactionalRegionInterface)
                connection.getHRegionConnection(location.getServerAddress());
        transactionalRegionServer.commit(
            location.getRegionInfo().getRegionName(), transactionState.getTransactionId());
      }
    } catch (Exception e) {
      LOG.info(
          "Commit of transaction [" + transactionState.getTransactionId() + "] was unsucsessful",
          e);
      // This happens on a NSRE that is triggered by a split
      try {
        abort(transactionState);
      } catch (Exception abortException) {
        LOG.warn("Exeption durring abort", abortException);
      }
      throw new CommitUnsuccessfulException(e);
    }
    transactionLogger.forgetTransaction(transactionState.getTransactionId());
  }
Esempio n. 7
0
 /**
  * Test get of root region fails properly if nothing to connect to.
  *
  * @throws IOException
  * @throws InterruptedException
  * @throws KeeperException
  */
 @Test
 public void testVerifyRootRegionLocationFails()
     throws IOException, InterruptedException, KeeperException {
   HConnection connection = Mockito.mock(HConnection.class);
   ConnectException connectException = new ConnectException("Connection refused");
   final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
   Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).thenThrow(connectException);
   Mockito.when(
           connection.getHRegionConnection(
               Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
       .thenReturn(implementation);
   final CatalogTracker ct = constructAndStartCatalogTracker(connection);
   RootLocationEditor.setRootLocation(
       this.watcher, new ServerName("example.com", 1234, System.currentTimeMillis()));
   Assert.assertFalse(ct.verifyRootRegionLocation(100));
 }
  /**
   * Abort a s transaction.
   *
   * @param transactionState
   * @throws IOException
   */
  public void abort(final TransactionState transactionState) throws IOException {
    transactionLogger.setStatusForTransaction(
        transactionState.getTransactionId(), TransactionLogger.TransactionStatus.ABORTED);

    for (HRegionLocation location : transactionState.getParticipatingRegions()) {
      if (transactionState.getRegionsToIngore().contains(location)) {
        continue;
      }
      try {
        TransactionalRegionInterface transactionalRegionServer =
            (TransactionalRegionInterface)
                connection.getHRegionConnection(location.getServerAddress());

        transactionalRegionServer.abortTransaction(
            location.getRegionInfo().getRegionName(), transactionState.getTransactionId());
      } catch (UnknownTransactionException e) {
        LOG.info(
            "Got unknown transaciton exception durring abort. Transaction: ["
                + transactionState.getTransactionId()
                + "], region: ["
                + location.getRegionInfo().getRegionNameAsString()
                + "]. Ignoring.");
      } catch (NotServingRegionException e) {
        LOG.info(
            "Got NSRE durring abort. Transaction: ["
                + transactionState.getTransactionId()
                + "], region: ["
                + location.getRegionInfo().getRegionNameAsString()
                + "]. Ignoring.");
      }
    }
    transactionLogger.forgetTransaction(transactionState.getTransactionId());
  }
Esempio n. 9
0
  /*
   * 为表添加数据(适合知道有多少列族的固定表)
   *
   * @rowKey rowKey
   *
   * @tableName 表名
   *
   * @column1 第一个列族列表
   *
   * @value1 第一个列的值的列表
   *
   * @column2 第二个列族列表
   *
   * @value2 第二个列的值的列表
   */
  public static void addData(
      String rowKey,
      String tableName,
      String[] column1,
      String[] value1,
      String[] column2,
      String[] value2)
      throws IOException {

    Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
    HTableInterface htable = conn.getTable(tableName);
    HColumnDescriptor[] columnFamilies =
        htable
            .getTableDescriptor() // 获取所有的列族
            .getColumnFamilies();

    for (int i = 0; i < columnFamilies.length; i++) {
      String familyName = columnFamilies[i].getNameAsString(); // 获取列族名
      if (familyName.equals("article")) { // article列族put数据
        for (int j = 0; j < column1.length; j++) {
          put.add(Bytes.toBytes(familyName), Bytes.toBytes(column1[j]), Bytes.toBytes(value1[j]));
        }
      }
      if (familyName.equals("author")) { // author列族put数据
        for (int j = 0; j < column2.length; j++) {
          put.add(Bytes.toBytes(familyName), Bytes.toBytes(column2[j]), Bytes.toBytes(value2[j]));
        }
      }
    }
    htable.put(put);
    // System.out.println("add data Success!");
  }
Esempio n. 10
0
 public HTableInterface getTable(String tablename) {
   try {
     if (connection == null) {
       connection = HConnectionManager.createConnection(config);
     }
     return connection.getTable(tablename);
   } catch (IOException e) {
     LOG.error(ExceptionUtils.getFullStackTrace(e));
     LOG.error("exception accurs when getting table from hconnection, will be retried");
     try {
       connection = HConnectionManager.createConnection(config);
       return connection.getTable(tablename);
     } catch (Throwable throwable) {
       throw new RuntimeException(throwable);
     }
   }
 }
Esempio n. 11
0
  public static void createTable(String tableName) {
    long start = System.currentTimeMillis();
    try {
      HTableInterface htable = conn.getTable(tableName);
      List<Put> putList = new ArrayList<Put>();
      HColumnDescriptor[] columnFamilies = htable.getTableDescriptor().getColumnFamilies();

      String[] family = {"article", "author"};
      creatTable(tableName, family);
      String[] column1 = {"title", "content", "tag"};
      String[] column2 = {"name", "nickname"};
      for (int i = 1; i <= 1000000; i++) {
        DecimalFormat format = new DecimalFormat("00000000");
        String rowKey = format.format(i);
        // System.out.println("==| insert"+rowKey);
        String[] valueOfArticle = {"title" + i, "content" + i, "tag" + i};
        String[] valueOfAuthor = {"name" + i, "nickname" + i};
        // addData(rowKey, tableName, column1, valueOfArticle, column2,
        // valueOfAuthor);

        Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey

        for (int colIndex = 0; colIndex < columnFamilies.length; colIndex++) {
          String familyName = columnFamilies[colIndex].getNameAsString(); // 获取列族名

          if (familyName.equals("article")) { // article列族put数据
            for (int k = 0; k < column1.length; k++) {
              put.add(
                  Bytes.toBytes(familyName),
                  Bytes.toBytes(column1[k]),
                  Bytes.toBytes(valueOfArticle[k]));
            }
          }
          if (familyName.equals("author")) { // author列族put数据
            for (int k = 0; k < column2.length; k++) {
              put.add(
                  Bytes.toBytes(familyName),
                  Bytes.toBytes(column2[k]),
                  Bytes.toBytes(valueOfAuthor[k]));
            }
          }
        }
        putList.add(put);
        if (i % 10000 == 0) {

          htable.put(putList);
          putList = new ArrayList<Put>();
        }
      }

      long end = System.currentTimeMillis();
      long used = end - start;
      System.out.println("data insert  finished used " + used + " ms");

    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Esempio n. 12
0
 public static HTableInterface getHBaseInstance(String tableName) {
   try {
     return hConnection.getTable(tableName);
   } catch (IOException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
     return null;
   }
 }
Esempio n. 13
0
    private void loadIPs() {
      dns = new HashMap(100000000); // ���貢��
      unknownHosts = new HashMap(1000000);
      querying = new HashMap(100000);

      try {
        int statsCommit = 500000;

        HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create());
        HTableInterface fetchFailTable = connection.getTable("fetchFail");
        Scan scan = new Scan();
        scan.setCaching(statsCommit);

        List<Filter> filters = new ArrayList<Filter>();
        Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip"));
        filters.add(filter);
        FilterList filterList = new FilterList(filters);
        scan.setFilter(filterList);

        ResultScanner rs = fetchFailTable.getScanner(scan);
        long cnt = 0;
        for (Result r : rs) {
          NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf"));
          String ip = Bytes.toString(map.get(Bytes.toBytes("ip")));
          String host = Bytes.toString(r.getRow()).split("��")[0];
          if (host != null && ip != null) {
            dns.put(host, ip);
          }

          if (++cnt % statsCommit == 0) {
            LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt);
          }
        }
        rs.close();
        fetchFailTable.close();
        LOG.info("load hostip cache=" + dns.size());

        connection.close();
      } catch (Exception e) {
        e.printStackTrace();
      } finally {
        //
      }
    }
Esempio n. 14
0
 /**
  * @param implementation An {@link HRegionInterface} instance; you'll likely want to pass a mocked
  *     HRS; can be null.
  * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
  *     {@link HConnection#getConfiguration()} is called, a 'location' when {@link
  *     HConnection#getRegionLocation(byte[], byte[], boolean)} is called, and that returns the
  *     passed {@link HRegionInterface} instance when {@link
  *     HConnection#getHRegionConnection(String, int)} is called (Be sure call {@link
  *     HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)} when done with
  *     this mocked Connection.
  * @throws IOException
  */
 private HConnection mockConnection(final HRegionInterface implementation) throws IOException {
   HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
   Mockito.doNothing().when(connection).close();
   // Make it so we return any old location when asked.
   final HRegionLocation anyLocation =
       new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(), SN.getPort());
   Mockito.when(
           connection.getRegionLocation(
               (byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean()))
       .thenReturn(anyLocation);
   Mockito.when(connection.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any()))
       .thenReturn(anyLocation);
   if (implementation != null) {
     // If a call to getHRegionConnection, return this implementation.
     Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt()))
         .thenReturn(implementation);
   }
   return connection;
 }
Esempio n. 15
0
  public static void scanTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] startRow = Bytes.toBytes("rowKey_0");
      byte[] stopRow = Bytes.toBytes("rowKey_6");
      byte[] family = f0;

      Scan scan = new Scan();
      scan.addFamily(family);
      scan.setMaxVersions(1);

      //            scan.setStartRow(startRow);
      //            scan.setStopRow(stopRow);
      int count = 0;
      ResultScanner scanner = table.getScanner(scan);
      Result result = scanner.next();
      while (result != null) {
        String rowKey = Bytes.toString(result.getRow());
        NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);

        if (m == null || m.isEmpty()) {
          System.err.println("Empty." + m);
          return;
        }
        for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
          String qualifier = Bytes.toString(entry.getKey());
          String value = Bytes.toString(entry.getValue());
          System.out.println(rowKey + ":" + qualifier + ":" + value);
        }
        result = scanner.next();
        count++;
        System.out.println("-----------------------------");
      }
      table.close(); // very important
      System.out.println("count:" + count);
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Esempio n. 16
0
 @Override
 protected void setup(Context context) throws IOException, InterruptedException {
   super.setup(context);
   configuration = context.getConfiguration();
   connection = HConnectionManager.createConnection(configuration);
   table = connection.getTable("");
   table.setAutoFlush(false, true);
   table.setWriteBufferSize(12 * 1024 * 1024);
   wal = true;
 }
Esempio n. 17
0
 /** Re-connect to HBase */
 private void startClient() {
   if (this.htable == null) {
     try {
       hconnection = HConnectionManager.createConnection(hconf);
       htable = hconnection.getTable(table);
     } catch (IOException e) {
       LOG.warn("Failed to create HBase connection. " + e.getMessage());
     }
   }
 }
Esempio n. 18
0
  /**
   * 向表中插入一条新数据
   *
   * @param tableName 表名
   * @param row 行键key
   * @param columnFamily 列族
   * @param column 列名
   * @param data 要插入的数据
   * @throws IOException
   */
  public static void putData(
      String tableName, String row, String columnFamily, String column, byte[] data)
      throws IOException {
    HTableInterface table = conn.getTable(tableName);
    Put put = new Put(Bytes.toBytes(row));

    put.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), data);
    table.put(put);
    System.out.println(
        "PUT:" + row + "','" + columnFamily + ":" + column + "','" + Bytes.toString(data) + "'");
  }
 public boolean validateObject(HConnection obj) {
   try {
     // TODO 只要連過線,之後斷線還是true
     if (!(obj.isClosed())) {
       return true;
     } else {
       return false;
     }
   } catch (Exception e) {
     return false;
   }
 }
Esempio n. 20
0
  public void testPool() throws IOException {
    PoolConfig config = new PoolConfig();
    config.setMaxTotal(20);
    config.setMaxIdle(5);
    config.setMaxWaitMillis(1000);
    config.setTestOnBorrow(true);

    /* properties */
    Properties props = new Properties();
    props.setProperty("hbase.zookeeper.quorum", "host1,host2,host3");
    props.setProperty("hbase.zookeeper.property.clientPort", "2181");
    props.setProperty("hbase.master", "host1:60000");
    props.setProperty("hbase.rootdir", "hdfs://host1:9000/hbase");

    /* connection pool */
    HbaseConnectionPool pool = new HbaseConnectionPool(config, props);
    HTableInterface table = null;

    HConnection conn = pool.getConnection();
    table = conn.getTable(TableName.valueOf("relation"));

    Get get = new Get(Bytes.toBytes("rowKey"));
    Result r = table.get(get);
    for (Cell cell : r.rawCells()) {
      System.out.println(
          "Rowkey : "
              + Bytes.toString(r.getRow())
              + " Familiy:Quilifier : "
              + Bytes.toString(CellUtil.cloneQualifier(cell))
              + " Value : "
              + Bytes.toString(CellUtil.cloneValue(cell)));
    }
    table.close();
    System.out.println(table);
    pool.returnConnection(conn);

    pool.close();
  }
Esempio n. 21
0
 /**
  * Contacts a region server and waits up to hbase.hbck.close.timeout ms (default 120s) to close
  * the region. This bypasses the active hmaster.
  */
 public static void closeRegionSilentlyAndWait(
     HBaseAdmin admin, ServerName server, HRegionInfo region)
     throws IOException, InterruptedException {
   HConnection connection = admin.getConnection();
   AdminService.BlockingInterface rs = connection.getAdmin(server);
   try {
     ProtobufUtil.closeRegion(rs, server, region.getRegionName(), false);
   } catch (IOException e) {
     LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
   }
   long timeout = admin.getConfiguration().getLong("hbase.hbck.close.timeout", 120000);
   long expiration = timeout + System.currentTimeMillis();
   while (System.currentTimeMillis() < expiration) {
     try {
       HRegionInfo rsRegion = ProtobufUtil.getRegionInfo(rs, region.getRegionName());
       if (rsRegion == null) return;
     } catch (IOException ioe) {
       return;
     }
     Thread.sleep(1000);
   }
   throw new IOException("Region " + region + " failed to close within" + " timeout " + timeout);
 }
Esempio n. 22
0
  /*
   * 遍历查询hbase表
   *
   * @tableName 表名
   */
  public static void getResultScann(String tableName) throws IOException {
    Scan scan = new Scan();
    ResultScanner rs = null;
    HTableInterface table = conn.getTable(tableName);
    try {
      //			String split = StringUtils.S001;
      //			QualifierFilter ff = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
      //					Bytes.toBytes("A")));
      //			scan.setFilter(ff);
      rs = table.getScanner(scan);
      int count = 0;
      for (Result r : rs) {
        count++;
        for (KeyValue kv : r.list()) {
          System.out.println("row:" + Bytes.toString(kv.getRow()));
          //					System.out.println("family:" + Bytes.toString(kv.getFamily()));
          //					System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
          System.out.println("value:" + Bytes.toString(kv.getValue()));
          //
          //					System.out.println("timestamp:" + kv.getTimestamp());

          //					StringBuilder sb = new StringBuilder();
          //					sb.append(Bytes.toString(r.getRow()));
          //					sb.append(split);
          //					sb.append(Bytes.toString(kv.getValue()));
          //					EntBaseinfo baseInfo = new EntBaseinfo();
          //					baseInfo.parseFromString(sb.toString());
          //					System.out.println(baseInfo.getENTNAME());
          //					if(baseInfo.getNAME()!=null&&baseInfo.getNAME().isEmpty()){
          //						System.out.println(baseInfo.getENTNAME());
          //					}
          //
          //
          //					if(baseInfo.getDOM()!=null&&baseInfo.getNAME().isEmpty()){
          //						System.out.println(baseInfo.getENTNAME());
          //					}

        }
        if (count > 1000) {
          return;
        }
      }
    } finally {
      rs.close();
    }
  }
Esempio n. 23
0
  CatalogTracker(
      final ZooKeeperWatcher zk,
      final Configuration conf,
      HConnection connection,
      Abortable abortable,
      final int defaultTimeout)
      throws IOException {
    this.connection = connection;
    if (abortable == null) {
      // A connection is abortable.
      this.abortable = this.connection;
    }
    Abortable throwableAborter =
        new Abortable() {

          @Override
          public void abort(String why, Throwable e) {
            throw new RuntimeException(why, e);
          }

          @Override
          public boolean isAborted() {
            return true;
          }
        };
    if (zk == null) {
      // Create our own.  Set flag so we tear it down on stop.
      this.zookeeper =
          new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(), abortable);
      instantiatedzkw = true;
    } else {
      this.zookeeper = zk;
    }
    this.rootRegionTracker = new RootRegionTracker(zookeeper, throwableAborter);
    final CatalogTracker ct = this;
    // Override nodeDeleted so we get notified when meta node deleted
    this.metaNodeTracker =
        new MetaNodeTracker(zookeeper, throwableAborter) {
          public void nodeDeleted(String path) {
            if (!path.equals(node)) return;
            ct.resetMetaLocation();
          }
        };
    this.defaultTimeout = defaultTimeout;
  }
Esempio n. 24
0
 /**
  * @param sn ServerName to get a connection against.
  * @return The HRegionInterface we got when we connected to <code>sn</code> May have come from
  *     cache, may not be good, may have been setup by this invocation, or may be null.
  * @throws IOException
  */
 private HRegionInterface getCachedConnection(ServerName sn) throws IOException {
   if (sn == null) {
     return null;
   }
   HRegionInterface protocol = null;
   try {
     protocol = connection.getHRegionConnection(sn.getHostname(), sn.getPort());
   } catch (RetriesExhaustedException e) {
     if (e.getCause() != null && e.getCause() instanceof ConnectException) {
       // Catch this; presume it means the cached connection has gone bad.
     } else {
       throw e;
     }
   } catch (SocketTimeoutException e) {
     LOG.debug("Timed out connecting to " + sn);
   } catch (NoRouteToHostException e) {
     LOG.debug("Connecting to " + sn, e);
   } catch (SocketException e) {
     LOG.debug("Exception connecting to " + sn);
   } catch (UnknownHostException e) {
     LOG.debug("Unknown host exception connecting to  " + sn);
   } catch (IOException ioe) {
     Throwable cause = ioe.getCause();
     if (ioe instanceof ConnectException) {
       // Catch. Connect refused.
     } else if (cause != null && cause instanceof EOFException) {
       // Catch. Other end disconnected us.
     } else if (cause != null
         && cause.getMessage() != null
         && cause.getMessage().toLowerCase().contains("connection reset")) {
       // Catch. Connection reset.
     } else {
       throw ioe;
     }
   }
   return protocol;
 }
  /**
   * Prepare to commit a transaction.
   *
   * @param transactionState
   * @return commitStatusCode (see {@link TransactionalRegionInterface})
   * @throws IOException
   * @throws CommitUnsuccessfulException
   */
  public int prepareCommit(final TransactionState transactionState)
      throws CommitUnsuccessfulException, IOException {
    boolean allReadOnly = true;
    try {

      for (HRegionLocation location : transactionState.getParticipatingRegions()) {

        TransactionalRegionInterface transactionalRegionServer =
            (TransactionalRegionInterface)
                connection.getHRegionConnection(location.getServerAddress());
        int commitStatus =
            transactionalRegionServer.commitRequest(
                location.getRegionInfo().getRegionName(), transactionState.getTransactionId());
        boolean canCommit = true;
        switch (commitStatus) {
          case TransactionalRegionInterface.COMMIT_OK:
            allReadOnly = false;
            break;
          case TransactionalRegionInterface.COMMIT_OK_READ_ONLY:
            transactionState.addRegionToIgnore(location); // No need to doCommit for read-onlys
            break;
          case TransactionalRegionInterface.COMMIT_UNSUCESSFUL:
            canCommit = false;
            transactionState.addRegionToIgnore(location); // No need to re-abort.
            break;
          default:
            throw new CommitUnsuccessfulException(
                "Unexpected return code from prepareCommit: " + commitStatus);
        }

        if (LOG.isTraceEnabled()) {
          LOG.trace(
              "Region ["
                  + location.getRegionInfo().getRegionNameAsString()
                  + "] votes "
                  + (canCommit ? "to commit" : "to abort")
                  + " transaction "
                  + transactionState.getTransactionId());
        }

        if (!canCommit) {
          LOG.debug("Aborting [" + transactionState.getTransactionId() + "]");
          abort(transactionState);
          return TransactionalRegionInterface.COMMIT_UNSUCESSFUL;
        }
      }
    } catch (Exception e) {
      LOG.debug(
          "Commit of transaction [" + transactionState.getTransactionId() + "] was unsucsessful",
          e);
      // This happens on a NSRE that is triggered by a split
      try {
        abort(transactionState);
      } catch (Exception abortException) {
        LOG.warn("Exeption durring abort", abortException);
      }
      throw new CommitUnsuccessfulException(e);
    }
    return allReadOnly
        ? TransactionalRegionInterface.COMMIT_OK_READ_ONLY
        : TransactionalRegionInterface.COMMIT_OK;
  }
Esempio n. 26
0
  /**
   * Test for HBASE-4288. Throw an IOE when trying to verify meta region and prove it doesn't cause
   * master shutdown.
   *
   * @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
   * @throws IOException
   * @throws InterruptedException
   * @throws KeeperException
   */
  @Test
  public void testServerNotRunningIOException()
      throws IOException, InterruptedException, KeeperException {
    // Mock an HRegionInterface.
    final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
    HConnection connection = mockConnection(implementation);

    // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
    // the first time.  'Succeed' the second time we are called.
    Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any()))
        .thenThrow(new IOException("Server not running, aborting"))
        .thenReturn(new HRegionInfo());

    // After we encounter the above 'Server not running', we should catch the
    // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
    // get new meta location.  Return something so this 'get' succeeds
    // (here we mock up getRegionServerWithRetries, the wrapper around
    // the actual get).

    // TODO: Refactor.  This method has been moved out of HConnection.
    // It works for now but has been deprecated.
    Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>) Mockito.any()))
        .thenReturn(getMetaTableRowResult());

    // Now start up the catalogtracker with our doctored Connection.
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    try {
      // Set a location for root and meta.
      RootLocationEditor.setRootLocation(this.watcher, SN);
      ct.setMetaLocation(SN);
      // Call the method that HBASE-4288 calls.  It will try and verify the
      // meta location and will fail on first attempt then go into a long wait.
      // So, do this in a thread and then reset meta location to break it out
      // of its wait after a bit of time.
      final AtomicBoolean metaSet = new AtomicBoolean(false);
      final CountDownLatch latch = new CountDownLatch(1);
      Thread t =
          new Thread() {
            @Override
            public void run() {
              try {
                latch.countDown();
                metaSet.set(ct.waitForMeta(100000) != null);
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          };
      t.start();
      latch.await();
      Threads.sleep(1);
      // Now reset the meta as though it were redeployed.
      ct.setMetaLocation(SN);
      t.join();
      Assert.assertTrue(metaSet.get());
    } finally {
      // Clean out root and meta locations or later tests will be confused...
      // they presume start fresh in zk.
      ct.resetMetaLocation();
    }
  }