Ejemplo n.º 1
0
 /**
  * rowFilter的使用
  *
  * @param tableName
  * @param reg
  * @throws Exception
  */
 public void getRowFilter(String tableName, String reg) throws Exception {
   Connection conn = null;
   HTable table = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     table = (HTable) conn.getTable(TableName.valueOf(tableName));
     Scan scan = new Scan();
     //			Filter
     RowFilter rowFilter = new RowFilter(CompareOp.NOT_EQUAL, new RegexStringComparator(reg));
     scan.setFilter(rowFilter);
     ResultScanner scanner = table.getScanner(scan);
     for (Result result : scanner) {
       System.out.println(new String(result.getRow()));
     }
   } catch (Exception e) {
     e.printStackTrace();
   } finally {
     if (null != table) {
       try {
         table.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (Exception e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
Ejemplo n.º 2
0
 /** Close the open connections and shutdown the batchpool */
 public void close() {
   synchronized (connectionsLock) {
     if (connections != null) {
       for (Connection conn : connections) {
         if (conn != null) {
           try {
             conn.close();
           } catch (IOException e) {
             LOG.info("Got exception in closing connection", e);
           } finally {
             conn = null;
           }
         }
       }
       connections = null;
     }
   }
   if (this.batchPool != null && !this.batchPool.isShutdown()) {
     this.batchPool.shutdown();
     try {
       if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
         this.batchPool.shutdownNow();
       }
     } catch (InterruptedException e) {
       this.batchPool.shutdownNow();
     }
   }
 }
Ejemplo n.º 3
0
 /**
  * 删除指定表名
  *
  * @param rowKey
  */
 public void deleteTable(byte[] rowKey) {
   Connection conn = null;
   HBaseAdmin admin = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     admin = (HBaseAdmin) conn.getAdmin();
     // 在删除一张表前,要使其失效
     admin.disableTable(rowKey);
     admin.deleteTable(rowKey);
     admin.enableTable(rowKey);
   } catch (Exception e) {
     logger.error("HBaseAdmin deleteTable exception, errMsg:{}", e.getMessage());
   } finally {
     try {
       if (null != admin) {
         admin.close();
       }
     } catch (IOException e) {
       logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
     }
     try {
       if (null != conn) {
         conn.close();
       }
     } catch (Exception e) {
       logger.error("Connection close exception, errMsg:{}", e.getMessage());
     }
   }
 }
Ejemplo n.º 4
0
 /**
  * 查询所有表
  *
  * @return 所以表 / null
  * @throws Exception
  */
 public static List<HTableDescriptor> queryALLTable() throws Exception {
   Connection conn = null;
   HBaseAdmin admin = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     admin = (HBaseAdmin) conn.getAdmin();
     if (admin != null) {
       HTableDescriptor[] listTables = admin.listTables();
       if (null != listTables && listTables.length > 0) {
         return Arrays.asList(listTables);
       }
     }
   } catch (Exception e) {
     e.printStackTrace();
   } finally {
     try {
       if (null != admin) {
         admin.close();
       }
     } catch (IOException e) {
       logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
     }
     try {
       if (null != conn) {
         conn.close();
       }
     } catch (Exception e) {
       logger.error("Connection close exception, errMsg:{}", e.getMessage());
     }
   }
   return null;
 }
Ejemplo n.º 5
0
 /**
  * 删除指定名称的列簇
  *
  * @param tableName 表名
  * @param columnFamilyName 列族
  */
 public static void deleteFamily(byte[] tableName, String columnFamilyName) {
   Connection conn = null;
   HBaseAdmin admin = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     admin = (HBaseAdmin) conn.getAdmin();
     admin.deleteColumn(tableName, columnFamilyName);
   } catch (Exception e) {
     logger.error("HBaseAdmin deleteColumn exception, errMsg:{}", e.getMessage());
   } finally {
     try {
       if (null != admin) {
         admin.close();
       }
     } catch (IOException e) {
       logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
     }
     try {
       if (null != conn) {
         conn.close();
       }
     } catch (Exception e) {
       logger.error("Connection close exception, errMsg:{}", e.getMessage());
     }
   }
 }
Ejemplo n.º 6
0
 /** 删除指定行 */
 public static void deleteRow(byte[] rowKey) {
   Connection conn = null;
   HTable table = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     table = (HTable) conn.getTable(TableName.valueOf("test"));
     table.delete(new Delete(rowKey));
   } catch (Exception e) {
     logger.error("HBaseAdmin deleteRow exception, errMsg:{}", e.getMessage());
   } finally {
     if (null != table) {
       try {
         table.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (Exception e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
Ejemplo n.º 7
0
 @Override
 protected int doWork() throws Exception {
   Connection connection = null;
   Admin admin = null;
   try {
     connection = ConnectionFactory.createConnection(getConf());
     admin = connection.getAdmin();
     HBaseProtos.SnapshotDescription.Type type = HBaseProtos.SnapshotDescription.Type.FLUSH;
     if (snapshotType != null) {
       type = ProtobufUtil.createProtosSnapShotDescType(snapshotName);
     }
     admin.snapshot(
         new SnapshotDescription(snapshotName, tableName, ProtobufUtil.createSnapshotType(type)));
   } catch (Exception e) {
     return -1;
   } finally {
     if (admin != null) {
       admin.close();
     }
     if (connection != null) {
       connection.close();
     }
   }
   return 0;
 }
Ejemplo n.º 8
0
 /**
  * 根据RowKey查询单行
  *
  * @param rowKey
  * @return
  */
 public static Result queryByRowKey(byte[] rowKey) {
   Result result = null;
   if (null == rowKey || rowKey.length < 0) {
     return result;
   }
   Connection conn = null;
   HTable table = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     table = (HTable) conn.getTable(TableName.valueOf("test"));
     result = table.get(new Get(rowKey));
   } catch (Exception e) {
     logger.error("queryByRowKey exception, rowKey:{}, errMsg:{}", new String(rowKey), e);
   } finally {
     if (null != table) {
       try {
         table.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (IOException e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
   return result;
 }
Ejemplo n.º 9
0
 /**
  * Constructor that creates a connection to the local ZooKeeper ensemble.
  *
  * @param conf Configuration to use
  * @throws IOException if an internal replication error occurs
  * @throws RuntimeException if replication isn't enabled.
  */
 public ReplicationAdmin(Configuration conf) throws IOException {
   if (!conf.getBoolean(
       HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT)) {
     throw new RuntimeException(
         "hbase.replication isn't true, please " + "enable it in order to use replication");
   }
   this.connection = ConnectionFactory.createConnection(conf);
   try {
     zkw = createZooKeeperWatcher();
     try {
       this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, this.connection);
       this.replicationPeers.init();
       this.replicationQueuesClient =
           ReplicationFactory.getReplicationQueuesClient(zkw, conf, this.connection);
       this.replicationQueuesClient.init();
     } catch (Exception exception) {
       if (zkw != null) {
         zkw.close();
       }
       throw exception;
     }
   } catch (Exception exception) {
     if (connection != null) {
       connection.close();
     }
     if (exception instanceof IOException) {
       throw (IOException) exception;
     } else if (exception instanceof RuntimeException) {
       throw (RuntimeException) exception;
     } else {
       throw new IOException("Error initializing the replication admin client.", exception);
     }
   }
 }
Ejemplo n.º 10
0
 /**
  * Sweeps the mob files on one column family. It deletes the unused mob files and merges the small
  * mob files into bigger ones.
  *
  * @param tableName The current table name in string format.
  * @param familyName The column family name.
  * @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other
  *     compaction,4 if mr job was unsuccessful
  * @throws IOException
  * @throws InterruptedException
  * @throws ClassNotFoundException
  * @throws KeeperException
  * @throws ServiceException
  */
 int sweepFamily(String tableName, String familyName)
     throws IOException, InterruptedException, ClassNotFoundException, KeeperException,
         ServiceException {
   Configuration conf = getConf();
   // make sure the target HBase exists.
   HBaseAdmin.checkHBaseAvailable(conf);
   Connection connection = ConnectionFactory.createConnection(getConf());
   Admin admin = connection.getAdmin();
   try {
     FileSystem fs = FileSystem.get(conf);
     TableName tn = TableName.valueOf(tableName);
     HTableDescriptor htd = admin.getTableDescriptor(tn);
     HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
     if (family == null || !family.isMobEnabled()) {
       throw new IOException("Column family " + familyName + " is not a MOB column family");
     }
     SweepJob job = new SweepJob(conf, fs);
     // Run the sweeping
     return job.sweep(tn, family);
   } catch (Exception e) {
     System.err.println("Job aborted due to exception " + e);
     return 2; // job failed
   } finally {
     try {
       admin.close();
     } catch (IOException e) {
       System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
     }
     try {
       connection.close();
     } catch (IOException e) {
       System.out.println("Failed to close the connection: " + e.getMessage());
     }
   }
 }
Ejemplo n.º 11
0
  public static boolean createTableOrOverwrite(String tableName, String[] columnFamily) {
    if (tableName == null && columnFamily == null) {
      return false;
    }
    Connection connection = null;
    try {
      connection = ConnectionFactory.createConnection(config);
      Admin admin = connection.getAdmin();
      if (admin.tableExists(TableName.valueOf(tableName))) {
        admin.disableTable(TableName.valueOf(tableName));
        admin.deleteTable(TableName.valueOf(tableName));
      }
      HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));

      for (String cf : columnFamily) {
        table.addFamily(new HColumnDescriptor(cf));
      }

      admin.createTable(table);
      System.out.println("create table successfully.");
      return true;
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    } finally {
      try {
        connection.close();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }
  }
Ejemplo n.º 12
0
  public static void main(String[] args) throws Exception {
    conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("demo"));

    Scan scan1 = new Scan();
    ResultScanner scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Put put = new Put("3".getBytes());

    put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
    put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());

    table.put(put);

    Get get = new Get("3".getBytes());
    Result getResult = table.get(get);
    System.out.println("Printing colunns for rowkey 3");
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));

    scanner1 = table.getScanner(scan1);
    System.out.println("Before Delete");
    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Delete del = new Delete("3".getBytes());
    table.delete(del);

    System.out.println("After Delete");

    scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();
    table.close();
    connection.close();
  }
 private static void cleanup() {
   try {
     table.close();
     connection.close();
   } catch (IOException e) {
     System.out.println("Error: while clean up database connection " + e);
     System.exit(-1);
   }
 }
    @Override
    protected void cleanup(
        Reducer<Key_IMOAndRecordTime, TextArrayWritable, NullWritable, NullWritable>.Context
            context)
        throws IOException, InterruptedException {
      // TODO Auto-generated method stub

      connection.close();
    }
 public void contextDestroyed(ServletContextEvent event) {
   // App Engine does not currently invoke this method.
   try {
     connection.close();
   } catch (IOException io) {
     sc.log("contextDestroyed ", io);
   }
   connection = null;
 }
  /**
   * Confirm ImportTsv via data in online table.
   *
   * @param dataAvailable
   */
  private static void validateTable(
      Configuration conf,
      TableName tableName,
      String family,
      int valueMultiplier,
      boolean dataAvailable)
      throws IOException {

    LOG.debug("Validating table.");
    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(tableName);
    boolean verified = false;
    long pause = conf.getLong("hbase.client.pause", 5 * 1000);
    int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
    for (int i = 0; i < numRetries; i++) {
      try {
        Scan scan = new Scan();
        // Scan entire family.
        scan.addFamily(Bytes.toBytes(family));
        if (dataAvailable) {
          ResultScanner resScanner = table.getScanner(scan);
          for (Result res : resScanner) {
            LOG.debug("Getting results " + res.size());
            assertTrue(res.size() == 2);
            List<Cell> kvs = res.listCells();
            assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
            assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
            assertTrue(
                CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
            assertTrue(
                CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
            // Only one result set is expected, so let it loop.
            verified = true;
          }
        } else {
          ResultScanner resScanner = table.getScanner(scan);
          Result[] next = resScanner.next(2);
          assertEquals(0, next.length);
          verified = true;
        }

        break;
      } catch (NullPointerException e) {
        // If here, a cell was empty. Presume its because updates came in
        // after the scanner had been opened. Wait a while and retry.
      }
      try {
        Thread.sleep(pause);
      } catch (InterruptedException e) {
        // continue
      }
    }
    table.close();
    connection.close();
    assertTrue(verified);
  }
  public static void main(String[] args) throws IOException {

    Configuration conf = HBaseClientHelper.loadDefaultConfiguration();

    Connection connection = ConnectionFactory.createConnection(conf);

    try {

      Table table = connection.getTable(TableName.valueOf("testtable"));
      try {
        // 1 Put
        Put p = new Put(Bytes.toBytes("row1"));
        p.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));

        table.put(p);

        // 2 Get
        Get g = new Get(Bytes.toBytes("row1"));
        Result r = table.get(g);
        byte[] value = r.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
        String valueStr = Bytes.toString(value);
        System.out.println("GET: " + valueStr);

        // 3 Scan
        Scan s = new Scan();
        s.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
        ResultScanner scanner = table.getScanner(s);
        try {
          for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
            System.out.println("Found row: " + rr);
          }

          // The other approach is to use a foreach loop. Scanners are
          // iterable!
          // for (Result rr : scanner) {
          // System.out.println("Found row: " + rr);
          // }
        } finally {
          scanner.close();
        }

        // Close your table and cluster connection.
      } finally {
        if (table != null) table.close();
      }
    } finally {
      connection.close();
    }
  }
  public static void main(String[] args) throws IOException {
    Configuration conf = HBaseConfiguration.create();

    HBaseHelper helper = HBaseHelper.getHelper(conf);
    helper.dropTable("testtable");
    helper.createTable("testtable", "colfam1");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("testtable"));

    List<Put> puts = new ArrayList<Put>();

    // vv PutListErrorExample2
    Put put1 = new Put(Bytes.toBytes("row1"));
    put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
    puts.add(put1);
    Put put2 = new Put(Bytes.toBytes("row2"));
    put2.addColumn(Bytes.toBytes("BOGUS"), Bytes.toBytes("qual1"), Bytes.toBytes("val2"));
    puts.add(put2);
    Put put3 = new Put(Bytes.toBytes("row2"));
    put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val3"));
    puts.add(put3);
    /*[*/ Put put4 = new Put(Bytes.toBytes("row2"));
    puts.add(
        put4); /*]*/ // co PutListErrorExample2-1-AddErrorPut Add put with no content at all to
                     // list.

    /*[*/ try {
        /*]*/
      table.put(puts);
      /*[*/ } catch (Exception e) {
      System.err.println("Error: " + e);
      // table.flushCommits();
      // todo: FIX!
      /*]*/
      // co PutListErrorExample2-2-Catch Catch local exception and commit queued updates.
      /*[*/ } /*]*/
    // ^^ PutListErrorExample2
    table.close();
    connection.close();
    helper.close();
  }
Ejemplo n.º 19
0
 /**
  * 添加多条记录到表中
  *
  * @param tableName
  * @param puts
  * @throws ParamIsNullException 参数为空
  * @throws TableNotFoundException 表不存在/获取表连接失败
  */
 public static void insertRecords(String tableName, List<Put> puts)
     throws ParamIsNullException, TableNotFoundException {
   if (null == tableName) {
     throw new ParamIsNullException("tableName不能为空");
   }
   if (null == puts) {
     throw new ParamIsNullException("Put不能为空");
   }
   logger.info(
       "insertRecords begin......, tableName:{}, List<Put> size:{}", tableName, puts.size());
   Connection conn = null;
   HTable hTable = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     hTable = (HTable) conn.getTable(TableName.valueOf(tableName));
     if (null != hTable) {
       hTable.put(puts);
       logger.info("hTable.put(puts) ......");
     } else {
       throw new TableNotFoundException(tableName);
     }
   } catch (Exception e) {
     logger.error("获取Hbase连接发生异常, errMsg:{}", e);
   } finally {
     if (null != hTable) {
       try {
         hTable.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e);
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (IOException e) {
         logger.error("Connection close exception, errMsg:{}", e);
       }
     }
   }
   logger.info("insertRecords end ......");
 }
Ejemplo n.º 20
0
 /**
  * 创建一张表
  *
  * @param table
  * @throws ParamIsNullException 参数为空
  * @throws TableExistsException 表已存在
  */
 public static void createTable(HTableDescriptor table)
     throws ParamIsNullException, TableExistsException {
   if (null == table) {
     throw new ParamIsNullException("参数不能为空");
   }
   logger.info("create table begin... , table:{}", table.toString());
   Connection conn = null;
   HBaseAdmin admin = null;
   String tableName = table.getNameAsString();
   try {
     logger.info("获取connection");
     conn = ConnectionFactory.createConnection(conf);
     logger.info("获取admin");
     admin = (HBaseAdmin) conn.getAdmin();
     /** 表已存在 */
     if (admin.tableExists(Bytes.toBytes(tableName))) {
       throw new TableExistsException(tableName);
     }
     logger.info("create...");
     admin.createTable(table);
     logger.info("table create success, tableName:{}", tableName);
   } catch (IOException e) {
     logger.error("table create fail, tableName:{}, errMsg:{}", tableName, e);
   } finally {
     if (null != admin) {
       try {
         admin.close();
       } catch (IOException e) {
         logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (IOException e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
Ejemplo n.º 21
0
  public static int getTable(Configuration config) throws IOException {
    /** Connection to the cluster. A single connection shared by all application threads. */
    Connection connection = null;
    /** A lightweight handle to a specific table. Used from a single thread. */
    Table table = null;
    try {
      // establish the connection to the cluster.
      connection = ConnectionFactory.createConnection(config);
      // retrieve a handle to the target table.
      table = connection.getTable(TABLE_NAME1);
      // describe the data we want to write.
      Get g = new Get(Bytes.toBytes("row4"));

      // send the data.
      table.get(g);
    } finally {
      // close everything down
      if (table != null) table.close();
      if (connection != null) connection.close();
    }
    return 0;
  }
 public static void main(String[] args) throws Exception {
   Configuration conf = HBaseConfiguration.create();
   IntegrationTestingUtility.setUseDistributedCluster(conf);
   IntegrationTestDDLMasterFailover masterFailover = new IntegrationTestDDLMasterFailover();
   Connection connection = null;
   int ret = 1;
   try {
     // Initialize connection once, then pass to Actions
     LOG.debug("Setting up connection ...");
     connection = ConnectionFactory.createConnection(conf);
     masterFailover.setConnection(connection);
     ret = ToolRunner.run(conf, masterFailover, args);
   } catch (IOException e) {
     LOG.fatal("Failed to establish connection. Aborting test ...", e);
   } finally {
     connection = masterFailover.getConnection();
     if (connection != null) {
       connection.close();
     }
     System.exit(ret);
   }
 }
Ejemplo n.º 23
0
 /**
  * Cleanup any state for this DB. Called once per DB instance; there is one DB instance per client
  * thread.
  */
 @Override
 public void cleanup() throws DBException {
   // Get the measurements instance as this is the only client that should
   // count clean up time like an update if client-side buffering is
   // enabled.
   Measurements measurements = Measurements.getMeasurements();
   try {
     long st = System.nanoTime();
     if (bufferedMutator != null) {
       bufferedMutator.close();
     }
     if (currentTable != null) {
       currentTable.close();
     }
     long en = System.nanoTime();
     final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
     measurements.measure(type, (int) ((en - st) / 1000));
     connection.close();
   } catch (IOException e) {
     throw new DBException(e);
   }
 }
Ejemplo n.º 24
0
 /**
  * 添加一个列族
  *
  * @param tableName
  * @param family 列族
  * @throws ParamIsNullException 参数为空
  * @throws TableNotFoundException 表不存在/获取表连接失败
  */
 public static void addColumnFamily(String tableName, HColumnDescriptor family)
     throws ParamIsNullException, TableNotFoundException {
   if (null == tableName) {
     throw new ParamIsNullException("tableName不能为空");
   }
   if (null == family) {
     throw new ParamIsNullException("HColumnDescriptor不能为空");
   }
   Connection conn = null;
   HBaseAdmin admin = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     admin = (HBaseAdmin) conn.getAdmin();
     /** 表不存在 */
     if (!admin.tableExists(Bytes.toBytes(tableName))) {
       throw new TableNotFoundException(tableName);
     }
     HTableDescriptor table = admin.getTableDescriptor(Bytes.toBytes(tableName));
     table.addFamily(family);
   } catch (IOException e) {
     logger.error("获取Hbase连接发生异常, errMsg:{}", e.getMessage());
   } finally {
     if (null != admin) {
       try {
         admin.close();
       } catch (IOException e) {
         logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (IOException e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
  @Override
  public void cleanUpCluster() throws Exception {
    if (!keepObjectsAtTheEnd) {
      Admin admin = util.getAdmin();
      admin.disableTables("ittable-\\d+");
      admin.deleteTables("ittable-\\d+");
      NamespaceDescriptor[] nsds = admin.listNamespaceDescriptors();
      for (NamespaceDescriptor nsd : nsds) {
        if (nsd.getName().matches("itnamespace\\d+")) {
          LOG.info("Removing namespace=" + nsd.getName());
          admin.deleteNamespace(nsd.getName());
        }
      }
    }

    enabledTables.clear();
    disabledTables.clear();
    deletedTables.clear();
    namespaceMap.clear();

    Connection connection = getConnection();
    connection.close();
    super.cleanUpCluster();
  }
Ejemplo n.º 26
0
 /**
  * 添加一条记录到表中
  *
  * @param tableName
  * @param put
  * @throws ParamIsNullException 参数为空
  * @throws IOException
  */
 public static void insertOneRecord(String tableName, Put put)
     throws ParamIsNullException, IOException {
   if (null == tableName) {
     throw new ParamIsNullException("tableName不能为空");
   }
   if (null == put) {
     throw new ParamIsNullException("Put不能为空");
   }
   logger.info("insertOneRecord ...");
   Connection conn = null;
   HTable hTable = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     hTable = (HTable) conn.getTable(TableName.valueOf(tableName));
     if (null != hTable) {
       hTable.put(put);
     } else {
       throw new TableNotFoundException(tableName);
     }
   } finally {
     if (null != hTable) {
       try {
         hTable.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (IOException e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
Ejemplo n.º 27
0
 @SuppressWarnings("deprecation")
 public static int putTable(Configuration config) throws IOException {
   /** Connection to the cluster. A single connection shared by all application threads. */
   Connection connection = null;
   /** A lightweight handle to a specific table. Used from a single thread. */
   Table table = null;
   try {
     // establish the connection to the cluster.
     connection = ConnectionFactory.createConnection(config);
     // retrieve a handle to the target table.
     table = connection.getTable(TABLE_NAME1);
     // describe the data we want to write.
     Put p = new Put(Bytes.toBytes("row4"));
     p.add(CF, Bytes.toBytes("b"), Bytes.toBytes("datos agregados desde java"));
     p.add(CF, Bytes.toBytes("qual"), Bytes.toBytes("segunda columna"));
     // send the data.
     table.put(p);
   } finally {
     // close everything down
     if (table != null) table.close();
     if (connection != null) connection.close();
   }
   return 0;
 }
Ejemplo n.º 28
0
  public void convertH()
      throws SQLException, IOException, ClassNotFoundException, IllegalAccessException,
          InstantiationException {
    /** create HBase connection */
    conf_h = HBaseConfiguration.create();
    conf_h.set("hbase.zookeeper.quorum", "localhost");
    conf_h.set("hbase.zookeeper.property.clientPort", "2181");
    try {
      conn_h = ConnectionFactory.createConnection(conf_h);
    } catch (IOException e) {
      e.printStackTrace();
    }
    /** create Mysql connection */
    try {
      Class.forName("com.mysql.jdbc.Driver").newInstance();
      // con_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera");
      conn_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera");
      if (!conn_sql.isClosed()) {
        System.out.println("Successfully connected to MySQL server...");
      }
    } catch (Exception e) {
      System.err.println(e.getMessage());
    }
    /** create table and family */
    Admin admin = conn_h.getAdmin();

    HTableDescriptor tableDesc = new HTableDescriptor(tableName_retail);
    // create Family Name
    String[] familyName = {"order_items", "orders", "products"};
    for (String name : familyName) {
      HColumnDescriptor colDesc = new HColumnDescriptor(name);
      tableDesc.addFamily(colDesc);
    }
    admin.createTable(tableDesc);

    /** query from mysql */
    if (conn_sql != null) {
      Statement stat = conn_sql.createStatement();
      stat.executeQuery(
          "select * from products "
              + "inner join order_items on product_id = order_item_product_id "
              + "inner join orders on order_item_order_id = order_id ");
      ResultSet rs = stat.getResultSet();
      while (rs.next()) {
        // order_items
        String rowKey = rs.getString("order_item_id");
        Put put = new Put(Bytes.toBytes(rowKey)); // new Put(rowKey)
        put.addColumn(
            Bytes.toBytes("order_items"), Bytes.toBytes("order_item_id"), Bytes.toBytes(rowKey));
        String order_item_order_id = rs.getString("order_item_order_id");
        put.addColumn(
            Bytes.toBytes("order_items"),
            Bytes.toBytes("order_item_order_id"),
            Bytes.toBytes(order_item_order_id));
        String order_item_product_id = rs.getString("order_item_product_id");
        put.addColumn(
            Bytes.toBytes("order_items"),
            Bytes.toBytes("order_item_product_id"),
            Bytes.toBytes(order_item_product_id));
        String order_item_quantity = rs.getString("order_item_quantity");
        put.addColumn(
            Bytes.toBytes("order_items"),
            Bytes.toBytes("order_item_quantity"),
            Bytes.toBytes(order_item_quantity));
        String order_item_subtotal = rs.getString("order_item_subtotal");
        put.addColumn(
            Bytes.toBytes("order_items"),
            Bytes.toBytes("order_item_subtotal"),
            Bytes.toBytes(order_item_subtotal));
        String order_item_product_price = rs.getString("order_item_product_price");
        put.addColumn(
            Bytes.toBytes("order_items"),
            Bytes.toBytes("order_item_product_price"),
            Bytes.toBytes(order_item_product_price));
        // orders
        String order_id = rs.getString("order_id");
        put.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_id"), Bytes.toBytes(order_id));
        String order_date = rs.getString("order_date");
        put.addColumn(
            Bytes.toBytes("orders"), Bytes.toBytes("order_date"), Bytes.toBytes(order_date));
        String order_customer_id = rs.getString("order_customer_id");
        put.addColumn(
            Bytes.toBytes("orders"),
            Bytes.toBytes("order_customer_id"),
            Bytes.toBytes(order_customer_id));
        String order_status = rs.getString("order_status");
        put.addColumn(
            Bytes.toBytes("orders"), Bytes.toBytes("order_status"), Bytes.toBytes(order_status));
        // products
        String product_id = rs.getString("product_id");
        put.addColumn(
            Bytes.toBytes("products"), Bytes.toBytes("product_id"), Bytes.toBytes(product_id));
        String product_category_id = rs.getString("product_category_id");
        put.addColumn(
            Bytes.toBytes("products"),
            Bytes.toBytes("product_category_id"),
            Bytes.toBytes(product_category_id));
        String product_name = rs.getString("product_name");
        put.addColumn(
            Bytes.toBytes("products"), Bytes.toBytes("product_name"), Bytes.toBytes(product_name));
        String product_description = rs.getString("product_description");
        put.addColumn(
            Bytes.toBytes("products"),
            Bytes.toBytes("product_description"),
            Bytes.toBytes(product_description));
        String product_price = rs.getString("product_price");
        put.addColumn(
            Bytes.toBytes("products"),
            Bytes.toBytes("product_price"),
            Bytes.toBytes(product_price));
        String product_image = rs.getString("product_image");
        put.addColumn(
            Bytes.toBytes("products"),
            Bytes.toBytes("product_image"),
            Bytes.toBytes(product_image));
        // put in table
        Table table = conn_h.getTable(tableName_retail);
        table.put(put);
        table.close();
      }
      rs.close();
      stat.close();
    }
    conn_h.close();
    conn_sql.close();
  }
 @After
 public void close() throws IOException {
   deleteTable();
   admin.close();
   conn.close();
 }
Ejemplo n.º 30
0
  public static boolean scanTable(Configuration config) throws IOException, URISyntaxException {

    /** Connection to the cluster. A single connection shared by all application threads. */
    Connection connection = null;
    /** A lightweight handle to a specific table. Used from a single thread. */
    Table table = null;

    Scan scan = null;

    ResultScanner scanner = null;

    //		String[] columns = null;

    try {
      // establish the connection to the cluster.
      connection = ConnectionFactory.createConnection(config);
      // retrieve a handle to the target table.
      table = connection.getTable(TABLE_NAME1);
      // Instantiating the Scan class
      scan = new Scan();
      // Recommended tuning Ch12 HBase Performance Tuning
      scan.setCaching(5000); // Defined rows returned from server during next()
      scan.setCacheBlocks(false); // Disable server side cache
      // Scanning the required columns
      scan.addFamily(CF);
      // Getting the scan result
      scanner = table.getScanner(scan);

      System.out.println("Rows founded");
      StringBuffer line = new StringBuffer();

      int counter = 0;
      // Reading values from scan result
      for (Result result = scanner.next(); result != null; result = scanner.next()) {
        columns = getColumnsInColumnFamily(result, CF);

        // clean line
        //		    	line.delete(0, line.length());

        line.append(Bytes.toString(result.getRow()));
        for (String column : columns) {
          line.append("^" + Bytes.toString(result.getValue(CF, Bytes.toBytes(column))));
        }
        line.append("\n");
        counter++;
        System.out.println("Linea: " + counter);

        //		    	line.append(""
        //		    		+ Bytes.toString(result.getRow()) + ","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("account"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("amount"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("aut"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("balance"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("card"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("commerce"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("commerceData"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("date"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("f"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("f2"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("f3"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("f4"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("hour"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("m"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("money"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("ms"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("r"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("ref"))) +","
        //		    		+ Bytes.toString(result.getValue(CF, Bytes.toBytes("trxcd")))  +"\n"
        //		    		);
        //		    	System.out.println(Bytes.toString(result.getRow()) +","+
        // Bytes.toString(result.getValue(CF, qualifier)));
      }

      DistributedFileSystem hdfs = new DistributedFileSystem();
      hdfs.initialize(new URI("hdfs://quickstart.cloudera:8020"), config);

      Path homeDir = hdfs.getHomeDirectory();
      // Print the home directory
      System.out.println("Home folder HDFS-" + homeDir);

      Path newFilePath = new Path(homeDir + "/datatest/" + TABLE_NAME + "/file.csv");

      byte[] byt = line.toString().getBytes();
      FSDataOutputStream fsOutStream = hdfs.create(newFilePath);
      fsOutStream.write(byt);

      fsOutStream.close();
      hdfs.close();

      System.out.println("File exported to " + homeDir + "/datatest/" + TABLE_NAME + "/file.csv");

    } finally {
      // close everything down
      if (table != null) table.close();
      if (connection != null) connection.close();
      if (scanner != null) scanner.close();
    }
    return true;
  }