Esempio n. 1
0
 public boolean fetchReadAllFieldLine(List<Line> lines, LineSender sender) throws IOException {
   if (null == this.rs) {
     throw new IllegalStateException("HBase Client try to fetch data failed .");
   }
   for (Result result = rs.next(); result != null; result = rs.next()) {
     Get get = new Get(result.getRow());
     for (int i = 0; i < this.families.length; i++) {
       get.addColumn(this.families[i].getBytes(), this.columns[i].getBytes());
     }
     gets.add(get);
     if (gets.size() > this.BUFFER_LINE) {
       Result[] getResults = this.htable.get(gets);
       for (Result resu : getResults) {
         if (null != resu) {
           Line line = sender.createLine();
           for (int i = 0; i < this.families.length; i++) {
             byte[] value = resu.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
             if (null == value) {
               line.addField(null);
             } else {
               line.addField(new String(value, encode));
             }
           }
           line.addField(new String(resu.getRow(), encode));
         }
       }
       return true;
     }
   }
   return false;
 }
  private void testSimpleScanInternal(
      long origKeyPrefix,
      Scan scan,
      int numValues,
      int startWithValue,
      int seekIntervalMinValue,
      int seekIntervalMaxValue)
      throws IOException {
    int valuesCountInSeekInterval =
        writeTestData(
            origKeyPrefix, numValues, startWithValue, seekIntervalMinValue, seekIntervalMaxValue);

    // TODO: add some filters to the scan for better testing
    ResultScanner distributedScanner = DistributedScanner.create(hTable, scan, keyDistributor);

    Result previous = null;
    int countMatched = 0;
    for (Result current : distributedScanner) {
      countMatched++;
      if (previous != null) {
        byte[] currentRowOrigKey = keyDistributor.getOriginalKey(current.getRow());
        byte[] previousRowOrigKey = keyDistributor.getOriginalKey(previous.getRow());
        Assert.assertTrue(Bytes.compareTo(currentRowOrigKey, previousRowOrigKey) >= 0);

        int currentValue = Bytes.toInt(current.getValue(CF, QUAL));
        Assert.assertTrue(currentValue >= seekIntervalMinValue);
        Assert.assertTrue(currentValue <= seekIntervalMaxValue);
      }
      previous = current;
    }

    Assert.assertEquals(valuesCountInSeekInterval, countMatched);
  }
  private HashMap<String, String[][]> getObjectList(
      ResultScanner resultScan, byte[][] family, List<byte[]> rowKeyList) {
    String tableFamilies; // = new table;
    String data[][];
    String rowKey;
    HashMap<String, String[][]> tableDataList = new HashMap<String, String[][]>();

    long k = 0;
    for (Result rows : resultScan) {
      System.out.println("Rows passed : " + (++k));
      for (int i = 0; i < family.length; i++) {

        tableFamilies = Bytes.toString(family[i]);
        rowKey = Bytes.toString(rows.getRow());

        rowKeyList.add(rows.getRow());

        if (tblMngr.getDataFiller(rows, tableFamilies)) {
          data = tblMngr.getResultMap();
          tableFamilies = tableFamilies + "," + rowKey;
          tableDataList.put(tableFamilies, data);
        }
      }
    }
    return tableDataList;
  }
 /**
  * Looks at every value of the mapreduce output and verifies that indeed the values have been
  * reversed.
  *
  * @param table Table to scan.
  * @throws IOException
  * @throws NullPointerException if we failed to find a cell value
  */
 private void verifyAttempt(final Table table) throws IOException, NullPointerException {
   Scan scan = new Scan();
   scan.addFamily(INPUT_FAMILY);
   scan.addFamily(OUTPUT_FAMILY);
   ResultScanner scanner = table.getScanner(scan);
   try {
     Iterator<Result> itr = scanner.iterator();
     assertTrue(itr.hasNext());
     while (itr.hasNext()) {
       Result r = itr.next();
       if (LOG.isDebugEnabled()) {
         if (r.size() > 2) {
           throw new IOException("Too many results, expected 2 got " + r.size());
         }
       }
       byte[] firstValue = null;
       byte[] secondValue = null;
       int count = 0;
       for (Cell kv : r.listCells()) {
         if (count == 0) {
           firstValue = CellUtil.cloneValue(kv);
         } else if (count == 1) {
           secondValue = CellUtil.cloneValue(kv);
         } else if (count == 2) {
           break;
         }
         count++;
       }
       String first = "";
       if (firstValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
       }
       first = Bytes.toString(firstValue);
       String second = "";
       if (secondValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
       }
       byte[] secondReversed = new byte[secondValue.length];
       for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
         secondReversed[i] = secondValue[j];
       }
       second = Bytes.toString(secondReversed);
       if (first.compareTo(second) != 0) {
         if (LOG.isDebugEnabled()) {
           LOG.debug(
               "second key is not the reverse of first. row="
                   + Bytes.toStringBinary(r.getRow())
                   + ", first value="
                   + first
                   + ", second value="
                   + second);
         }
         fail();
       }
     }
   } finally {
     scanner.close();
   }
 }
Esempio n. 5
0
  public static void main(String[] args) throws Exception {
    conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("demo"));

    Scan scan1 = new Scan();
    ResultScanner scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Put put = new Put("3".getBytes());

    put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
    put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());

    table.put(put);

    Get get = new Get("3".getBytes());
    Result getResult = table.get(get);
    System.out.println("Printing colunns for rowkey 3");
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));

    scanner1 = table.getScanner(scan1);
    System.out.println("Before Delete");
    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Delete del = new Delete("3".getBytes());
    table.delete(del);

    System.out.println("After Delete");

    scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();
    table.close();
    connection.close();
  }
Esempio n. 6
0
 @Override
 protected SizeResult createResult(Result result) {
   if (result.getRow() != null) {
     SizeResult size = new SizeResult(result.getRow());
     size.setProbability(result.getValue(Bytes.toBytes("bp"), Bytes.toBytes("prob")));
     return size;
   }
   return null;
 }
    // Get all events with exit at last location
    public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
        Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
      Scan getAllEventsWithExistAtLastLocation = new Scan();
      getAllEventsWithExistAtLastLocation
          .setStartRow(
              Bytes.toBytes(
                  IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
          .setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
          .addColumn(details, exittime);
      getAllEventsWithExistAtLastLocation.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.GREATER_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
      getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);

      ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);

      Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();

      for (Result res : Result_event) {

        Get get = new Get(res.getRow());
        get.addColumn(details, entrytime);
        get.addColumn(details, entrycoordinates);

        Result result = VTEvent_Table.get(get);
        String rowkey = Bytes.toString(result.getRow());
        String polygonid = rowkey.substring(26);

        VesselEvent VE = new VesselEvent();
        VE.exittime = location.recordtime;
        VE.exitcoordinates = location.coordinates;
        VE.destination = location.destination;
        VE.polygonid = Integer.parseInt(polygonid);

        for (Cell cell : result.rawCells()) {
          String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
          String Value = Bytes.toString(CellUtil.cloneValue(cell));

          if (Qualifier.equals("entertime")) {
            VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
          } else if (Qualifier.equals("entercoordinates")) {
            VE.entrycoordinates = Value;
          }
        }

        events.put(VE.polygonid, VE);
      }

      Result_event.close();
      return events;
    }
    public void doAnAction() throws Exception {
      Scan s = new Scan();
      for (byte[] family : targetFamilies) {
        s.addFamily(family);
      }
      ResultScanner scanner = table.getScanner(s);

      for (Result res : scanner) {
        byte[] lastRow = null, lastFam = null, lastQual = null;
        byte[] gotValue = null;
        for (byte[] family : targetFamilies) {
          byte qualifier[] = QUAL;
          byte thisValue[] = res.getValue(family, qualifier);
          if (gotValue != null && thisValue != null && !Bytes.equals(gotValue, thisValue)) {

            StringBuilder msg = new StringBuilder();
            msg.append("Failed on scan ")
                .append(numScans)
                .append(" after scanning ")
                .append(numRowsScanned)
                .append(" rows!\n");
            msg.append(
                "Current  was "
                    + Bytes.toString(res.getRow())
                    + "/"
                    + Bytes.toString(family)
                    + ":"
                    + Bytes.toString(qualifier)
                    + " = "
                    + Bytes.toString(thisValue)
                    + "\n");
            msg.append(
                "Previous  was "
                    + Bytes.toString(lastRow)
                    + "/"
                    + Bytes.toString(lastFam)
                    + ":"
                    + Bytes.toString(lastQual)
                    + " = "
                    + Bytes.toString(gotValue));
            throw new RuntimeException(msg.toString());
          }

          lastFam = family;
          lastQual = qualifier;
          lastRow = res.getRow();
          gotValue = thisValue;
        }
        numRowsScanned.getAndIncrement();
      }
      numScans.getAndIncrement();
    }
Esempio n. 9
0
  public Blog(String blogid) throws IOException {

    Configuration conf = HBaseConfiguration.create();
    table = new HTable(conf, "blogs");

    // 1. Get the row whose row key is blogid from above
    Get g = new Get(Bytes.toBytes(blogid));
    Result r = table.get(g);

    // 2. Extract the rowkey, blog text (column "body") and blog title
    // (column "meta:title")
    key = r.getRow();
    keyStr = Bytes.toString(key);
    blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
    blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
    Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
    Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
    dateOfPost = new Date(epoch);

    // Get an iterator for the comments
    Scan s = new Scan();
    s.addFamily(Bytes.toBytes("comment"));
    // Use a PrefixFilter
    PrefixFilter filter = new PrefixFilter(key);
    s.setFilter(filter);
    scanner = table.getScanner(s);
    resultIterator = scanner.iterator();
  }
    public static List<Delete> GetDeleteEventsBetween(
        Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
        throws IOException {
      // scan
      // 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
      Scan GetEventsBetween = new Scan();
      GetEventsBetween.setStartRow(
              Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
          .setStopRow(
              Bytes.toBytes(
                  imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
          .addColumn(details, exittime);
      GetEventsBetween.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.LESS_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
      GetEventsBetween.setFilter(ExistTimeValuefilter);

      ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
      List<Delete> deletes = new ArrayList<Delete>();

      for (Result res : Result_ExistingEvents) {
        deletes.add(new Delete(res.getRow()));
      }

      Result_ExistingEvents.close();
      return deletes;
    }
Esempio n. 11
0
  /**
   * 遍历多行
   *
   * @param tableName 表名
   * @param start_rowkey 开始行键
   * @param stop_rowkey 结束行键
   * @return 行列表
   */
  public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
    ResultScanner rowstmp = null;
    ArrayList<HbaseRow> rows = null;

    try {
      Scan scan = new Scan();
      scan.setStartRow(Bytes.toBytes(start_rowkey));
      scan.setStopRow(Bytes.toBytes(stop_rowkey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      rowstmp = table.getScanner(scan);
      rows = new ArrayList<>();

      for (Result rowtmp : rowstmp) {
        HbaseRow row = new HbaseRow();
        row.rowkey = Bytes.toString(rowtmp.getRow());
        for (Cell cell : rowtmp.listCells()) {
          HbaseColumn col = new HbaseColumn(cell);
          row.cols.add(col);
        }
        rows.add(row);
      }
    } catch (Exception e) {
      logger.error("scanRows failed", e);
    } finally {
      rowstmp.close();
    }
    return rows;
  }
Esempio n. 12
0
  @TimeDepend
  @Test
  public void testScan_ts_same() throws Exception {

    recreateTable();

    Date ts = parse("2000-01-01", "yyyy-MM-dd");

    Put put = new Put(rowKey_ForTest);

    put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));

    table.put(put);

    Set<String> resultRowKeys = new HashSet<String>();
    Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
    scan.setTimeRange(ts.getTime(), ts.getTime());

    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
      resultRowKeys.add(Bytes.toString(result.getRow()));
    }

    close(resultScanner);

    Assert.assertTrue(resultRowKeys.size() == 0);

    recreateTable();
  }
  public static List<String> getBooksbyPrice(String min, String max) throws IOException {

    List list = new ArrayList<String>();
    Scan scan = new Scan();
    scan.setMaxVersions();

    FilterList filterList = new FilterList();
    Filter maxFilter =
        new SingleColumnValueFilter(
            Bytes.toBytes("statics"),
            Bytes.toBytes("price"),
            CompareOp.GREATER_OR_EQUAL,
            Bytes.toBytes(min));
    Filter minFilter =
        new SingleColumnValueFilter(
            Bytes.toBytes("statics"),
            Bytes.toBytes("price"),
            CompareOp.LESS_OR_EQUAL,
            Bytes.toBytes(max));
    filterList.addFilter(maxFilter);
    filterList.addFilter(minFilter);
    scan.setFilter(filterList);
    ResultScanner rs = table.getScanner(scan);
    for (Result r : rs) {
      String str = "title: " + Bytes.toString(r.getRow());
      for (KeyValue kv : r.raw()) {

        str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
        str += Bytes.toString(kv.getValue());
      }
      System.out.println(str);
      list.add(str);
    }
    return list;
  }
  public static List<String> getFreeBooks() throws IOException {

    List list = new ArrayList<String>();
    Scan scan = new Scan();
    scan.setMaxVersions();
    Filter filter =
        new SingleColumnValueFilter(
            Bytes.toBytes("statics"),
            Bytes.toBytes("price"),
            CompareOp.EQUAL,
            Bytes.toBytes("0.00"));
    scan.setFilter(filter);
    ResultScanner rs = table.getScanner(scan);
    for (Result r : rs) {
      String str = "title: " + Bytes.toString(r.getRow());
      for (KeyValue kv : r.raw()) {

        str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
        str += Bytes.toString(kv.getValue());
      }
      /*
       * for (KeyValue kv : r.raw()) { System.out.println(String.format(
       * "row:%s, family:%s, qualifier:%s, qualifiervalue:%s, timestamp:%s."
       * , Bytes.toString(kv.getRow()), Bytes.toString(kv.getFamily()),
       * Bytes.toString(kv.getQualifier()), Bytes.toString(kv.getValue()),
       * kv.getTimestamp())); }
       */
      System.out.println(str);
      list.add(str);
    }
    return list;
  }
  @Override
  public void emitTuples() {
    try {
      Scan scan = nextScan();
      if (scan == null) return;

      ResultScanner resultScanner = store.getTable().getScanner(scan);

      while (true) {
        Result result = resultScanner.next();
        if (result == null) break;

        String readRow = Bytes.toString(result.getRow());
        if (readRow.equals(lastReadRow)) continue;

        Object instance = pojoType.newInstance();
        rowSetter.set(instance, readRow);

        List<Cell> cells = result.listCells();

        for (Cell cell : cells) {
          String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
          byte[] value = CellUtil.cloneValue(cell);
          fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
        }

        outputPort.emit(instance);
        lastReadRow = readRow;
      }

    } catch (Exception e) {
      throw new RuntimeException(e.getMessage());
    }
  }
Esempio n. 16
0
  @Override
  protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
    assert resPath.startsWith("/");
    String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
    byte[] startRow = Bytes.toBytes(lookForPrefix);
    byte[] endRow = Bytes.toBytes(lookForPrefix);
    endRow[endRow.length - 1]++;

    ArrayList<String> result = new ArrayList<String>();

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    Scan scan = new Scan(startRow, endRow);
    scan.setFilter(new KeyOnlyFilter());
    try {
      ResultScanner scanner = table.getScanner(scan);
      for (Result r : scanner) {
        String path = Bytes.toString(r.getRow());
        assert path.startsWith(lookForPrefix);
        int cut = path.indexOf('/', lookForPrefix.length());
        String child = cut < 0 ? path : path.substring(0, cut);
        if (result.contains(child) == false) result.add(child);
      }
    } finally {
      IOUtils.closeQuietly(table);
    }
    // return null to indicate not a folder
    return result.isEmpty() ? null : result;
  }
Esempio n. 17
0
  @Override
  protected List<RawResource> getAllResources(String rangeStart, String rangeEnd)
      throws IOException {
    byte[] startRow = Bytes.toBytes(rangeStart);
    byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));

    Scan scan = new Scan(startRow, endRow);
    scan.addColumn(B_FAMILY, B_COLUMN_TS);
    scan.addColumn(B_FAMILY, B_COLUMN);

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    List<RawResource> result = Lists.newArrayList();
    try {
      ResultScanner scanner = table.getScanner(scan);
      for (Result r : scanner) {
        result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
      }
    } catch (IOException e) {
      for (RawResource rawResource : result) {
        IOUtils.closeQuietly(rawResource.resource);
      }
      throw e;
    } finally {
      IOUtils.closeQuietly(table);
    }
    return result;
  }
Esempio n. 18
0
 /**
  * This utility method creates a list of Thrift TRowResult "struct" based on an Hbase RowResult
  * object. The empty list is returned if the input is null.
  *
  * @param in Hbase RowResult object
  * @param sortColumns This boolean dictates if row data is returned in a sorted order sortColumns
  *     = True will set TRowResult's sortedColumns member which is an ArrayList of TColumn struct
  *     sortColumns = False will set TRowResult's columns member which is a map of columnName and
  *     TCell struct
  * @return Thrift TRowResult array
  */
 public static List<TRowResult> rowResultFromHBase(Result[] in, boolean sortColumns) {
   List<TRowResult> results = new ArrayList<TRowResult>();
   for (Result result_ : in) {
     if (result_ == null || result_.isEmpty()) {
       continue;
     }
     TRowResult result = new TRowResult();
     result.row = ByteBuffer.wrap(result_.getRow());
     if (sortColumns) {
       result.sortedColumns = new ArrayList<TColumn>();
       for (Cell kv : result_.rawCells()) {
         result.sortedColumns.add(
             new TColumn(
                 ByteBuffer.wrap(
                     KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
                 new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp())));
       }
     } else {
       result.columns = new TreeMap<ByteBuffer, TCell>();
       for (Cell kv : result_.rawCells()) {
         result.columns.put(
             ByteBuffer.wrap(
                 KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
             new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp()));
       }
     }
     results.add(result);
   }
   return results;
 }
Esempio n. 19
0
 /* (non-Javadoc)
  * @see com.hazelcast.core.MapLoader#loadAllKeys()
  */
 @Override
 public Set<String> loadAllKeys() {
   Set<String> keySet = null;
   if (allowLoadAll) {
     keySet = new HashSet<String>();
     HTableInterface hti = null;
     try {
       hti = pool.getTable(tableName);
       Scan s = new Scan();
       s.addColumn(family, qualifier);
       ResultScanner rs = hti.getScanner(s);
       Result r = null;
       while ((r = rs.next()) != null) {
         String k = new String(r.getRow());
         keySet.add(k);
       }
     } catch (IOException e) {
       LOG.error("IOException while loading all keys", e);
     } finally {
       if (hti != null) {
         pool.putTable(hti);
       }
     }
   }
   return keySet;
 }
Esempio n. 20
0
 /**
  * rowFilter的使用
  *
  * @param tableName
  * @param reg
  * @throws Exception
  */
 public void getRowFilter(String tableName, String reg) throws Exception {
   Connection conn = null;
   HTable table = null;
   try {
     conn = ConnectionFactory.createConnection(conf);
     table = (HTable) conn.getTable(TableName.valueOf(tableName));
     Scan scan = new Scan();
     //			Filter
     RowFilter rowFilter = new RowFilter(CompareOp.NOT_EQUAL, new RegexStringComparator(reg));
     scan.setFilter(rowFilter);
     ResultScanner scanner = table.getScanner(scan);
     for (Result result : scanner) {
       System.out.println(new String(result.getRow()));
     }
   } catch (Exception e) {
     e.printStackTrace();
   } finally {
     if (null != table) {
       try {
         table.close();
       } catch (IOException e) {
         logger.error("HTable close exception, errMsg:{}", e.getMessage());
       }
     }
     if (null != conn) {
       try {
         conn.close();
       } catch (Exception e) {
         logger.error("Connection close exception, errMsg:{}", e.getMessage());
       }
     }
   }
 }
Esempio n. 21
0
  public static void deleteTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);

      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] startRow = Bytes.toBytes("rowKey_1");
      byte[] stopRow = Bytes.toBytes("rowKey_3");
      byte[] family = f0;

      Scan scan = new Scan();
      scan.addFamily(family);
      scan.setMaxVersions(1);

      //            scan.setStartRow(startRow);
      //            scan.setStopRow(stopRow);

      ResultScanner scanner = table.getScanner(scan);
      Result result = scanner.next();
      List<Delete> delete = new ArrayList<Delete>();
      while (result != null) {
        Delete del = new Delete(result.getRow());
        delete.add(del);
        result = scanner.next();
      }
      table.delete(delete);
      System.out.println("delete done");
      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Esempio n. 22
0
  @Override
  public GCResult createResult(Result result) {
    if (result.getRow() != null) {
      GCResult gcResult = new GCResult(result.getRow());
      gcResult.setChromosome(result.getValue(Bytes.toBytes("chr"), Bytes.toBytes("name")));

      byte[] family = Bytes.toBytes("gc");
      gcResult.setMin(result.getValue(family, Bytes.toBytes("min")));
      gcResult.setMax(result.getValue(family, Bytes.toBytes("max")));

      gcResult.setTotalFragments(result.getValue(Bytes.toBytes("frag"), Bytes.toBytes("total")));

      return gcResult;
    }
    return null;
  }
 public void map(ImmutableBytesWritable row, Result value, Context context)
     throws InterruptedException, IOException {
   /* BERLIN SPARQL BENHCMARK QUERY 11
     ----------------------------------------
   SELECT ?property ?hasValue ?isValueOf
   WHERE {
   [TP-01]	{ %OfferXYZ% ?property ?hasValue }
   		UNION
   [TP-02]	{ ?isValueOf ?property %OfferXYZ% }
   }
     ---------------------------------------
   */
   // TP-01
   if (isPartOfFirstUnion(value)) {
     List<KeyValue> entireRowAsList = value.list();
     KeyValue[] kvsAsArray = new KeyValue[entireRowAsList.size()];
     for (int i = 0; i < entireRowAsList.size(); i++) {
       kvsAsArray[i] = entireRowAsList.get(i);
     }
     context.write(
         new CompositeKeyWritable(new String(value.getRow()), 1),
         new KeyValueArrayWritable(kvsAsArray));
     return;
   }
   // TP-02
   else {
     List<KeyValue> entireRowAsList = value.list();
     List<KeyValue> kvsToTransmit = new LinkedList<KeyValue>();
     // Check all cells and see if the OFFER is part of the value
     for (KeyValue kv : entireRowAsList) {
       if (new String(kv.getValue()).equals(OfferXYZ)) {
         kvsToTransmit.add(kv);
       }
     }
     KeyValue[] kvsAsArray = new KeyValue[kvsToTransmit.size()];
     for (int i = 0; i < kvsToTransmit.size(); i++) {
       kvsAsArray[i] = kvsToTransmit.get(i);
     }
     if (kvsAsArray.length > 0) {
       context.write(
           new CompositeKeyWritable(new String(value.getRow()), 2),
           new KeyValueArrayWritable(kvsAsArray));
     } else {
       return;
     }
   }
 }
Esempio n. 24
0
  /**
   * Get the field out of the row without checking whether parsing is needed. This is called by both
   * getField and getFieldsAsList.
   *
   * @param fieldID The id of the field starting from 0.
   * @return The value of the field
   */
  private Object uncheckedGetField(int fieldID) {

    LazyObjectBase[] fields = getFields();
    boolean[] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      fieldsInited[fieldID] = true;

      ColumnMapping colMap = columnsMapping[fieldID];

      if (!colMap.hbaseRowKey && !colMap.hbaseTimestamp && colMap.qualifierName == null) {
        // it is a column family
        // primitive type for Map<Key, Value> can be stored in binary format. Pass in the
        // qualifier prefix to cherry pick the qualifiers that match the prefix instead of picking
        // up everything
        ((LazyHBaseCellMap) fields[fieldID])
            .init(
                result,
                colMap.familyNameBytes,
                colMap.binaryStorage,
                colMap.qualifierPrefixBytes,
                colMap.isDoPrefixCut());
        return fields[fieldID].getObject();
      }

      if (colMap.hbaseTimestamp) {
        // Get the latest timestamp of all the cells as the row timestamp
        long timestamp = result.rawCells()[0].getTimestamp(); // from hbase-0.96.0
        for (int i = 1; i < result.rawCells().length; i++) {
          timestamp = Math.max(timestamp, result.rawCells()[i].getTimestamp());
        }
        LazyObjectBase lz = fields[fieldID];
        if (lz instanceof LazyTimestamp) {
          ((LazyTimestamp) lz).getWritableObject().setTime(timestamp);
        } else {
          ((LazyLong) lz).getWritableObject().set(timestamp);
        }
        return lz.getObject();
      }

      byte[] bytes;
      if (colMap.hbaseRowKey) {
        bytes = result.getRow();
      } else {
        // it is a column i.e. a column-family with column-qualifier
        bytes = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);
      }
      if (bytes == null || isNull(oi.getNullSequence(), bytes, 0, bytes.length)) {
        fields[fieldID].setNull();
      } else {
        ByteArrayRef ref = new ByteArrayRef();
        ref.setData(bytes);
        fields[fieldID].init(ref, 0, bytes.length);
      }
    }

    return fields[fieldID].getObject();
  }
Esempio n. 25
0
    private void loadIPs() {
      dns = new HashMap(100000000); // ���貢��
      unknownHosts = new HashMap(1000000);
      querying = new HashMap(100000);

      try {
        int statsCommit = 500000;

        HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create());
        HTableInterface fetchFailTable = connection.getTable("fetchFail");
        Scan scan = new Scan();
        scan.setCaching(statsCommit);

        List<Filter> filters = new ArrayList<Filter>();
        Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip"));
        filters.add(filter);
        FilterList filterList = new FilterList(filters);
        scan.setFilter(filterList);

        ResultScanner rs = fetchFailTable.getScanner(scan);
        long cnt = 0;
        for (Result r : rs) {
          NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf"));
          String ip = Bytes.toString(map.get(Bytes.toBytes("ip")));
          String host = Bytes.toString(r.getRow()).split("��")[0];
          if (host != null && ip != null) {
            dns.put(host, ip);
          }

          if (++cnt % statsCommit == 0) {
            LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt);
          }
        }
        rs.close();
        fetchFailTable.close();
        LOG.info("load hostip cache=" + dns.size());

        connection.close();
      } catch (Exception e) {
        e.printStackTrace();
      } finally {
        //
      }
    }
Esempio n. 26
0
 private void printLocations(Result r) {
   RegionLocations rl = null;
   if (r == null) {
     LOG.info("FAILED FOR null Result");
     return;
   }
   LOG.info("FAILED FOR " + resultToString(r) + " Stale " + r.isStale());
   if (r.getRow() == null) {
     return;
   }
   try {
     rl = ((ClusterConnection) connection).locateRegion(tableName, r.getRow(), true, true);
   } catch (IOException e) {
     LOG.warn("Couldn't get locations for row " + Bytes.toString(r.getRow()));
   }
   HRegionLocation locations[] = rl.getRegionLocations();
   for (HRegionLocation h : locations) {
     LOG.info("LOCATION " + h);
   }
 }
 @Override
 public Map<String, Set<String>> claimQueues(String regionserver) {
   Map<String, Set<String>> queues = new HashMap<>();
   if (isThisOurRegionServer(regionserver)) {
     return queues;
   }
   ResultScanner queuesToClaim = null;
   try {
     queuesToClaim = getAllQueuesScanner(regionserver);
     for (Result queue : queuesToClaim) {
       if (attemptToClaimQueue(queue, regionserver)) {
         String rowKey = Bytes.toString(queue.getRow());
         ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(rowKey);
         if (replicationState.peerExists(replicationQueueInfo.getPeerId())) {
           Set<String> sortedLogs = new HashSet<String>();
           List<String> logs = getLogsInQueue(queue.getRow());
           for (String log : logs) {
             sortedLogs.add(log);
           }
           queues.put(rowKey, sortedLogs);
           LOG.info(serverName + " has claimed queue " + rowKey + " from " + regionserver);
         } else {
           // Delete orphaned queues
           removeQueue(Bytes.toString(queue.getRow()));
           LOG.info(
               serverName + " has deleted abandoned queue " + rowKey + " from " + regionserver);
         }
       }
     }
   } catch (IOException | KeeperException e) {
     String errMsg = "Failed claiming queues for regionserver=" + regionserver;
     abortable.abort(errMsg, e);
     queues.clear();
   } finally {
     if (queuesToClaim != null) {
       queuesToClaim.close();
     }
   }
   return queues;
 }
Esempio n. 28
0
 private MetaRecord getMetaRecord(Result r) {
   Value[] data = new Value[4];
   // id
   // data[0] = ValueInt.get(Bytes.toInt(r.getValue(FAMILY, ID)));
   data[0] = ValueInt.get(Bytes.toInt(r.getRow()));
   // head 未使用
   // data[1] = null;
   // type
   data[2] = ValueInt.get(Bytes.toInt(r.getValue(FAMILY, OBJECT_TYPE)));
   // sql
   data[3] = ValueString.get(Bytes.toString(r.getValue(FAMILY, SQL)));
   return new MetaRecord(new SimpleRow(data));
 }
 protected void wipeOutMeta() throws IOException {
   // Mess it up by blowing up meta.
   Admin admin = TEST_UTIL.getHBaseAdmin();
   Scan s = new Scan();
   Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
   ResultScanner scanner = meta.getScanner(s);
   List<Delete> dels = new ArrayList<Delete>();
   for (Result r : scanner) {
     HRegionInfo info = HRegionInfo.getHRegionInfo(r);
     if (info != null
         && !info.getTable()
             .getNamespaceAsString()
             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
       Delete d = new Delete(r.getRow());
       dels.add(d);
       admin.unassign(r.getRow(), true);
     }
   }
   meta.delete(dels);
   scanner.close();
   meta.close();
 }
Esempio n. 30
0
  /*
   * (non-Javadoc)
   *
   * @see com.hazelcast.core.MapLoader#loadAll(java.util.Collection)
   */
  @Override
  public Map<String, String> loadAll(Collection<String> keys) {
    if (allowLoadAll) {
      List<Get> gets = new ArrayList<Get>(keys.size());
      for (String k : keys) {
        Get g = new Get(Bytes.toBytes(k));
        gets.add(g);
      }

      HTableInterface table = null;
      Map<String, String> kvMap = new HashMap<String, String>();
      try {
        table = pool.getTable(tableName);
        table.get(gets);
        Result[] result = table.get(gets);
        for (Result r : result) {
          byte[] value = r.getValue(family, qualifier);
          if (value != null) {
            if (outputFormatType == StoreFormatType.SMILE) {
              kvMap.put(
                  new String(r.getRow()),
                  jsonSmileConverter.convertFromSmile(r.getValue(family, qualifier)));
            } else {
              kvMap.put(new String(r.getRow()), new String(r.getValue(family, qualifier)));
            }
          }
        }
      } catch (IOException e) {
        LOG.error("IOException while getting values", e);
      } finally {
        if (pool != null && table != null) {
          pool.putTable(table);
        }
      }
    }

    return null;
  }