Ejemplo n.º 1
0
  private static void waitUntil(long until) {
    long now = System.currentTimeMillis();

    while (now <= until) {
      try {
        Thread.sleep(1L);
        now = System.currentTimeMillis();
      } catch (InterruptedException e) {
        throw new RuntimeException(e);
      }
    }
  }
Ejemplo n.º 2
0
  private void ensureColumnFamilyExists(String tableName, String columnFamily)
      throws StorageException {
    HBaseAdmin adm = getAdminInterface();
    HTableDescriptor desc = ensureTableExists(tableName);

    Preconditions.checkNotNull(desc);

    HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());

    // Create our column family, if necessary
    if (cf == null) {
      try {
        adm.disableTable(tableName);
        desc.addFamily(
            new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ));
        adm.modifyTable(tableName.getBytes(), desc);

        try {
          logger.debug(
              "Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
          Thread.sleep(1000L);
        } catch (InterruptedException ie) {
          throw new TemporaryStorageException(ie);
        }

        adm.enableTable(tableName);
      } catch (TableNotFoundException ee) {
        logger.error("TableNotFoundException", ee);
        throw new PermanentStorageException(ee);
      } catch (org.apache.hadoop.hbase.TableExistsException ee) {
        logger.debug("Swallowing exception {}", ee);
      } catch (IOException ee) {
        throw new TemporaryStorageException(ee);
      }
    } else { // check if compression was enabled, if not - enable it
      if (cf.getCompressionType() == null
          || cf.getCompressionType() == Compression.Algorithm.NONE) {
        try {
          adm.disableTable(tableName);

          adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ));

          adm.enableTable(tableName);
        } catch (IOException e) {
          throw new TemporaryStorageException(e);
        }
      }
    }
  }
Ejemplo n.º 3
0
  private static String lock(String lock) {
    String realPath = "";
    String parent = "/lock";
    String lockName = parent + "/" + lock;

    logger.debug("Getting lock " + lockName);

    try {
      if (zkInstance.exists(parent, false) == null)
        zkInstance.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.fromFlag(0));
    } catch (Exception E) {
      logger.error("Error creating lock node: " + E.toString());
      return null;
    }

    List<String> children = new LinkedList<String>();
    try {
      // List <ACL> ACLList = zkInstance.getACL(lockName, zkInstance.exists(lock, false));

      realPath =
          zkInstance.create(
              lockName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
      // children = zkInstance.getChildren(realPath, false);
      checkLock:
      while (true) {
        children = zkInstance.getChildren(parent, false);
        for (String curChild : children) {
          String child = parent + "/" + curChild;
          // System.out.println(child + " " + realPath + " " +
          // Integer.toString(child.compareTo(realPath)));
          if (child.compareTo(realPath) < 0
              && child.length() == realPath.length()
              && curChild.startsWith(lock)) {
            // System.out.println(child + " cmp to " + realPath);
            Thread.sleep(300);
            continue checkLock;
          }
        }
        logger.info("Got lock " + lockName);
        return realPath;
      }
    } catch (Exception E) {
      logger.error("Exception while trying to get lock " + lockName + " :" + E.toString());
      E.printStackTrace();
      return null;
    }
  }
Ejemplo n.º 4
0
  @Test
  public void insert_multithread() throws InterruptedException {

    for (int k = 1; k <= 10; k++) {
      final int j = k;
      new Thread(
              new Runnable() {
                public void run() {
                  try {

                    //                    List<Put> puts = new ArrayList<Put>();
                    Configuration conf = new Configuration();
                    conf.set("hbase.zookeeper.quorum", "192.168.1.160");
                    HTable table = new HTable(conf, "wf:error");
                    table.setAutoFlushTo(false);
                    long t1 = System.currentTimeMillis();
                    for (int i = 0; i < 1000000; i++) {
                      String uuid =
                          UUID.randomUUID().toString().replaceAll("-", "").substring(0, 8);
                      Put put = new Put(Bytes.toBytes(uuid + "_" + "2015070" + j));
                      put.add(
                          fBytes,
                          Bytes.toBytes("stacktrace"),
                          Bytes.toBytes(
                              "java.io.IOException:file not found" + UUID.randomUUID().toString()));
                      //                        puts.add(put);
                      table.put(put);
                      if (i % 10000 == 0) {
                        table.flushCommits();
                      }
                    }
                    table.close();
                    long t2 = System.currentTimeMillis();
                    System.out.println(Thread.currentThread() + ",t2-t1=" + (t2 - t1));
                  } catch (IOException e) {

                  }
                }
              })
          .start();
    }

    System.out.println("waiting.....");
    Thread.sleep(1000 * 60 * 10);
    System.out.println("completing.......");
  }
Ejemplo n.º 5
0
  public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
    List<KeyValue> result = new ArrayList<>();
    String strJson = null;
    BufferedWriter writer = null;
    Table table = null;
    try (Connection connect = ConnectionFactory.createConnection(config);
        Admin admin = connect.getAdmin()) {
      TableName tablename = TableName.valueOf(TABLE_NAME);
      if (!admin.tableExists(tablename)) {
        System.out.println("Table does not exist.");
        return null;
      }

      table = connect.getTable(tablename);
      Put put = null;
      String start = null;
      String end = null;
      writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
      for (KeyValue item : lstAllProcessStation) {
        start = (String) item.getKey();
        end = (String) item.getValue();
        try {
          try {
            Thread.sleep(200);
          } catch (InterruptedException e1) {
            e1.printStackTrace();
          }
          System.out.println("process : " + start + ":" + end);
          strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
          writer.write(start + ":" + end);
          writer.newLine();

        } catch (Exception e) {
          System.out.println(start + ":" + end + "error");
          e.printStackTrace();
          break;
        }
        JSONObject jo = new JSONObject(strJson);

        if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
          JSONObject joData = jo.getJSONObject("data");
          if (joData.has("flag") && joData.getBoolean("flag")) {
            result.add(new DefaultKeyValue(start, end));
            // 插入到hbase
            String rowkey = start + ":" + end;
            put = new Put(rowkey.getBytes());
            put.addColumn(
                CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
            table.put(put);

            System.out.println("start " + start + "\t end " + end + "\t has ticket");
          }
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      if (writer != null) {
        try {
          writer.flush();
          writer.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (table != null) {
        try {
          table.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    }

    return result;
  }