Example #1
0
    public void prepare(String query, InetAddress toExclude) throws InterruptedException {
      for (Map.Entry<Host, HostConnectionPool> entry : pools.entrySet()) {
        if (entry.getKey().getAddress().equals(toExclude)) continue;

        // Let's not wait too long if we can't get a connection. Things
        // will fix themselves once the user tries a query anyway.
        Connection c = null;
        try {
          c = entry.getValue().borrowConnection(200, TimeUnit.MILLISECONDS);
          c.write(new PrepareMessage(query)).get();
        } catch (ConnectionException e) {
          // Again, not being able to prepare the query right now is no big deal, so just ignore
        } catch (BusyConnectionException e) {
          // Same as above
        } catch (TimeoutException e) {
          // Same as above
        } catch (ExecutionException e) {
          // We shouldn't really get exception while preparing a
          // query, so log this (but ignore otherwise as it's not a big deal)
          logger.error(
              String.format(
                  "Unexpected error while preparing query (%s) on %s", query, entry.getKey()),
              e);
        } finally {
          if (c != null) entry.getValue().returnConnection(c);
        }
      }
    }
Example #2
0
 public void sendResponse(ClientEndpoint endpoint, Object response) {
   if (response instanceof Throwable) {
     response =
         ClientExceptionConverters.get(endpoint.getClientType()).convert((Throwable) response);
   }
   final Data resultData = response != null ? serializationService.toData(response) : NULL;
   Connection conn = endpoint.getConnection();
   conn.write(new DataAdapter(resultData, serializationService.getSerializationContext()));
 }
Example #3
0
 void bind(final ClientEndpoint endpoint) {
   final Connection conn = endpoint.getConnection();
   if (conn instanceof TcpIpConnection) {
     Address address = new Address(conn.getRemoteSocketAddress());
     TcpIpConnectionManager connectionManager =
         (TcpIpConnectionManager) node.getConnectionManager();
     connectionManager.bind((TcpIpConnection) conn, address, null, false);
   }
   sendClientEvent(endpoint);
 }
 private boolean discardAvailableConnections(long timeout, TimeUnit unit)
     throws InterruptedException {
   long start = System.nanoTime();
   boolean success = true;
   for (Connection connection : connections) {
     success &= connection.close(timeout - Cluster.timeSince(start, unit), unit);
     open.decrementAndGet();
   }
   return success;
 }
Example #5
0
  public void connectionRemoved(Connection connection) {
    if (connection.isClient() && connection instanceof TcpIpConnection && nodeEngine.isActive()) {
      final ClientEndpoint endpoint = endpoints.get(connection);
      if (endpoint != null
          && node.getLocalMember().getUuid().equals(endpoint.getPrincipal().getOwnerUuid())) {
        removeEndpoint(connection, true);
        if (!endpoint.isFirstConnection()) {
          return;
        }
        NodeEngine nodeEngine = node.nodeEngine;
        final Collection<MemberImpl> memberList = nodeEngine.getClusterService().getMemberList();
        for (MemberImpl member : memberList) {
          final ClientDisconnectionOperation op =
              new ClientDisconnectionOperation(endpoint.getUuid());
          op.setNodeEngine(nodeEngine)
              .setServiceName(SERVICE_NAME)
              .setService(this)
              .setResponseHandler(ResponseHandlerFactory.createEmptyResponseHandler());

          if (member.localMember()) {
            nodeEngine.getOperationService().runOperation(op);
          } else {
            nodeEngine.getOperationService().send(op, member.getAddress());
          }
        }
      }
    }
  }
  public Connection borrowConnection(long timeout, TimeUnit unit)
      throws ConnectionException, TimeoutException {
    if (isShutdown.get())
      // Note: throwing a ConnectionException is probably fine in practice as it will trigger the
      // creation of a new host.
      // That being said, maybe having a specific exception could be cleaner.
      throw new ConnectionException(host.getAddress(), "Pool is shutdown");

    if (connections.isEmpty()) {
      for (int i = 0; i < options().getCoreConnectionsPerHost(hostDistance); i++) {
        // We don't respect MAX_SIMULTANEOUS_CREATION here because it's  only to
        // protect against creating connection in excess of core too quickly
        scheduledForCreation.incrementAndGet();
        manager.executor().submit(newConnectionTask);
      }
      Connection c = waitForConnection(timeout, unit);
      c.setKeyspace(manager.poolsState.keyspace);
      return c;
    }

    int minInFlight = Integer.MAX_VALUE;
    Connection leastBusy = null;
    for (Connection connection : connections) {
      int inFlight = connection.inFlight.get();
      if (inFlight < minInFlight) {
        minInFlight = inFlight;
        leastBusy = connection;
      }
    }

    if (minInFlight >= options().getMaxSimultaneousRequestsPerConnectionThreshold(hostDistance)
        && connections.size() < options().getMaxConnectionsPerHost(hostDistance))
      maybeSpawnNewConnection();

    while (true) {
      int inFlight = leastBusy.inFlight.get();

      if (inFlight >= Connection.MAX_STREAM_PER_CONNECTION) {
        leastBusy = waitForConnection(timeout, unit);
        break;
      }

      if (leastBusy.inFlight.compareAndSet(inFlight, inFlight + 1)) break;
    }
    leastBusy.setKeyspace(manager.poolsState.keyspace);
    return leastBusy;
  }
Example #7
0
  /**
   * Remove RabbitReconnector if it's not null (to prevent reconnect). Close connection if it's not
   * null. Swallow IOExceptions.
   *
   * @param conn
   * @param reconnector
   */
  public static void closeConnectionAndRemoveReconnector(
      Connection conn, RabbitReconnector reconnector) {
    if (conn == null) {
      return;
    }

    if (reconnector != null) {
      conn.removeShutdownListener(reconnector);
    }

    try {
      conn.close();
    } catch (IOException e) {
      e.printStackTrace();
      // throw new RuntimeException(e);
    }
  }
Example #8
0
  private Connection _newConnection(int numThreads) throws IOException {
    ConnectionFactory connFactory = new ConnectionFactory();
    ThreadFactory threadFactory = DaemonThreadFactory.getInstance("RabbitMQ-ConsumerThread", true);
    final ExecutorService executor = Executors.newFixedThreadPool(numThreads, threadFactory);
    Address[] array = addresses.toArray(new Address[0]);
    Connection conn = connFactory.newConnection(executor, array);

    conn.addShutdownListener(
        new ShutdownListener() {
          @Override
          public void shutdownCompleted(ShutdownSignalException sse) {
            executor.shutdown();
          }
        });

    return conn;
  }
Example #9
0
 public void shutdown() {
   for (ClientEndpoint endpoint : endpoints.values()) {
     try {
       endpoint.destroy();
     } catch (LoginException e) {
       logger.finest(e.getMessage());
     }
     try {
       final Connection conn = endpoint.getConnection();
       if (conn.live()) {
         conn.close();
       }
     } catch (Exception e) {
       logger.finest(e);
     }
   }
   endpoints.clear();
 }
  public void returnConnection(Connection connection) {
    int inFlight = connection.inFlight.decrementAndGet();

    if (connection.isDefunct()) {
      if (host.getMonitor().signalConnectionFailure(connection.lastException())) shutdown();
      else replace(connection);
    } else {

      if (trash.contains(connection) && inFlight == 0) {
        if (trash.remove(connection)) close(connection);
        return;
      }

      if (connections.size() > options().getCoreConnectionsPerHost(hostDistance)
          && inFlight <= options().getMinSimultaneousRequestsPerConnectionThreshold(hostDistance)) {
        trashConnection(connection);
      } else {
        signalAvailableConnection();
      }
    }
  }
Example #11
0
  private void destroyEndpoint(ClientEndpoint endpoint, boolean closeImmediately) {
    if (endpoint != null) {
      logger.info("Destroying " + endpoint);
      try {
        endpoint.destroy();
      } catch (LoginException e) {
        logger.warning(e);
      }

      final Connection connection = endpoint.getConnection();
      if (closeImmediately) {
        try {
          connection.close();
        } catch (Throwable e) {
          logger.warning("While closing client connection: " + connection, e);
        }
      } else {
        nodeEngine
            .getExecutionService()
            .schedule(
                new Runnable() {
                  public void run() {
                    if (connection.live()) {
                      try {
                        connection.close();
                      } catch (Throwable e) {
                        logger.warning("While closing client connection: " + e.toString());
                      }
                    }
                  }
                },
                1111,
                TimeUnit.MILLISECONDS);
      }
      sendClientEvent(endpoint);
    }
  }
Example #12
0
 public ClientEndpoint createNew(Connection conn) {
   return new ClientEndpoint(
       ClientEngineImpl.this, conn, UuidUtil.createClientUuid(conn.getEndPoint()));
 }
  public void loadTables(String tableNames, String procNames)
      throws SQLException, IOException, InterruptedException, ExecutionException {
    String[] tableNameArray =
        tableNames != null && !"".equals(tableNames) ? tableNames.split(",") : null;
    String[] procNameArray =
        procNames != null && !"".equals(procNames) ? procNames.split(",") : null;

    ExecutorService executor = Executors.newFixedThreadPool(tableNameArray.length * 3);
    CompletionService completion = new ExecutorCompletionService(executor);

    for (int j = 0; j < tableNameArray.length && tableNameArray != null; j++) {
      String tableName = tableNameArray[j];
      String procName = procNameArray != null ? procNameArray[j] : "";

      // if procName not provided, use the default VoltDB TABLENAME.insert procedure
      if (procName.length() == 0) {
        if (tableName.contains("..")) {
          procName = tableName.split("\\.\\.")[1].toUpperCase() + ".insert";
        } else {
          procName = tableName.toUpperCase() + ".insert";
        }
      }

      // query the table
      String jdbcSelect = "SELECT * FROM " + tableName + ";";

      // create query to find count
      String countquery = jdbcSelect.replace("*", "COUNT(*)");
      int pages = 1;
      String base = "";
      if (config.srisvoltdb) {
        if (config.isPaginated) {
          try {
            // find count
            if (countquery.contains("<") || countquery.contains(">")) {
              int bracketOpen = countquery.indexOf("<");
              int bracketClose = countquery.indexOf(">");
              String orderCol = countquery.substring(bracketOpen + 1, bracketClose);
              countquery = countquery.replace("<" + orderCol + ">", "");
            }
            VoltTable vcount = client.callProcedure("@AdHoc", countquery).getResults()[0];
            int count = Integer.parseInt(vcount.toString());
            // determine number of pages from total data and page size
            pages = (int) Math.ceil((double) count / config.pageSize);
            System.out.println(pages);
          } catch (Exception e) {
            System.out.println("Count formation failure!");
          }
        }
      } else {
        // find count
        Connection conn =
            DriverManager.getConnection(config.jdbcurl, config.jdbcuser, config.jdbcpassword);
        base = conn.getMetaData().getDatabaseProductName().toLowerCase();
        Statement jdbcStmt =
            conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
        if (countquery.contains("<") || countquery.contains(">")) {
          int bracketOpen = countquery.indexOf("<");
          int bracketClose = countquery.indexOf(">");
          String orderCol = countquery.substring(bracketOpen + 1, bracketClose);
          countquery = countquery.replace("<" + orderCol + ">", "");
        }
        ResultSet rcount =
            jdbcStmt.executeQuery(
                countquery); // determine number of pages from total data and page size
        if (base.contains("postgres") && config.isPaginated) {
          int count = Integer.parseInt(rcount.toString());
          pages = (int) Math.ceil((double) count / config.pageSize);
        }
      }

      // establish new SourceReaders and DestinationWriters for pages
      SourceReader[] sr = new SourceReader[pages];
      DestinationWriter[] cr = new DestinationWriter[pages];
      for (int i = 0; i < pages; i++) {
        sr[i] = new SourceReader();
        cr[i] = new DestinationWriter();
      }
      Controller processor =
          new Controller<ArrayList<Object[]>>(
              client, sr, cr, jdbcSelect, procName, config, pages, base);
      completion.submit(processor);
    }

    // wait for all tasks to complete.
    for (int i = 0; i < tableNameArray.length; ++i) {
      logger.info(
          "****************"
              + completion.take().get()
              + " completed *****************"); // will block until the next sub task has
      // completed.
    }
    executor.shutdown();
  }
  public void load(String queryFile, String modules, String tables)
      throws SQLException, IOException, InterruptedException, ExecutionException {
    Properties properties = new Properties();
    properties.load(new FileInputStream(queryFile));

    Collection<String> keys = properties.stringPropertyNames();

    // Filtering by validating if property starts with any of the module names
    if (!Config.ALL.equalsIgnoreCase(modules)) {
      keys =
          Util.filter(
              keys, "^(" + modules.replaceAll(Config.COMMA_SEPARATOR, Config.MODULE_SUFFIX) + ")");
    }

    // Filtering by table names
    if (!Config.ALL.equalsIgnoreCase(tables)) {
      keys =
          Util.filter(
              keys, "(" + tables.replaceAll(Config.COMMA_SEPARATOR, Config.TABLE_SUFFIX) + ")$");
    }

    logger.info("The final modules and tables that are being considered" + keys.toString());

    ExecutorService executor = Executors.newFixedThreadPool(keys.size() * 3);
    CompletionService completion = new ExecutorCompletionService(executor);

    for (String key : keys) {
      String query = properties.getProperty(key);
      key =
          (key.contains(Config.DOT_SEPARATOR)
              ? key.substring(key.indexOf(Config.DOT_SEPARATOR) + 1)
              : key);

      while (query.contains("[:")) {
        String param = query.substring(query.indexOf("[:") + 2, query.indexOf("]"));

        query = query.replaceFirst("\\[\\:" + param + "\\]", properties.getProperty(param));
      }
      int pages = 1;
      String base = "";
      if (config.srisvoltdb) {
        if (config.isPaginated) {
          try {
            // find count
            String countquery = query;
            if (countquery.contains("<") || countquery.contains(">")) {
              int bracketOpen = countquery.indexOf("<");
              int bracketClose = countquery.indexOf(">");
              String orderCol = countquery.substring(bracketOpen + 1, bracketClose);
              countquery = countquery.replace("<" + orderCol + ">", "");
            }
            VoltTable vcount = client.callProcedure("@AdHoc", countquery).getResults()[0];
            int count = vcount.getRowCount();
            pages = (int) Math.ceil((double) count / config.pageSize);
          } catch (Exception e) {
            System.out.println("Count formation failure!");
          }
        }
        // set up data in order
      } else {
        // find count
        String countquery = query.replace("*", "COUNT(*)");
        Connection conn =
            DriverManager.getConnection(config.jdbcurl, config.jdbcuser, config.jdbcpassword);
        base = conn.getMetaData().getDatabaseProductName().toLowerCase();
        System.out.println("BASE: " + base);
        Statement jdbcStmt =
            conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
        if (countquery.contains("<") || countquery.contains(">")) {
          int bracketOpen = countquery.indexOf("<");
          int bracketClose = countquery.indexOf(">");
          String orderCol = countquery.substring(bracketOpen + 1, bracketClose);
          countquery = countquery.replace("<" + orderCol + ">", "");
        }
        ResultSet rcount = jdbcStmt.executeQuery(countquery);
        rcount.next();
        int count = Integer.parseInt(rcount.getArray(1).toString());

        // THIS IF NEEDS A WAY TO DETERMINE IF POSTGRES
        if (base.contains("postgres") && config.isPaginated) {
          pages = (int) Math.ceil((double) count / config.pageSize);
        }
        // set up data in order
      }
      // establish new SourceReaders and DestinationWriters for pages
      SourceReader[] sr = new SourceReader[pages];
      DestinationWriter[] cr = new DestinationWriter[pages];
      for (int i = 0; i < pages; i++) {
        sr[i] = new SourceReader();
        cr[i] = new DestinationWriter();
      }
      Controller processor =
          new Controller<ArrayList<Object[]>>(
              client, sr, cr, query, key.toUpperCase() + ".insert", config, pages, base);
      completion.submit(processor);
    }

    // wait for all tasks to complete.
    for (int i = 0; i < keys.size(); ++i) {
      logger.info(
          "****************"
              + completion.take().get()
              + " completed *****************"); // will block until the next sub task has
      // completed.
    }

    executor.shutdown();
  }