/**
* Wait until futures are complete or the supplied timeout is reached.
*
* @param timeout Maximum time to wait for futures to complete.
* @param unit Unit of time for the timeout.
* @param futures Futures to wait for.
* @return True if all futures complete in time.
*/
public boolean awaitAll(long timeout, TimeUnit unit, Future<?>... futures) {
boolean complete;
try {
// 将timeout 转换为微秒
long nanos = unit.toNanos(timeout);
// 获取系统当前时间的微秒
long time = System.nanoTime();
for (Future<?> f : futures) {
if (nanos < 0) return false;
// 此处f的示例为Command
f.get(nanos, TimeUnit.NANOSECONDS);
long now = System.nanoTime();
nanos -= now - time;
time = now;
}
complete = true;
} catch (TimeoutException e) {
complete = false;
} catch (Exception e) {
throw new RedisCommandInterruptedException(e);
}
return complete;
}
public static void main(String[] args) {
try {
int port = DEFAULT_SERVER_PORT;
File systemDir = null;
if (args.length > 0) {
try {
port = Integer.parseInt(args[0]);
} catch (NumberFormatException e) {
System.err.println("Error parsing port: " + e.getMessage());
System.exit(-1);
}
systemDir = new File(args[1]);
}
final Server server =
new Server(
systemDir, System.getProperty(GlobalOptions.USE_MEMORY_TEMP_CACHE_OPTION) != null);
initLoggers();
server.start(port);
System.out.println("Server classpath: " + System.getProperty("java.class.path"));
System.err.println(SERVER_SUCCESS_START_MESSAGE + port);
} catch (Throwable e) {
System.err.println(SERVER_ERROR_START_MESSAGE + e.getMessage());
e.printStackTrace(System.err);
System.exit(-1);
}
}
/**
* Send to server Payload
*
* @param payloadSend
* @throws IOException
* @see com.reversemind.hypergate.Payload
*/
public void send(Payload payloadSend) throws IOException {
this.setPayload(null);
// clean & start a timer
if (this.channel != null && this.channel.isConnected()) {
LOG.info("Connected:" + this.channel.isConnected());
if (payloadSend != null) {
LOG.info("Send from HyperGateClient payload:" + payloadSend.toString());
payloadSend.setClientTimestamp(System.currentTimeMillis());
// client is occupied
this.occupied = true;
// TODO make it #6
this.channel.write(payloadSend);
this.shutDownExecutor();
this.executor = this.getExecutor();
this.executor.execute(this.createFutureTask());
return;
}
}
throw new IOException("Channel is closed or even not opened");
}
static {
long ping = -1L;
try {
ping = Long.parseLong(System.getProperty(GlobalOptions.PING_INTERVAL_MS_OPTION, "-1"));
} catch (NumberFormatException ignored) {
}
PING_INTERVAL = ping;
int builds = 1;
try {
builds =
Math.min(
Integer.parseInt(
System.getProperty(GlobalOptions.MAX_SIMULTANEOUS_BUILDS_OPTION, "1")),
Runtime.getRuntime().availableProcessors());
} catch (NumberFormatException ignored) {
}
MAX_SIMULTANEOUS_BUILD_SESSIONS = builds;
}
/**
* Netty + Java provide WRAPPER FACADES for all of the underlying functionality of this program
*/
public static void main(String[] args) throws Exception {
if (args.length == 1) {
try {
int port = Integer.parseInt(args[0]);
EchoServer.accept(port);
} catch (NumberFormatException e) {
System.err.println("Argument" + args[0] + " must be an integer.");
System.exit(1);
}
}
}
private void handleIncomingConfirmableCoapRequest(ChannelHandlerContext ctx, MessageEvent me) {
InetSocketAddress remoteEndpoint = (InetSocketAddress) me.getRemoteAddress();
CoapMessage coapMessage = (CoapMessage) me.getMessage();
IncomingReliableMessageExchange newMessageExchange =
new IncomingReliableMessageExchange(remoteEndpoint, coapMessage.getMessageID());
IncomingMessageExchange oldMessageExchange =
ongoingMessageExchanges.get(remoteEndpoint, coapMessage.getMessageID());
// Check if there is an ongoing
if (oldMessageExchange != null) {
if (oldMessageExchange instanceof IncomingReliableMessageExchange) {
// if the old message exchange is reliable and the empty ACK was already sent send another
// empty ACK
if (((IncomingReliableMessageExchange) oldMessageExchange).isAcknowledgementSent())
writeEmptyAcknowledgement(remoteEndpoint, coapMessage.getMessageID());
}
// if the old message was unreliable and the duplicate message is confirmable send empty ACK
else writeEmptyAcknowledgement(remoteEndpoint, coapMessage.getMessageID());
// As the message is already being processed there is nothing more to do
return;
}
// try to add new reliable message exchange
boolean added = false;
synchronized (monitor) {
Long time = System.currentTimeMillis() + MIN_EMPTY_ACK_DELAY_MILLIS;
// Add message exchange to set of ongoing exchanges to detect duplicates
if (!ongoingMessageExchanges.contains(remoteEndpoint, coapMessage.getMessageID())) {
ongoingMessageExchanges.put(remoteEndpoint, coapMessage.getMessageID(), newMessageExchange);
added = true;
}
// If the scheduling of the empty ACK does not work then it was already scheduled
if (!emptyAcknowledgementSchedule.put(time, newMessageExchange)) {
log.error("Could not schedule empty ACK for message: {}", coapMessage);
ongoingMessageExchanges.remove(remoteEndpoint, coapMessage.getMessageID());
added = false;
}
}
// everything is fine, so further process message
if (added) ctx.sendUpstream(me);
}
@Override
public void run() {
try {
long now = System.currentTimeMillis();
synchronized (monitor) {
// Send due empty acknowledgements
Iterator<Map.Entry<Long, Collection<IncomingReliableMessageExchange>>>
dueAcknowledgements =
emptyAcknowledgementSchedule.asMap().headMap(now, true).entrySet().iterator();
while (dueAcknowledgements.hasNext()) {
Map.Entry<Long, Collection<IncomingReliableMessageExchange>> part =
dueAcknowledgements.next();
for (IncomingReliableMessageExchange messageExchange : part.getValue()) {
if (!messageExchange.isAcknowledgementSent()) {
InetSocketAddress remoteEndpoint = messageExchange.getRemoteEndpoint();
int messageID = messageExchange.getMessageID();
writeEmptyAcknowledgement(remoteEndpoint, messageID);
messageExchange.setAcknowledgementSent();
}
}
dueAcknowledgements.remove();
}
// Retire open NON messages
}
} catch (Exception e) {
log.error("Error in reliability task for incoming message exchanges!", e);
}
}
public void pingReceived() {
myLastPingTime = System.currentTimeMillis();
}
// IDEA-91436 idea <121 binds to 127.0.0.1, but >=121 must be available not only from localhost
// but if we bind only to any local port (0.0.0.0), instance of idea <121 can bind to our ports
// and any request to us will be intercepted
// so, we bind to 127.0.0.1 and 0.0.0.0
private int bind(int firstPort, int portsCount, boolean tryAnyPort, ServerBootstrap bootstrap) {
String property = System.getProperty(PROPERTY_ONLY_ANY_HOST);
boolean onlyAnyHost =
property == null
? (SystemInfo.isLinux || SystemInfo.isWindows && !SystemInfo.isWinVistaOrNewer)
: (property.isEmpty() || Boolean.valueOf(property));
boolean portChecked = false;
for (int i = 0; i < portsCount; i++) {
int port = firstPort + i;
ChannelException channelException = null;
try {
openChannels.add(bootstrap.bind(new InetSocketAddress(port)));
if (!onlyAnyHost) {
InetSocketAddress localAddress = null;
try {
localAddress = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port);
openChannels.add(bootstrap.bind(localAddress));
} catch (UnknownHostException ignored) {
return port;
} catch (ChannelException e) {
channelException = e;
if (!portChecked) {
portChecked = true;
assert localAddress != null;
if (checkPortSafe(localAddress)) {
return port;
}
}
}
}
} catch (ChannelException e) {
channelException = e;
}
if (channelException == null) {
return port;
} else {
if (!openChannels.isEmpty()) {
openChannels.close();
openChannels.clear();
}
if (portsCount == 1) {
throw channelException;
} else if (!tryAnyPort && i == (portsCount - 1)) {
LOG.error(channelException);
}
}
}
if (tryAnyPort) {
LOG.info("We cannot bind to our default range, so, try to bind to any free port");
try {
Channel channel = bootstrap.bind(new InetSocketAddress(0));
openChannels.add(channel);
return ((InetSocketAddress) channel.getLocalAddress()).getPort();
} catch (ChannelException e) {
LOG.error(e);
}
}
return -1;
}
public static void main(String[] args) {
final int payloadSize = 100;
int CYCLE_SIZE = 50000;
final long NUMBER_OF_ITERATIONS = 500000;
ChannelBuffer message = ChannelBuffers.buffer(100);
for (int i = 0; i < message.capacity(); i++) {
message.writeByte((byte) i);
}
// Configure the server.
ServerBootstrap serverBootstrap =
new ServerBootstrap(
new NioServerSocketChannelFactory(
Executors.newCachedThreadPool(), Executors.newCachedThreadPool()));
// Set up the pipeline factory.
serverBootstrap.setPipelineFactory(
new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(new EchoServerHandler());
}
});
// Bind and start to accept incoming connections.
serverBootstrap.bind(new InetSocketAddress(9000));
ClientBootstrap clientBootstrap =
new ClientBootstrap(
new NioClientSocketChannelFactory(
Executors.newCachedThreadPool(), Executors.newCachedThreadPool()));
// ClientBootstrap clientBootstrap = new ClientBootstrap(
// new OioClientSocketChannelFactory(Executors.newCachedThreadPool()));
// Set up the pipeline factory.
final EchoClientHandler clientHandler = new EchoClientHandler();
clientBootstrap.setPipelineFactory(
new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
return Channels.pipeline(clientHandler);
}
});
// Start the connection attempt.
ChannelFuture future = clientBootstrap.connect(new InetSocketAddress("localhost", 9000));
future.awaitUninterruptibly();
Channel clientChannel = future.getChannel();
System.out.println("Warming up...");
for (long i = 0; i < 10000; i++) {
clientHandler.latch = new CountDownLatch(1);
clientChannel.write(message);
try {
clientHandler.latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("Warmed up");
long start = System.currentTimeMillis();
long cycleStart = System.currentTimeMillis();
for (long i = 1; i < NUMBER_OF_ITERATIONS; i++) {
clientHandler.latch = new CountDownLatch(1);
clientChannel.write(message);
try {
clientHandler.latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
if ((i % CYCLE_SIZE) == 0) {
long cycleEnd = System.currentTimeMillis();
System.out.println(
"Ran 50000, TPS " + (CYCLE_SIZE / ((double) (cycleEnd - cycleStart) / 1000)));
cycleStart = cycleEnd;
}
}
long end = System.currentTimeMillis();
long seconds = (end - start) / 1000;
System.out.println(
"Ran ["
+ NUMBER_OF_ITERATIONS
+ "] iterations, payload ["
+ payloadSize
+ "]: took ["
+ seconds
+ "], TPS: "
+ ((double) NUMBER_OF_ITERATIONS) / seconds);
clientChannel.close().awaitUninterruptibly();
clientBootstrap.releaseExternalResources();
serverBootstrap.releaseExternalResources();
}
private Process launchBuildProcess(Project project, final int port, final UUID sessionId)
throws ExecutionException {
// choosing sdk with which the build process should be run
final Sdk internalJdk = JavaAwareProjectJdkTableImpl.getInstanceEx().getInternalJdk();
Sdk projectJdk = internalJdk;
final String versionString = projectJdk.getVersionString();
JavaSdkVersion sdkVersion =
versionString != null
? ((JavaSdk) projectJdk.getSdkType()).getVersion(versionString)
: null;
if (sdkVersion != null) {
final Set<Sdk> candidates = new HashSet<Sdk>();
for (Module module : ModuleManager.getInstance(project).getModules()) {
final Sdk sdk = ModuleRootManager.getInstance(module).getSdk();
if (sdk != null && sdk.getSdkType() instanceof JavaSdk) {
candidates.add(sdk);
}
}
// now select the latest version from the sdks that are used in the project, but not older
// than the internal sdk version
for (Sdk candidate : candidates) {
final String vs = candidate.getVersionString();
if (vs != null) {
final JavaSdkVersion candidateVersion = ((JavaSdk) candidate.getSdkType()).getVersion(vs);
if (candidateVersion != null) {
if (candidateVersion.compareTo(sdkVersion) > 0) {
sdkVersion = candidateVersion;
projectJdk = candidate;
}
}
}
}
}
// validate tools.jar presence
final File compilerPath;
if (projectJdk.equals(internalJdk)) {
final JavaCompiler systemCompiler = ToolProvider.getSystemJavaCompiler();
if (systemCompiler == null) {
throw new ExecutionException(
"No system java compiler is provided by the JRE. Make sure tools.jar is present in IntelliJ IDEA classpath.");
}
compilerPath = ClasspathBootstrap.getResourcePath(systemCompiler.getClass());
} else {
final String path = ((JavaSdk) projectJdk.getSdkType()).getToolsPath(projectJdk);
if (path == null) {
throw new ExecutionException(
"Cannot determine path to 'tools.jar' library for "
+ projectJdk.getName()
+ " ("
+ projectJdk.getHomePath()
+ ")");
}
compilerPath = new File(path);
}
final GeneralCommandLine cmdLine = new GeneralCommandLine();
final String vmExecutablePath =
((JavaSdkType) projectJdk.getSdkType()).getVMExecutablePath(projectJdk);
cmdLine.setExePath(vmExecutablePath);
cmdLine.addParameter("-XX:MaxPermSize=150m");
cmdLine.addParameter("-XX:ReservedCodeCacheSize=64m");
final int heapSize = Registry.intValue("compiler.process.heap.size");
final int xms = heapSize / 2;
if (xms > 32) {
cmdLine.addParameter("-Xms" + xms + "m");
}
cmdLine.addParameter("-Xmx" + heapSize + "m");
if (SystemInfo.isMac
&& sdkVersion != null
&& JavaSdkVersion.JDK_1_6.equals(sdkVersion)
&& Registry.is("compiler.process.32bit.vm.on.mac")) {
// unfortunately -d32 is supported on jdk 1.6 only
cmdLine.addParameter("-d32");
}
cmdLine.addParameter("-Djava.awt.headless=true");
final String shouldGenerateIndex =
System.getProperty(GlobalOptions.GENERATE_CLASSPATH_INDEX_OPTION);
if (shouldGenerateIndex != null) {
cmdLine.addParameter(
"-D" + GlobalOptions.GENERATE_CLASSPATH_INDEX_OPTION + "=" + shouldGenerateIndex);
}
final String additionalOptions = Registry.stringValue("compiler.process.vm.options");
if (!StringUtil.isEmpty(additionalOptions)) {
final StringTokenizer tokenizer = new StringTokenizer(additionalOptions, " ", false);
while (tokenizer.hasMoreTokens()) {
cmdLine.addParameter(tokenizer.nextToken());
}
}
// debugging
final int debugPort = Registry.intValue("compiler.process.debug.port");
if (debugPort > 0) {
cmdLine.addParameter("-XX:+HeapDumpOnOutOfMemoryError");
cmdLine.addParameter(
"-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=" + debugPort);
}
if (Registry.is("compiler.process.use.memory.temp.cache")) {
cmdLine.addParameter("-D" + GlobalOptions.USE_MEMORY_TEMP_CACHE_OPTION);
}
if (Registry.is("compiler.process.use.external.javac")) {
cmdLine.addParameter("-D" + GlobalOptions.USE_EXTERNAL_JAVAC_OPTION);
}
final String host = NetUtils.getLocalHostString();
cmdLine.addParameter("-D" + GlobalOptions.HOSTNAME_OPTION + "=" + host);
// javac's VM should use the same default locale that IDEA uses in order for javac to print
// messages in 'correct' language
final String lang = System.getProperty("user.language");
if (lang != null) {
//noinspection HardCodedStringLiteral
cmdLine.addParameter("-Duser.language=" + lang);
}
final String country = System.getProperty("user.country");
if (country != null) {
//noinspection HardCodedStringLiteral
cmdLine.addParameter("-Duser.country=" + country);
}
//noinspection HardCodedStringLiteral
final String region = System.getProperty("user.region");
if (region != null) {
//noinspection HardCodedStringLiteral
cmdLine.addParameter("-Duser.region=" + region);
}
cmdLine.addParameter("-classpath");
final List<File> cp = ClasspathBootstrap.getBuildProcessApplicationClasspath();
cp.add(compilerPath);
cp.addAll(myClasspathManager.getCompileServerPluginsClasspath());
cmdLine.addParameter(classpathToString(cp));
cmdLine.addParameter(BuildMain.class.getName());
cmdLine.addParameter(host);
cmdLine.addParameter(Integer.toString(port));
cmdLine.addParameter(sessionId.toString());
final File workDirectory = new File(mySystemDirectory, SYSTEM_ROOT);
workDirectory.mkdirs();
ensureLogConfigExists(workDirectory);
cmdLine.addParameter(FileUtil.toSystemIndependentName(workDirectory.getPath()));
cmdLine.setWorkDirectory(workDirectory);
return cmdLine.createProcess();
}
@Inject
public NettyHttpServerTransport(
Settings settings, NetworkService networkService, BigArrays bigArrays) {
super(settings);
this.networkService = networkService;
this.bigArrays = bigArrays;
if (settings.getAsBoolean("netty.epollBugWorkaround", false)) {
System.setProperty("org.jboss.netty.epollBugWorkaround", "true");
}
ByteSizeValue maxContentLength =
settings.getAsBytesSize(
"http.netty.max_content_length",
settings.getAsBytesSize(
"http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB)));
this.maxChunkSize =
settings.getAsBytesSize(
"http.netty.max_chunk_size",
settings.getAsBytesSize("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB)));
this.maxHeaderSize =
settings.getAsBytesSize(
"http.netty.max_header_size",
settings.getAsBytesSize("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB)));
this.maxInitialLineLength =
settings.getAsBytesSize(
"http.netty.max_initial_line_length",
settings.getAsBytesSize(
"http.max_initial_line_length", new ByteSizeValue(4, ByteSizeUnit.KB)));
// don't reset cookies by default, since I don't think we really need to
// note, parsing cookies was fixed in netty 3.5.1 regarding stack allocation, but still,
// currently, we don't need cookies
this.resetCookies =
settings.getAsBoolean(
"http.netty.reset_cookies", settings.getAsBoolean("http.reset_cookies", false));
this.maxCumulationBufferCapacity =
settings.getAsBytesSize("http.netty.max_cumulation_buffer_capacity", null);
this.maxCompositeBufferComponents =
settings.getAsInt("http.netty.max_composite_buffer_components", -1);
this.workerCount =
settings.getAsInt(
"http.netty.worker_count", EsExecutors.boundedNumberOfProcessors(settings) * 2);
this.blockingServer =
settings.getAsBoolean(
"http.netty.http.blocking_server",
settings.getAsBoolean(TCP_BLOCKING_SERVER, settings.getAsBoolean(TCP_BLOCKING, false)));
this.port = settings.get("http.netty.port", settings.get("http.port", "9200-9300"));
this.bindHost =
settings.get(
"http.netty.bind_host", settings.get("http.bind_host", settings.get("http.host")));
this.publishHost =
settings.get(
"http.netty.publish_host",
settings.get("http.publish_host", settings.get("http.host")));
this.publishPort =
settings.getAsInt("http.netty.publish_port", settings.getAsInt("http.publish_port", 0));
this.tcpNoDelay = settings.get("http.netty.tcp_no_delay", settings.get(TCP_NO_DELAY, "true"));
this.tcpKeepAlive =
settings.get("http.netty.tcp_keep_alive", settings.get(TCP_KEEP_ALIVE, "true"));
this.reuseAddress =
settings.getAsBoolean(
"http.netty.reuse_address",
settings.getAsBoolean(TCP_REUSE_ADDRESS, NetworkUtils.defaultReuseAddress()));
this.tcpSendBufferSize =
settings.getAsBytesSize(
"http.netty.tcp_send_buffer_size",
settings.getAsBytesSize(TCP_SEND_BUFFER_SIZE, TCP_DEFAULT_SEND_BUFFER_SIZE));
this.tcpReceiveBufferSize =
settings.getAsBytesSize(
"http.netty.tcp_receive_buffer_size",
settings.getAsBytesSize(TCP_RECEIVE_BUFFER_SIZE, TCP_DEFAULT_RECEIVE_BUFFER_SIZE));
this.detailedErrorsEnabled = settings.getAsBoolean(SETTING_HTTP_DETAILED_ERRORS_ENABLED, true);
long defaultReceiverPredictor = 512 * 1024;
if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
// we can guess a better default...
long l =
(long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / workerCount);
defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
}
// See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use
// higher ones for us, even fixed one
ByteSizeValue receivePredictorMin =
settings.getAsBytesSize(
"http.netty.receive_predictor_min",
settings.getAsBytesSize(
"http.netty.receive_predictor_size", new ByteSizeValue(defaultReceiverPredictor)));
ByteSizeValue receivePredictorMax =
settings.getAsBytesSize(
"http.netty.receive_predictor_max",
settings.getAsBytesSize(
"http.netty.receive_predictor_size", new ByteSizeValue(defaultReceiverPredictor)));
if (receivePredictorMax.bytes() == receivePredictorMin.bytes()) {
receiveBufferSizePredictorFactory =
new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.bytes());
} else {
receiveBufferSizePredictorFactory =
new AdaptiveReceiveBufferSizePredictorFactory(
(int) receivePredictorMin.bytes(),
(int) receivePredictorMin.bytes(),
(int) receivePredictorMax.bytes());
}
this.compression = settings.getAsBoolean(SETTING_HTTP_COMPRESSION, false);
this.compressionLevel = settings.getAsInt(SETTING_HTTP_COMPRESSION_LEVEL, 6);
this.pipelining = settings.getAsBoolean(SETTING_PIPELINING, DEFAULT_SETTING_PIPELINING);
this.pipeliningMaxEvents =
settings.getAsInt(SETTING_PIPELINING_MAX_EVENTS, DEFAULT_SETTING_PIPELINING_MAX_EVENTS);
// validate max content length
if (maxContentLength.bytes() > Integer.MAX_VALUE) {
logger.warn(
"maxContentLength[" + maxContentLength + "] set to high value, resetting it to [100mb]");
maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
}
this.maxContentLength = maxContentLength;
logger.debug(
"using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], receive_predictor[{}->{}], pipelining[{}], pipelining_max_events[{}]",
maxChunkSize,
maxHeaderSize,
maxInitialLineLength,
this.maxContentLength,
receivePredictorMin,
receivePredictorMax,
pipelining,
pipeliningMaxEvents);
}
