@Override
public SubPlanBuilder visitUnion(UnionNode node, Void context) {
if (createSingleNodePlan) {
ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder();
for (PlanNode source : node.getSources()) {
sourceBuilder.add(source.accept(this, context).getRoot());
}
UnionNode unionNode =
new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping());
return createSingleNodePlan(unionNode);
} else {
ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder();
ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder();
for (int i = 0; i < node.getSources().size(); i++) {
PlanNode subPlan = node.getSources().get(i);
SubPlanBuilder current = subPlan.accept(this, context);
current.setRoot(
new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i)));
fragmentIdBuilder.add(current.getId());
sourceBuilder.add(current.build());
}
ExchangeNode exchangeNode =
new ExchangeNode(
idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols());
return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build());
}
}
private List<KnapsackState> get(String name) throws IOException {
ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
try {
logger.debug("get knapsack states: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
GetResponse getResponse =
client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
if (!getResponse.isExists()) {
return builder.build();
}
XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
while (parser.nextToken() != START_ARRAY) {
// forward
}
while (parser.nextToken() != END_ARRAY) {
KnapsackState state = new KnapsackState();
builder.add(state.fromXContent(parser));
}
return builder.build();
} catch (Throwable t) {
logger.error("get settings failed", t);
return null;
}
}
@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
buildColumnInfo(tableMetadata, columnNames, columnTypes);
ImmutableList.Builder<FieldSchema> partitionKeys = ImmutableList.builder();
ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder();
List<String> names = columnNames.build();
List<String> typeNames =
columnTypes
.build()
.stream()
.map(HiveType::toHiveType)
.map(HiveType::getHiveTypeName)
.collect(toList());
for (int i = 0; i < names.size(); i++) {
if (tableMetadata.getColumns().get(i).isPartitionKey()) {
partitionKeys.add(new FieldSchema(names.get(i), typeNames.get(i), null));
} else {
columns.add(new FieldSchema(names.get(i), typeNames.get(i), null));
}
}
Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);
SerDeInfo serdeInfo = new SerDeInfo();
serdeInfo.setName(tableName);
serdeInfo.setSerializationLib(hiveStorageFormat.getSerDe());
StorageDescriptor sd = new StorageDescriptor();
sd.setLocation(targetPath.toString());
sd.setCols(columns.build());
sd.setSerdeInfo(serdeInfo);
sd.setInputFormat(hiveStorageFormat.getInputFormat());
sd.setOutputFormat(hiveStorageFormat.getOutputFormat());
Table table = new Table();
table.setDbName(schemaName);
table.setTableName(tableName);
table.setOwner(tableMetadata.getOwner());
table.setTableType(TableType.MANAGED_TABLE.toString());
String tableComment = "Created by Presto";
table.setParameters(ImmutableMap.of("comment", tableComment));
table.setPartitionKeys(partitionKeys.build());
table.setSd(sd);
metastore.createTable(table);
}
private ImmutableList<ColumnIdent> getPrimaryKey() {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
Object pKeys = metaMap.get("primary_keys");
if (pKeys != null) {
if (pKeys instanceof String) {
builder.add(ColumnIdent.fromPath((String) pKeys));
return builder.build();
} else if (pKeys instanceof Collection) {
Collection keys = (Collection) pKeys;
if (!keys.isEmpty()) {
for (Object pkey : keys) {
builder.add(ColumnIdent.fromPath(pkey.toString()));
}
return builder.build();
}
}
}
}
if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
hasAutoGeneratedPrimaryKey = true;
return ImmutableList.of(ID_IDENT);
}
return ImmutableList.of();
}
private static ImmutableList<Token> buildTokens(List<Tok> toks) {
ImmutableList.Builder<Token> tokens = ImmutableList.builder();
int k = 0;
int kN = toks.size();
while (k < kN) {
// Remaining non-tokens before the token go here.
ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder();
while (!toks.get(k).isToken()) {
toksBefore.add(toks.get(k++));
}
Tok tok = toks.get(k++);
// Non-tokens starting on the same line go here too.
ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder();
while (k < kN && !"\n".equals(toks.get(k).getText()) && !toks.get(k).isToken()) {
Tok nonTokenAfter = toks.get(k++);
toksAfter.add(nonTokenAfter);
if (nonTokenAfter.getText().contains("\n")) {
break;
}
}
tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
}
return tokens.build();
}
private <T> CachedRuleSource doExtract(final Class<T> source) {
final ModelType<T> type = ModelType.of(source);
DefaultMethodModelRuleExtractionContext context =
new DefaultMethodModelRuleExtractionContext(type, this);
// TODO - exceptions thrown here should point to some extensive documentation on the concept of
// class rule sources
StructSchema<T> schema = getSchema(source, context);
if (schema == null) {
throw new InvalidModelRuleDeclarationException(context.problems.format());
}
// sort for determinism
Set<Method> methods = new TreeSet<Method>(Ordering.usingToString());
methods.addAll(Arrays.asList(source.getDeclaredMethods()));
ImmutableList.Builder<ModelProperty<?>> implicitInputs = ImmutableList.builder();
ModelProperty<?> target = null;
for (ModelProperty<?> property : schema.getProperties()) {
if (property.isAnnotationPresent(RuleTarget.class)) {
target = property;
} else if (property.isAnnotationPresent(RuleInput.class)
&& !(property.getSchema() instanceof ScalarValueSchema)) {
implicitInputs.add(property);
}
for (WeaklyTypeReferencingMethod<?, ?> method : property.getAccessors()) {
methods.remove(method.getMethod());
}
}
ImmutableList.Builder<ExtractedRuleDetails> rules = ImmutableList.builder();
for (Method method : methods) {
MethodRuleDefinition<?, ?> ruleDefinition =
DefaultMethodRuleDefinition.create(source, method);
ExtractedModelRule rule = getMethodHandler(ruleDefinition, method, context);
if (rule != null) {
rules.add(new ExtractedRuleDetails(ruleDefinition, rule));
}
}
if (context.hasProblems()) {
throw new InvalidModelRuleDeclarationException(context.problems.format());
}
StructBindings<T> bindings = structBindingsStore.getBindings(schema);
if (schema.getProperties().isEmpty()) {
return new StatelessRuleSource(
rules.build(),
Modifier.isAbstract(source.getModifiers())
? new AbstractRuleSourceFactory<T>(schema, bindings, proxyFactory)
: new ConcreteRuleSourceFactory<T>(type));
} else {
return new ParameterizedRuleSource(
rules.build(), target, implicitInputs.build(), schema, bindings, proxyFactory);
}
}
private void assertCompDir(Path compDir, Optional<String> failure) throws Exception {
ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath());
CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig()));
// Build up the paths to various files the archive step will use.
ImmutableList<String> compiler =
platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver()));
Path output = filesystem.resolve(Paths.get("output.o"));
Path relativeInput = Paths.get("input.c");
Path input = filesystem.resolve(relativeInput);
filesystem.writeContentsToPath("int main() {}", relativeInput);
ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder();
preprocessorCommand.addAll(compiler);
ImmutableList.Builder<String> compilerCommand = ImmutableList.builder();
compilerCommand.addAll(compiler);
compilerCommand.add("-g");
DebugPathSanitizer sanitizer =
new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of());
// Build an archive step.
CxxPreprocessAndCompileStep step =
new CxxPreprocessAndCompileStep(
CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO,
output,
relativeInput,
CxxSource.Type.C,
Optional.of(preprocessorCommand.build()),
Optional.of(compilerCommand.build()),
ImmutableMap.<Path, Path>of(),
sanitizer);
// Execute the archive step and verify it ran successfully.
ExecutionContext executionContext =
TestExecutionContext.newBuilder()
.setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath()))
.build();
TestConsole console = (TestConsole) executionContext.getConsole();
int exitCode = step.execute(executionContext);
if (failure.isPresent()) {
assertNotEquals("compile step succeeded", 0, exitCode);
assertThat(
console.getTextWrittenToStdErr(),
console.getTextWrittenToStdErr(),
Matchers.containsString(failure.get()));
} else {
assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode);
// Verify that we find the expected compilation dir embedded in the file.
String contents = new String(Files.readAllBytes(output));
assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory()));
}
// Cleanup.
Files.delete(input);
Files.deleteIfExists(output);
}
private ImmutableList<SourcePath> generateCCompilation(ImmutableList<SourcePath> cInput) {
ImmutableList.Builder<SourcePath> objects = ImmutableList.builder();
ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder();
cCompileFlags.addAll(ocamlContext.getCCompileFlags());
cCompileFlags.addAll(ocamlContext.getCommonCFlags());
CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput();
for (SourcePath cSrc : cInput) {
String name = pathResolver.getAbsolutePath(cSrc).toFile().getName();
BuildTarget target = createCCompileBuildTarget(params.getBuildTarget(), name);
BuildRuleParams cCompileParams =
params.copyWithChanges(
target,
/* declaredDeps */ Suppliers.ofInstance(
ImmutableSortedSet.<BuildRule>naturalOrder()
// Depend on the rule that generates the sources and headers we're compiling.
.addAll(
pathResolver.filterBuildRuleInputs(
ImmutableList.<SourcePath>builder()
.add(cSrc)
.addAll(
cxxPreprocessorInput
.getIncludes()
.getNameToPathMap()
.values())
.build()))
// Also add in extra deps from the preprocessor input, such as the symlink
// tree rules.
.addAll(
BuildRules.toBuildRulesFor(
params.getBuildTarget(), resolver, cxxPreprocessorInput.getRules()))
.addAll(params.getDeclaredDeps().get())
.build()),
/* extraDeps */ params.getExtraDeps());
Path outputPath = ocamlContext.getCOutput(pathResolver.getRelativePath(cSrc));
OCamlCCompile compileRule =
new OCamlCCompile(
cCompileParams,
pathResolver,
new OCamlCCompileStep.Args(
cCompiler.getEnvironment(pathResolver),
cCompiler.getCommandPrefix(pathResolver),
ocamlContext.getOcamlCompiler().get(),
outputPath,
cSrc,
cCompileFlags.build(),
ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap())));
resolver.addToIndex(compileRule);
objects.add(new BuildTargetSourcePath(compileRule.getBuildTarget()));
}
return objects.build();
}
@Override
public HiveOutputTableHandle beginCreateTable(
ConnectorSession session, ConnectorTableMetadata tableMetadata) {
verifyJvmTimeZone();
checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
// get the root directory for the database
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
buildColumnInfo(tableMetadata, columnNames, columnTypes);
Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);
if (!useTemporaryDirectory(targetPath)) {
return new HiveOutputTableHandle(
connectorId,
schemaName,
tableName,
columnNames.build(),
columnTypes.build(),
tableMetadata.getOwner(),
targetPath.toString(),
targetPath.toString(),
session,
hiveStorageFormat);
}
// use a per-user temporary directory to avoid permission problems
// TODO: this should use Hadoop UserGroupInformation
String temporaryPrefix = "/tmp/presto-" + StandardSystemProperty.USER_NAME.value();
// create a temporary directory on the same filesystem
Path temporaryRoot = new Path(targetPath, temporaryPrefix);
Path temporaryPath = new Path(temporaryRoot, randomUUID().toString());
createDirectories(temporaryPath);
return new HiveOutputTableHandle(
connectorId,
schemaName,
tableName,
columnNames.build(),
columnTypes.build(),
tableMetadata.getOwner(),
targetPath.toString(),
temporaryPath.toString(),
session,
hiveStorageFormat);
}
/**
* Splits a constructor in two if the constructor contains a parameter of type UUID. The first
* constructor is unchanged, while the second constructor has all UUID parameters converted into
* strings. If the constructor does not have a UUID parameter, then the original constructor is
* returned.
*/
protected static ImmutableList<RequestConstructor> splitUuidConstructor(
final RequestConstructor constructor) {
final ImmutableList.Builder<RequestConstructor> builder = ImmutableList.builder();
builder.add(constructor);
if (!containsType(constructor.getParameters(), "UUID")) {
return builder.build();
}
builder.add(convertUuidConstructorToStringConstructor(constructor));
return builder.build();
}
static {
ImmutableList.Builder<SqlAggFunction> builder = ImmutableList.builder();
builder.add(SqlStdOperatorTable.COUNT);
builder.add(SqlStdOperatorTable.SUM);
builder.add(SqlStdOperatorTable.SUM0);
builder.add(SqlStdOperatorTable.MIN);
builder.add(SqlStdOperatorTable.MAX);
AGG_FUNCS = builder.build();
builder.add(SqlStdOperatorTable.SINGLE_VALUE);
MYSQL_AGG_FUNCS = builder.build();
}
@Override
protected RelationPlan visitUnnest(Unnest node, Void context) {
TupleDescriptor descriptor = analysis.getOutputDescriptor(node);
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder();
for (Field field : descriptor.getVisibleFields()) {
Symbol symbol = symbolAllocator.newSymbol(field);
outputSymbolsBuilder.add(symbol);
}
List<Symbol> unnestedSymbols = outputSymbolsBuilder.build();
// If we got here, then we must be unnesting a constant, and not be in a join (where there could
// be column references)
ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder();
ImmutableList.Builder<Expression> values = ImmutableList.builder();
ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder();
Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator();
for (Expression expression : node.getExpressions()) {
Object constantValue =
evaluateConstantExpression(expression, analysis.getCoercions(), metadata, session);
Type type = analysis.getType(expression);
values.add(LiteralInterpreter.toExpression(constantValue, type));
Symbol inputSymbol = symbolAllocator.newSymbol(expression, type);
argumentSymbols.add(inputSymbol);
if (type instanceof ArrayType) {
unnestSymbols.put(inputSymbol, ImmutableList.of(unnestedSymbolsIterator.next()));
} else if (type instanceof MapType) {
unnestSymbols.put(
inputSymbol,
ImmutableList.of(unnestedSymbolsIterator.next(), unnestedSymbolsIterator.next()));
} else {
throw new IllegalArgumentException("Unsupported type for UNNEST: " + type);
}
}
Optional<Symbol> ordinalitySymbol =
node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty();
checkState(
!unnestedSymbolsIterator.hasNext(),
"Not all output symbols were matched with input symbols");
ValuesNode valuesNode =
new ValuesNode(
idAllocator.getNextId(),
argumentSymbols.build(),
ImmutableList.<List<Expression>>of(values.build()));
UnnestNode unnestNode =
new UnnestNode(
idAllocator.getNextId(),
valuesNode,
ImmutableList.<Symbol>of(),
unnestSymbols.build(),
ordinalitySymbol);
return new RelationPlan(unnestNode, descriptor, unnestedSymbols, Optional.empty());
}
public SetMessagesResponse build() {
return new SetMessagesResponse(
accountId,
oldState,
newState,
created.build(),
updated.build(),
destroyed.build(),
notCreated.build(),
notUpdated.build(),
notDestroyed.build());
}
@Override
public void generateMethods(
ClassDefinition classDefinition,
CallSiteBinder callSiteBinder,
RowExpression filter,
List<RowExpression> projections) {
CachedInstanceBinder cachedInstanceBinder =
new CachedInstanceBinder(classDefinition, callSiteBinder);
ImmutableList.Builder<MethodDefinition> projectMethods = ImmutableList.builder();
ImmutableList.Builder<MethodDefinition> projectColumnarMethods = ImmutableList.builder();
ImmutableList.Builder<MethodDefinition> projectDictionaryMethods = ImmutableList.builder();
for (int i = 0; i < projections.size(); i++) {
MethodDefinition project =
generateProjectMethod(
classDefinition,
callSiteBinder,
cachedInstanceBinder,
"project_" + i,
projections.get(i));
MethodDefinition projectColumnar =
generateProjectColumnarMethod(
classDefinition, callSiteBinder, "projectColumnar_" + i, projections.get(i), project);
MethodDefinition projectDictionary =
generateProjectDictionaryMethod(
classDefinition,
"projectDictionary_" + i,
projections.get(i),
project,
projectColumnar);
projectMethods.add(project);
projectColumnarMethods.add(projectColumnar);
projectDictionaryMethods.add(projectDictionary);
}
List<MethodDefinition> projectMethodDefinitions = projectMethods.build();
List<MethodDefinition> projectColumnarMethodDefinitions = projectColumnarMethods.build();
List<MethodDefinition> projectDictionaryMethodDefinitions = projectDictionaryMethods.build();
generateProcessMethod(classDefinition, filter, projections, projectMethodDefinitions);
generateGetNonLazyPageMethod(classDefinition, filter, projections);
generateProcessColumnarMethod(classDefinition, projections, projectColumnarMethodDefinitions);
generateProcessColumnarDictionaryMethod(
classDefinition,
projections,
projectColumnarMethodDefinitions,
projectDictionaryMethodDefinitions);
generateFilterPageMethod(classDefinition, filter);
generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filter);
generateConstructor(classDefinition, cachedInstanceBinder, projections.size());
}
public ParquetPageSource(
ParquetReader parquetReader,
ParquetDataSource dataSource,
MessageType fileSchema,
MessageType requestedSchema,
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
TypeManager typeManager,
boolean useParquetColumnNames) {
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(columns, "columns is null");
requireNonNull(effectivePredicate, "effectivePredicate is null");
this.parquetReader = parquetReader;
this.dataSource = dataSource;
this.requestedSchema = requestedSchema;
this.totalBytes = totalBytes;
int size = columns.size();
this.constantBlocks = new Block[size];
this.hiveColumnIndexes = new int[size];
ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
for (int columnIndex = 0; columnIndex < size; columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
checkState(column.getColumnType() == REGULAR, "column type must be regular");
String name = column.getName();
Type type = typeManager.getType(column.getTypeSignature());
namesBuilder.add(name);
typesBuilder.add(type);
hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();
if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
BlockBuilder blockBuilder =
type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
blockBuilder.appendNull();
}
constantBlocks[columnIndex] = blockBuilder.build();
}
}
types = typesBuilder.build();
columnNames = namesBuilder.build();
}
public OperatorFactory createTableScanOperator(
Session session, int operatorId, String tableName, String... columnNames) {
checkArgument(session.getCatalog().isPresent(), "catalog not set");
checkArgument(session.getSchema().isPresent(), "schema not set");
// look up the table
QualifiedTableName qualifiedTableName =
new QualifiedTableName(session.getCatalog().get(), session.getSchema().get(), tableName);
TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null);
checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName);
// lookup the columns
Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle);
ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypesBuilder = ImmutableList.builder();
for (String columnName : columnNames) {
ColumnHandle columnHandle = allColumnHandles.get(columnName);
checkArgument(
columnHandle != null, "Table %s does not have a column %s", tableName, columnName);
columnHandlesBuilder.add(columnHandle);
ColumnMetadata columnMetadata =
metadata.getColumnMetadata(session, tableHandle, columnHandle);
columnTypesBuilder.add(columnMetadata.getType());
}
List<ColumnHandle> columnHandles = columnHandlesBuilder.build();
List<Type> columnTypes = columnTypesBuilder.build();
// get the split for this table
List<TableLayoutResult> layouts =
metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty());
Split split = getLocalQuerySplit(layouts.get(0).getLayout().getHandle());
return new OperatorFactory() {
@Override
public List<Type> getTypes() {
return columnTypes;
}
@Override
public Operator createOperator(DriverContext driverContext) {
OperatorContext operatorContext =
driverContext.addOperatorContext(operatorId, "BenchmarkSource");
ConnectorPageSource pageSource =
pageSourceManager.createPageSource(session, split, columnHandles);
return new PageSourceOperator(pageSource, columnTypes, operatorContext);
}
@Override
public void close() {}
};
}
IndexShardRoutingTable(
ShardId shardId, List<ShardRouting> shards, boolean primaryAllocatedPostApi) {
this.shardId = shardId;
this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt());
this.shards = ImmutableList.copyOf(shards);
this.primaryAllocatedPostApi = primaryAllocatedPostApi;
ShardRouting primary = null;
ImmutableList.Builder<ShardRouting> replicas = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> activeShards = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> assignedShards = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> allInitializingShards = ImmutableList.builder();
boolean allShardsStarted = true;
for (ShardRouting shard : shards) {
if (shard.primary()) {
primary = shard;
} else {
replicas.add(shard);
}
if (shard.active()) {
activeShards.add(shard);
}
if (shard.initializing()) {
allInitializingShards.add(shard);
}
if (shard.relocating()) {
// create the target initializing shard routing on the node the shard is relocating to
allInitializingShards.add(shard.buildTargetRelocatingShard());
}
if (shard.assignedToNode()) {
assignedShards.add(shard);
}
if (shard.state() != ShardRoutingState.STARTED) {
allShardsStarted = false;
}
}
this.allShardsStarted = allShardsStarted;
this.primary = primary;
if (primary != null) {
this.primaryAsList = ImmutableList.of(primary);
} else {
this.primaryAsList = ImmutableList.of();
}
this.replicas = replicas.build();
this.activeShards = activeShards.build();
this.assignedShards = assignedShards.build();
this.allInitializingShards = allInitializingShards.build();
}
/** @return the current stack trace as a list of functions. */
public ImmutableList<BaseFunction> getStackTrace() {
ImmutableList.Builder<BaseFunction> builder = new ImmutableList.Builder<>();
for (Continuation k = continuation; k != null; k = k.continuation) {
builder.add(k.function);
}
return builder.build().reverse();
}
/**
* Parse a string delimited by comma into a list of object instances depending on
*
* <pre>itemParser</pre>
*
* .
*
* @param str String delimited by comma
* @param itemParser A function to transform a string to an object.
* @param <T> Type to be transformed from a string
* @return List of object instances
*/
static <T> List<T> parseList(String str, Function<String, T> itemParser) {
if (!str.contains(",")) { // if just one item
return ImmutableList.of(itemParser.apply(str));
}
final ImmutableList.Builder<T> fields = ImmutableList.builder();
final Stack<Character> stack = new Stack<>();
int paramStartIdx = 0;
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);
if (c == '<' || c == '[' || c == '(') {
stack.push(c);
} else if (c == '>') {
Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", str);
} else if (c == ']') {
Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", str);
} else if (c == ')') {
Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", str);
} else if (c == ',') {
if (stack.isEmpty()) { // ensure outermost type parameters
fields.add(itemParser.apply(str.substring(paramStartIdx, i)));
paramStartIdx = i + 1;
}
}
}
Assert.assertCondition(stack.empty(), "Bad signature: '%s'", str);
if (paramStartIdx < str.length()) {
fields.add(itemParser.apply(str.substring(paramStartIdx, str.length())));
}
return fields.build();
}
/**
* Gets the list of Arguments needed to create the request constructor. This includes all non-void
* required parameters, and arguments described within the request header.
*/
@Override
public ImmutableList<Arguments> toConstructorArgumentsList(final Ds3Request ds3Request) {
final ImmutableList.Builder<Arguments> builder = ImmutableList.builder();
builder.addAll(getRequiredArgsFromRequestHeader(ds3Request));
builder.addAll(removeVoidArguments(toArgumentsList(ds3Request.getRequiredQueryParams())));
return builder.build();
}
public static ImmutableList<Mod> split(String cs) {
ImmutableList.Builder<Mod> builder = ImmutableList.builder();
for (String ms : Splitters.csplitter(cs)) {
builder.add(Mod.parseMod(ms));
}
return builder.build();
}
private List<JavaType> convertAsmTypes(org.objectweb.asm.Type[] asmTypes) {
ImmutableList.Builder<JavaType> result = ImmutableList.builder();
for (org.objectweb.asm.Type asmType : asmTypes) {
result.add(convertAsmType(asmType));
}
return result.build();
}
/**
* @author gabizou - February 7th, 2016
* <p>This will short circuit all other patches such that we control the entities being loaded
* by chunkloading and can throw our bulk entity event. This will bypass Forge's hook for
* individual entity events, but the SpongeModEventManager will still successfully throw the
* appropriate event and cancel the entities otherwise contained.
* @param entities The entities being loaded
* @param callbackInfo The callback info
*/
@Final
@Inject(method = "loadEntities", at = @At("HEAD"), cancellable = true)
private void spongeLoadEntities(
Collection<net.minecraft.entity.Entity> entities, CallbackInfo callbackInfo) {
if (entities.isEmpty()) {
// just return, no entities to load!
callbackInfo.cancel();
return;
}
List<Entity> entityList = new ArrayList<>();
ImmutableList.Builder<EntitySnapshot> snapshotBuilder = ImmutableList.builder();
for (net.minecraft.entity.Entity entity : entities) {
entityList.add((Entity) entity);
snapshotBuilder.add(((Entity) entity).createSnapshot());
}
SpawnCause cause = SpawnCause.builder().type(InternalSpawnTypes.CHUNK_LOAD).build();
List<NamedCause> causes = new ArrayList<>();
causes.add(NamedCause.source(cause));
causes.add(NamedCause.of("World", this));
SpawnEntityEvent.ChunkLoad chunkLoad =
SpongeEventFactory.createSpawnEntityEventChunkLoad(
Cause.of(causes), entityList, snapshotBuilder.build(), this);
SpongeImpl.postEvent(chunkLoad);
if (!chunkLoad.isCancelled()) {
for (Entity successful : chunkLoad.getEntities()) {
this.loadedEntityList.add((net.minecraft.entity.Entity) successful);
this.onEntityAdded((net.minecraft.entity.Entity) successful);
}
}
callbackInfo.cancel();
}
private void testCorrectnessOfErrorFunction(List<Number> inputList) throws Exception {
int inRange = 0;
int numberOfRuns = 1000;
double sampleRatio = 1 / (double) WEIGHT;
double actual = getExpectedValue(inputList);
Random rand = new Random(1);
for (int i = 0; i < numberOfRuns; i++) {
// Compute Sampled Value using sampledList (numberOfRuns times)
ImmutableList.Builder<Number> sampledList = ImmutableList.builder();
for (Number x : inputList) {
if (rand.nextDouble() < sampleRatio) {
sampledList.add(x);
}
}
BlockBuilder builder = getType().createBlockBuilder(new BlockBuilderStatus());
for (Number sample : sampledList.build()) {
if (getType() == BIGINT) {
BIGINT.writeLong(builder, sample.longValue());
} else if (getType() == DOUBLE) {
DOUBLE.writeDouble(builder, sample.doubleValue());
} else {
throw new AssertionError("Can only handle longs and doubles");
}
}
Page page = new Page(builder.build());
page = OperatorAssertion.appendSampleWeight(ImmutableList.of(page), WEIGHT).get(0);
Accumulator accumulator =
getFunction()
.bind(
ImmutableList.of(0),
Optional.<Integer>absent(),
Optional.of(page.getChannelCount() - 1),
getConfidence())
.createAccumulator();
accumulator.addInput(page);
Block result = accumulator.evaluateFinal();
String approxValue =
BlockAssertions.toValues(accumulator.getFinalType(), result).get(0).toString();
double approx = Double.parseDouble(approxValue.split(" ")[0]);
double error = Double.parseDouble(approxValue.split(" ")[2]);
// Check if actual answer lies within [approxAnswer - error, approxAnswer + error]
if (Math.abs(approx - actual) <= error) {
inRange++;
}
}
BinomialDistribution binomial = new BinomialDistribution(numberOfRuns, getConfidence());
int lowerBound = binomial.inverseCumulativeProbability(0.01);
int upperBound = binomial.inverseCumulativeProbability(0.99);
assertTrue(
lowerBound < inRange && inRange < upperBound,
String.format(
"%d out of %d passed. Expected [%d, %d]",
inRange, numberOfRuns, lowerBound, upperBound));
}
private int executeCCompilation(
ExecutionContext context, ImmutableList.Builder<Path> linkerInputs)
throws IOException, InterruptedException {
ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder();
cCompileFlags.addAll(ocamlContext.getCCompileFlags());
cCompileFlags.addAll(ocamlContext.getCommonCFlags());
CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput();
for (SourcePath cSrc : ocamlContext.getCInput()) {
Path outputPath = ocamlContext.getCOutput(resolver.getAbsolutePath(cSrc));
linkerInputs.add(outputPath);
Step compileStep =
new OCamlCCompileStep(
resolver,
filesystem.getRootPath(),
new OCamlCCompileStep.Args(
cCompilerEnvironment,
cCompiler,
ocamlContext.getOcamlCompiler().get(),
outputPath,
cSrc,
cCompileFlags.build(),
ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap())));
int compileExitCode = compileStep.execute(context);
if (compileExitCode != 0) {
return compileExitCode;
}
}
return 0;
}
private void logEndpoints() {
final StringBuilder stringBuilder = new StringBuilder(1024).append("\n\n");
final ImmutableList.Builder<Class<?>> builder = ImmutableList.builder();
for (Object o : config.getSingletons()) {
if (o.getClass().isAnnotationPresent(Path.class)) {
builder.add(o.getClass());
}
}
for (Class<?> klass : config.getClasses()) {
if (klass.isAnnotationPresent(Path.class)) {
builder.add(klass);
}
}
for (Class<?> klass : builder.build()) {
final String path = klass.getAnnotation(Path.class).value();
final ImmutableList.Builder<String> endpoints = ImmutableList.builder();
for (AnnotatedMethod method : annotatedMethods(klass)) {
for (HttpMethod verb : method.getMetaMethodAnnotations(HttpMethod.class)) {
endpoints.add(
String.format(" %-7s %s (%s)", verb.value(), path, klass.getCanonicalName()));
}
}
for (String line : Ordering.natural().sortedCopy(endpoints.build())) {
stringBuilder.append(line).append('\n');
}
}
LOG.info(stringBuilder.toString());
}
public static Optional<TypeSignature> getCommonSuperTypeSignature(
TypeSignature firstType, TypeSignature secondType) {
// Special handling for UnknownType is necessary because we forbid cast between types with
// different number of type parameters.
// Without this, cast from null to map<bigint, bigint> will not be allowed.
if (UnknownType.NAME.equals(firstType.getBase())) {
return Optional.of(secondType);
}
if (UnknownType.NAME.equals(secondType.getBase())) {
return Optional.of(firstType);
}
List<TypeSignatureParameter> firstTypeTypeParameters = firstType.getParameters();
List<TypeSignatureParameter> secondTypeTypeParameters = secondType.getParameters();
if (firstTypeTypeParameters.size() != secondTypeTypeParameters.size()) {
return Optional.empty();
}
Optional<String> commonSuperTypeBase =
getCommonSuperTypeBase(firstType.getBase(), secondType.getBase());
if (!commonSuperTypeBase.isPresent()) {
return Optional.empty();
}
ImmutableList.Builder<TypeSignatureParameter> typeParameters = ImmutableList.builder();
for (int i = 0; i < firstTypeTypeParameters.size(); i++) {
TypeSignatureParameter firstParameter = firstTypeTypeParameters.get(i);
TypeSignatureParameter secondParameter = secondTypeTypeParameters.get(i);
if (firstParameter.getKind() == secondParameter.getKind()
&& firstParameter.getKind() == ParameterKind.LONG_LITERAL) {
typeParameters.add(
TypeSignatureParameter.of(
Math.max(firstParameter.getLongLiteral(), secondParameter.getLongLiteral())));
} else if (isCovariantParameterPosition(commonSuperTypeBase.get(), i)) {
Optional<TypeSignature> firstParameterSignature =
firstParameter.getTypeSignatureOrNamedTypeSignature();
Optional<TypeSignature> secondParameterSignature =
secondParameter.getTypeSignatureOrNamedTypeSignature();
if (!firstParameterSignature.isPresent() || !secondParameterSignature.isPresent()) {
return Optional.empty();
}
Optional<TypeSignature> commonSuperType =
getCommonSuperTypeSignature(
firstParameterSignature.get(), secondParameterSignature.get());
if (!commonSuperType.isPresent()) {
return Optional.empty();
}
typeParameters.add(TypeSignatureParameter.of(commonSuperType.get()));
} else {
if (!firstParameter.equals(secondParameter)) {
return Optional.empty();
}
typeParameters.add(firstParameter);
}
}
return Optional.of(new TypeSignature(commonSuperTypeBase.get(), typeParameters.build()));
}
protected List<String> buildArguments() {
ImmutableList.Builder<String> builder = ImmutableList.builder();
builder.add("-host").add(settings.getHost());
builder.add("-port").add(String.format("%s", launcherPort));
if (settings.getDisplay() != null && settings.getDisplay() > 0) {
builder.add("-display").add(String.format(":%s", settings.getDisplay()));
}
if (settings.logging().getLevel() != Level.OFF) {
builder.add("-console"); // TODO(andreastt): Allow for file logging
builder
.add("-verbosity")
.add(toLauncherLoggingLevel(settings.logging().getLevel()).toString());
}
if (settings.getProduct() != OperaProduct.ALL) {
builder.add("-profile").add(settings.getProduct().getDescriptionString());
}
if (settings.getBackend() != null && !settings.getBackend().isEmpty()) {
builder.add("-backend").add(settings.getBackend());
}
if (settings.hasDetach()) {
builder.add("-noquit");
}
builder.add("-bin").add(settings.getBinary().getAbsolutePath());
// The launcher will pass on any extra arguments to Opera
for (OperaArgument argument : settings.arguments()) {
builder.add(settings.arguments().sign() + argument.getArgument());
if (argument.getValue() != null && !argument.getValue().isEmpty()) {
builder.add(argument.getValue());
}
}
return builder.build();
}
private ImmutableList<Artifact> generatedOutputArtifacts(FileType newFileType) {
ImmutableList.Builder<Artifact> builder = new ImmutableList.Builder<>();
for (Artifact protoFile : getFilteredProtoSources()) {
String protoFileName = FileSystemUtils.removeExtension(protoFile.getFilename());
String generatedOutputName;
if (attributes.outputsCpp()) {
generatedOutputName = protoFileName;
} else if (usesProtobufLibrary()) {
// The protobuf library generates filenames with some slight modifications.
generatedOutputName = generateProtobufFilename(protoFileName);
} else {
String lowerUnderscoreBaseName = protoFileName.replace('-', '_').toLowerCase();
generatedOutputName = LOWER_UNDERSCORE.to(UPPER_CAMEL, lowerUnderscoreBaseName);
}
PathFragment generatedFilePath =
new PathFragment(
protoFile.getRootRelativePath().getParentDirectory(),
new PathFragment(generatedOutputName));
PathFragment outputFile =
FileSystemUtils.appendExtension(generatedFilePath, newFileType.getExtensions().get(0));
if (outputFile != null) {
builder.add(
ruleContext.getUniqueDirectoryArtifact(
UNIQUE_DIRECTORY_NAME, outputFile, ruleContext.getBinOrGenfilesDirectory()));
}
}
return builder.build();
}
@Override
public Records getNext(int maxNumberOfRecords) {
ensureBuffered();
if (!it.hasNext() && buffer.isEndOfShard()) {
return new Records(ImmutableList.<Record>of(), true);
}
ImmutableList.Builder<Record> recs = new ImmutableList.Builder<>();
int recsSize = 0;
while (recsSize < maxNumberOfRecords) {
if (it.hasNext()) {
recs.add(it.next());
recsSize++;
} else if (!it.hasNext() && !buffer.isEndOfShard()) {
rebuffer();
// No more data in shard.
if (!it.hasNext()) {
break;
}
} else {
// No more records, end of shard.
break;
}
}
return new Records(recs.build(), false);
}
