@Override
public void configure(YarnAppmasterConfigurer master) throws Exception {
master
.appmasterClass(
syap.getAppmasterClass() != null ? syap.getAppmasterClass() : appmasterClass)
.containerCommands(createContainerCommands(syalcp));
MasterContainerAllocatorConfigurer containerAllocatorConfigurer =
master.withContainerAllocator();
containerAllocatorConfigurer
.locality(syalcp.isLocality())
.memory(syarp.getMemory())
.priority(syarp.getPriority())
.virtualCores(syarp.getVirtualCores());
if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) {
for (java.util.Map.Entry<String, ContainerClustersProperties> entry :
syap.getContainercluster().getClusters().entrySet()) {
SpringYarnAppmasterResourceProperties resource = entry.getValue().getResource();
SpringYarnAppmasterLaunchContextProperties launchcontext =
entry.getValue().getLaunchcontext();
master.containerCommands(entry.getKey(), createContainerCommands(launchcontext));
containerAllocatorConfigurer
.withCollection(entry.getKey())
.priority(resource != null ? resource.getPriority() : null)
.memory(resource != null ? resource.getMemory() : null)
.virtualCores(resource != null ? resource.getVirtualCores() : null)
.locality(launchcontext != null ? launchcontext.isLocality() : false);
}
}
}
@Override
public void configure(YarnEnvironmentConfigurer environment) throws Exception {
environment
.includeLocalSystemEnv(syalcp.isIncludeLocalSystemEnv())
.withClasspath()
.includeBaseDirectory(syalcp.isIncludeBaseDirectory())
.useYarnAppClasspath(syalcp.isUseYarnAppClasspath())
.useMapreduceAppClasspath(syalcp.isUseMapreduceAppClasspath())
.siteYarnAppClasspath(syp.getSiteYarnAppClasspath())
.siteMapreduceAppClasspath(syp.getSiteMapreduceAppClasspath())
.delimiter(syalcp.getPathSeparator())
.entries(syalcp.getContainerAppClasspath())
.entry(explodedEntryIfZip(syalcp));
if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) {
for (java.util.Map.Entry<String, ContainerClustersProperties> entry :
syap.getContainercluster().getClusters().entrySet()) {
SpringYarnAppmasterLaunchContextProperties props = entry.getValue().getLaunchcontext();
environment
.withClasspath(entry.getKey())
.includeBaseDirectory(props.isIncludeBaseDirectory())
.useYarnAppClasspath(props.isUseYarnAppClasspath())
.useMapreduceAppClasspath(props.isUseMapreduceAppClasspath())
.siteYarnAppClasspath(syp.getSiteYarnAppClasspath())
.siteMapreduceAppClasspath(syp.getSiteMapreduceAppClasspath())
.delimiter(props.getPathSeparator())
.entries(props.getContainerAppClasspath())
.entry(explodedEntryIfZip(props));
}
}
}
@Bean
@ConditionalOnMissingBean(AppmasterLauncherRunner.class)
@ConditionalOnBean(YarnAppmaster.class)
public AppmasterLauncherRunner appmasterLauncherRunner() {
AppmasterLauncherRunner runner = new AppmasterLauncherRunner();
runner.setWaitLatch(syap.isKeepContextAlive());
runner.setContainerCount(syap.getContainerCount());
return runner;
}
@Bean
@ConditionalOnMissingBean(LocalResourcesSelector.class)
public LocalResourcesSelector localResourcesSelector() {
Map<String, LocalResourcesSelector> selectors = new HashMap<String, LocalResourcesSelector>();
if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) {
for (java.util.Map.Entry<String, ContainerClustersProperties> entry :
syap.getContainercluster().getClusters().entrySet()) {
SpringYarnAppmasterLocalizerProperties props = entry.getValue().getLocalizer();
if (props == null) {
continue;
}
BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.CONTAINER);
if (StringUtils.hasText(props.getZipPattern())) {
selector.setZipArchivePattern(props.getZipPattern());
}
if (props.getPropertiesNames() != null) {
selector.setPropertiesNames(props.getPropertiesNames());
}
if (props.getPropertiesSuffixes() != null) {
selector.setPropertiesSuffixes(props.getPropertiesSuffixes());
}
selector.addPatterns(props.getPatterns());
selectors.put(entry.getKey(), selector);
}
}
BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.CONTAINER);
if (StringUtils.hasText(syalp.getZipPattern())) {
selector.setZipArchivePattern(syalp.getZipPattern());
}
if (syalp.getPropertiesNames() != null) {
selector.setPropertiesNames(syalp.getPropertiesNames());
}
if (syalp.getPropertiesSuffixes() != null) {
selector.setPropertiesSuffixes(syalp.getPropertiesSuffixes());
}
selector.addPatterns(syalp.getPatterns());
return new BootMultiLocalResourcesSelector(selector, selectors);
}
@Override
public void configure(YarnResourceLocalizerConfigurer localizer) throws Exception {
String applicationDir = SpringYarnBootUtils.resolveApplicationdir(syp);
localizer.stagingDirectory(syp.getStagingDir());
LocalResourcesHdfsConfigurer withHdfs = localizer.withHdfs();
for (Entry e : localResourcesSelector.select(applicationDir != null ? applicationDir : "/")) {
withHdfs.hdfs(e.getPath(), e.getType(), applicationDir == null);
}
if (syap.getContainercluster() != null
&& localResourcesSelector instanceof MultiLocalResourcesSelector
&& syap.getContainercluster().getClusters() != null) {
MultiLocalResourcesSelector selector =
((MultiLocalResourcesSelector) localResourcesSelector);
for (java.util.Map.Entry<String, ContainerClustersProperties> entry :
syap.getContainercluster().getClusters().entrySet()) {
withHdfs = localizer.withHdfs(entry.getKey());
for (Entry e :
selector.select(entry.getKey(), applicationDir != null ? applicationDir : "/")) {
withHdfs.hdfs(e.getPath(), e.getType(), applicationDir == null);
}
}
}
}
@Bean
public ProjectionDataRegistry projectionDataRegistry() {
Map<String, ProjectionData> projections = new HashMap<String, ProjectionData>();
Map<String, ContainerClustersProperties> clusterProps =
syap.getContainercluster().getClusters();
if (clusterProps != null) {
for (java.util.Map.Entry<String, ContainerClustersProperties> entry :
clusterProps.entrySet()) {
ProjectionData data =
new ProjectionData(
entry.getValue().getProjectionAny(),
entry.getValue().getProjectionHosts(),
entry.getValue().getProjectionRacks());
data.setType(entry.getValue().getProjectionType());
SpringYarnAppmasterResourceProperties resource = entry.getValue().getResource();
if (resource != null) {
data.setPriority(resource.getPriority());
}
projections.put(entry.getKey(), data);
}
}
return new ProjectionDataRegistry(projections);
}