private void mountFiles(String path, Class<?> clazz) { try { List<Resource> list = new ArrayList<>(); String packagePath = clazz.getPackage().getName().replace('.', '/'); PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); Resource[] res = resolver.getResources("classpath:" + packagePath + "/*.png"); if (res != null) { list.addAll(Arrays.asList(res)); } res = resolver.getResources("classpath:" + packagePath + "/*.gif"); if (res != null) { list.addAll(Arrays.asList(res)); } for (Resource resource : list) { URI uri = resource.getURI(); File file = new File(uri.toString()); mountResource( path + "/" + file.getName(), new SharedResourceReference(clazz, file.getName())); } } catch (Exception ex) { LoggingUtils.logUnexpectedException(LOGGER, "Couldn't mount files", ex); } }
/* * (non-Javadoc) * * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() */ @Override public void afterPropertiesSet() throws Exception { Set<String> l = new HashSet<String>(); String i18nPattern = basePath + "/**/*.properties"; String i18nFileREG = "(" + StringUtils.join(languages, '|') + ").properties$"; if (defaultLocale != null) { Locale.setDefault(this.defaultLocale); } try { Resource[] resources = applicationContext.getResources(i18nPattern); for (Resource resource : resources) { String path = resource.getURI().toString(); if (CapString.checkRegularMatch(path, i18nFileREG)) { path = path.replaceAll(i18nFileREG, "") .replaceAll(".*/i18n/", "classpath:/i18n/") // for windows .replaceAll(".*\\\\i18n\\\\", "classpath:\\\\i18n\\\\") .replaceAll("\\\\", "/"); if (!l.contains(path)) { l.add(path); logger.debug("set message path:" + path); } } } } catch (Exception e) { logger.error(e.getMessage(), e); } super.setBasenames(l.toArray(new String[l.size()])); }
@Test public void testWorkflow() throws Exception { GenericXmlApplicationContext ctx = new GenericXmlApplicationContext("/org/springframework/data/hadoop/batch/in-do-out.xml"); ctx.registerShutdownHook(); FileSystem fs = FileSystem.get(ctx.getBean(Configuration.class)); System.out.println("FS is " + fs.getClass().getName()); HdfsResourceLoader hrl = ctx.getBean(HdfsResourceLoader.class); Resource resource = hrl.getResource("/ide-test/output/word/"); assertTrue(ctx.isPrototype("script-tasklet")); fs.delete(new Path(resource.getURI().toString()), true); JobsTrigger.startJobs(ctx); Path p = new Path("/ide-test/output/word/"); Job job = (Job) ctx.getBean("mr-job"); Configuration c = job.getConfiguration(); FileSystem fs2 = p.getFileSystem(c); System.out.println("FS is " + fs2.getClass().getName()); fs2.exists(p); ctx.close(); }
/** * Compensate for the fact that a Resource <i>may</i> not be a File or even addressable through a * URI. If it is, we want the created StreamSource to read other resources relative to the * provided one. If it isn't, it loads from the default path. */ private static StreamSource createStreamSourceOnResource(Resource xslResource) throws IOException { try { String systemId = xslResource.getURI().toString(); return new StreamSource(xslResource.getInputStream(), systemId); } catch (IOException e) { return new StreamSource(xslResource.getInputStream()); } }
@Override public void afterPropertiesSet() throws IOException { toolBoxConfigurationPath = toolBoxConfigLocation.getFile().getAbsolutePath(); if (logger.isInfoEnabled()) { logger.info( "Resource loader path '{}' resolved to file '{}'", toolBoxConfigLocation.getURI(), toolBoxConfigurationPath); } }
/** {@inheritDoc} */ @Override public RouteBuilder load() { try { final Resource script = getScript(); final String bootstrapCode = IOUtils.toString(script.getInputStream()); log.debug( "Applying {}:{}{}", new Object[] {script.getURI(), Environment.NEWLINE, bootstrapCode}); return scriptEvaluator.configure(bootstrapCode); } catch (IOException e) { throw new LifecycleException(e.getLocalizedMessage(), e); } }
@Test public void jCacheCacheWithConfig() throws IOException { String cachingProviderFqn = MockCachingProvider.class.getName(); String configLocation = "org/springframework/boot/autoconfigure/cache/hazelcast-specific.xml"; load( JCacheCustomConfiguration.class, "spring.cache.type=jcache", "spring.cache.jcache.provider=" + cachingProviderFqn, "spring.cache.jcache.config=" + configLocation); JCacheCacheManager cacheManager = validateCacheManager(JCacheCacheManager.class); Resource configResource = new ClassPathResource(configLocation); assertThat(cacheManager.getCacheManager().getURI(), equalTo(configResource.getURI())); }
@Test public void infinispanAsJCacheWithConfig() throws IOException { String cachingProviderFqn = JCachingProvider.class.getName(); String configLocation = "infinispan.xml"; load( DefaultCacheConfiguration.class, "spring.cache.type=jcache", "spring.cache.jcache.provider=" + cachingProviderFqn, "spring.cache.jcache.config=" + configLocation); JCacheCacheManager cacheManager = validateCacheManager(JCacheCacheManager.class); Resource configResource = new ClassPathResource(configLocation); assertThat(cacheManager.getCacheManager().getURI(), equalTo(configResource.getURI())); }
@Test public void readXmlRootElementExternalEntityDisabled() throws Exception { Resource external = new ClassPathResource("external.txt", getClass()); String content = "<!DOCTYPE root SYSTEM \"http://192.168.28.42/1.jsp\" [" + " <!ELEMENT external ANY >\n" + " <!ENTITY ext SYSTEM \"" + external.getURI() + "\" >]>" + " <rootElement><external>&ext;</external></rootElement>"; MockHttpInputMessage inputMessage = new MockHttpInputMessage(content.getBytes("UTF-8")); converter.setSupportDtd(true); RootElement rootElement = (RootElement) converter.read(RootElement.class, inputMessage); assertEquals("", rootElement.external); }
@Test public void readXmlRootElementExternalEntityEnabled() throws Exception { Resource external = new ClassPathResource("external.txt", getClass()); String content = "<!DOCTYPE root [" + " <!ELEMENT external ANY >\n" + " <!ENTITY ext SYSTEM \"" + external.getURI() + "\" >]>" + " <rootElement><external>&ext;</external></rootElement>"; MockHttpInputMessage inputMessage = new MockHttpInputMessage(content.getBytes("UTF-8")); this.converter.setProcessExternalEntities(true); RootElement rootElement = (RootElement) converter.read(RootElement.class, inputMessage); assertEquals("Foo Bar", rootElement.external); }
@Override protected String parseItemName(ParserContext parserContext, Element element, BeanDefinition bd) { String resourceName = assertNotNull( trimToNull(element.getAttribute(SCRIPT_SOURCE_ATTRIBUTE)), "Missing Attribute: %s", SCRIPT_SOURCE_ATTRIBUTE); Resource resource = parserContext.getReaderContext().getResourceLoader().getResource(resourceName); try { return resource.getURI().normalize().toString(); } catch (IOException e) { parserContext.getReaderContext().error("Failed to get resource: " + resourceName, element, e); return null; } }
/** * Scans the configured base package for files matching the configured mapping file name pattern. * Will simply return an empty {@link Set} in case no {@link ResourceLoader} or mapping file name * pattern was configured. Resulting paths are resource-loadable from the application classpath * according to the JPA spec. * * @see javax.persistence.spi.PersistenceUnitInfo.PersistenceUnitInfo#getMappingFileNames() * @return */ private Set<String> scanForMappingFileLocations() { if (resolver == null || !StringUtils.hasText(mappingFileNamePattern)) { return Collections.emptySet(); } /* * Note that we cannot use File.pathSeparator here since resourcePath uses a forward slash path ('/') separator * being an URI, while basePackagePathComponent has system dependent separator (on windows it's the backslash separator). * * @see DATAJPA-407 */ char slash = '/'; String basePackagePathComponent = basePackage.replace('.', slash); String path = ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX + basePackagePathComponent + slash + mappingFileNamePattern; Set<String> mappingFileUris = new HashSet<String>(); Resource[] scannedResources = new Resource[0]; try { scannedResources = resolver.getResources(path); } catch (IOException e) { throw new IllegalStateException( String.format("Cannot load mapping files from path %s!", path), e); } for (Resource resource : scannedResources) { try { String resourcePath = getResourcePath(resource.getURI()); String resourcePathInClasspath = resourcePath.substring(resourcePath.indexOf(basePackagePathComponent)); mappingFileUris.add(resourcePathInClasspath); } catch (IOException e) { throw new IllegalStateException( String.format("Couldn't get URI for %s!", resource.toString()), e); } } return mappingFileUris; }
@Autowired public WarConfigDataController( @Value("${custom.message1}") String message1, @Value("${custom.message2}") String message2, @Value("${custom.message3}") String message3, @Value("${custom.message4}") String message4, @Value("${custom.message5}") String message5) throws IOException { byte[] bytes = Files.readAllBytes(Paths.get(configDataTemplate.getURI())); String configDataTemplateString = new String(bytes); Map<String, String> data = new HashMap<>(); data.put("message1", message1); data.put("message2", message2); data.put("message3", message3); data.put("message4", message4); data.put("message5", message5); this.configData = StrSubstitutor.replace(configDataTemplateString, data); }
/** * 设置BaseNames。 * * @param baseNames String[] */ public void setBaseNames(String[] baseNames) { List<String> baseNameList = new ArrayList<String>(); try { for (String baseName : baseNames) { Resource[] resources = resolver.getResources(baseName); for (Resource resource : resources) { String fileName = resource.getURI().toString(); baseNameList.add(fileName.substring(0, fileName.indexOf(PROPERTY_POSTFIX))); if (log.isInfoEnabled()) { log.info("Add properties file: [" + resource.getDescription() + "]"); } } } } catch (Exception e) { throw new RuntimeException(e); } this.rMessageSource.setCacheSeconds(this.getCacheMillis()); this.rMessageSource.setBasenames(baseNameList.toArray(new String[baseNameList.size()])); }
/** * Scans the configured base package for files matching the configured mapping file name pattern. * Will simply return an empty {@link Set} in case no {@link ResourceLoader} or mapping file name * pattern was configured. Resulting paths are resource-loadable from the application classpath * according to the JPA spec. * * @see javax.persistence.spi.PersistenceUnitInfo.PersistenceUnitInfo#getMappingFileNames() * @return */ private Set<String> scanForMappingFileLocations() { if (resolver == null || !StringUtils.hasText(mappingFileNamePattern)) { return Collections.emptySet(); } String basePackagePathComponent = basePackage.replace('.', File.separatorChar); String path = ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX + basePackagePathComponent + File.separator + mappingFileNamePattern; Set<String> mappingFileUris = new HashSet<String>(); Resource[] scannedResources = new Resource[0]; try { scannedResources = resolver.getResources(path); } catch (IOException e) { throw new IllegalStateException( String.format("Cannot load mapping files from path %s!", path), e); } for (Resource resource : scannedResources) { try { String resourcePath = resource.getURI().getPath(); String resourcePathInClasspath = resourcePath.substring(resourcePath.indexOf(basePackagePathComponent)); mappingFileUris.add(resourcePathInClasspath); } catch (IOException e) { throw new IllegalStateException( String.format("Couldn't get URI for %s!", resource.toString()), e); } } return mappingFileUris; }
@Override public URI getURI() throws IOException { return delegate.getURI(); }
@SuppressWarnings("rawtypes") public void afterPropertiesSet() throws Exception { final Configuration cfg = ConfigurationUtils.createFrom(configuration, properties); buildGenericOptions(cfg); if (StringUtils.hasText(user)) { UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); ugi.doAs( new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { job = new Job(cfg); return null; } }); } else { job = new Job(cfg); } ClassLoader loader = (beanClassLoader != null ? beanClassLoader : org.springframework.util.ClassUtils.getDefaultClassLoader()); if (jar != null) { JobConf conf = (JobConf) job.getConfiguration(); conf.setJar(jar.getURI().toString()); loader = ExecutionUtils.createParentLastClassLoader(jar, beanClassLoader, cfg); conf.setClassLoader(loader); } // set first to enable auto-detection of K/V to skip the key/value types to be specified if (mapper != null) { Class<? extends Mapper> mapperClass = resolveClass(mapper, loader, Mapper.class); job.setMapperClass(mapperClass); configureMapperTypesIfPossible(job, mapperClass); } if (reducer != null) { Class<? extends Reducer> reducerClass = resolveClass(reducer, loader, Reducer.class); job.setReducerClass(reducerClass); configureReducerTypesIfPossible(job, reducerClass); } if (StringUtils.hasText(name)) { job.setJobName(name); } if (combiner != null) { job.setCombinerClass(resolveClass(combiner, loader, Reducer.class)); } if (groupingComparator != null) { job.setGroupingComparatorClass(resolveClass(groupingComparator, loader, RawComparator.class)); } if (inputFormat != null) { job.setInputFormatClass(resolveClass(inputFormat, loader, InputFormat.class)); } if (mapKey != null) { job.setMapOutputKeyClass(resolveClass(mapKey, loader, Object.class)); } if (mapValue != null) { job.setMapOutputValueClass(resolveClass(mapValue, loader, Object.class)); } if (numReduceTasks != null) { job.setNumReduceTasks(numReduceTasks); } if (key != null) { job.setOutputKeyClass(resolveClass(key, loader, Object.class)); } if (value != null) { job.setOutputValueClass(resolveClass(value, loader, Object.class)); } if (outputFormat != null) { job.setOutputFormatClass(resolveClass(outputFormat, loader, OutputFormat.class)); } if (partitioner != null) { job.setPartitionerClass(resolveClass(partitioner, loader, Partitioner.class)); } if (sortComparator != null) { job.setSortComparatorClass(resolveClass(sortComparator, loader, RawComparator.class)); } if (StringUtils.hasText(workingDir)) { job.setWorkingDirectory(new Path(workingDir)); } if (jarClass != null) { job.setJarByClass(jarClass); } if (!CollectionUtils.isEmpty(inputPaths)) { for (String path : inputPaths) { FileInputFormat.addInputPath(job, new Path(path)); } } if (StringUtils.hasText(outputPath)) { FileOutputFormat.setOutputPath(job, new Path(outputPath)); } if (compressOutput != null) { FileOutputFormat.setCompressOutput(job, compressOutput); } if (codecClass != null) { FileOutputFormat.setOutputCompressorClass( job, resolveClass(codecClass, loader, CompressionCodec.class)); } processJob(job); }
@Override protected void doParse( Element element, ParserContext parserContext, BeanDefinitionBuilder builder) { Map<String, ParsingModuleInfo> scripts = parseSpecificBeans(element, parserContext, builder.getRawBeanDefinition(), ns(LANG_URI)); ElementSelector searchFolders = and(sameNs(element), name("search-folders")); ElementSelector searchFiles = and(sameNs(element), name("search-files")); for (Element subElement : subElements(element)) { String prefix = null; String typeName = null; String moduleName = null; Pattern scriptNamePattern = null; String scriptResourceName = null; String language = null; if (searchFolders.accept(subElement)) { String folderName = assertNotNull( normalizePathName(subElement.getAttribute("folders")), "no folder name provided for search-folders"); // 取prefix prefix = getPrefix(folderName); if (prefix != null) { folderName = folderName.substring(prefix.length() + 1); } // folderName不以/开始 if (folderName.startsWith("/")) { folderName = folderName.substring(1); } scriptNamePattern = compilePathName(folderName); typeName = assertNotNull(trimToNull(subElement.getAttribute("type")), "no type name provided"); language = trimToNull(subElement.getAttribute("language")); scriptResourceName = folderName + "/**/*.*"; log.trace( "Searching in folders: {}, moduleType={}, language={}", new Object[] {folderName, typeName, language == null ? "auto" : language}); } else if (searchFiles.accept(subElement)) { String fileName = assertNotNull( normalizePathName(subElement.getAttribute("files")), "no script file name provided for search-files"); // fileName不以/结尾 assertTrue(!fileName.endsWith("/"), "invalid script file name: %s", fileName); // 取prefix prefix = getPrefix(fileName); if (prefix != null) { fileName = fileName.substring(prefix.length() + 1); } // fileName不以/开始 if (fileName.startsWith("/")) { fileName = fileName.substring(1); } scriptNamePattern = compilePathName(fileName); typeName = assertNotNull(trimToNull(subElement.getAttribute("type")), "no type name provided"); moduleName = assertNotNull(trimToNull(subElement.getAttribute("name")), "no module name provided"); language = trimToNull(subElement.getAttribute("language")); scriptResourceName = fileName; log.trace( "Searching for script files: {}, moduleType={}, moduleName={}, language={}", new Object[] {fileName, typeName, moduleName, language == null ? "auto" : language}); } if (scriptResourceName != null) { scriptResourceName = prefix == null ? scriptResourceName : prefix + ":" + scriptResourceName; // 扫描scripts ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver( parserContext.getReaderContext().getResourceLoader()); int found = 0; try { Resource[] resources = resolver.getResources(scriptResourceName.replace('?', '*')); BeanDefinitionDefaults defaults = getBeanDefinitionDefaults(subElement, parserContext); ParsingModuleMatcher matcher = new ParsingModuleMatcher(scripts, scriptNamePattern, typeName, moduleName) { @Override protected String getName(String name, String itemName) { String ext = getExt(itemName); if (ext != null && name.endsWith("." + ext)) { return name.substring(0, name.length() - ext.length() - 1); } return name; } }; for (Resource resource : resources) { if (resource.isReadable()) { URI uri = resource.getURI(); if (uri == null) { continue; } String resourceName = uri.normalize().toString(); if (matcher.doMatch(resourceName)) { BeanDefinition scriptBean = createScriptBean(subElement, parserContext, resourceName, language, defaults); String beanName = matcher.generateBeanName(resourceName, parserContext.getRegistry()); parserContext.getRegistry().registerBeanDefinition(beanName, scriptBean); found++; } } } } catch (IOException e) { parserContext .getReaderContext() .error("Failed to scan resources: " + scriptResourceName, subElement, e); return; } log.debug("Found {} module scripts with pattern: {}", found, scriptResourceName); } } postProcessItems(element, parserContext, builder, scripts, "search-folders or search-files"); }
@Override HadoopFlow createFlow() throws IOException { // copy flowDef FlowDef def = FlowDef.flowDef(); if (flowDef != null) { def.addSinks(flowDef.getSinksCopy()) .addSources(flowDef.getSourcesCopy()) .addTraps(flowDef.getTrapsCopy()) .addTails(flowDef.getTailsArray()) .setAssertionLevel(flowDef.getAssertionLevel()) .setDebugLevel(flowDef.getDebugLevel()) .addCheckpoints(flowDef.getCheckpointsCopy()) .addTags(flowDef.getTags()) .setName(flowDef.getName()); } Set<Pipe> heads = new LinkedHashSet<Pipe>(); if (tails != null) { for (Pipe pipe : tails) { Collections.addAll(heads, pipe.getHeads()); } } Pipe pipe = null; if (heads.size() == 1) { pipe = heads.iterator().next(); } if (sources != null && sources.size() == 1) { Tap tap = sources.remove(MARKER); if (tap != null) { sources.put(pipe.getName(), tap); } } if (sinks != null && sinks.size() == 1) { Tap tap = sinks.remove(MARKER); if (tap != null) { sinks.put(pipe.getName(), tap); } } def.addSources(sources).addSinks(sinks).addTraps(traps); if (tails != null) { def.addTails(tails); } if (StringUtils.hasText(beanName)) { def.addTag(beanName); if (!StringUtils.hasText(def.getName())) { def.setName(beanName); } } Configuration cfg = ConfigurationUtils.createFrom(configuration, properties); Properties props = ConfigurationUtils.asProperties(cfg); if (jarSetup) { if (jar != null) { AppProps.setApplicationJarPath(props, ResourceUtils.decode(jar.getURI().toString())); } else if (jarClass != null) { AppProps.setApplicationJarClass(props, jarClass); } else { // auto-detection based on the classpath ClassLoader cascadingCL = Cascade.class.getClassLoader(); Resource cascadingCore = ResourceUtils.findContainingJar(Cascade.class); Resource cascadingHadoop = ResourceUtils.findContainingJar(cascadingCL, "cascading/flow/hadoop/HadoopFlow.class"); // find jgrapht Resource jgrapht = ResourceUtils.findContainingJar(cascadingCL, "org/jgrapht/Graph.class"); Assert.notNull(cascadingCore, "Cannot find cascading-core.jar"); Assert.notNull(cascadingHadoop, "Cannot find cascading-hadoop.jar"); Assert.notNull(jgrapht, "Cannot find jgraphts-jdk.jar"); if (log.isDebugEnabled()) { log.debug( "Auto-detecting Cascading Libs [" + Arrays.toString(new Resource[] {cascadingCore, cascadingHadoop, jgrapht}) + "]"); } ConfigurationUtils.addLibs(cfg, cascadingCore, cascadingHadoop, jgrapht); // config changed, reinit properties props = ConfigurationUtils.asProperties(cfg); } } if (jobPoolingInterval != null) { FlowProps.setJobPollingInterval(props, jobPoolingInterval); } if (maxConcurrentSteps != null) { FlowProps.setMaxConcurrentSteps(props, maxConcurrentSteps); } HadoopFlow flow = (HadoopFlow) new HadoopFlowConnector(props).connect(def); return flow; }
@Override public List<ModuleResource> findModuleResources(LoadScope scope) throws IOException { Local local = new Local(); String[] controllersScope = scope.getScope("controllers"); if (logger.isInfoEnabled()) { logger.info("[moduleResource] starting ..."); logger.info( "[moduleResource] call 'findFiles':" + " to find classes or jar files by scope " // + Arrays.toString(controllersScope)); } List<ResourceRef> refers = RoseScanner.getInstance().getJarOrClassesFolderResources(controllersScope); if (logger.isInfoEnabled()) { logger.info("[moduleResource] exits from 'findFiles'"); logger.info( "[moduleResource] going to scan controllers" + " from these folders or jar files:" + refers); } FileSystemManager fileSystem = new FileSystemManager(); for (ResourceRef refer : refers) { Resource resource = refer.getResource(); if (!refer.hasModifier("controllers")) { if (logger.isDebugEnabled()) { logger.debug( "[moduleResource] Ignored because not marked as 'controllers'" + " in META-INF/rose.properties or META-INF/MANIFEST.MF: " + resource.getURI()); } continue; } File resourceFile = resource.getFile(); String urlString; if ("jar".equals(refer.getProtocol())) { urlString = ResourceUtils.URL_PROTOCOL_JAR + ":" + resourceFile.toURI() + ResourceUtils.JAR_URL_SEPARATOR; } else { urlString = resourceFile.toURI().toString(); } FileObject rootObject = fileSystem.resolveFile(urlString); if (rootObject == null || !rootObject.exists()) { if (logger.isDebugEnabled()) { logger.debug("[moduleResource] Ignored because not exists: " + urlString); } continue; } if (logger.isInfoEnabled()) { logger.info("[moduleResource] start to scan moduleResource in file: " + rootObject); } try { int oldSize = local.moduleResourceList.size(); deepScanImpl(local, rootObject, rootObject); int newSize = local.moduleResourceList.size(); if (logger.isInfoEnabled()) { logger.info("[moduleResource] got " + (newSize - oldSize) + " modules in " + rootObject); } } catch (Exception e) { logger.error("[moduleResource] error happend when scanning " + rootObject, e); } fileSystem.clearCache(); } afterScanning(local); logger.info("[moduleResource] found " + local.moduleResourceList.size() + " module resources "); return local.moduleResourceList; }