/**
* 消费者基础实现
*
* @author Peter Zhang
*/
public abstract class BaseConsumer implements MessageListenerConcurrently {
protected final Logger logger = LoggerFactory.getLogger(getClass());
protected DefaultMQPushConsumer consumer;
protected String nameServer;
protected int minConsumeThread = 2;
protected int maxConsumeThread = 5;
protected String group;
protected String subExpression;
protected Topic topic;
protected Class<? extends Topic> topicType;
// 定时消息相关
// 线上环境：messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 40m 50m 1h 2h 6h
// 开发环境：messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 40m 50m 1h 2h 6h
// 12h 1d
private static final int[] DELAY_LEVELS = new int[] {3, 5, 9, 14, 15, 16, 17, 18, 19, 20, 21};
protected int maxRetryCount = 10;
public void init() throws MQClientException {
nameServer = PropertyFileUtil.get("rocketmq.namesrv.domain");
if (StringUtils.isBlank(nameServer)) {
logger.warn("【MQ init】property rocketmq.namesrv.domain not found");
return;
}
if ("localTest".equals(nameServer)) {
logger.warn("【MQ init】localTest");
return;
}
if (StringUtils.isBlank(System.getProperty("rocketmq.namesrv.domain"))) {
System.setProperty("rocketmq.namesrv.domain", nameServer);
}
topicType = getTopic();
topic = RocketMqUtils.getTopic(topicType);
if (StringUtils.isBlank(group)) {
group = "S_" + topic.getTopic() + "_" + topic.getTags();
}
consumer = new DefaultMQPushConsumer(group);
consumer.setNamesrvAddr(nameServer);
consumer.setMessageModel(getMessageModel());
consumer.setConsumeThreadMin(minConsumeThread);
consumer.setConsumeThreadMax(maxConsumeThread);
// 可以不设置 设置后可以起多个 消费端
try {
consumer.setInstanceName("DEFAULT_CONSUMER-" + InetAddress.getLocalHost().getHostName());
} catch (UnknownHostException e) {
logger.error("getHostName error", e);
}
// 设置订阅的topic 设置订阅过滤表达式
if (StringUtils.isBlank(subExpression)) {
subExpression = topic.getTags();
consumer.subscribe(topic.getTopic(), subExpression);
} else {
consumer.subscribe(topic.getTopic(), subExpression);
}
try {
consumer.registerMessageListener(this);
consumer.start();
} catch (MQClientException e) {
logger.error(
"consumer start error!topic={},subExpression={},group={}",
topic.getTopic(),
subExpression,
group,
e);
}
logger.info(
"consumer start! topic={},subExpression={},group={}",
topic.getTopic(),
subExpression,
group);
}
protected MessageModel getMessageModel() {
return MessageModel.CLUSTERING;
}
public void destroy() {
if (consumer != null) {
consumer.shutdown();
logger.info(
"consumer shutdown! topic={0},subExpression={1},group={2}",
topic.getTopic(), subExpression, group);
}
}
public abstract Class<? extends Topic> getTopic();
/**
* 子类实现的，用于实际记录错误的代码
*
* @param msgObj 消息对象列表
* @return 消费状态
*/
public abstract void doLogErrorConsumeMessage(MsgObj msgObj);
/**
* 子类实现的，用于实际消费的代码
*
* @param msgObj 消息对象列表
* @return 消费状态
*/
public abstract ConsumeConcurrentlyStatus doConsumeMessage(MsgObj msgObj);
/** 基类实现消息监听接口，加上打印metaq监控日志的方法 */
@Override
public ConsumeConcurrentlyStatus consumeMessage(
List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
long startTime = System.currentTimeMillis();
logger.info("receive_message:{}", msgs.toString());
if (msgs == null || msgs.size() < 1) {
logger.error("receive empty msg!");
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
List<Serializable> msgList = new ArrayList<>();
for (MessageExt message : msgs) {
msgList.add(decodeMsg(message));
}
final int reconsumeTimes = msgs.get(0).getReconsumeTimes();
MsgObj msgObj = new MsgObj();
msgObj.setReconsumeTimes(reconsumeTimes);
msgObj.setMsgList(msgList);
msgObj.setContext(context);
context.setDelayLevelWhenNextConsume(getDelayLevelWhenNextConsume(reconsumeTimes));
ConsumeConcurrentlyStatus status = doConsumeMessage(msgObj);
logger.info(
"ConsumeConcurrentlyStatus:{}|cost:{}", status, System.currentTimeMillis() - startTime);
return status;
}
/**
* 根据重试次数设置重新消费延迟时间 1s 10s 30s 2m 10m 30m 1h 2h 12h 1d
*
* @param reconsumeTimes 重试的次数
* @return level级别
*/
public int getDelayLevelWhenNextConsume(int reconsumeTimes) {
if (reconsumeTimes >= DELAY_LEVELS.length) {
return DELAY_LEVELS[DELAY_LEVELS.length - 1];
}
return DELAY_LEVELS[reconsumeTimes];
}
private Serializable decodeMsg(MessageExt msg) {
if (msg == null) {
return null;
}
// 1.反序列化
try {
return HessianUtils.decode(msg.getBody());
} catch (IOException e) {
logger.error("反序列化出错!" + e.getMessage(), e);
return null;
}
}
protected ConsumeConcurrentlyStatus exceptionConsumeConcurrentlyStatus(
TransactionStatus status, Throwable e, MsgObj msgObj, int maxRetryCount) {
logger.error("mq consume failed", e);
status.setRollbackOnly();
if (msgObj.getReconsumeTimes() >= maxRetryCount) {
logger.error(
"retryCount: {}, msgs: {}, context: {}", maxRetryCount, msgObj, msgObj.getContext());
msgObj.setErrorMsg(e.getMessage());
doLogErrorConsumeMessage(msgObj);
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
} else {
return ConsumeConcurrentlyStatus.RECONSUME_LATER;
}
}
public void setMinConsumeThread(int minConsumeThread) {
this.minConsumeThread = minConsumeThread;
}
public void setMaxConsumeThread(int maxConsumeThread) {
this.maxConsumeThread = maxConsumeThread;
}
public void setGroup(String group) {
this.group = group;
}
public void setSubExpression(String subExpression) {
this.subExpression = subExpression;
}
}
public class NetUtils {
private static Logger logger = LoggerFactory.createLogger(NetUtils.class);
public static final String DYNAMIC_RESOURCES_SESSION_KEY = "orbeon.resources.dynamic.";
// Resources are served by the XForms server. It is not ideal to refer to XForms-related
// functionality from here.
public static final String DYNAMIC_RESOURCES_PATH = "/xforms-server/dynamic/";
private static final Pattern PATTERN_NO_AMP;
private static final Pattern PATTERN_AMP;
// private static final Pattern PATTERN_AMP_AMP;
public static final String STANDARD_PARAMETER_ENCODING = "utf-8";
private static final SimpleDateFormat dateHeaderFormats[] = {
new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US),
new SimpleDateFormat("EEEEEE, dd-MMM-yy HH:mm:ss zzz", Locale.US),
new SimpleDateFormat("EEE MMMM d HH:mm:ss yyyy", Locale.US)
};
private static final TimeZone gmtZone = TimeZone.getTimeZone("GMT");
private static FileItemFactory fileItemFactory;
public static final int REQUEST_SCOPE = 0;
public static final int SESSION_SCOPE = 1;
public static final int APPLICATION_SCOPE = 2;
// Default HTTP 1.1 charset for text/* mediatype
public static final String DEFAULT_HTTP_TEXT_READING_ENCODING = "iso-8859-1";
// Default RFC 3023 default charset for txt/xml mediatype
public static final String DEFAULT_TEXT_XML_READING_ENCODING = "us-ascii";
public static final String APPLICATION_SOAP_XML = "application/soap+xml";
static {
// Set timezone to GMT as required for HTTP headers
for (SimpleDateFormat dateHeaderFormat : dateHeaderFormats)
dateHeaderFormat.setTimeZone(gmtZone);
final String notEqNorAmpChar = "[^=&]";
final String token = notEqNorAmpChar + "+";
PATTERN_NO_AMP = Pattern.compile("(" + token + ")=(" + token + ")(?:&|(?<!&)\\z)");
PATTERN_AMP = Pattern.compile("(" + token + ")=(" + token + ")(?:&|&|(?<!&|&)\\z)");
// PATTERN_AMP_AMP = Pattern.compile( "(" + token + ")=(" + token +
// ")(?:&amp;|&|(?<!&amp;|&)\\z)" );
}
public static long getDateHeader(String stringValue) throws ParseException {
for (SimpleDateFormat dateHeaderFormat : dateHeaderFormats) {
try {
Date date = dateHeaderFormat.parse(stringValue);
return date.getTime();
} catch (
Exception
e) { // used to be ParseException, but NumberFormatException may be thrown as well
// Ignore and try next
}
}
throw new ParseException(stringValue, 0);
}
/**
* Return true if the document was modified since the given date, based on the If-Modified-Since
* header. If the request method was not "GET", or if no valid lastModified value was provided,
* consider the document modified.
*/
public static boolean checkIfModifiedSince(HttpServletRequest request, long lastModified) {
// Do the check only for the GET method
if (!"GET".equals(request.getMethod()) || lastModified <= 0) return true;
// Check dates
String ifModifiedHeader = request.getHeader("If-Modified-Since");
if (logger.isDebugEnabled()) logger.debug("Found If-Modified-Since header");
if (ifModifiedHeader != null) {
try {
long dateTime = getDateHeader(ifModifiedHeader);
if (lastModified <= (dateTime + 1000)) {
if (logger.isDebugEnabled()) logger.debug("Sending SC_NOT_MODIFIED response");
return false;
}
} catch (
Exception
e) { // used to be ParseException, but NumberFormatException may be thrown as well
// Ignore
}
}
return true;
}
/**
* Return a request path info that looks like what one would expect. The path starts with a "/",
* relative to the servlet context. If the servlet was included or forwarded to, return the path
* by which the *current* servlet was invoked, NOT the path of the calling servlet.
*
* <p>Request path = servlet path + path info.
*
* @param request servlet HTTP request
* @return path
*/
public static String getRequestPathInfo(HttpServletRequest request) {
// NOTE: Servlet 2.4 spec says: "These attributes [javax.servlet.include.*] are accessible from
// the included
// servlet via the getAttribute method on the request object and their values must be equal to
// the request URI,
// context path, servlet path, path info, and query string of the included servlet,
// respectively."
// NOTE: This is very different from the similarly-named forward attributes!
// Get servlet path
String servletPath = (String) request.getAttribute("javax.servlet.include.servlet_path");
if (servletPath == null) {
servletPath = request.getServletPath();
if (servletPath == null) servletPath = "";
}
// Get path info
String pathInfo = (String) request.getAttribute("javax.servlet.include.path_info");
if (pathInfo == null) {
pathInfo = request.getPathInfo();
if (pathInfo == null) pathInfo = "";
}
// Concatenate servlet path and path info, avoiding a double slash
String requestPath =
servletPath.endsWith("/") && pathInfo.startsWith("/")
? servletPath + pathInfo.substring(1)
: servletPath + pathInfo;
// Add starting slash if missing
if (!requestPath.startsWith("/")) requestPath = "/" + requestPath;
return requestPath;
}
/**
* Return the last modification date of the given absolute URL if it is "fast" to do so, i.e. if
* it is an "oxf:" or a "file:" protocol.
*
* @param absoluteURL absolute URL to check
* @return last modification date if "fast" or 0 if not fast or if an error occurred
*/
public static long getLastModifiedIfFast(String absoluteURL) {
final long lastModified;
if (absoluteURL.startsWith("oxf:") || absoluteURL.startsWith("file:")) {
try {
lastModified = getLastModified(URLFactory.createURL(absoluteURL));
} catch (IOException e) {
throw new OXFException(e);
}
} else {
// Value of 0 for lastModified will cause XFormsResourceServer to set Last-Modified and
// Expires properly to "now".
lastModified = 0;
}
return lastModified;
}
/**
* Get the last modification date of a URL.
*
* @return last modified timestamp, null if le 0
*/
public static Long getLastModifiedAsLong(URL url) throws IOException {
final long connectionLastModified = getLastModified(url);
// Zero and negative values often have a special meaning, make sure to normalize here
return connectionLastModified <= 0 ? null : connectionLastModified;
}
/**
* Get the last modification date of a URL.
*
* @return last modified timestamp "as is"
*/
public static long getLastModified(URL url) throws IOException {
if ("file".equals(url.getProtocol())) {
// Optimize file: access. Also, this prevents throwing an exception if the file doesn't exist
// as we try to close the stream below.
return new File(URLDecoder.decode(url.getFile(), STANDARD_PARAMETER_ENCODING)).lastModified();
} else {
// Use URLConnection
final URLConnection urlConnection = url.openConnection();
if (urlConnection instanceof HttpURLConnection)
((HttpURLConnection) urlConnection).setRequestMethod("HEAD");
try {
return getLastModified(urlConnection);
} finally {
final InputStream is = urlConnection.getInputStream();
if (is != null) is.close();
}
}
}
/**
* Get the last modification date of an open URLConnection.
*
* <p>This handles the (broken at some point in the Java libraries) case of the file: protocol.
*
* @return last modified timestamp, null if le 0
*/
public static Long getLastModifiedAsLong(URLConnection urlConnection) {
final long connectionLastModified = getLastModified(urlConnection);
// Zero and negative values often have a special meaning, make sure to normalize here
return connectionLastModified <= 0 ? null : connectionLastModified;
}
/**
* Get the last modification date of an open URLConnection.
*
* <p>This handles the (broken at some point in the Java libraries) case of the file: protocol.
*
* @return last modified timestamp "as is"
*/
public static long getLastModified(URLConnection urlConnection) {
try {
long lastModified = urlConnection.getLastModified();
if (lastModified == 0 && "file".equals(urlConnection.getURL().getProtocol()))
lastModified =
new File(
URLDecoder.decode(
urlConnection.getURL().getFile(), STANDARD_PARAMETER_ENCODING))
.lastModified();
return lastModified;
} catch (UnsupportedEncodingException e) {
// Should not happen as we are using a required encoding
throw new OXFException(e);
}
}
/** Check if an URL is relative to another URL. */
public static boolean relativeURL(URL url1, URL url2) {
return ((url1.getProtocol() == null && url2.getProtocol() == null)
|| url1.getProtocol().equals(url2.getProtocol()))
&& ((url1.getAuthority() == null && url2.getAuthority() == null)
|| url1.getAuthority().equals(url2.getAuthority()))
&& ((url1.getPath() == null && url2.getPath() == null)
|| url2.getPath().startsWith(url1.getPath()));
}
public static void copyStream(InputStream is, OutputStream os) throws IOException {
int count;
byte[] buffer = new byte[1024];
while ((count = is.read(buffer)) > 0) os.write(buffer, 0, count);
}
public static void copyStream(Reader reader, Writer writer) throws IOException {
int count;
char[] buffer = new char[1024];
while ((count = reader.read(buffer)) > 0) writer.write(buffer, 0, count);
}
public static String readStreamAsString(Reader reader) throws IOException {
final StringBuilderWriter writer = new StringBuilderWriter();
copyStream(reader, writer);
return writer.toString();
}
public static String getContentTypeCharset(String contentType) {
final Map<String, String> parameters = getContentTypeParameters(contentType);
return (parameters == null) ? null : parameters.get("charset");
}
public static Map<String, String> getContentTypeParameters(String contentType) {
if (contentType == null) return null;
// Check whether there may be parameters
final int semicolonIndex = contentType.indexOf(";");
if (semicolonIndex == -1) return null;
// Tokenize
final StringTokenizer st = new StringTokenizer(contentType, ";");
if (!st.hasMoreTokens())
return null; // should not happen as there should be at least the content type
st.nextToken();
// No parameters
if (!st.hasMoreTokens()) return null;
// Parse parameters
final Map<String, String> parameters = new HashMap<String, String>();
while (st.hasMoreTokens()) {
final String parameter = st.nextToken().trim();
final int equalIndex = parameter.indexOf('=');
if (equalIndex == -1) continue;
final String name = parameter.substring(0, equalIndex).trim();
final String value = parameter.substring(equalIndex + 1).trim();
parameters.put(name, value);
}
return parameters;
}
public static Map<String, String> getCharsetHeaderCharsets(String header) {
if (header == null) return null;
int semicolonIndex = header.indexOf(";");
final String charsets;
if (semicolonIndex == -1) charsets = header.trim();
else charsets = header.substring(0, semicolonIndex).trim();
final StringTokenizer st = new StringTokenizer(charsets, ",");
final Map<String, String> charsetsMap = new HashMap<String, String>();
while (st.hasMoreTokens()) {
charsetsMap.put(st.nextToken(), "");
}
return charsetsMap;
}
public static String getContentTypeMediaType(String contentType) {
if (contentType == null || contentType.equalsIgnoreCase("content/unknown")) return null;
int semicolonIndex = contentType.indexOf(";");
if (semicolonIndex == -1) return contentType;
return contentType.substring(0, semicolonIndex).trim();
}
/**
* @param queryString a query string of the form n1=v1&n2=v2&... to decode. May be null.
* @param acceptAmp -> "&" if true, "&" if false
* @return a Map of String[] indexed by name, an empty Map if the query string was null
*/
public static Map<String, String[]> decodeQueryString(
final CharSequence queryString, final boolean acceptAmp) {
final Map<String, String[]> result = new TreeMap<String, String[]>();
if (queryString != null) {
final Matcher matcher =
acceptAmp ? PATTERN_AMP.matcher(queryString) : PATTERN_NO_AMP.matcher(queryString);
int matcherEnd = 0;
while (matcher.find()) {
matcherEnd = matcher.end();
try {
// Group 0 is the whole match, e.g. a=b, while group 1 is the first group
// denoted ( with parens ) in the expression. Hence we start with group 1.
final String name =
URLDecoder.decode(matcher.group(1), NetUtils.STANDARD_PARAMETER_ENCODING);
final String value =
URLDecoder.decode(matcher.group(2), NetUtils.STANDARD_PARAMETER_ENCODING);
StringUtils.addValueToStringArrayMap(result, name, value);
} catch (UnsupportedEncodingException e) {
// Should not happen as we are using a required encoding
throw new OXFException(e);
}
}
if (queryString.length() != matcherEnd) {
// There was garbage at the end of the query.
throw new OXFException("Malformed URL: " + queryString);
}
}
return result;
}
/** Encode a query string. The input Map contains names indexing Object[]. */
public static String encodeQueryString(Map parameters) {
final StringBuilder sb = new StringBuilder(100);
boolean first = true;
try {
for (Object o : parameters.keySet()) {
final String name = (String) o;
final Object[] values = (Object[]) parameters.get(name);
for (final Object currentValue : values) {
if (currentValue instanceof String) {
if (!first) sb.append('&');
sb.append(URLEncoder.encode(name, NetUtils.STANDARD_PARAMETER_ENCODING));
sb.append('=');
sb.append(
URLEncoder.encode((String) currentValue, NetUtils.STANDARD_PARAMETER_ENCODING));
first = false;
}
}
}
} catch (UnsupportedEncodingException e) {
// Should not happen as we are using a required encoding
throw new OXFException(e);
}
return sb.toString();
}
/** Combine a path info and a parameters map to form a path info with a query string. */
public static String pathInfoParametersToPathInfoQueryString(String pathInfo, Map parameters)
throws IOException {
final StringBuilder redirectURL = new StringBuilder(pathInfo);
if (parameters != null) {
boolean first = true;
for (Object o : parameters.keySet()) {
final String name = (String) o;
final Object[] values = (Object[]) parameters.get(name);
for (final Object currentValue : values) {
if (currentValue instanceof String) {
redirectURL.append(first ? "?" : "&");
redirectURL.append(URLEncoder.encode(name, NetUtils.STANDARD_PARAMETER_ENCODING));
redirectURL.append("=");
redirectURL.append(
URLEncoder.encode((String) currentValue, NetUtils.STANDARD_PARAMETER_ENCODING));
first = false;
}
}
}
}
return redirectURL.toString();
}
/**
* Append a query string to an URL. This adds a '?' or a '&' or nothing, as needed.
*
* @param urlString existing URL string
* @param queryString query string, or null
* @return resulting URL
*/
public static String appendQueryString(String urlString, String queryString) {
if (org.apache.commons.lang.StringUtils.isBlank(queryString)) {
return urlString;
} else {
final StringBuilder updatedActionStringBuilder = new StringBuilder(urlString);
updatedActionStringBuilder.append((urlString.indexOf('?') == -1) ? '?' : '&');
updatedActionStringBuilder.append(queryString);
return updatedActionStringBuilder.toString();
}
}
/**
* Check whether a URL starts with a protocol.
*
* <p>We consider that a protocol consists only of ASCII letters and must be at least two
* characters long, to avoid confusion with Windows drive letters.
*/
public static boolean urlHasProtocol(String urlString) {
int colonIndex = urlString.indexOf(":");
// No protocol is there is no colon or if there is only one character in the protocol
if (colonIndex == -1 || colonIndex == 1) return false;
// Check that there is a protocol
boolean allChar = true;
for (int i = 0; i < colonIndex; i++) {
char c = urlString.charAt(i);
if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z')) {
allChar = false;
break;
}
}
return allChar;
}
/**
* Resolve a URI against a base URI. (Be sure to pay attention to the order or parameters.)
*
* @param href URI to resolve (accept human-readable URI)
* @param base URI base (accept human-readable URI)
* @return resolved URI
*/
public static String resolveURI(String href, String base) {
final String resolvedURIString;
if (base != null) {
final URI baseURI;
try {
baseURI = new URI(encodeHRRI(base, true));
} catch (URISyntaxException e) {
throw new OXFException(e);
}
resolvedURIString =
baseURI
.resolve(encodeHRRI(href, true))
.normalize()
.toString(); // normalize to remove "..", etc.
} else {
resolvedURIString = encodeHRRI(href, true);
}
return resolvedURIString;
}
public static String headersToString(HttpServletRequest request) {
final StringBuffer sb = new StringBuffer();
for (Enumeration e = request.getHeaderNames(); e.hasMoreElements(); ) {
final String name = (String) e.nextElement();
sb.append(name);
sb.append("=");
for (Enumeration f = request.getHeaders(name); f.hasMoreElements(); ) {
final String value = (String) f.nextElement();
sb.append(value);
if (f.hasMoreElements()) sb.append(",");
}
if (e.hasMoreElements()) sb.append("|");
}
return sb.toString();
}
public static String readURIToLocalURI(String uri) throws URISyntaxException, IOException {
final PipelineContext pipelineContext =
StaticExternalContext.getStaticContext().getPipelineContext();
final URLConnection urlConnection = new URI(uri).toURL().openConnection();
InputStream inputStream = null;
try {
inputStream = urlConnection.getInputStream();
return inputStreamToAnyURI(pipelineContext, inputStream, REQUEST_SCOPE);
} finally {
if (inputStream != null) inputStream.close();
}
}
public static byte[] base64StringToByteArray(String base64String) {
return Base64.decode(base64String);
}
/**
* Convert a String in xs:base64Binary to an xs:anyURI.
*
* <p>NOTE: The implementation creates a temporary file. The Pipeline Context is required so that
* the file can be deleted when no longer used.
*/
public static String base64BinaryToAnyURI(
PipelineContext pipelineContext, String value, int scope) {
// Convert Base64 to binary first
final byte[] bytes = base64StringToByteArray(value);
return inputStreamToAnyURI(pipelineContext, new ByteArrayInputStream(bytes), scope);
}
/**
* Read an InputStream into a byte array.
*
* @param is InputStream
* @return byte array
*/
public static byte[] inputStreamToByteArray(InputStream is) {
try {
final ByteArrayOutputStream os = new ByteArrayOutputStream();
copyStream(new BufferedInputStream(is), os);
os.close();
return os.toByteArray();
} catch (Exception e) {
throw new OXFException(e);
}
}
/**
* Read a URI into a byte array.
*
* @param uri URI to read
* @return byte array
*/
public static byte[] uriToByteArray(String uri) {
InputStream is = null;
try {
is = new URI(uri).toURL().openStream();
return inputStreamToByteArray(is);
} catch (Exception e) {
throw new OXFException(e);
} finally {
try {
if (is != null) is.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
/**
* Convert a URI to a FileItem.
*
* <p>The implementation creates a temporary file. The PipelineContext is required so that the
* file can be deleted when no longer used.
*/
public static FileItem anyURIToFileItem(PipelineContext pipelineContext, String uri, int scope) {
InputStream inputStream = null;
try {
inputStream = new URI(uri).toURL().openStream();
// Get FileItem
return prepareFileItemFromInputStream(pipelineContext, inputStream, scope);
} catch (Exception e) {
throw new OXFException(e);
} finally {
try {
if (inputStream != null) inputStream.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
/**
* Convert an InputStream to an xs:anyURI.
*
* <p>The implementation creates a temporary file. The PipelineContext is required so that the
* file can be deleted when no longer used.
*/
public static String inputStreamToAnyURI(
PipelineContext pipelineContext, InputStream inputStream, int scope) {
// Get FileItem
final FileItem fileItem = prepareFileItemFromInputStream(pipelineContext, inputStream, scope);
// Return a file URL
final File storeLocation = ((DiskFileItem) fileItem).getStoreLocation();
return storeLocation.toURI().toString();
}
private static FileItem prepareFileItemFromInputStream(
PipelineContext pipelineContext, InputStream inputStream, int scope) {
// Get FileItem
final FileItem fileItem = prepareFileItem(pipelineContext, scope);
// Write to file
OutputStream os = null;
try {
os = fileItem.getOutputStream();
copyStream(inputStream, os);
} catch (IOException e) {
throw new OXFException(e);
} finally {
if (os != null) {
try {
os.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
// Create file if it doesn't exist (necessary when the file size is 0)
final File storeLocation = ((DiskFileItem) fileItem).getStoreLocation();
try {
storeLocation.createNewFile();
} catch (IOException e) {
throw new OXFException(e);
}
return fileItem;
}
/**
* Return a FileItem which is going to be automatically destroyed upon destruction of the request,
* session or application.
*/
public static FileItem prepareFileItem(PipelineContext pipelineContext, int scope) {
// We use the commons file upload utilities to save a file
if (fileItemFactory == null)
fileItemFactory = new DiskFileItemFactory(0, SystemUtils.getTemporaryDirectory());
final FileItem fileItem = fileItemFactory.createItem("dummy", "dummy", false, null);
// Make sure the file is deleted appropriately
if (scope == REQUEST_SCOPE) {
deleteFileOnRequestEnd(pipelineContext, fileItem);
} else if (scope == SESSION_SCOPE) {
deleteFileOnSessionTermination(pipelineContext, fileItem);
} else if (scope == APPLICATION_SCOPE) {
deleteFileOnContextDestroyed(pipelineContext, fileItem);
} else {
throw new OXFException("Invalid context requested: " + scope);
}
// Return FileItem object
return fileItem;
}
/**
* Add listener to fileItem which is going to be automatically destroyed at the end of request
*
* @param pipelineContext PipelineContext
* @param fileItem FileItem
*/
public static void deleteFileOnRequestEnd(
PipelineContext pipelineContext, final FileItem fileItem) {
// Make sure the file is deleted at the end of request
pipelineContext.addContextListener(
new PipelineContext.ContextListenerAdapter() {
public void contextDestroyed(boolean success) {
deleteFileItem(fileItem, REQUEST_SCOPE);
}
});
}
/**
* Add listener to fileItem which is going to be automatically destroyed on session destruction
*
* @param pipelineContext PipelineContext
* @param fileItem FileItem
*/
public static void deleteFileOnSessionTermination(
PipelineContext pipelineContext, final FileItem fileItem) {
// Try to delete the file on exit and on session termination
final ExternalContext externalContext =
(ExternalContext) pipelineContext.getAttribute(PipelineContext.EXTERNAL_CONTEXT);
final ExternalContext.Session session = externalContext.getSession(false);
if (session != null) {
session.addListener(
new ExternalContext.Session.SessionListener() {
public void sessionDestroyed() {
deleteFileItem(fileItem, SESSION_SCOPE);
}
});
} else {
logger.debug(
"No existing session found so cannot register temporary file deletion upon session destruction: "
+ fileItem.getName());
}
}
/**
* Add listener to fileItem which is going to be automatically destroyed when the servlet is
* destroyed
*
* @param pipelineContext PipelineContext
* @param fileItem FileItem
*/
public static void deleteFileOnContextDestroyed(
PipelineContext pipelineContext, final FileItem fileItem) {
// Try to delete the file on exit and on session termination
final ExternalContext externalContext =
(ExternalContext) pipelineContext.getAttribute(PipelineContext.EXTERNAL_CONTEXT);
ExternalContext.Application application = externalContext.getApplication();
if (application != null) {
application.addListener(
new ExternalContext.Application.ApplicationListener() {
public void servletDestroyed() {
deleteFileItem(fileItem, APPLICATION_SCOPE);
}
});
} else {
logger.debug(
"No application object found so cannot register temporary file deletion upon session destruction: "
+ fileItem.getName());
}
}
private static void deleteFileItem(FileItem fileItem, int scope) {
if (logger.isDebugEnabled() && fileItem instanceof DiskFileItem) {
final File storeLocation = ((DiskFileItem) fileItem).getStoreLocation();
if (storeLocation != null) {
final String temporaryFileName = storeLocation.getAbsolutePath();
final String scopeString =
(scope == REQUEST_SCOPE)
? "request"
: (scope == SESSION_SCOPE) ? "session" : "application";
logger.debug("Deleting temporary " + scopeString + "-scoped file: " + temporaryFileName);
}
}
fileItem.delete();
}
/**
* Convert a String in xs:anyURI to an xs:base64Binary.
*
* <p>The URI has to be a URL. It is read entirely
*/
public static String anyURIToBase64Binary(String value) {
InputStream is = null;
try {
// Read from URL and convert to Base64
is = URLFactory.createURL(value).openStream();
final StringBuffer sb = new StringBuffer();
XMLUtils.inputStreamToBase64Characters(
is,
new ContentHandlerAdapter() {
public void characters(char ch[], int start, int length) {
sb.append(ch, start, length);
}
});
// Return Base64 String
return sb.toString();
} catch (IOException e) {
throw new OXFException(e);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
}
public static void anyURIToOutputStream(String value, OutputStream outputStream) {
InputStream is = null;
try {
is = URLFactory.createURL(value).openStream();
copyStream(is, outputStream);
} catch (IOException e) {
throw new OXFException(e);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
}
/**
* Return the charset associated with a text/* Content-Type header. If a charset is present,
* return it. Otherwise, guess depending on whether the mediatype is text/xml or not.
*
* @param contentType Content-Type header value
* @return charset
*/
public static String getTextCharsetFromContentType(String contentType) {
final String charset;
final String connectionCharset = getContentTypeCharset(contentType);
if (connectionCharset != null) {
charset = connectionCharset;
} else {
// RFC 3023: "Conformant with [RFC2046], if a text/xml entity is
// received with the charset parameter omitted, MIME processors and
// XML processors MUST use the default charset value of
// "us-ascii"[ASCII]. In cases where the XML MIME entity is
// transmitted via HTTP, the default charset value is still
// "us-ascii". (Note: There is an inconsistency between this
// specification and HTTP/1.1, which uses ISO-8859-1[ISO8859] as the
// default for a historical reason. Since XML is a new format, a new
// default should be chosen for better I18N. US-ASCII was chosen,
// since it is the intersection of UTF-8 and ISO-8859-1 and since it
// is already used by MIME.)"
if (XMLUtils.isXMLMediatype(contentType)) charset = DEFAULT_TEXT_XML_READING_ENCODING;
else charset = DEFAULT_HTTP_TEXT_READING_ENCODING;
}
return charset;
}
/**
* Remove the first path element of a path. Return null if there is only one path element
*
* <p>E.g. /foo/bar => /bar?a=b
*
* @param path path to modify
* @return modified path or null
*/
public static String removeFirstPathElement(String path) {
final int secondSlashIndex = path.indexOf('/', 1);
if (secondSlashIndex == -1) return null;
return path.substring(secondSlashIndex);
}
/**
* Return the first path element of a path. If there is only one path element, return the entire
* path.
*
* <p>E.g. /foo/bar => /foo
*
* @param path path to analyze
* @return first path element
*/
public static String getFirstPathElement(String path) {
final int secondSlashIndex = path.indexOf('/', 1);
if (secondSlashIndex == -1) return path;
return path.substring(0, secondSlashIndex);
}
/**
* Transform an URI accessible from the server into a URI accessible from the client. The mapping
* expires with the session.
*
* @param propertyContext context to obtain session
* @param uri server URI to transform
* @param filename file name
* @param contentType type of the content referred to by the URI, or null if unknown
* @param lastModified last modification timestamp
* @return client URI
*/
public static String proxyURI(
PropertyContext propertyContext,
String uri,
String filename,
String contentType,
long lastModified) {
// Create a digest, so that for a given URI we always get the same key
final String digest = SecureUtils.digestString(uri, "MD5", "hex");
// Get session
final ExternalContext externalContext =
(ExternalContext) propertyContext.getAttribute(PipelineContext.EXTERNAL_CONTEXT);
final ExternalContext.Session session =
externalContext.getSession(
true); // NOTE: We force session creation here. Should we? What's the alternative?
if (session != null) {
// Store mapping into session
session
.getAttributesMap(ExternalContext.Session.APPLICATION_SCOPE)
.put(
DYNAMIC_RESOURCES_SESSION_KEY + digest,
new DynamicResource(uri, filename, contentType, -1, lastModified));
}
// Rewrite new URI to absolute path without the context
return DYNAMIC_RESOURCES_PATH + digest;
}
/**
* Utility method to decode a multipart/fomr-data stream and return a Map of parameters of type
* Object[], each of which can be a String or FileData.
*/
public static Map<String, Object[]> getParameterMapMultipart(
PipelineContext pipelineContext,
final ExternalContext.Request request,
String headerEncoding) {
final Map<String, Object[]> uploadParameterMap = new HashMap<String, Object[]>();
try {
// Setup commons upload
// Read properties
// NOTE: We use properties scoped in the Request generator for historical reasons. Not too
// good.
int maxSize = RequestGenerator.getMaxSizeProperty();
int maxMemorySize = RequestGenerator.getMaxMemorySizeProperty();
final DiskFileItemFactory diskFileItemFactory =
new DiskFileItemFactory(maxMemorySize, SystemUtils.getTemporaryDirectory());
final ServletFileUpload upload =
new ServletFileUpload(diskFileItemFactory) {
protected FileItem createItem(Map headers, boolean isFormField)
throws FileUploadException {
if (isFormField) {
// Handle externalized values
final String externalizeFormValuesPrefix =
org.orbeon.oxf.properties.Properties.instance()
.getPropertySet()
.getString(ServletExternalContext.EXTERNALIZE_FORM_VALUES_PREFIX_PROPERTY);
final String fieldName = getFieldName(headers);
if (externalizeFormValuesPrefix != null
&& fieldName.startsWith(externalizeFormValuesPrefix)) {
// In this case, we do as if the value content is an uploaded file so that it can
// be externalized
return super.createItem(headers, false);
} else {
// Just create the FileItem using the default way
return super.createItem(headers, isFormField);
}
} else {
// Just create the FileItem using the default way
return super.createItem(headers, isFormField);
}
}
};
upload.setHeaderEncoding(headerEncoding);
upload.setSizeMax(maxSize);
// Add a listener to destroy file items when the pipeline context is destroyed
pipelineContext.addContextListener(
new PipelineContext.ContextListenerAdapter() {
public void contextDestroyed(boolean success) {
for (final String name : uploadParameterMap.keySet()) {
final Object values[] = uploadParameterMap.get(name);
for (final Object currentValue : values) {
if (currentValue instanceof FileItem) {
final FileItem fileItem = (FileItem) currentValue;
fileItem.delete();
}
}
}
}
});
// Wrap and implement just the required methods for the upload code
final InputStream inputStream;
try {
inputStream = request.getInputStream();
} catch (IOException e) {
throw new OXFException(e);
}
final RequestContext requestContext =
new RequestContext() {
public int getContentLength() {
return request.getContentLength();
}
public InputStream getInputStream() {
// NOTE: The upload code does not actually check that it doesn't read more than the
// content-length
// sent by the client! Maybe here would be a good place to put an interceptor and make
// sure we
// don't read too much.
return new InputStream() {
public int read() throws IOException {
return inputStream.read();
}
};
}
public String getContentType() {
return request.getContentType();
}
public String getCharacterEncoding() {
return request.getCharacterEncoding();
}
};
// Parse the request and add file information
try {
for (Object o : upload.parseRequest(requestContext)) {
final FileItem fileItem = (FileItem) o;
// Add value to existing values if any
if (fileItem.isFormField()) {
// Simple form field
// Assume that form fields are in UTF-8. Can they have another encoding? If so, how is
// it specified?
StringUtils.addValueToObjectArrayMap(
uploadParameterMap,
fileItem.getFieldName(),
fileItem.getString(STANDARD_PARAMETER_ENCODING));
} else {
// File
StringUtils.addValueToObjectArrayMap(
uploadParameterMap, fileItem.getFieldName(), fileItem);
}
}
} catch (FileUploadBase.SizeLimitExceededException e) {
// Should we do something smart so we can use the Presentation
// Server error page anyway? Right now, this is going to fail
// miserably with an error.
throw e;
} catch (UnsupportedEncodingException e) {
// Should not happen
throw new OXFException(e);
} finally {
// Close the input stream; if we don't nobody does, and if this stream is
// associated with a temporary file, that file may resist deletion
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
throw new OXFException(e);
}
}
}
return uploadParameterMap;
} catch (FileUploadException e) {
throw new OXFException(e);
}
}
/**
* Encode a Human Readable Resource Identifier to a URI. Leading and trailing spaces are removed
* first.
*
* <p>NOTE: See more recent W3C note: http://www.w3.org/TR/2008/NOTE-leiri-20081103/
*
* @param uriString URI to encode
* @param processSpace whether to process the space character or leave it unchanged
* @return encoded URI, or null if uriString was null
*/
public static String encodeHRRI(String uriString, boolean processSpace) {
if (uriString == null) return null;
// Note that the XML Schema spec says "Spaces are, in principle, allowed in the ·lexical space·
// of anyURI,
// however, their use is highly discouraged (unless they are encoded by %20).".
// We assume that we never want leading or trailing spaces. You can use %20 if you really want
// this.
uriString = uriString.trim();
// We try below to follow the "Human Readable Resource Identifiers" RFC, in draft as of
// 2007-06-06.
// * the control characters #x0 to #x1F and #x7F to #x9F
// * space #x20
// * the delimiters "<" #x3C, ">" #x3E, and """ #x22
// * the unwise characters "{" #x7B, "}" #x7D, "|" #x7C, "\" #x5C, "^" #x5E, and "`" #x60
final StringBuilder sb = new StringBuilder(uriString.length() * 2);
for (int i = 0; i < uriString.length(); i++) {
final char currentChar = uriString.charAt(i);
if (currentChar >= 0
&& (currentChar <= 0x1f
|| (processSpace && currentChar == 0x20)
|| currentChar == 0x22
|| currentChar == 0x3c
|| currentChar == 0x3e
|| currentChar == 0x5c
|| currentChar == 0x5e
|| currentChar == 0x60
|| (currentChar >= 0x7b && currentChar <= 0x7d)
|| (currentChar >= 0x7f && currentChar <= 0x9f))) {
sb.append('%');
sb.append(NumberUtils.toHexString((byte) currentChar).toUpperCase());
} else {
sb.append(currentChar);
}
}
return sb.toString();
}
public static class DynamicResource {
private String uri;
private String filename;
private String contentType;
private long size;
private long lastModified;
public DynamicResource(
String uri, String filename, String contentType, long size, long lastModified) {
this.uri = uri;
this.filename = filename;
this.contentType = contentType;
this.size = size;
this.lastModified = lastModified;
}
public String getURI() {
return uri;
}
public String getFilename() {
return filename;
}
public String getContentType() {
return contentType;
}
public long getSize() {
return size;
}
public long getLastModified() {
return lastModified;
}
}
}
