private static String computeKeyHash(String memcachedPrefix, NamedKey key) {
// hash keys to keep things under 250 characters for memcached
return memcachedPrefix
+ ":"
+ DigestUtils.sha1Hex(key.namespace)
+ ":"
+ DigestUtils.sha1Hex(key.key);
}
private String getUserId() {
final String computerName = getComputerName();
final String userName = getSystemProperty("user.name"); // $NON-NLS-1$
final String fakeUserId =
MessageFormat.format("{0}@{1}", userName, computerName); // $NON-NLS-1$
return DigestUtils.sha1Hex(fakeUserId);
}
public Feed findByUrl(String url) {
List<Feed> feeds = findByField(Feed_.urlHash, DigestUtils.sha1Hex(url));
Feed feed = Iterables.getFirst(feeds, null);
if (feed != null && StringUtils.equals(url, feed.getUrl())) {
return feed;
}
return null;
}
/**
* 签名字符串
*
* @param text 需要签名的字符串
* @param sign 签名结果
* @param key 密钥
* @param input_charset 编码格式
* @return 签名结果
*/
public static boolean verify(String text, String sign, String key, String input_charset) {
text = text + key;
String mysign = DigestUtils.sha1Hex(getContentBytes(text, input_charset));
if (mysign.equals(sign)) {
return true;
} else {
return false;
}
}
@Override
public void execute(String folder) {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
// dbf.setNamespaceAware(true);
// dbf.setValidating(true);
// dbf.setAttribute(JAXP_SCHEMA_LANGUAGE, W3C_XML_SCHEMA);
try {
DocumentBuilder db = dbf.newDocumentBuilder();
// db.setErrorHandler(new DefaultHandler());
Document document = db.parse(Paths.get(folder, "changelog.xml").toFile());
NodeList nodeList = document.getElementsByTagName("include");
for (int i = 0; i < nodeList.getLength(); i++) {
final Node node = nodeList.item(i);
if (node.getNodeType() == Node.ELEMENT_NODE) {
final String mode = node.getAttributes().getNamedItem("mode").getTextContent();
final String filePath = node.getAttributes().getNamedItem("file").getTextContent();
final File file = Paths.get(folder, filePath).toFile();
final String checksum = DigestUtils.sha1Hex(new FileInputStream(file));
final String author = System.getProperty("user.name");
final String options = node.getAttributes().getNamedItem("options").getTextContent();
// row = getODChangeLog(where id = i)
// if ( != checksum.equals(row.getDisplayValue("checksum")) {
// System.err.println("Error file has been modified since previous migration: " +
// filePath);
// System.err.println("Invalid checksum: item n° "+ i + " file computed checksum: " +
// checksum + " doesn't match previous checksum in od: " +
// row.getDisplayValue("checksum"));
// System.exit(-1);
// }
final String validateCommand =
new StringBuilder("arsxml validate -i ").append(file.getAbsolutePath()).toString();
exec(validateCommand);
final String arsxmlCommand =
new StringBuilder("arsxml ")
.append(mode)
.append(" ")
.append(options)
.append(" ")
.append(file.getAbsolutePath())
.toString();
exec(arsxmlCommand);
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-hhmmss.S");
String timestamp = formatter.format(new Date());
// insert in ODCHANGELOG (i, checksum, filePath, author, timestamp, Version.VERSION)
}
}
} catch (IOException | ParserConfigurationException | SAXException | InterruptedException e) {
throw new RuntimeException(e);
}
}
// run to get a sense of cache key distribution for different ketama reps / hash functions
@Test
public void testDistribution() throws Exception {
KetamaNodeLocator locator =
new KetamaNodeLocator(
ImmutableList.of(
dummyNode("druid-cache.0001", 11211),
dummyNode("druid-cache.0002", 11211),
dummyNode("druid-cache.0003", 11211),
dummyNode("druid-cache.0004", 11211),
dummyNode("druid-cache.0005", 11211)),
hash,
new DefaultKetamaNodeLocatorConfiguration() {
@Override
public int getNodeRepetitions() {
return reps;
}
});
Map<MemcachedNode, AtomicLong> counter = Maps.newHashMap();
long t = 0;
for (int i = 0; i < KEY_COUNT; ++i) {
final String k = DigestUtils.sha1Hex("abc" + i) + ":" + DigestUtils.sha1Hex("xyz" + i);
long t0 = System.nanoTime();
MemcachedNode node = locator.getPrimary(k);
t += System.nanoTime() - t0;
if (counter.containsKey(node)) {
counter.get(node).incrementAndGet();
} else {
counter.put(node, new AtomicLong(1));
}
}
long min = Long.MAX_VALUE;
long max = 0;
System.out.printf("%25s\t%5d\t", hash, reps);
for (AtomicLong count : counter.values()) {
System.out.printf("%10d\t", count.get());
min = Math.min(min, count.get());
max = Math.max(max, count.get());
}
System.out.printf("%7.2f\t%5.0f\n", (double) min / (double) max, (double) t / KEY_COUNT);
}
private void assertSHA1(String path, Response dataResponse) {
String sha1Path = path + ".sha1";
Response sha1Response = context.client().target(sha1Path).request().get();
assertResponse(sha1Path, sha1Response, MediaType.TEXT_PLAIN);
try (InputStream is = dataResponse.readEntity(InputStream.class)) {
byte[] data = IOUtils.toByteArray(is);
String sha1 = sha1Response.readEntity(String.class);
assertEquals("SHA-1 " + path, sha1, DigestUtils.sha1Hex(data));
} catch (IOException ex) {
throw new RuntimeException("Error checking SHA-1 of " + path, ex);
}
}
/**
* 支付签名
*
* @param timestamp
* @param noncestr
* @param packages
* @return
* @throws UnsupportedEncodingException
*/
public static String paySign(String timestamp, String noncestr, String packages)
throws UnsupportedEncodingException {
Map<String, String> paras = new HashMap<String, String>();
paras.put("appid", ConfKit.get("AppId"));
paras.put("timestamp", timestamp);
paras.put("noncestr", noncestr);
paras.put("package", packages);
paras.put("appkey", ConfKit.get("paySignKey"));
// appid、timestamp、noncestr、package 以及 appkey。
String string1 = createSign(paras, false);
String paySign = DigestUtils.sha1Hex(string1);
return paySign;
}
@SuppressWarnings("unchecked")
public ImportRecords(final Config c) {
random = new SecureRandom();
recordID = DigestUtils.sha1Hex(new BigInteger(16, random).toString(16)).substring(0, 16);
record = new JSONObject();
record.put("record_id", recordID);
record.put("first_name", "First");
record.put("last_name", "Last");
record.put("address", "123 Cherry Lane\nNashville, TN 37015");
record.put("telephone", "(615) 255-4000");
record.put("email", "*****@*****.**");
record.put("dob", "1972-08-10");
record.put("age", "43");
record.put("ethnicity", "1");
record.put("race", "4");
record.put("sex", "1");
record.put("height", "180");
record.put("weight", "105");
record.put("bmi", "31.4");
record.put("comments", "comments go here");
record.put("redcap_event_name", "events_2_arm_1");
record.put("basic_demography_form_complete", "2");
data = new JSONArray();
data.add(record);
params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("token", c.API_TOKEN));
params.add(new BasicNameValuePair("content", "record"));
params.add(new BasicNameValuePair("format", "json"));
params.add(new BasicNameValuePair("type", "flat"));
params.add(new BasicNameValuePair("data", data.toJSONString()));
post = new HttpPost(c.API_URL);
post.setHeader("Content-Type", "application/x-www-form-urlencoded");
try {
post.setEntity(new UrlEncodedFormEntity(params));
} catch (final Exception e) {
e.printStackTrace();
}
result = new StringBuffer();
client = HttpClientBuilder.create().build();
respCode = -1;
reader = null;
line = null;
}
/**
* 支付回调校验签名
*
* @param timestamp
* @param noncestr
* @param openid
* @param issubscribe
* @param appsignature
* @return
* @throws UnsupportedEncodingException
*/
public static boolean verifySign(
long timestamp, String noncestr, String openid, int issubscribe, String appsignature)
throws UnsupportedEncodingException {
Map<String, String> paras = new HashMap<String, String>();
paras.put("appid", ConfKit.get("AppId"));
paras.put("appkey", ConfKit.get("paySignKey"));
paras.put("timestamp", String.valueOf(timestamp));
paras.put("noncestr", noncestr);
paras.put("openid", openid);
paras.put("issubscribe", String.valueOf(issubscribe));
// appid、appkey、productid、timestamp、noncestr、openid、issubscribe
String string1 = createSign(paras, false);
String paySign = DigestUtils.sha1Hex(string1);
return paySign.equalsIgnoreCase(appsignature);
}
private boolean addEntry(
final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
boolean success = false;
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty("" + feed.getId());
// lock on content, make sure we are not updating the same entry
// twice at the same time
FeedEntryContent content = entry.getContent();
String key2 =
DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent() + content.getTitle()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
feedUpdateService.updateEntry(feed, entry);
List<User> users = Lists.newArrayList();
for (FeedSubscription sub : subscriptions) {
users.add(sub.getUser());
}
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
metricsBean.entryInserted();
success = true;
} else {
log.error("lock timeout for " + feed.getUrl() + " - " + key1);
}
} catch (InterruptedException e) {
log.error(
"interrupted while waiting for lock for " + feed.getUrl() + " : " + e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return success;
}
/**
* Constructs an internal node based on its left and right child.
*
* @param childL left child for the current node (always not null)
* @param childR right child for the current node (can be null)
* @return internal node
*/
private Node constructInternalNode(Node childL, Node childR) {
Node parent = new Node();
if (childR == null) {
parent.digest = childL.digest;
} else {
parent.digest = DigestUtils.sha1Hex(childL.digest + childR.digest);
childR.parent = parent;
}
childL.parent = parent;
parent.left = childL;
parent.right = childR;
return parent;
}
public FetchedFeed fetch(
String feedUrl,
boolean extractFeedUrlFromHtml,
String lastModified,
String eTag,
Date lastPublishedDate,
String lastContentHash)
throws FeedException, ClientProtocolException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(StringUtils.newStringUtf8(result.getContent()), feedUrl);
if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
result = getter.getBinary(extractedUrl, lastModified, eTag);
feedUrl = extractedUrl;
}
}
byte[] content = result.getContent();
if (content == null) {
throw new IOException("Feed content is empty.");
}
String hash = DigestUtils.sha1Hex(content);
if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException();
}
fetchedFeed = parser.parse(feedUrl, content);
if (lastPublishedDate != null
&& fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException();
}
Feed feed = fetchedFeed.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.geteTag(), 255));
feed.setLastContentHash(hash);
fetchedFeed.setFetchDuration(result.getDuration());
return fetchedFeed;
}
@Override
public void processMeta(FileMeta meta) {
try {
FileInputStream fis = new FileInputStream(meta.getFilePointer());
String sha1 = org.apache.commons.codec.digest.DigestUtils.sha1Hex(fis);
Hash result = new Hash();
result.setHashType(HashType.SHA1);
result.setHash(sha1);
meta.getDecorations().add(result);
if (LOG.isDebugEnabled()) {
LOG.debug("File: " + meta.getFilePointer().getAbsolutePath() + " : SHA1: " + sha1);
}
chainDecorators(meta);
} catch (Exception e) {
LOG.error("Exception generating MD5.", e);
}
}
public void onData(JID jid, String id, byte[] avatarData) {
LOGGER.info("new avatar, jid: " + jid + " id: " + id);
if (avatarData.length > MAX_SIZE) LOGGER.info("avatar data too long: " + avatarData.length);
Contact contact = ContactList.getInstance().get(jid).orElse(null);
if (contact == null) {
LOGGER.warning("can't find contact with jid:" + jid);
return;
}
if (!id.equals(DigestUtils.sha1Hex(avatarData))) {
LOGGER.warning("this is not what we wanted");
return;
}
BufferedImage img = MediaUtils.readImage(avatarData).orElse(null);
if (img == null) return;
contact.setAvatar(new Avatar(id, img));
}
@RequestMapping(method = RequestMethod.GET)
public String verify(
@RequestParam String signature,
@RequestParam String timestamp,
@RequestParam String nonce,
@RequestParam String echostr,
ModelMap model) {
try {
List<String> strList = Arrays.asList(TOKEN, timestamp, nonce);
Collections.sort(strList);
String verifyCode = DigestUtils.sha1Hex(StringUtils.join(strList, ""));
if (verifyCode.equals(signature)) {
model.addAttribute("echostr", echostr);
} else {
model.addAttribute("echostr", "failed to verify");
}
} catch (Exception e) {
e.printStackTrace();
}
return "verify";
}
/**
* 检查消息验证
*
* @param signature
* @param timestamp
* @param nonce
* @param echostr
* @param token
* @return
*/
public static boolean checkAuthentication(
String signature, String timestamp, String nonce, String echostr, String token) {
// String result = "";
boolean result = false;
// 将获取到的参数放入数组
String[] ArrTmp = {token, timestamp, nonce};
// 按微信提供的方法，对数据内容进行排序
Arrays.sort(ArrTmp);
StringBuffer sb = new StringBuffer();
for (int i = 0; i < ArrTmp.length; i++) {
sb.append(ArrTmp[i]);
}
// 对排序后的字符串进行SHA-1加密
String pwd = "";
try {
pwd = DigestUtils.sha1Hex(sb.toString());
} catch (Exception e) {
e.printStackTrace();
}
result = pwd.equals(signature);
return result;
}
public List<Feed> findByTopic(String topic) {
return findByField(Feed_.pushTopicHash, DigestUtils.sha1Hex(topic));
}
/**
* 发货通知签名
*
* @param paras
* @return
* @throws UnsupportedEncodingException @参数
* appid、appkey、openid、transid、out_trade_no、deliver_timestamp、deliver_status、deliver_msg；
*/
private static String deliverSign(Map<String, String> paras) throws UnsupportedEncodingException {
paras.put("appkey", ConfKit.get("paySignKey"));
String string1 = createSign(paras, false);
String paySign = DigestUtils.sha1Hex(string1);
return paySign;
}
public static String toHex(MessageDigest crypt) {
String hash = DigestUtils.sha1Hex(crypt.digest());
return hash;
}
public String generateApiKey(User user) {
byte[] key =
encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return DigestUtils.sha1Hex(key);
}
/**
* Returns a random token not guaranteed to be unique.
*
* @return token
*/
public static String generateToken() {
String randomString =
"somesaltymcsalt" + String.valueOf(Math.random()) + String.valueOf((new Date()).getTime());
return DigestUtils.sha1Hex(randomString);
}
public DBObject buildDbObject(JSONObject jsonObject) {
if (!check(jsonObject)) {
return null;
}
BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
JSONObject mobile = jsonObject.getJSONObject(FieldName.Tanx.mobile);
JSONObject device = mobile.getJSONObject(FieldName.Tanx.device);
String deviceId = device.getString(FieldName.Tanx.device_id);
// 过滤非法deviceId
if (deviceId.contains("000000000000000") || deviceId.contains(":")) {
logger.info("=== ignore deviceId:{}", deviceId);
return null;
}
// 新增， 不更新的
BasicDBObject setObj =
new BasicDBObject(FieldName.field_deviceId, deviceId)
.append(FieldName.field_deviceIdMd5, DigestUtils.md5Hex(deviceId))
.append(FieldName.field_deviceIdSha1, DigestUtils.sha1Hex(deviceId));
// brand
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.brand))) {
setObj.append(FieldName.field_brand, device.getString(FieldName.Tanx.brand));
}
// model
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.model))) {
setObj.append(FieldName.field_model, device.getString(FieldName.Tanx.model));
}
// platform
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.platform))) {
setObj.append(FieldName.field_platform, device.getString(FieldName.Tanx.platform));
}
// os
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.os))) {
setObj.append(FieldName.field_os, device.getString(FieldName.Tanx.os));
}
// os_version
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.os_version))) {
setObj.append(FieldName.field_osVersion, device.getString(FieldName.Tanx.os_version));
}
// mac
String mac = device.getString(FieldName.Tanx.mac);
if (StringUtils.isNotBlank(mac) && mac.contains(":")) {
setObj
.append(FieldName.field_mac, mac)
.append(FieldName.field_macMd5, DigestUtils.md5Hex(mac))
.append(FieldName.field_macSha1, DigestUtils.sha1Hex(mac));
}
// device_size
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.device_size))) {
setObj.append(FieldName.field_deviceSize, device.getString(FieldName.Tanx.device_size));
}
// builder.add(MongoSink.OP_SET_ON_INSERT, setObj);
BasicDBObject addToSetObj = new BasicDBObject();
// ip
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.Tanx.ip))) {
addToSetObj.append(FieldName.field_ipList, jsonObject.getString(FieldName.Tanx.ip));
}
// 城市名称
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.Tanx.city))) {
addToSetObj.append(FieldName.field_cityList, jsonObject.getString(FieldName.Tanx.city));
}
// 追加的数据，geo
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.latitude))
&& StringUtils.isNotBlank(device.getString(FieldName.Tanx.longitude))) {
addToSetObj.append(
FieldName.field_geoList,
new BasicDBObject(FieldName.field_latitude, device.getString(FieldName.Tanx.latitude))
.append(FieldName.field_longitude, device.getString(FieldName.Tanx.longitude)));
}
builder.add(MongoSink.OP_ADD_TO_SET, addToSetObj);
// app
if (StringUtils.isNotBlank(mobile.getString(FieldName.Tanx.package_name))) {
BasicDBObject appObj = new BasicDBObject();
appObj.append(FieldName.field_packageName, mobile.getString(FieldName.Tanx.package_name));
if (StringUtils.isNotBlank(mobile.getString(FieldName.Tanx.app_name))) {
appObj.append(FieldName.field_appName, mobile.getString(FieldName.Tanx.app_name));
}
// app分类
if (mobile.getJSONArray(FieldName.Tanx.app_categories) != null
&& !mobile.getJSONArray(FieldName.Tanx.app_categories).isEmpty()) {
appObj.append(
FieldName.field_appCategorys,
mobile
.getJSONArray(FieldName.Tanx.app_categories)
.getJSONObject(0)
.getString(FieldName.Tanx.id));
}
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.bidRequestTime))) {
appObj.append(FieldName.lastUseTime, jsonObject.getString(FieldName.bidRequestTime));
}
// mongo 不支持包含点. 的做key， 替换为_下划线
setObj.append(
FieldName.field_appList + mobile.getString(FieldName.Tanx.package_name).replace(".", "_"),
appObj);
}
if (addToSetObj.isEmpty()) {
return null;
}
// app需要记录最后使用日期， 每次都更新
builder.add(MongoSink.OP_SET, setObj);
return builder.get();
}
/** Find a leaf node by a line from input file */
public Node findLeafNode(String str) throws Exception {
return this.getLeafMap().get(DigestUtils.sha1Hex(str));
}
