private void deleteIndexesAll(List<CNSSubscription> subscriptionList)
throws PersistenceException {
List<String> subArnList = new LinkedList<String>();
List<String> userIdList = new LinkedList<String>();
List<String> tokenList = new LinkedList<String>();
for (CNSSubscription sub : subscriptionList) {
subArnList.add(sub.getArn());
userIdList.add(sub.getUserId());
tokenList.add(sub.getToken());
}
cassandraHandler.deleteBatch(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsIndex,
subArnList,
null,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.deleteBatch(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsUserIndex,
userIdList,
subArnList,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.deleteBatch(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsTokenIndex,
tokenList,
subArnList,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
@Override
public void setRawMessageDelivery(String subscriptionArn, boolean rawMessageDelivery)
throws Exception {
CNSSubscription sub;
sub = getSubscription(subscriptionArn);
if (sub != null) {
sub.setRawMessageDelivery(rawMessageDelivery);
insertOrUpdateSubsAndIndexes(sub, null);
}
}
private void insertOrUpdateSubsAndIndexes(final CNSSubscription subscription, Integer ttl)
throws Exception {
subscription.checkIsValid();
CmbComposite columnName =
cassandraHandler.getCmbComposite(
subscription.getEndpoint(), subscription.getProtocol().name());
cassandraHandler.update(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
subscription.getTopicArn(),
columnName,
getColumnValuesJSON(subscription),
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.COMPOSITE_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
ttl);
cassandraHandler.insertRow(
AbstractDurablePersistence.CNS_KEYSPACE,
subscription.getArn(),
columnFamilySubscriptionsIndex,
getIndexColumnValues(subscription.getEndpoint(), subscription.getProtocol()),
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
ttl);
cassandraHandler.insertRow(
AbstractDurablePersistence.CNS_KEYSPACE,
subscription.getUserId(),
columnFamilySubscriptionsUserIndex,
new HashMap<String, String>() {
{
put(subscription.getArn(), "");
}
},
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
ttl);
cassandraHandler.insertRow(
AbstractDurablePersistence.CNS_KEYSPACE,
subscription.getToken(),
columnFamilySubscriptionsTokenIndex,
new HashMap<String, String>() {
{
put(subscription.getArn(), "");
}
},
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
ttl);
}
@Override
public CNSSubscription getSubscription(String arn) throws Exception {
// read form index to get composite col-name
CmbColumnSlice<String, String> slice =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsIndex,
arn,
null,
null,
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (slice != null) {
// get Column from main table
String colName = slice.getColumns().get(0).getName();
CnsSubscriptionProtocol protocol = getEndpointAndProtoIndexValProtocol(colName);
String endpoint = getEndpointAndProtoIndexValEndpoint(colName);
CmbComposite columnName = cassandraHandler.getCmbComposite(endpoint, protocol.name());
CmbColumn<CmbComposite, String> column =
cassandraHandler.readColumn(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
Util.getCnsTopicArn(arn),
columnName,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.COMPOSITE_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (column != null) {
CNSSubscription s = extractSubscriptionFromColumn(column, Util.getCnsTopicArn(arn));
s.checkIsValid();
return s;
}
}
return null;
}
@Override
public void unsubscribe(String arn) throws Exception {
CNSSubscription s = getSubscription(arn);
if (s != null) {
deleteIndexes(arn, s.getUserId(), s.getToken());
CmbComposite columnName =
cassandraHandler.getCmbComposite(s.getEndpoint(), s.getProtocol().name());
cassandraHandler.delete(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
Util.getCnsTopicArn(arn),
columnName,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.COMPOSITE_SERIALIZER);
if (s.isConfirmed()) {
cassandraHandler.decrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
s.getTopicArn(),
"subscriptionConfirmed",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
} else {
cassandraHandler.decrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
s.getTopicArn(),
"subscriptionPending",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
s.getTopicArn(),
"subscriptionDeleted",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
}
@Override
public CNSSubscription confirmSubscription(
boolean authenticateOnUnsubscribe, String token, String topicArn) throws Exception {
// get Sub-arn given token
CmbColumnSlice<String, String> slice =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsTokenIndex,
token,
null,
null,
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (slice == null) {
throw new CMBException(CMBErrorCodes.NotFound, "Resource not found.");
}
// get Column from main table
String subArn = slice.getColumns().get(0).getName();
// get Subscription given subArn
final CNSSubscription s = getSubscription(subArn);
if (s == null) {
throw new SubscriberNotFoundException(
"Could not find subscription given subscription arn " + subArn);
}
s.setAuthenticateOnUnsubscribe(authenticateOnUnsubscribe);
s.setConfirmed(true);
s.setConfirmDate(new Date());
// re-insert with no TTL. will clobber the old one which had ttl
insertOrUpdateSubsAndIndexes(s, null);
cassandraHandler.decrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
s.getTopicArn(),
"subscriptionPending",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
s.getTopicArn(),
"subscriptionConfirmed",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
return s;
}
@Override
public void unsubscribeAll(String topicArn) throws Exception {
int pageSize = 1000;
String nextToken = null;
List<CNSSubscription> subs =
listSubscriptionsByTopic(nextToken, topicArn, null, pageSize, false);
// Note: for pagination to work we need the nextToken's corresponding sub to not be deleted.
CNSSubscription nextTokenSub = null;
while (subs.size() > 0) {
// if retrieve subscription is less than page size, delete all index.
if (subs.size() < pageSize) {
deleteIndexesAll(subs);
break;
} else {
// keep the last subscription for pagination purpose.
nextTokenSub = subs.get(subs.size() - 1);
nextToken = nextTokenSub.getArn();
subs.remove(subs.size() - 1);
deleteIndexesAll(subs);
subs = listSubscriptionsByTopic(nextToken, topicArn, null, pageSize, false);
deleteIndexes(nextTokenSub.getArn(), nextTokenSub.getUserId(), nextTokenSub.getToken());
}
}
// int subscriptionConfirmedNum = (int)cassandraHandler.getCounter(CNS_KEYSPACE,
// columnFamilyTopicStats, topicArn, "subscriptionConfirmed", CMB_SERIALIZER.STRING_SERIALIZER,
// CMB_SERIALIZER.STRING_SERIALIZER);
// int subscriptionPendingNum = (int)cassandraHandler.getCounter(CNS_KEYSPACE,
// columnFamilyTopicStats, topicArn, "subscriptionPending", CMB_SERIALIZER.STRING_SERIALIZER,
// CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
topicArn,
"subscriptionDeleted",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.decrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
topicArn,
"subscriptionConfirmed",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.decrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
topicArn,
"subscriptionPending",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
cassandraHandler.delete(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
topicArn,
null,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
private String getColumnValuesJSON(CNSSubscription s) throws JSONException {
Writer writer = new StringWriter();
JSONWriter jw = new JSONWriter(writer);
jw = jw.object();
if (s.getEndpoint() != null) {
jw.key("endPoint").value(s.getEndpoint());
}
if (s.getToken() != null) {
jw.key("token").value(s.getToken());
}
if (s.getArn() != null) {
jw.key("subArn").value(s.getArn());
}
if (s.getUserId() != null) {
jw.key("userId").value(s.getUserId());
}
if (s.getConfirmDate() != null) {
jw.key("confirmDate").value(s.getConfirmDate().getTime() + "");
}
if (s.getProtocol() != null) {
jw.key("protocol").value(s.getProtocol().toString());
}
if (s.getRequestDate() != null) {
jw.key("requestDate").value(s.getRequestDate().getTime() + "");
}
jw.key("authenticateOnSubscribe").value(s.isAuthenticateOnUnsubscribe() + "");
jw.key("isConfirmed").value(s.isConfirmed() + "");
jw.key("rawMessageDelivery").value(s.getRawMessageDelivery() + "");
jw.endObject();
return writer.toString();
}
/**
* Enumerate all subs in a topic
*
* @param nextToken The ARN of the last sub-returned or null is first time call.
* @param topicArn
* @param protocol
* @param pageSize
* @param hidePendingArn
* @return The list of subscriptions given a topic. Note: if nextToken is provided, the returned
* list will not contain it for convenience
* @throws Exception
*/
public List<CNSSubscription> listSubscriptionsByTopic(
String nextToken,
String topicArn,
CnsSubscriptionProtocol protocol,
int pageSize,
boolean hidePendingArn)
throws Exception {
if (nextToken != null) {
if (getSubscription(nextToken) == null) {
throw new SubscriberNotFoundException("Subscriber not found for arn " + nextToken);
}
}
// read from index to get composite-col-name corresponding to nextToken
CmbComposite nextTokenComposite = null;
if (nextToken != null) {
CmbColumnSlice<String, String> slice =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsIndex,
nextToken,
null,
null,
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (slice == null) {
throw new IllegalArgumentException("Could not find any subscription with arn " + nextToken);
}
// get Column from main table
String colName = slice.getColumns().get(0).getName();
CnsSubscriptionProtocol tokProtocol = getEndpointAndProtoIndexValProtocol(colName);
String endpoint = getEndpointAndProtoIndexValEndpoint(colName);
nextTokenComposite = cassandraHandler.getCmbComposite(endpoint, tokProtocol.name());
}
List<CNSSubscription> l = new ArrayList<CNSSubscription>();
CNSTopic t = PersistenceFactory.getTopicPersistence().getTopic(topicArn);
if (t == null) {
throw new TopicNotFoundException("Resource not found.");
}
// read pageSize at a time
CmbColumnSlice<CmbComposite, String> cols =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
topicArn,
nextTokenComposite,
null,
pageSize,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.COMPOSITE_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (nextToken != null && cols.size() > 0) {
cols.getColumns().remove(0);
}
while (l.size() < pageSize) {
if (cols == null || cols.size() == 0) {
return l;
}
for (CmbColumn<CmbComposite, String> col : cols.getColumns()) {
CNSSubscription sub = extractSubscriptionFromColumn(col, topicArn);
// ignore invalid subscriptions coming from Cassandra
try {
sub.checkIsValid();
} catch (CMBException ex) {
logger.error("event=invalid_subscription " + sub.toString(), ex);
continue;
}
if (protocol != null && protocol != sub.getProtocol()) {
continue;
}
if (hidePendingArn) {
if (sub.isConfirmed()) {
l.add(sub);
} else {
sub.setArn("PendingConfirmation");
l.add(sub);
}
} else {
l.add(sub);
}
if (l.size() == pageSize) {
return l;
}
}
nextTokenComposite = cols.getColumns().get(cols.size() - 1).getName();
cols =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptions,
topicArn,
nextTokenComposite,
null,
pageSize,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.COMPOSITE_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (cols.size() > 0) {
cols.getColumns().remove(0);
}
}
return l;
}
private List<CNSSubscription> listSubscriptions(
String nextToken, CnsSubscriptionProtocol protocol, String userId, boolean hidePendingArn)
throws Exception {
if (nextToken != null) {
if (getSubscription(nextToken) == null) {
throw new SubscriberNotFoundException("Subscriber not found for arn " + nextToken);
}
}
// Algorithm is to keep reading in chunks of 100 till we've seen 500 from the nextToken-ARN
List<CNSSubscription> l = new ArrayList<CNSSubscription>();
// read form index to get sub-arn
while (l.size() < 100) {
CmbColumnSlice<String, String> slice =
cassandraHandler.readColumnSlice(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilySubscriptionsUserIndex,
userId,
nextToken,
null,
500,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
if (slice == null) {
return l;
}
// get Column from main table
List<CmbColumn<String, String>> cols = slice.getColumns();
if (nextToken != null) {
cols.remove(0);
}
if (cols.size() == 0) {
return l;
}
for (CmbColumn<String, String> col : cols) {
String subArn = col.getName();
CNSSubscription subscription = getSubscription(subArn);
if (subscription == null) {
throw new IllegalStateException(
"Subscriptions-user-index contains subscription-arn which doesn't exist in subscriptions-index. subArn:"
+ subArn);
}
// ignore invalid subscriptions coming from Cassandra
try {
subscription.checkIsValid();
} catch (CMBException ex) {
logger.error("event=invalid_subscription " + subscription.toString(), ex);
continue;
}
if (protocol != null && subscription.getProtocol() != protocol) {
continue;
}
if (hidePendingArn) {
if (subscription.isConfirmed()) {
l.add(subscription);
} else {
subscription.setArn("PendingConfirmation");
l.add(subscription);
}
} else {
l.add(subscription);
}
if (l.size() == 100) {
return l;
}
}
nextToken = cols.get(cols.size() - 1).getName();
}
return l;
}
private static CNSSubscription extractSubscriptionFromColumn(
CmbColumn<CmbComposite, String> column, String topicArn) throws JSONException {
JSONObject json = new JSONObject(column.getValue());
CNSSubscription s = new CNSSubscription(json.getString("subArn"));
s.setEndpoint(json.getString("endPoint"));
s.setUserId(json.getString("userId"));
if (json.has("confirmDate")) {
s.setConfirmDate(new Date(json.getLong("confirmDate")));
}
if (json.has("requestDate")) {
s.setRequestDate(new Date(json.getLong("requestDate")));
}
if (json.has("protocol")) {
s.setProtocol(CnsSubscriptionProtocol.valueOf(json.getString("protocol")));
}
if (json.has("isConfirmed")) {
s.setConfirmed(json.getBoolean("isConfirmed"));
}
s.setToken(json.getString("token"));
if (json.has("authenticateOnSubscribe")) {
s.setAuthenticateOnUnsubscribe(json.getBoolean("authenticateOnSubscribe"));
}
if (json.has("rawMessageDelivery")) {
s.setRawMessageDelivery(json.getBoolean("rawMessageDelivery"));
}
s.setTopicArn(topicArn);
return s;
}
@Override
public CNSSubscription subscribe(
String endpoint, CnsSubscriptionProtocol protocol, String topicArn, String userId)
throws Exception {
// subscription is unique by protocol + endpoint + topic
final CNSSubscription subscription = new CNSSubscription(endpoint, protocol, topicArn, userId);
CNSTopic t = PersistenceFactory.getTopicPersistence().getTopic(topicArn);
if (t == null) {
throw new TopicNotFoundException("Resource not found.");
}
// check if queue exists for cqs endpoints
if (protocol.equals(CnsSubscriptionProtocol.cqs)) {
CQSQueue queue =
PersistenceFactory.getQueuePersistence()
.getQueue(com.comcast.cqs.util.Util.getRelativeQueueUrlForArn(endpoint));
if (queue == null) {
throw new CMBException(
CMBErrorCodes.NotFound, "Queue with arn " + endpoint + " does not exist.");
}
}
subscription.setArn(Util.generateCnsTopicSubscriptionArn(topicArn, protocol, endpoint));
// attempt to delete existing subscription
/*Composite superColumnName = new Composite(subscription.getEndpoint(), subscription.getProtocol().name());
HSuperColumn<Composite, String, String> superCol = readColumnFromSuperColumnFamily(columnFamilySubscriptions, subscription.getTopicArn(), superColumnName, new StringSerializer(), new CompositeSerializer(), StringSerializer.get(), StringSerializer.get(), CMBProperties.getInstance().getReadConsistencyLevel());
if (superCol != null) {
CNSSubscription exisitingSub = extractSubscriptionFromSuperColumn(superCol, topicArn);
deleteIndexes(exisitingSub.getArn(), exisitingSub.getUserId(), exisitingSub.getToken());
deleteSuperColumn(subscriptionsTemplate, exisitingSub.getTopicArn(), superColumnName);
}*/
// then set confirmation stuff and update cassandra
CNSSubscription retrievedSubscription = getSubscription(subscription.getArn());
if (!CMBProperties.getInstance().getCNSRequireSubscriptionConfirmation()) {
subscription.setConfirmed(true);
subscription.setConfirmDate(new Date());
insertOrUpdateSubsAndIndexes(subscription, null);
if (retrievedSubscription == null) {
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
subscription.getTopicArn(),
"subscriptionConfirmed",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
} else {
// protocols that cannot confirm subscriptions (e.g. redisPubSub)
// get an automatic confirmation here
if (!protocol.canConfirmSubscription()) {
subscription.setConfirmed(true);
subscription.setConfirmDate(new Date());
insertOrUpdateSubsAndIndexes(subscription, null);
// auto confirm subscription to cqs queue by owner
} else if (protocol.equals(CnsSubscriptionProtocol.cqs)) {
String queueOwner = com.comcast.cqs.util.Util.getQueueOwnerFromArn(endpoint);
if (queueOwner != null && queueOwner.equals(userId)) {
subscription.setConfirmed(true);
subscription.setConfirmDate(new Date());
insertOrUpdateSubsAndIndexes(subscription, null);
if (retrievedSubscription == null) {
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
subscription.getTopicArn(),
"subscriptionConfirmed",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
} else {
// use cassandra ttl to implement expiration after 3 days
insertOrUpdateSubsAndIndexes(subscription, 3 * 24 * 60 * 60);
if (retrievedSubscription == null) {
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
subscription.getTopicArn(),
"subscriptionPending",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
}
} else {
// use cassandra ttl to implement expiration after 3 days
insertOrUpdateSubsAndIndexes(subscription, 3 * 24 * 60 * 60);
if (retrievedSubscription == null) {
cassandraHandler.incrementCounter(
AbstractDurablePersistence.CNS_KEYSPACE,
columnFamilyTopicStats,
subscription.getTopicArn(),
"subscriptionPending",
1,
CMB_SERIALIZER.STRING_SERIALIZER,
CMB_SERIALIZER.STRING_SERIALIZER);
}
}
}
CNSSubscriptionAttributes attributes =
new CNSSubscriptionAttributes(topicArn, subscription.getArn(), userId);
PersistenceFactory.getCNSAttributePersistence()
.setSubscriptionAttributes(attributes, subscription.getArn());
return subscription;
}