/**
* Updates (renames) an element, attribute or pi name.
*
* @param pre pre value
* @param kind node kind
* @param name new tag, attribute or pi name
* @param uri uri
*/
public final void update(final int pre, final int kind, final byte[] name, final byte[] uri) {
meta.update();
if (kind == PI) {
updateText(pre, trim(concat(name, SPACE, atom(pre))), kind);
} else {
// update/set namespace reference
final int ouri = nspaces.uri(name, pre, this);
final boolean ne = ouri == 0 && uri.length != 0;
final int npre = kind == ATTR ? parent(pre, kind) : pre;
final int nuri =
ne
? nspaces.add(npre, npre, prefix(name), uri, this)
: ouri != 0 && eq(nspaces.uri(ouri), uri) ? ouri : 0;
// write namespace uri reference
table.write1(pre, kind == ELEM ? 3 : 11, nuri);
// write name reference
table.write2(
pre,
1,
(nsFlag(pre) ? 1 << 15 : 0)
| (kind == ELEM ? tagindex : atnindex).index(name, null, false));
// write namespace flag
table.write2(npre, 1, (ne || nsFlag(npre) ? 1 << 15 : 0) | name(npre));
}
}
Namespace getNamespaceDeclaration(String prefix) {
if (prefix.equals("") && dom instanceof Attr) {
// Default namespaces do not apply to attributes; see XML Namespaces section 5.2
return Namespace.create("", "");
}
Namespaces rv = getAllNamespaces();
return rv.getNamespace(prefix);
}
/**
* Returns all namespace keys and values. Should be only called for element nodes.
*
* @param pre pre value
* @return key and value ids
*/
public final Atts ns(final int pre) {
final Atts as = new Atts();
if (nsFlag(pre)) {
final int[] nsp = nspaces.get(pre, this);
for (int n = 0; n < nsp.length; n += 2)
as.add(nspaces.prefix(nsp[n]), nspaces.uri(nsp[n + 1]));
}
return as;
}
Namespace[] getNamespaceDeclarations() {
// ECMA357 13.4.4.24
if (this.dom instanceof Element) {
Namespaces rv = new Namespaces();
addNamespaces(rv, (Element) this.dom);
return rv.getNamespaces();
} else {
return new Namespace[0];
}
}
/**
* Prints the node structure for debugging purposes.
*
* @param tb token builder
* @param level level
* @param ns namespace reference
* @param start start pre value
* @param end end pre value
*/
private void print(
final TokenBuilder tb, final int level, final Namespaces ns, final int start, final int end) {
if (pr >= start && pr <= end) {
tb.add(NL);
for (int i = 0; i < level; ++i) tb.add(" ");
tb.add(toString() + ' ');
for (int i = 0; i < values.length; i += 2) {
tb.add("xmlns");
final byte[] p = ns.prefix(values[i]);
if (p.length != 0) tb.add(':');
tb.add(p).add("=\"").add(ns.uri(values[i + 1])).add("\" ");
}
}
for (int c = 0; c < sz; ++c) children[c].print(tb, level + 1, ns, start, end);
}
@Override
public boolean exists(String key, boolean isFile) throws IOException {
if (isFile) {
Preconditions.checkArgument(
key.charAt(key.length() - 1) != '/', "File should not end in %s: %s", '/', key);
} else {
Preconditions.checkArgument(
key.charAt(key.length() - 1) == '/', "Dir should end in %s: %s", '/', key);
}
ObjectListing listing;
try {
listing = s3Client.listObjects(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
if (listing != null) {
for (S3ObjectSummary s3os : listing.getObjectSummaries()) {
String object = s3os.getKey();
if (object.equals(key) || (!isFile && object.startsWith(key))) {
// Counts if the keys match exactly, or, the match continues with a subdir path in which
// case this must be a directory
return true;
}
}
}
return false;
}
private Namespaces getAllNamespaces() {
Namespaces rv = new Namespaces();
Node target = this.dom;
if (target instanceof Attr) {
target = ((Attr) target).getOwnerElement();
}
while (target != null) {
if (target instanceof Element) {
addNamespaces(rv, (Element) target);
}
target = target.getParentNode();
}
// Fallback in case no namespace was declared
rv.declare(Namespace.create("", ""));
return rv;
}
@Override
public void delete(String key) throws IOException {
try {
s3Client.deleteObject(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
}
/* */ public static boolean fileExists(String name, String namespace, Namespaces namespaces)
/* */ throws IOException
/* */ {
/* 640 */ String packageName = namespaces.getAsDir(namespace);
/* 641 */ String fullName = packageName + name;
/* */
/* 643 */ return new File(fullName).exists();
/* */ }
@Override
public void copy(String from, String to) throws IOException {
String bucket = Namespaces.get().getBucket();
try {
s3Client.copyObject(bucket, from, bucket, to);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
}
OWL2Datatype(
Namespaces namespace, String shortName, Category category, boolean finite, String regEx) {
iri = IRI.create(namespace.toString(), shortName);
this.shortName = shortName;
this.category = category;
this.finite = finite;
if (regEx != null) {
pattern = Pattern.compile(regEx, Pattern.DOTALL);
}
}
@Override
public void touch(String key) throws IOException {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);
try {
s3Client.putObject(Namespaces.get().getBucket(), key, NullInputStream.INSTANCE, metadata);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
}
private void addNamespaces(Namespaces rv, Element element) {
if (element == null) throw new RuntimeException("element must not be null");
String myDefaultNamespace = toUri(element.lookupNamespaceURI(null));
String parentDefaultNamespace = "";
if (element.getParentNode() != null) {
parentDefaultNamespace = toUri(element.getParentNode().lookupNamespaceURI(null));
}
if (!myDefaultNamespace.equals(parentDefaultNamespace)
|| !(element.getParentNode() instanceof Element)) {
rv.declare(Namespace.create("", myDefaultNamespace));
}
NamedNodeMap attributes = element.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
Attr attr = (Attr) attributes.item(i);
if (attr.getPrefix() != null && attr.getPrefix().equals("xmlns")) {
rv.declare(Namespace.create(attr.getLocalName(), attr.getValue()));
}
}
}
private static SortedMap<String, String> getMap(Instance instance, boolean nameAsKey) {
ZooCache zc = getZooCache(instance);
List<String> tableIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTABLES);
TreeMap<String, String> tableMap = new TreeMap<String, String>();
Map<String, String> namespaceIdToNameMap = new HashMap<String, String>();
for (String tableId : tableIds) {
byte[] tableName =
zc.get(
ZooUtil.getRoot(instance)
+ Constants.ZTABLES
+ "/"
+ tableId
+ Constants.ZTABLE_NAME);
byte[] nId =
zc.get(
ZooUtil.getRoot(instance)
+ Constants.ZTABLES
+ "/"
+ tableId
+ Constants.ZTABLE_NAMESPACE);
String namespaceName = Constants.DEFAULT_NAMESPACE;
// create fully qualified table name
if (nId != null) {
String namespaceId = new String(nId, Constants.UTF8);
if (!namespaceId.equals(Constants.DEFAULT_NAMESPACE_ID)) {
try {
namespaceName = namespaceIdToNameMap.get(namespaceId);
if (namespaceName == null) {
namespaceName = Namespaces.getNamespaceName(instance, namespaceId);
namespaceIdToNameMap.put(namespaceId, namespaceName);
}
} catch (NamespaceNotFoundException e) {
log.error(
"Table ("
+ tableId
+ ") contains reference to namespace ("
+ namespaceId
+ ") that doesn't exist");
continue;
}
}
}
if (tableName != null) {
String tableNameStr = qualified(new String(tableName, Constants.UTF8), namespaceName);
if (nameAsKey) tableMap.put(tableNameStr, tableId);
else tableMap.put(tableId, tableNameStr);
}
}
return tableMap;
}
@Override
public long getLastModified(String key) throws IOException {
ObjectMetadata metadata;
try {
metadata = s3Client.getObjectMetadata(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
if (metadata == null) {
throw new FileNotFoundException(key);
}
return metadata.getLastModified().getTime();
}
/**
* Deletes a node and its descendants.
*
* @param pre pre value of the node to delete
*/
public final void delete(final int pre) {
meta.update();
// size of the subtree to delete
int k = kind(pre);
final int s = size(pre, k);
resources.delete(pre, s);
if (meta.updindex) {
// delete child records from indexes
indexDelete(pre, s);
}
/// explicitly delete text or attribute value
if (k != DOC && k != ELEM) delete(pre, k != ATTR);
// reduce size of ancestors
int par = pre;
// check if we are an attribute (different size counters)
if (k == ATTR) {
par = parent(par, ATTR);
attSize(par, ELEM, attSize(par, ELEM) - 1);
size(par, ELEM, size(par, ELEM) - 1);
k = kind(par);
}
// reduce size of ancestors
while (par > 0 && k != DOC) {
par = parent(par, k);
k = kind(par);
size(par, k, size(par, k) - s);
}
// preserve empty root node
if (kind(pre) == DOC) --meta.ndocs;
if (meta.updindex) {
// delete node and descendants from ID -> PRE map:
idmap.delete(pre, id(pre), -s);
}
// delete node from table structure and reduce document size
table.delete(pre, s);
if (!cache) updateDist(pre, -s);
// propagate PRE value shifts to namespaces
nspaces.delete(pre, s, this);
}
@Override
public void download(String key, File file) throws IOException {
S3Object s3Object;
try {
s3Object = s3Client.getObject(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
file.delete();
InputStream in = s3Object.getObjectContent();
try {
IOUtils.copyStreamToFile(in, file);
} finally {
in.close();
}
}
@Override
public BufferedReader streamFrom(String key) throws IOException {
S3Object s3Object;
try {
s3Object = s3Client.getObject(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
String[] contentTypeEncoding = guessContentTypeEncodingFromExtension(key);
String encoding = contentTypeEncoding[1];
InputStream in = s3Object.getObjectContent();
try {
if ("gzip".equals(encoding)) {
in = new GZIPInputStream(in);
} else if ("zip".equals(encoding)) {
in = new ZipInputStream(in);
} else if ("deflate".equals(encoding)) {
in = new InflaterInputStream(in);
} else if ("bzip2".equals(encoding)) {
in = new BZip2CompressorInputStream(in);
} else {
ObjectMetadata metadata = s3Object.getObjectMetadata();
if (metadata != null) {
String contentEncoding = metadata.getContentEncoding();
if ("gzip".equals(contentEncoding)) {
in = new GZIPInputStream(in);
} else if ("zip".equals(contentEncoding)) {
in = new ZipInputStream(in);
} else if ("deflate".equals(contentEncoding)) {
in = new InflaterInputStream(in);
} else if ("bzip2".equals(contentEncoding)) {
in = new BZip2CompressorInputStream(in);
}
}
}
} catch (IOException ioe) {
// Be extra sure this doesn't like in case of an error
in.close();
throw ioe;
}
return new BufferedReader(new InputStreamReader(in, Charsets.UTF_8), 1 << 20); // ~1MB
}
@Override
public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {
if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
// Need to delete the dir too, and to list it, can't specify its trailing slash
keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
}
boolean truncated = true;
String marker = null;
String bucket = Namespaces.get().getBucket();
while (truncated) {
ListObjectsRequest listRequest =
new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
if (summaries.isEmpty()) {
return;
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete =
Lists.newArrayListWithCapacity(summaries.size());
for (S3ObjectSummary summary : summaries) {
String key = summary.getKey();
if (except == null || !except.accept(null, key)) {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
}
}
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
deleteObjectsRequest.setKeys(keysToDelete);
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
}
}
@Override
protected void doUpload(String key, File file, boolean overwrite) throws IOException {
String bucket = Namespaces.get().getBucket();
String[] contentTypeEncoding = guessContentTypeEncodingFromExtension(file.getName());
ObjectMetadata metadata = new ObjectMetadata();
if (contentTypeEncoding[0] != null) {
metadata.setContentType(contentTypeEncoding[0]);
}
if (contentTypeEncoding[1] != null) {
metadata.setContentEncoding(contentTypeEncoding[1]);
}
long contentLength = file.length();
try {
if (contentLength > PARALLEL_UPLOAD_SIZE) {
doParallelUpload(bucket, key, file, metadata);
} else {
doSerialUpload(bucket, key, file, metadata);
}
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
}
@Override
public List<String> list(String prefix, boolean files) throws IOException {
boolean truncated = true;
String marker = null;
List<String> result = Lists.newArrayList();
while (truncated) {
if (marker != null) {
log.info("Next page after {}", marker);
}
ListObjectsRequest listRequest =
new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
if (!truncated) {
if (files) {
for (S3ObjectSummary summary : listing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
} else {
for (String key : listing.getCommonPrefixes()) {
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
}
}
}
Collections.sort(result);
return result;
}
/**
* Inserts a data instance at the specified pre value. Note that the specified data instance must
* differ from this instance.
*
* @param ipre value at which to insert new data
* @param ipar parent pre value of node
* @param clip data clip
*/
public final void insert(final int ipre, final int ipar, final DataClip clip) {
meta.update();
// update value and document indexes
if (meta.updindex) indexBegin();
resources.insert(ipre, clip);
final int dsize = clip.size();
final int buf = Math.min(dsize, IO.BLOCKSIZE >> IO.NODEPOWER);
// resize buffer to cache more entries
buffer(buf);
// find all namespaces in scope to avoid duplicate declarations
final TokenMap nsScope = nspaces.scope(ipar, this);
// loop through all entries
final IntList preStack = new IntList();
final NSNode nsRoot = nspaces.current();
final HashSet<NSNode> newNodes = new HashSet<NSNode>();
final IntList flagPres = new IntList();
// indicates if database only contains a dummy node
final Data data = clip.data;
int c = 0;
for (int dpre = clip.start; dpre < clip.end; ++dpre, ++c) {
if (c != 0 && c % buf == 0) insert(ipre + c - buf);
final int pre = ipre + c;
final int dkind = data.kind(dpre);
final int dpar = data.parent(dpre, dkind);
// ipar < 0 if document nodes on top level are added
final int dis = dpar >= 0 ? dpre - dpar : ipar >= 0 ? pre - ipar : 0;
final int par = dis == 0 ? -1 : pre - dis;
if (c == 0) nspaces.root(par, this);
while (!preStack.isEmpty() && preStack.peek() > par) nspaces.close(preStack.pop());
switch (dkind) {
case DOC:
// add document
nspaces.prepare();
final int s = data.size(dpre, dkind);
doc(pre, s, data.text(dpre, true));
meta.ndocs++;
preStack.push(pre);
break;
case ELEM:
// add element
nspaces.prepare();
boolean ne = false;
if (data.nsFlag(dpre)) {
final Atts at = data.ns(dpre);
for (int a = 0; a < at.size(); ++a) {
// see if prefix has been declared/ is part of current ns scope
final byte[] old = nsScope.get(at.name(a));
if (old == null || !eq(old, at.value(a))) {
// we have to keep track of all new NSNodes that are added
// to the Namespace structure, as their pre values must not
// be updated. I.e. if an NSNode N with pre value 3 existed
// prior to inserting and two new nodes are inserted at
// location pre == 3 we have to make sure N and only N gets
// updated.
newNodes.add(nspaces.add(at.name(a), at.value(a), pre));
ne = true;
}
}
}
byte[] nm = data.name(dpre, dkind);
elem(
dis,
tagindex.index(nm, null, false),
data.attSize(dpre, dkind),
data.size(dpre, dkind),
nspaces.uri(nm, true),
ne);
preStack.push(pre);
break;
case TEXT:
case COMM:
case PI:
// add text
text(pre, dis, data.text(dpre, true), dkind);
break;
case ATTR:
// add attribute
nm = data.name(dpre, dkind);
// check if prefix already in nsScope or not
final byte[] attPref = prefix(nm);
// check if prefix of attribute has already been declared, otherwise
// add declaration to parent node
if (data.nsFlag(dpre) && nsScope.get(attPref) == null) {
nspaces.add(
par,
preStack.isEmpty() ? -1 : preStack.peek(),
attPref,
data.nspaces.uri(data.uri(dpre, dkind)),
this);
// save pre value to set ns flag later for this node. can't be done
// here as direct table access would interfere with the buffer
flagPres.add(par);
}
attr(
pre,
dis,
atnindex.index(nm, null, false),
data.text(dpre, false),
nspaces.uri(nm, false),
false);
break;
}
}
// finalize and update namespace structure
while (!preStack.isEmpty()) nspaces.close(preStack.pop());
nspaces.root(nsRoot);
if (bp != 0) insert(ipre + c - 1 - (c - 1) % buf);
// reset buffer to old size
buffer(1);
// set ns flags
for (int f = 0; f < flagPres.size(); f++) {
final int fl = flagPres.get(f);
table.write2(fl, 1, name(fl) | 1 << 15);
}
// increase size of ancestors
int p = ipar;
while (p >= 0) {
final int k = kind(p);
size(p, k, size(p, k) + dsize);
p = parent(p, k);
}
if (meta.updindex) {
// add the entries to the ID -> PRE mapping:
idmap.insert(ipre, id(ipre), dsize);
indexEnd();
}
if (!cache) updateDist(ipre + dsize, dsize);
// propagate PRE value shifts to namespaces
if (ipar != -1) nspaces.insert(ipre, dsize, newNodes);
}
/**
* Replaces parts of the database with the specified data instance.
*
* @param rpre pre value to be replaced
* @param clip data clip
*/
public final void replace(final int rpre, final DataClip clip) {
meta.update();
final int dsize = clip.size();
final Data data = clip.data;
final int rkind = kind(rpre);
final int rsize = size(rpre, rkind);
final int rpar = parent(rpre, rkind);
final int diff = dsize - rsize;
buffer(dsize);
resources.replace(rpre, rsize, clip);
if (meta.updindex) {
// update index
indexDelete(rpre, rsize);
indexBegin();
}
for (int dpre = clip.start; dpre < clip.end; ++dpre) {
final int dkind = data.kind(dpre);
final int dpar = data.parent(dpre, dkind);
final int pre = rpre + dpre - clip.start;
final int dis = dpar >= 0 ? dpre - dpar : pre - rpar;
switch (dkind) {
case DOC:
// add document
doc(pre, data.size(dpre, dkind), data.text(dpre, true));
meta.ndocs++;
break;
case ELEM:
// add element
byte[] nm = data.name(dpre, dkind);
elem(
dis,
tagindex.index(nm, null, false),
data.attSize(dpre, dkind),
data.size(dpre, dkind),
nspaces.uri(nm, true),
false);
break;
case TEXT:
case COMM:
case PI:
// add text
text(pre, dis, data.text(dpre, true), dkind);
break;
case ATTR:
// add attribute
nm = data.name(dpre, dkind);
attr(
pre,
dis,
atnindex.index(nm, null, false),
data.text(dpre, false),
nspaces.uri(nm, false),
false);
break;
}
}
if (meta.updindex) {
indexEnd();
// update ID -> PRE map:
idmap.delete(rpre, id(rpre), -rsize);
idmap.insert(rpre, meta.lastid - dsize + 1, dsize);
}
// update table:
table.replace(rpre, buffer(), rsize);
buffer(1);
// no distance/size update if the two subtrees are of equal size
if (diff == 0) return;
// increase/decrease size of ancestors, adjust distances of siblings
int p = rpar;
while (p >= 0) {
final int k = kind(p);
size(p, k, size(p, k) + diff);
p = parent(p, k);
}
if (!cache) updateDist(rpre + dsize, diff);
// adjust attribute size of parent if attributes inserted. attribute size
// of parent cannot be reduced via a replace expression.
int dpre = clip.start;
if (data.kind(dpre) == ATTR) {
int d = 0;
while (dpre < clip.end && data.kind(dpre++) == ATTR) d++;
if (d > 1) attSize(rpar, kind(rpar), d + 1);
}
}
Namespace[] getInScopeNamespaces() {
Namespaces rv = getAllNamespaces();
return rv.getNamespaces();
}
