// Private Method for checking forward linking
private void checkForForwardLinking() {
for (String uri : uriSet) {
CachedHTTPResource httpResource =
(CachedHTTPResource)
DiachronCacheManager.getInstance()
.getFromCache(DiachronCacheManager.HTTP_RESOURCE_CACHE, uri);
if (httpResource == null
|| (httpResource.getResponses() == null
&& httpResource.getDereferencabilityStatusCode() != StatusCode.BAD)) {
this.notFetchedQueue.add(uri);
} else {
logger.info("Checking resource: {}. URIs left: {}.", httpResource.getUri(), uriSet.size());
// We perform a semantic lookup using heuristics to check if we
// really need to try parsing or not
if (HTTPResourceUtils.semanticURILookup(httpResource)) {
logger.info(
"Trying to find any dereferencable forward links for {}.", httpResource.getUri());
if (Dereferencer.hasValidDereferencability(httpResource)) {
logger.info("Dereferencable resource {}.", httpResource.getUri());
// boolean isValid = ModelParser.snapshotParserForForwardDereference(httpResource,
// (Lang) null, httpResource.getUri());
// if (isValid){
// //ok
// logger.info("A description exists for resource {}.", httpResource.getUri());
//
// totalDerefDataWithSub++;
// } else {
// //not ok
// this.createNotValidForwardLink(httpResource.getUri());
// }
Model m = RDFDataMgr.loadModel(httpResource.getUri()); // load partial model
Resource r = m.createResource(httpResource.getUri());
List<Statement> stmtList =
m.listStatements(r, (Property) null, (RDFNode) null).toList();
if (stmtList.size() > 1) {
// ok
logger.info("A description exists for resource {}.", httpResource.getUri());
totalDerefDataWithSub++;
} else {
// not ok
this.createNotValidForwardLink(httpResource.getUri());
}
}
} else {
logger.info("Non-meaningful dereferencable resource {}.", httpResource.getUri());
this.createNotValidForwardLink(httpResource.getUri());
}
}
}
}
/**
* Tries to dereference all the URIs contained in the parameter, by retrieving them from the
* cache. URIs not found in the cache are added to the queue containing the URIs to be fetched by
* the async HTTP retrieval process
*
* @param uriSet Set of URIs to be dereferenced
* @return list with the results of the dereferenceability operations, for those URIs that were
* found in the cache
*/
private List<DerefResult> deReferenceUris(List<String> uriSet) {
// Start the dereferencing process, which will be run in parallel
httpRetriever.addListOfResourceToQueue(uriSet);
httpRetriever.start();
List<DerefResult> lstDerefUris = new ArrayList<DerefResult>();
List<String> lstToDerefUris = new ArrayList<String>(uriSet);
// Dereference each and every one of the URIs contained in the specified set
while (lstToDerefUris.size() > 0) {
// Remove the URI at the head of the queue of URIs to be dereferenced
String headUri = lstToDerefUris.remove(0);
// First, search for the URI in the cache
CachedHTTPResource httpResource =
(CachedHTTPResource)
dcmgr.getFromCache(DiachronCacheManager.HTTP_RESOURCE_CACHE, headUri);
if (httpResource == null || httpResource.getStatusLines() == null) {
// URIs not found in the cache, is still to be fetched via HTTP, add it to the end of the
// list
lstToDerefUris.add(headUri);
} else {
// URI found in the cache (which means that was fetched at some point), check if
// successfully dereferenced
DerefResult curUrlResult = new DerefResult(headUri, false, false);
lstDerefUris.add(curUrlResult);
if (this.isDereferenceable(httpResource)) {
curUrlResult.isDeref = true;
if (this.is200AnRDF(httpResource)) {
curUrlResult.isRdfXml = true;
} else this.createProblemQuad(httpResource.getUri(), DQM.NotMeaningful);
} else if (httpResource.getDereferencabilityStatusCode() == StatusCode.SC200) {
curUrlResult.isDeref = true;
// Check if the resource contains RDF on XML
if (this.is200AnRDF(httpResource)) {
curUrlResult.isRdfXml = true;
}
}
logger.trace(
"Resource fetched: {}. Deref. status: {}. Is RDF: {}",
headUri,
httpResource.getDereferencabilityStatusCode(),
curUrlResult.isRdfXml);
}
}
return lstDerefUris;
}
/**
* @author Jeremy Debatista
* <p>This metric calculates an estimation of the number of valid redirects (303) or hashed
* links according to LOD Principles. Makes use of statistical sampling techniques to remain
* scalable to datasets of big-data proportions
* <p>Based on: <a href="http://www.hyperthing.org/">Hyperthing - A linked data Validator</a>
* @see <a href="http://dl.dropboxusercontent.com/u/4138729/paper/dereference_iswc2011.pdf">
* Dereferencing Semantic Web URIs: What is 200 OK on the Semantic Web? - Yang et al.</a>
*/
public class EstimatedDereferenceabilityByTld implements QualityMetric {
private final Resource METRIC_URI = DQM.DereferenceabilityMetric;
static final Logger logger = LoggerFactory.getLogger(EstimatedDereferenceability.class);
/**
* Constants controlling the maximum number of elements in the reservoir of Top-level Domains and
* Fully Qualified URIs of each TLD, respectively
*/
private static int MAX_TLDS = 20;
private static int MAX_FQURIS_PER_TLD = 500;
/** Performs HTTP requests, used to try to fetch identified URIs */
private HTTPRetriever httpRetriever = new HTTPRetriever();
/**
* Holds the set of dereferenceable top-level domains found among the subjects and objects of the
* triples, as a reservoir sampler, if its number of items grows beyond the limit (MAX_TLDS) items
* will be replaced randomly upon forthcoming insertions. Moreover, the items will be indexed so
* that search operations are O(1)
*/
private ReservoirSampler<Tld> tldsReservoir = new ReservoirSampler<Tld>(MAX_TLDS, true);
private DiachronCacheManager dcmgr = DiachronCacheManager.getInstance();
private double metricValue = 0.0;
private boolean metricCalculated = false;
private List<Quad> _problemList = new ArrayList<Quad>();
/**
* Processes each triple obtained from the dataset to be assessed (instance declarations, that is,
* triples with predicate rdf:type are ignored). Identifies URIs appearing in both, the subject
* and object of the triple and adds them to the set of URIs to be evaluated for
* dereferenceability
*
* @param quad Triple (in quad format) to be evaluated
*/
public void compute(Quad quad) {
logger.debug("Assessing {}", quad.asTriple().toString());
// we are currently ignoring triples ?s a ?o
if (!(quad.getPredicate().getURI().equals(RDF.type.getURI()))) {
String subject = quad.getSubject().toString();
if (httpRetriever.isPossibleURL(subject)) {
logger.trace("URI found on subject: {}", subject);
addUriToDereference(subject);
}
String object = quad.getObject().toString();
if (httpRetriever.isPossibleURL(object)) {
logger.trace("URI found on object: {}", object);
addUriToDereference(object);
}
}
}
/**
* Initiates the dereferencing process of some of the URIs identified in the dataset, chosen in
* accordance with a statistical sampling method, in order to compute the estimated
* dereferenceability of the whole dataset
*
* @return estimated dereferencibility, computed as aforementioned
*/
public double metricValue() {
if (!this.metricCalculated) {
// Collect the list of URIs of the TLDs, to be dereferenced
List<String> lstUrisToDeref = new ArrayList<String>(this.tldsReservoir.size());
for (Tld tld : this.tldsReservoir.getItems()) {
lstUrisToDeref.add(tld.getUri());
}
// Dereference all TLD URIs
List<DerefResult> lstDeRefTlds = this.deReferenceUris(lstUrisToDeref);
long totalDerefUris = 0;
long totalUris = 0;
for (DerefResult curTldDeRefRes : lstDeRefTlds) {
// Obtain the TLD corresponding to the URI whose result currently is being examined
Tld derefTld = this.tldsReservoir.findItem(new Tld(curTldDeRefRes.uri, MAX_FQURIS_PER_TLD));
totalUris += ((derefTld.getfqUris() != null) ? (derefTld.getfqUris().size()) : (0));
// Only URIs comprised by dereferenceable TLDs are subject to be counted as successfully
// dereferenced
if (curTldDeRefRes.isDeref && derefTld.getfqUris() != null) {
// Dereference all URIs part of the TLD
List<DerefResult> lstDeRefUris = this.deReferenceUris(derefTld.getfqUris().getItems());
// Count those successfully dereferenced
for (DerefResult curUriDeRefRes : lstDeRefUris) {
if (curUriDeRefRes.isDeref && curUriDeRefRes.isRdfXml) {
logger.debug("-- URI successfully dereferenced: {}", curUriDeRefRes.uri);
totalDerefUris += 1;
} else {
logger.debug("-- URI: {} failed to be dereferenced", curUriDeRefRes.uri);
}
}
logger.debug(
"TLD: {} successfully dereferenced, sampling from: {} URIs",
curTldDeRefRes.uri,
derefTld.countFqUris());
} else {
logger.debug("TLD: {} non-dereferenced", curTldDeRefRes.uri);
}
}
this.metricValue = (double) totalDerefUris / (double) totalUris;
}
return this.metricValue;
}
public Resource getMetricURI() {
return this.METRIC_URI;
}
public ProblemList<?> getQualityProblems() {
ProblemList<Quad> pl = null;
try {
if (this._problemList != null && this._problemList.size() > 0) {
pl = new ProblemList<Quad>(this._problemList);
} else {
pl = new ProblemList<Quad>();
}
} catch (ProblemListInitialisationException e) {
logger.error(e.getMessage());
}
return pl;
}
/* ------------------------------------ Private Methods ------------------------------------------------ */
/**
* Checks and properly processes an URI found as subject or object of a triple, adding it to the
* set of TLDs and fully-qualified URIs
*
* @param uri URI to be processed
*/
private void addUriToDereference(String uri) {
// Extract the top-level domain and look for it within the reservoir
String uriTLD = httpRetriever.extractTopLevelDomainURI(uri);
Tld newTld = new Tld(uriTLD, MAX_TLDS);
Tld foundTld = this.tldsReservoir.findItem(newTld);
if (foundTld == null) {
logger.trace("New TLD found and recorded: {}...", uriTLD);
// Add the new TLD to the reservoir
this.tldsReservoir.add(newTld);
// Add new fully qualified URI to those of the new TLD
newTld.addFqUri(uri);
} else {
// The identified TLD was found, it already exists on the reservoir, just add the fqdn to it
foundTld.addFqUri(uri);
}
}
/**
* Tries to dereference all the URIs contained in the parameter, by retrieving them from the
* cache. URIs not found in the cache are added to the queue containing the URIs to be fetched by
* the async HTTP retrieval process
*
* @param uriSet Set of URIs to be dereferenced
* @return list with the results of the dereferenceability operations, for those URIs that were
* found in the cache
*/
private List<DerefResult> deReferenceUris(List<String> uriSet) {
// Start the dereferencing process, which will be run in parallel
httpRetriever.addListOfResourceToQueue(uriSet);
httpRetriever.start();
List<DerefResult> lstDerefUris = new ArrayList<DerefResult>();
List<String> lstToDerefUris = new ArrayList<String>(uriSet);
// Dereference each and every one of the URIs contained in the specified set
while (lstToDerefUris.size() > 0) {
// Remove the URI at the head of the queue of URIs to be dereferenced
String headUri = lstToDerefUris.remove(0);
// First, search for the URI in the cache
CachedHTTPResource httpResource =
(CachedHTTPResource)
dcmgr.getFromCache(DiachronCacheManager.HTTP_RESOURCE_CACHE, headUri);
if (httpResource == null || httpResource.getStatusLines() == null) {
// URIs not found in the cache, is still to be fetched via HTTP, add it to the end of the
// list
lstToDerefUris.add(headUri);
} else {
// URI found in the cache (which means that was fetched at some point), check if
// successfully dereferenced
DerefResult curUrlResult = new DerefResult(headUri, false, false);
lstDerefUris.add(curUrlResult);
if (this.isDereferenceable(httpResource)) {
curUrlResult.isDeref = true;
if (this.is200AnRDF(httpResource)) {
curUrlResult.isRdfXml = true;
} else this.createProblemQuad(httpResource.getUri(), DQM.NotMeaningful);
} else if (httpResource.getDereferencabilityStatusCode() == StatusCode.SC200) {
curUrlResult.isDeref = true;
// Check if the resource contains RDF on XML
if (this.is200AnRDF(httpResource)) {
curUrlResult.isRdfXml = true;
}
}
logger.trace(
"Resource fetched: {}. Deref. status: {}. Is RDF: {}",
headUri,
httpResource.getDereferencabilityStatusCode(),
curUrlResult.isRdfXml);
}
}
return lstDerefUris;
}
private boolean isDereferenceable(CachedHTTPResource httpResource) {
if (httpResource.getDereferencabilityStatusCode() == null) {
List<Integer> statusCode = this.getStatusCodes(httpResource.getStatusLines());
if (httpResource.getUri().contains("#") && statusCode.contains(200))
httpResource.setDereferencabilityStatusCode(StatusCode.HASH);
else if (statusCode.contains(200)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC200);
if (statusCode.contains(303)) httpResource.setDereferencabilityStatusCode(StatusCode.SC303);
else {
if (statusCode.contains(301)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC301);
this.createProblemQuad(httpResource.getUri(), DQM.SC301MovedPermanently);
} else if (statusCode.contains(302)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC302);
this.createProblemQuad(httpResource.getUri(), DQM.SC302Found);
} else if (statusCode.contains(307)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC307);
this.createProblemQuad(httpResource.getUri(), DQM.SC307TemporaryRedirectory);
} else {
if (hasBad3xxCode(statusCode))
this.createProblemQuad(httpResource.getUri(), DQM.SC3XXRedirection);
}
}
}
if (has4xxCode(statusCode)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC4XX);
this.createProblemQuad(httpResource.getUri(), DQM.SC4XXClientError);
}
if (has5xxCode(statusCode)) {
httpResource.setDereferencabilityStatusCode(StatusCode.SC5XX);
this.createProblemQuad(httpResource.getUri(), DQM.SC5XXServerError);
}
}
StatusCode scode = httpResource.getDereferencabilityStatusCode();
return this.mapDerefStatusCode(scode);
}
private List<Integer> getStatusCodes(List<StatusLine> statusLines) {
ArrayList<Integer> codes = new ArrayList<Integer>();
if (statusLines != null) {
synchronized (statusLines) {
for (StatusLine s : statusLines) {
codes.add(s.getStatusCode());
}
}
}
return codes;
}
private boolean mapDerefStatusCode(StatusCode statusCode) {
if (statusCode == null) {
return false;
} else {
switch (statusCode) {
case SC303:
case HASH:
return true;
default:
return false;
}
}
}
private boolean is200AnRDF(CachedHTTPResource resource) {
if (resource.isContainsRDF() != null) return resource.isContainsRDF();
if (resource != null && resource.getResponses() != null) {
for (SerialisableHttpResponse response : resource.getResponses()) {
if (response != null && response.getHeaders("Content-Type") != null) {
if (LinkedDataContent.contentTypes.contains(response.getHeaders("Content-Type"))) {
if (response.getHeaders("Content-Type").equals(WebContent.contentTypeTextPlain)) {
Model m = this.tryRead(resource.getUri());
if (m != null && m.size() == 0) {
this.createProblemQuad(resource.getUri(), DQM.SC200WithoutRDF);
resource.setContainsRDF(false);
return false;
}
}
this.createProblemQuad(resource.getUri(), DQM.SC200WithRDF);
resource.setContainsRDF(true);
return true;
}
}
}
}
this.createProblemQuad(resource.getUri(), DQM.SC200WithoutRDF);
resource.setContainsRDF(false);
return false;
}
private boolean hasBad3xxCode(List<Integer> statusCode) {
for (int i : statusCode) {
if ((i == 300)
|| (i == 304)
|| (i == 305)
|| (i == 306)
|| (i == 308)
|| ((i >= 308) && (i < 399))) return true;
else continue;
}
return false;
}
private boolean has4xxCode(List<Integer> statusCode) {
for (int i : statusCode) {
if ((i >= 400) && (i < 499)) return true;
else continue;
}
return false;
}
private boolean has5xxCode(List<Integer> statusCode) {
for (int i : statusCode) {
if ((i >= 500) && (i < 599)) return true;
else continue;
}
return false;
}
/**
* Inner class, with the purpose of coupling an URI with the result of its dereferencing process
* It's basically a pair establishing a relation between an URI and its dereferenceability
*
* @author slondono
*/
private class DerefResult {
private String uri;
private boolean isDeref;
private boolean isRdfXml;
private DerefResult(String uri, boolean isDeref, boolean isRdfXml) {
this.uri = uri;
this.isDeref = isDeref;
this.isRdfXml = isRdfXml;
}
}
public static int getMAX_TLDS() {
return MAX_TLDS;
}
public static void setMAX_TLDS(int mAX_TLDS) {
MAX_TLDS = mAX_TLDS;
}
public static int getMAX_FQURIS_PER_TLD() {
return MAX_FQURIS_PER_TLD;
}
public static void setMAX_FQURIS_PER_TLD(int mAX_FQURIS_PER_TLD) {
MAX_FQURIS_PER_TLD = mAX_FQURIS_PER_TLD;
}
private void createProblemQuad(String resource, Resource problem) {
Quad q =
new Quad(
null,
ModelFactory.createDefaultModel().createResource(resource).asNode(),
QPRO.exceptionDescription.asNode(),
problem.asNode());
this._problemList.add(q);
}
/**
* Try Read content returned by text/plain
*
* @param uri
* @return
*/
private Model tryRead(String uri) {
Model m = ModelFactory.createDefaultModel();
try {
m = RDFDataMgr.loadModel(uri, Lang.NTRIPLES);
} catch (RiotException r) {
Log.debug("Resource could not be parsed:", r.getMessage());
}
return m;
}
@Override
public boolean isEstimate() {
return true;
}
@Override
public Resource getAgentURI() {
return DQM.LuzzuProvenanceAgent;
}
}
