public static void printElement(Site s, int index) {
if (s == null || index >= s.site().length) {
ps.print("(null)");
} else {
StackTraceElement e = s.site()[index];
ps.print(e.getMethodName() + " (" + e.getFileName() + ":" + e.getLineNumber() + ")");
}
}
/**
* update work site
*
* @param site
* @return
*/
public boolean update(Site site) {
if (site == null || !site.isWork()) {
return false;
}
SiteHost host = site.getHost();
Logger.info("WorkPool.update, work site %s", host);
remove(host, false);
return add(site, true);
}
/*
* Iterate over the sites in the given configuration and remove the one which
* has a url matching the given location.
*/
public boolean removeSite(Configuration configuration, String location)
throws IOException, URISyntaxException {
File left = new File(new URI(location)).getCanonicalFile();
List sites = configuration.getSites();
for (Iterator iter = sites.iterator(); iter.hasNext(); ) {
Site tempSite = (Site) iter.next();
String siteURL = tempSite.getUrl();
File right = new File(new URI(siteURL)).getCanonicalFile();
if (left.equals(right)) {
return configuration.removeSite(tempSite);
}
}
return false;
}
/** Test of run method, of class FileContentLoaderImpl. */
public void testRun1() {
System.out.println("run");
Site mockSite = new SiteImpl();
Page mockPage1 = new PageImpl();
mockPage1.setURL("http://my.testUrl1.org");
Page mockPage2 = new PageImpl();
mockPage2.setURL("http://my.testUrl2.org");
mockSite.addChild(mockPage1);
mockSite.addChild(mockPage2);
ContentFactory mockContentFactory = EasyMock.createMock(ContentFactory.class);
SSP mockSSP = EasyMock.createMock(SSP.class);
Date date = new Date();
DateFactory mockDateFactory = EasyMock.createMock(DateFactory.class);
EasyMock.expect(mockDateFactory.createDate()).andReturn(date).times(2);
EasyMock.expect(
mockContentFactory.createSSP(
date, "http://my.testUrl1.org", "My Page Content 1", mockPage1, HttpStatus.SC_OK))
.andReturn(mockSSP)
.once();
EasyMock.expect(
mockContentFactory.createSSP(
date, "http://my.testUrl2.org", "My Page Content 2", mockPage2, HttpStatus.SC_OK))
.andReturn(mockSSP)
.once();
Map<String, String> fileMap = new HashMap<String, String>();
fileMap.put("http://my.testUrl1.org", "My Page Content 1");
fileMap.put("http://my.testUrl2.org", "My Page Content 2");
EasyMock.replay(mockSSP);
EasyMock.replay(mockContentFactory);
EasyMock.replay(mockDateFactory);
FileContentLoaderImpl instance =
new FileContentLoaderImpl(mockContentFactory, fileMap, mockDateFactory);
instance.setWebResource(mockSite);
instance.run();
assertTrue(instance.getResult().contains(mockSSP));
EasyMock.verify(mockSSP);
EasyMock.verify(mockContentFactory);
EasyMock.verify(mockDateFactory);
}
public static void printSite(Site s, StringBuilder out) {
if (s == null) {
out.append("(null)");
return;
}
StackTraceElement[] st = s.site();
int interestingSitesPrinted = 0;
for (int i = s.offset; i < st.length; i++) {
StackTraceElement e = st[i];
String fileName = e.getFileName();
if (THIS_FILE_NAME.equals(fileName)) {
continue;
}
out.append(" " + e.getMethodName() + "(" + e.getFileName() + ":" + e.getLineNumber() + ")");
interestingSitesPrinted++;
if (interestingSitesPrinted <= SITES_TO_PRINT) {
continue;
}
if (fileName == null || "View.java".equals(fileName)) {
continue;
}
String methodName = e.getMethodName();
if (skipMethods.contains(methodName)) {
continue;
}
break;
}
}
/*
* Assert that a feature with the given id exists in the configuration. If
* a version is specified then match the version, otherwise any version will
* do.
*/
public void assertFeatureExists(
String message, Configuration configuration, String id, String version) {
List sites = configuration.getSites();
assertNotNull(message, sites);
boolean found = false;
for (Iterator iter = sites.iterator(); iter.hasNext(); ) {
Site site = (Site) iter.next();
Feature[] features = site.getFeatures();
for (int i = 0; features != null && i < features.length; i++) {
if (id.equals(features[i].getId())) {
if (version == null) found = true;
else if (version.equals(features[i].getVersion())) found = true;
}
}
}
assertTrue(message, found);
}
public static void printElements(Site s, int index, int nelems) {
if (s == null || index >= s.site().length) {
ps.print("(null)");
} else {
for (int i = 0; i < nelems; i++) {
int j = index + i;
if (j >= s.site().length) {
break;
}
if (i > 0) {
ps.print(", ");
}
StackTraceElement e = s.site()[j];
ps.print(e.getMethodName() + " (" + e.getFileName() + ":" + e.getLineNumber() + ")");
}
}
}
@Override
public boolean equals(Object o) {
if (o == this) {
return true;
}
if (!(o instanceof Site)) {
return false;
}
Site other = (Site) o;
if (other.hashCode() != hashCode()) {
return false;
}
StackTraceElement[] osite = other.site();
if (site.length != osite.length) {
return false;
}
for (int i = offset; i < site.length; i++) {
if (!site[i].equals(osite[i])) {
return false;
}
}
return true;
}
/** @author user */
public ParseXindm() {
enumName = "XINDM";
parserName = this.getClass().getName();
regexs = new String[] {"(?s).*www.xindm.cn/(?s).*"};
downloadBefore = true;
siteID = Site.formString("XINDM");
siteName = "Xindm";
indexName = Common.getStoredFileName(SetUp.getTempDirectory(), "index_xindm_parse_", "html");
indexEncodeName =
Common.getStoredFileName(SetUp.getTempDirectory(), "index_xindm_encode_parse_", "html");
jsName = "index_xindm.js";
radixNumber = 185273; // default value, not always be useful!!
}
private Site updateUseSite() {
if (TRACE_USE_SITES) {
Site here = new Site();
useSites.add(here);
return here;
} else if (TRACE_SINGLE_USE_SITE) {
if (multipleUseSites) {
return null;
} else {
Site here = new Site();
if (singleUseSite == null) {
singleUseSite = here;
} else {
if (!here.equals(singleUseSite)) {
multipleUseSites = true;
singleUseSite = null;
}
}
return here;
}
} else {
return null;
}
}
/*
* Create and return a new site object with the given parameters.
*/
public Site createSite(
String policy, boolean enabled, boolean updateable, String uri, String[] plugins) {
Site result = new Site();
result.setPolicy(policy);
result.setEnabled(enabled);
result.setUpdateable(updateable);
result.setUrl(uri);
if (plugins != null) for (int i = 0; i < plugins.length; i++) result.addPlugin(plugins[i]);
return result;
}
/** @author user */
public ParseKUKU() {
enumName = "KUKU";
regexs =
new String[] {
"(?s).*kukudm.com(?s).*", "(?s).*socomic.com(?s).*", "(?s).*socomic.net(?s).*"
};
parserName = this.getClass().getName();
downloadBefore = true;
siteID = Site.formString("KUKU");
siteName = "kuku";
indexName = Common.getStoredFileName(SetUp.getTempDirectory(), "index_kuku_parse_", "html");
indexEncodeName =
Common.getStoredFileName(SetUp.getTempDirectory(), "index_kuku_encode_parse_", "html");
baseURL = "http://n.kukudm.com/";
}
/**
* Generic find for when the precise type of a DSO is not known, just the a pair of type number
* and database ID.
*
* @param context - the context
* @param type - type number
* @param id - id within table of type'd objects
* @return the object found, or null if it does not exist.
* @throws SQLException only upon failure accessing the database.
*/
public static DSpaceObject find(Context context, int type, int id) throws SQLException {
switch (type) {
case Constants.BITSTREAM:
return Bitstream.find(context, id);
case Constants.BUNDLE:
return Bundle.find(context, id);
case Constants.ITEM:
return Item.find(context, id);
case Constants.COLLECTION:
return Collection.find(context, id);
case Constants.COMMUNITY:
return Community.find(context, id);
case Constants.GROUP:
return Group.find(context, id);
case Constants.EPERSON:
return EPerson.find(context, id);
case Constants.SITE:
return Site.find(context, id);
}
return null;
}
private boolean add(Site object, boolean notify) {
if (object == null || !object.isWork()) {
return false;
}
WorkSite site = (WorkSite) object;
SiteHost host = site.getHost();
Logger.info("WorkPool.add, work site %s", host);
boolean success = false;
this.lockSingle();
try {
if (mapSite.containsKey(host)) {
return false;
}
// save site
mapSite.put(host, site);
mapTime.put(host, System.currentTimeMillis());
for (Naming naming : site.list()) {
SiteSet set = mapNaming.get(naming);
if (set == null) {
set = new SiteSet();
mapNaming.put(naming, set);
}
set.add(host);
}
success = true;
if (notify) {
CallPool.getInstance().refreshWorkSite();
}
} catch (Throwable exp) {
Logger.error(exp);
} finally {
this.unlockSingle();
}
return success;
}
/** @author user */
public ParseCK() {
regexs =
new String[] {
"(?s).*comic101.com(?s).*",
"(?s).*comic.101.com(?s).*",
"(?s).*mh.ck101.com(?s).*",
"(?s).*comic.ck101.com(?s).*",
"(?s).*.com/vols/\\d+/\\d+(?s).*"
};
enumName = "CK";
parserName = this.getClass().getName();
downloadBefore = true;
siteID = Site.formString("CK");
siteName = "CK101_Comic";
indexName = Common.getStoredFileName(SetUp.getTempDirectory(), "index_ck_parse_", "html");
indexEncodeName =
Common.getStoredFileName(SetUp.getTempDirectory(), "index_ck_encode_parse_", "html");
jsName = "index_ck.js";
radixNumber = 151261471; // default value, not always be useful!!
baseURL = "http://comic.ck101.com"; // "http://comic101.com";
}
private static void dumpView(int depth, ViewTrace trace) {
printedIndividualViews.add(trace);
ps.println(trace.realView + " size = " + trace.realView.size());
if (TRACE_ALLOCATION_SITE) {
indent(depth);
ps.print(" allocationSite =");
Site.printSite(trace.allocationSite);
ps.println();
}
int unused = trace.unusedElements();
int redundant = trace.redundantGets();
boolean singleUse;
Site[] useSites;
if (TRACE_USE_SITES) {
useSites = trace.useSites.toArray(new Site[trace.useSites.size()]);
singleUse = (useSites.length == 1);
} else if (TRACE_SINGLE_USE_SITE) {
useSites = null;
singleUse = !trace.multipleUseSites;
} else {
useSites = null;
singleUse = false;
}
if (singleUse) {
indent(depth);
ps.print(" singleUseSite = US");
Site.printSite(useSites != null ? useSites[0] : trace.singleUseSite);
if (trace.getCount > 0) {
ps.println(" (get)");
} else if (trace.sumCount > 0) {
ps.println(" (sum)");
} else {
ps.println(" (materialize)");
}
} else if (trace.getCount > 0) {
if (TRACE_FIRST_GET_SITE) {
indent(depth);
ps.print(" firstGetSite =");
Site.printSite(trace.firstGetSite);
ps.println();
}
if (trace.materializeCount == 0 && trace.sumCount == 0) {
if (unused > 0) {
indent(depth);
ps.println(" unusedElements = " + unused);
}
if (redundant > 0) {
indent(depth);
ps.println(" redundantGets = " + redundant + " (no materialize, sum)");
}
}
} else {
if (trace.materializeCount == 0 && trace.sumCount == 0) {
indent(depth);
ps.println(" UNUSED");
} else {
indent(depth);
ps.println(
" materializeCount = "
+ trace.materializeCount
+ " sumCount = "
+ trace.sumCount
+ " getCount = "
+ trace.getCount);
}
}
if (TRACE_FIRST_MATERIALIZE_SITE && trace.materializeCount > 0 && !singleUse) {
indent(depth);
ps.print(" firstMaterializeSite =");
Site.printSite(trace.firstMaterializeSite);
ps.println();
}
if (TRACE_FIRST_SUM_SITE && trace.sumCount > 0 && !singleUse) {
indent(depth);
ps.print(" firstSumSite =");
Site.printSite(trace.firstSumSite);
ps.println();
}
if (TRACE_USE_SITES) {
if (useSites.length != 1) {
indent(depth);
ps.println(" useSites (" + useSites.length + "):");
for (Site s : useSites) {
indent(depth);
ps.print(" US");
Site.printSite(s);
ps.println();
}
}
}
ps.println();
RArray view = trace.realView;
Class viewClass = view.getClass();
Field[] fields = getAllFields(viewClass);
boolean printedField = false;
for (Field f : fields) {
if (f.isSynthetic()) {
continue;
}
Class fieldClass = f.getType();
if (RArray.class.isAssignableFrom(fieldClass)) {
continue; // these later
}
indent(depth);
ps.print(" " + f.getName() + " ");
try {
f.setAccessible(true);
ps.println(f.get(view));
printedField = true;
} catch (IllegalAccessException e) {
assert Utils.check(false, "can't read a view field " + e);
}
}
boolean printNewline = printedField;
for (Field f : fields) {
if (f.isSynthetic()) {
continue;
}
Class fieldClass = f.getType();
if (!RArray.class.isAssignableFrom(fieldClass)) {
continue;
}
if (printNewline) {
ps.println();
printNewline = false;
}
indent(depth);
ps.print(" " + f.getName() + " ");
try {
f.setAccessible(true);
Object o = f.get(view);
if (o instanceof TracingView) {
ps.print("VIEW ");
TracingView child = (TracingView) o;
dumpView(depth + 2, child.getTrace());
} else {
ps.print("ARRAY " + o + " size = " + ((RArray) o).size());
if (o instanceof View) {
ps.println("MISSED VIEW " + o);
}
}
ps.println();
} catch (IllegalAccessException e) {
assert Utils.check(false, "can't read a view field " + e);
}
}
}
private static void extractViewPattern(int depth, ViewTrace trace, StringBuilder p) {
if (!processedViewsForPatterns.add(trace)) {
p.append("(ALIASED) ");
}
p.append(trace.realView.getClass() + " size = " + trace.realView.size() + "\n");
indent(depth, p);
p.append(" use:");
if (trace.materializeCount == 0 && trace.sumCount == 0 && trace.getCount == 0) {
p.append(" UNUSED");
} else {
if (trace.getCount > 0) {
p.append(" get");
}
if (trace.materializeCount > 0) {
p.append(" materialize");
}
if (trace.sumCount > 0) {
p.append(" sum");
}
}
p.append("\n");
if (false) {
p.append(" allocationSite =");
Site.printSite(trace.allocationSite, p);
p.append("\n");
indent(depth, p);
}
RArray view = trace.realView;
Class viewClass = view.getClass();
Field[] fields = getAllFields(viewClass);
boolean printedField = false;
for (Field f : fields) {
if (f.isSynthetic()) {
continue;
}
Class fieldClass = f.getType();
if (RArray.class.isAssignableFrom(fieldClass)) {
continue; // these later
}
indent(depth, p);
p.append(" " + f.getName() + " ");
try {
f.setAccessible(true);
Object o = f.get(view);
p.append(o == null ? "null (" + fieldClass + ")" : o.getClass());
p.append("\n");
printedField = true;
} catch (IllegalAccessException e) {
assert Utils.check(false, "can't read a view field " + e);
}
}
boolean printNewline = printedField;
for (Field f : fields) {
if (f.isSynthetic()) {
continue;
}
Class fieldClass = f.getType();
if (!RArray.class.isAssignableFrom(fieldClass)) {
continue;
}
if (printNewline) {
p.append("\n");
printNewline = false;
}
indent(depth, p);
p.append(" " + f.getName() + " ");
try {
f.setAccessible(true);
Object o = f.get(view);
if (o instanceof TracingView) {
p.append("VIEW ");
TracingView child = (TracingView) o;
extractViewPattern(depth + 2, child.getTrace(), p);
} else {
p.append("ARRAY " + o.getClass() + " size = " + ((RArray) o).size());
if (o instanceof View) {
ps.println("MISSED VIEW " + o);
}
}
p.append("\n");
} catch (IllegalAccessException e) {
assert Utils.check(false, "can't read a view field " + e);
}
}
}
/**
* Implements a bidirectional breadth-first search to find the shortest path between Sites from
* and to. Currently not completely implemented. More detailed notes in the code for the method.
* If it worked it would be quadratic (O(n^2)), but would still be a bit faster than a non-bidi
* search.
*
* @param from Starting position
* @param to Ending position
* @return Stack of Sites containing the shortest path in order to be taken.
*/
public Stack<Site> bidiShortestSiteStack(Site from, Site to) {
boolean fFound = false, tFound = false;
Site fConnectSite = null, tConnectSite = null;
Queue<Site[]> fTemp = new LinkedList<Site[]>(), tTemp = new LinkedList<Site[]>();
ArrayList<Site> fromList = new ArrayList<Site>(), toList = new ArrayList<Site>();
Stack<Site[]> fMoves = new Stack<Site[]>(), tMoves = new Stack<Site[]>();
Stack<Site> moves = new Stack<Site>();
graph.mark(from);
for (Site site : graph.neighbors(from)) {
if (site.equals(to)) {}
fTemp.add(new Site[] {from, site});
graph.mark(site);
fromList.add(site);
}
graph.mark(to);
for (Site site : graph.neighbors(to)) {
tTemp.add(new Site[] {to, site});
if (graph.isMarked(site) && fromList.contains(site)) {
tFound = true;
break;
}
toList.add(site);
graph.mark(site);
}
if (tFound) {
moves.add(to);
return moves;
}
bigBreak:
while (!fTemp.isEmpty() && !tTemp.isEmpty()) {
Site[] fSites = fTemp.poll();
fMoves.push(fSites);
Site[] tSites = tTemp.poll();
tMoves.push(tSites);
for (Site site : graph.neighbors(fSites[1])) {
fTemp.add(new Site[] {fSites[1], site});
if (graph.isMarked(site) && toList.contains(site)) {
fromList.add(site);
fConnectSite = site;
fFound = true;
break bigBreak;
} else graph.mark(site);
}
for (Site site : graph.neighbors(tSites[1])) {
tTemp.add(new Site[] {tSites[1], site}); // I chose to put them in the same way.
// While this does make it a bit more difficult to code, the consistency makes it less
// confusing.
if (graph.isMarked(site) && fromList.contains(site)) {
toList.add(site);
tConnectSite = site;
tFound = true;
break bigBreak;
} else graph.mark(site);
}
}
if (fFound) {
Stack<Site> fTempMoves = pathFromStack(from, fMoves);
Site[] sites = null;
while (!tMoves.isEmpty()) {
sites = tMoves.pop();
if (sites[1].equals(fConnectSite)) break;
}
fMoves.push(sites);
Stack<Site> tTempMoves = pathFromStack(to, tMoves);
// need to combine them
} else if (tFound) {
Stack<Site> tTempMoves = pathFromStack(to, tMoves);
Site[] sites = null;
while (!fMoves.isEmpty()) {
sites = fMoves.pop();
if (sites[1].equals(tConnectSite)) break;
}
tMoves.push(sites);
Stack<Site> fTempMoves = pathFromStack(from, fMoves);
// need to combine them
}
return moves;
}
