public boolean insertData(ArchiveRecord rec) throws IOException {
final Properties prop = getProperties();
String url = prop.getProperty("datastore.remote.url");
if (url == null || url.trim().length() == 0) {
log.fine("No REST service configured");
return false;
}
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(url);
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String timestamp = dateFormat.format(rec.getTimestamp());
StringBuilder buf = new StringBuilder();
String clientKey = prop.getProperty("datastore.remote.client.key");
String clientSecret = prop.getProperty("datastore.remote.client.secret");
buf.append("{\n");
buf.append(" \"clientKey\": \"" + clientKey + "\",\n");
buf.append(" \"clientSecret\": \"" + clientSecret + "\",\n");
buf.append(" \"data\": [\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "outsideTemperature\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getOutsideTemperature() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "outsideHumidity\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getOutsideHumidity() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "windSpeed\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getWindSpeedAvg() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "windDirection\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getWindDirection() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "barometer\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getBarometer() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "rain\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getRainFall() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "sun\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getSolarRadiation() + "\n");
buf.append(" },\n");
buf.append(" {\n");
buf.append(" \"sid\": \"" + name + "uv\",\n");
buf.append(" \"timestamp\": \"" + timestamp + "\",\n");
buf.append(" \"value\": " + rec.getUvIndex() + "\n");
buf.append(" }\n");
buf.append(" ]\n");
buf.append("}\n");
httpPost.setEntity(new StringEntity(buf.toString(), ContentType.create("application/json")));
CloseableHttpResponse response = httpclient.execute(httpPost);
try {
HttpEntity entity = response.getEntity();
// do something useful with the response body
// and ensure it is fully consumed
EntityUtils.consume(entity);
} finally {
response.close();
}
log.fine("Weather data for " + timestamp + " sent to " + url);
return false;
}
/** Explode the archive into its constituent elements */
public void explode() throws CacheException {
CachedUrl cachedUrl = null;
int goodEntries = 0;
int badEntries = 0;
int ignoredEntries = 0;
int entriesBetweenSleep = 0;
ArchiveReader arcReader = null;
logger.info(
(storeArchive ? "Storing" : "Fetching") + " WARC file: " + origUrl + " will explode");
try {
if (storeArchive) {
UrlCacher uc = au.makeUrlCacher(new UrlData(arcStream, arcProps, fetchUrl));
BitSet bs = new BitSet();
bs.set(UrlCacher.DONT_CLOSE_INPUT_STREAM_FLAG);
uc.setFetchFlags(bs);
uc.storeContent();
archiveData.resetInputStream();
arcStream = archiveData.input;
}
// Wrap it in an ArchiveReader
logger.debug3("About to wrap stream");
arcReader = wrapStream(fetchUrl, arcStream);
logger.debug3("wrapStream() returns " + (arcReader == null ? "null" : "non-null"));
// Explode it
if (arcReader == null) {
throw new CacheException.ExploderException("no WarcReader for " + origUrl);
}
ArchivalUnit au = crawlFacade.getAu();
Set stemSet = new HashSet();
logger.debug("Exploding " + fetchUrl);
// Iterate through the elements in the WARC file, except the first
Iterator i = arcReader.iterator();
// Skip first record
for (i.next(); i.hasNext(); ) {
// XXX probably not necessary
helper.pokeWDog();
if ((++entriesBetweenSleep % sleepAfter) == 0) {
long pauseTime =
CurrentConfig.getTimeIntervalParam(PARAM_RETRY_PAUSE, DEFAULT_RETRY_PAUSE);
Deadline pause = Deadline.in(pauseTime);
logger.debug3("Sleeping for " + StringUtil.timeIntervalToString(pauseTime));
while (!pause.expired()) {
try {
pause.sleep();
} catch (InterruptedException ie) {
// no action
}
}
}
ArchiveRecord element = (ArchiveRecord) i.next();
// Each element is a URL to be cached in a suitable AU
ArchiveRecordHeader elementHeader = element.getHeader();
String elementUrl = elementHeader.getUrl();
String elementMimeType = elementHeader.getMimetype();
long elementLength = elementHeader.getLength();
logger.debug2("WARC url " + elementUrl + " mime " + elementMimeType);
if (elementUrl.startsWith("http:")) {
ArchiveEntry ae =
new ArchiveEntry(
elementUrl,
elementLength,
0, // XXX need to convert getDate string to long
element, // ArchiveRecord extends InputStream
this,
fetchUrl);
ae.setHeaderFields(makeCIProperties(elementHeader));
long bytesStored = elementLength;
logger.debug3("ArchiveEntry: " + ae.getName() + " bytes " + bytesStored);
try {
helper.process(ae);
} catch (PluginException ex) {
throw new CacheException.ExploderException("helper.process() threw", ex);
}
if (ae.getBaseUrl() != null) {
if (ae.getRestOfUrl() != null && ae.getHeaderFields() != null) {
storeEntry(ae);
handleAddText(ae);
goodEntries++;
crawlFacade.getCrawlerStatus().addContentBytesFetched(bytesStored);
} else {
ignoredEntries++;
}
} else {
badEntries++;
logger.debug2("Can't map " + elementUrl + " from " + archiveUrl);
}
}
}
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
} finally {
if (arcReader != null)
try {
arcReader.close();
arcReader = null;
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
}
if (cachedUrl != null) {
cachedUrl.release();
}
IOUtil.safeClose(arcStream);
}
if (badEntries == 0 && goodEntries > 0) {
// Make it look like a new crawl finished on each AU to which
// URLs were added.
for (Iterator it = touchedAus.iterator(); it.hasNext(); ) {
ArchivalUnit au = (ArchivalUnit) it.next();
logger.debug3(archiveUrl + " touching " + au.toString());
AuUtil.getDaemon(au).getNodeManager(au).newContentCrawlFinished();
}
} else {
ArchivalUnit au = crawlFacade.getAu();
String msg = archiveUrl + ": " + badEntries + "/" + goodEntries + " bad entries";
throw new CacheException.UnretryableException(msg);
}
}
