public void uploadBundle(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException, DotDataException, FileUploadException {
FileItemFactory factory = new DiskFileItemFactory();
ServletFileUpload upload = new ServletFileUpload(factory);
@SuppressWarnings("unchecked")
List<FileItem> items = (List<FileItem>) upload.parseRequest(request);
InputStream bundle = items.get(0).getInputStream();
String bundleName = items.get(0).getName();
String bundlePath = ConfigUtils.getBundlePath() + File.separator;
String bundleFolder = bundleName.substring(0, bundleName.indexOf(".tar.gz"));
String endpointId = getUser().getUserId();
response.setContentType("text/html; charset=utf-8");
PrintWriter out = response.getWriter();
PublishAuditStatus status;
try {
status = PublishAuditAPI.getInstance().updateAuditTable(endpointId, null, bundleFolder);
// Write file on FS
FileUtil.writeToFile(bundle, bundlePath + bundleName);
if (!status.getStatus().equals(Status.PUBLISHING_BUNDLE)) {
new Thread(new PublishThread(bundleName, null, endpointId, status)).start();
}
out.print(
"<html><head><script>isLoaded = true;</script></head><body><textarea>{'status':'success'}</textarea></body></html>");
} catch (DotPublisherException e) {
// TODO Auto-generated catch block
out.print(
"<html><head><script>isLoaded = true;</script></head><body><textarea>{'status':'error'}</textarea></body></html>");
}
}
@Override
public PublisherConfig process(final PublishStatus status) throws DotPublishingException {
if (LicenseUtil.getLevel() < 400)
throw new RuntimeException("need an enterprise prime license to run this bundler");
PublishAuditHistory currentStatusHistory = null;
try {
// Compressing bundle
File bundleRoot = BundlerUtil.getBundleRoot(config);
ArrayList<File> list = new ArrayList<File>(1);
list.add(bundleRoot);
File bundle =
new File(
bundleRoot + File.separator + ".." + File.separator + config.getId() + ".tar.gz");
PushUtils.compressFiles(list, bundle, bundleRoot.getAbsolutePath());
// Retriving enpoints and init client
List<PublishingEndPoint> endpoints = ((PushPublisherConfig) config).getEndpoints();
Map<String, List<PublishingEndPoint>> endpointsMap =
new HashMap<String, List<PublishingEndPoint>>();
List<PublishingEndPoint> buffer = null;
// Organize the endpoints grouping them by groupId
for (PublishingEndPoint pEndPoint : endpoints) {
String gid =
UtilMethods.isSet(pEndPoint.getGroupId()) ? pEndPoint.getGroupId() : pEndPoint.getId();
if (endpointsMap.get(gid) == null) buffer = new ArrayList<PublishingEndPoint>();
else buffer = endpointsMap.get(gid);
buffer.add(pEndPoint);
// put in map with either the group key or the id if no group is set
endpointsMap.put(gid, buffer);
}
ClientConfig cc = new DefaultClientConfig();
if (Config.getStringProperty("TRUSTSTORE_PATH") != null
&& !Config.getStringProperty("TRUSTSTORE_PATH").trim().equals(""))
cc.getProperties()
.put(
HTTPSProperties.PROPERTY_HTTPS_PROPERTIES,
new HTTPSProperties(tFactory.getHostnameVerifier(), tFactory.getSSLContext()));
Client client = Client.create(cc);
// Updating audit table
currentStatusHistory = pubAuditAPI.getPublishAuditStatus(config.getId()).getStatusPojo();
currentStatusHistory.setPublishStart(new Date());
pubAuditAPI.updatePublishAuditStatus(
config.getId(), PublishAuditStatus.Status.SENDING_TO_ENDPOINTS, currentStatusHistory);
// Increment numTries
currentStatusHistory.addNumTries();
boolean hasError = false;
int errorCounter = 0;
for (String group : endpointsMap.keySet()) {
List<PublishingEndPoint> groupList = endpointsMap.get(group);
boolean sent = false;
for (PublishingEndPoint endpoint : groupList) {
EndpointDetail detail = new EndpointDetail();
try {
FormDataMultiPart form = new FormDataMultiPart();
form.field(
"AUTH_TOKEN",
retriveKeyString(
PublicEncryptionFactory.decryptString(endpoint.getAuthKey().toString())));
form.field(
"GROUP_ID",
UtilMethods.isSet(endpoint.getGroupId())
? endpoint.getGroupId()
: endpoint.getId());
form.field("ENDPOINT_ID", endpoint.getId());
form.bodyPart(
new FileDataBodyPart("bundle", bundle, MediaType.MULTIPART_FORM_DATA_TYPE));
// Sending bundle to endpoint
WebResource resource =
client.resource(endpoint.toURL() + "/api/bundlePublisher/publish");
ClientResponse response =
resource.type(MediaType.MULTIPART_FORM_DATA).post(ClientResponse.class, form);
if (response.getClientResponseStatus().getStatusCode() == HttpStatus.SC_OK) {
detail.setStatus(PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY.getCode());
detail.setInfo("Everything ok");
sent = true;
} else {
detail.setStatus(PublishAuditStatus.Status.FAILED_TO_SENT.getCode());
detail.setInfo(
"Returned "
+ response.getClientResponseStatus().getStatusCode()
+ " status code "
+ "for the endpoint "
+ endpoint.getId()
+ "with address "
+ endpoint.getAddress());
}
} catch (Exception e) {
hasError = true;
detail.setStatus(PublishAuditStatus.Status.FAILED_TO_SENT.getCode());
String error =
"An error occured for the endpoint "
+ endpoint.getId()
+ " with address "
+ endpoint.getAddress()
+ ". Error: "
+ e.getMessage();
detail.setInfo(error);
Logger.error(this.getClass(), error);
}
currentStatusHistory.addOrUpdateEndpoint(group, endpoint.getId(), detail);
if (sent) break;
}
if (!sent) {
hasError = true;
errorCounter++;
}
}
if (!hasError) {
// Updating audit table
currentStatusHistory.setPublishEnd(new Date());
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY,
currentStatusHistory);
// Deleting queue records
// pubAPI.deleteElementsFromPublishQueueTable(config.getId());
} else {
if (errorCounter == endpointsMap.size()) {
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.FAILED_TO_SEND_TO_ALL_GROUPS,
currentStatusHistory);
} else {
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.FAILED_TO_SEND_TO_SOME_GROUPS,
currentStatusHistory);
}
}
return config;
} catch (Exception e) {
// Updating audit table
try {
pubAuditAPI.updatePublishAuditStatus(
config.getId(), PublishAuditStatus.Status.FAILED_TO_PUBLISH, currentStatusHistory);
} catch (DotPublisherException e1) {
throw new DotPublishingException(e.getMessage());
}
Logger.error(this.getClass(), e.getMessage(), e);
throw new DotPublishingException(e.getMessage());
}
}
public class PushPublisher extends Publisher {
private PublishAuditAPI pubAuditAPI = PublishAuditAPI.getInstance();
private TrustFactory tFactory;
@Override
public PublisherConfig init(PublisherConfig config) throws DotPublishingException {
if (LicenseUtil.getLevel() < 400)
throw new RuntimeException("need an enterprise prime license to run this bundler");
this.config = super.init(config);
tFactory = new TrustFactory();
return this.config;
}
@Override
public PublisherConfig process(final PublishStatus status) throws DotPublishingException {
if (LicenseUtil.getLevel() < 400)
throw new RuntimeException("need an enterprise prime license to run this bundler");
PublishAuditHistory currentStatusHistory = null;
try {
// Compressing bundle
File bundleRoot = BundlerUtil.getBundleRoot(config);
ArrayList<File> list = new ArrayList<File>(1);
list.add(bundleRoot);
File bundle =
new File(
bundleRoot + File.separator + ".." + File.separator + config.getId() + ".tar.gz");
PushUtils.compressFiles(list, bundle, bundleRoot.getAbsolutePath());
// Retriving enpoints and init client
List<PublishingEndPoint> endpoints = ((PushPublisherConfig) config).getEndpoints();
Map<String, List<PublishingEndPoint>> endpointsMap =
new HashMap<String, List<PublishingEndPoint>>();
List<PublishingEndPoint> buffer = null;
// Organize the endpoints grouping them by groupId
for (PublishingEndPoint pEndPoint : endpoints) {
String gid =
UtilMethods.isSet(pEndPoint.getGroupId()) ? pEndPoint.getGroupId() : pEndPoint.getId();
if (endpointsMap.get(gid) == null) buffer = new ArrayList<PublishingEndPoint>();
else buffer = endpointsMap.get(gid);
buffer.add(pEndPoint);
// put in map with either the group key or the id if no group is set
endpointsMap.put(gid, buffer);
}
ClientConfig cc = new DefaultClientConfig();
if (Config.getStringProperty("TRUSTSTORE_PATH") != null
&& !Config.getStringProperty("TRUSTSTORE_PATH").trim().equals(""))
cc.getProperties()
.put(
HTTPSProperties.PROPERTY_HTTPS_PROPERTIES,
new HTTPSProperties(tFactory.getHostnameVerifier(), tFactory.getSSLContext()));
Client client = Client.create(cc);
// Updating audit table
currentStatusHistory = pubAuditAPI.getPublishAuditStatus(config.getId()).getStatusPojo();
currentStatusHistory.setPublishStart(new Date());
pubAuditAPI.updatePublishAuditStatus(
config.getId(), PublishAuditStatus.Status.SENDING_TO_ENDPOINTS, currentStatusHistory);
// Increment numTries
currentStatusHistory.addNumTries();
boolean hasError = false;
int errorCounter = 0;
for (String group : endpointsMap.keySet()) {
List<PublishingEndPoint> groupList = endpointsMap.get(group);
boolean sent = false;
for (PublishingEndPoint endpoint : groupList) {
EndpointDetail detail = new EndpointDetail();
try {
FormDataMultiPart form = new FormDataMultiPart();
form.field(
"AUTH_TOKEN",
retriveKeyString(
PublicEncryptionFactory.decryptString(endpoint.getAuthKey().toString())));
form.field(
"GROUP_ID",
UtilMethods.isSet(endpoint.getGroupId())
? endpoint.getGroupId()
: endpoint.getId());
form.field("ENDPOINT_ID", endpoint.getId());
form.bodyPart(
new FileDataBodyPart("bundle", bundle, MediaType.MULTIPART_FORM_DATA_TYPE));
// Sending bundle to endpoint
WebResource resource =
client.resource(endpoint.toURL() + "/api/bundlePublisher/publish");
ClientResponse response =
resource.type(MediaType.MULTIPART_FORM_DATA).post(ClientResponse.class, form);
if (response.getClientResponseStatus().getStatusCode() == HttpStatus.SC_OK) {
detail.setStatus(PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY.getCode());
detail.setInfo("Everything ok");
sent = true;
} else {
detail.setStatus(PublishAuditStatus.Status.FAILED_TO_SENT.getCode());
detail.setInfo(
"Returned "
+ response.getClientResponseStatus().getStatusCode()
+ " status code "
+ "for the endpoint "
+ endpoint.getId()
+ "with address "
+ endpoint.getAddress());
}
} catch (Exception e) {
hasError = true;
detail.setStatus(PublishAuditStatus.Status.FAILED_TO_SENT.getCode());
String error =
"An error occured for the endpoint "
+ endpoint.getId()
+ " with address "
+ endpoint.getAddress()
+ ". Error: "
+ e.getMessage();
detail.setInfo(error);
Logger.error(this.getClass(), error);
}
currentStatusHistory.addOrUpdateEndpoint(group, endpoint.getId(), detail);
if (sent) break;
}
if (!sent) {
hasError = true;
errorCounter++;
}
}
if (!hasError) {
// Updating audit table
currentStatusHistory.setPublishEnd(new Date());
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.BUNDLE_SENT_SUCCESSFULLY,
currentStatusHistory);
// Deleting queue records
// pubAPI.deleteElementsFromPublishQueueTable(config.getId());
} else {
if (errorCounter == endpointsMap.size()) {
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.FAILED_TO_SEND_TO_ALL_GROUPS,
currentStatusHistory);
} else {
pubAuditAPI.updatePublishAuditStatus(
config.getId(),
PublishAuditStatus.Status.FAILED_TO_SEND_TO_SOME_GROUPS,
currentStatusHistory);
}
}
return config;
} catch (Exception e) {
// Updating audit table
try {
pubAuditAPI.updatePublishAuditStatus(
config.getId(), PublishAuditStatus.Status.FAILED_TO_PUBLISH, currentStatusHistory);
} catch (DotPublisherException e1) {
throw new DotPublishingException(e.getMessage());
}
Logger.error(this.getClass(), e.getMessage(), e);
throw new DotPublishingException(e.getMessage());
}
}
private String retriveKeyString(String token) throws IOException {
String key = null;
if (token.contains(File.separator)) {
File tokenFile = new File(token);
if (tokenFile != null && tokenFile.exists())
key = FileUtils.readFileToString(tokenFile, "UTF-8").trim();
} else {
key = token;
}
return PublicEncryptionFactory.encryptString(key);
}
@SuppressWarnings("rawtypes")
@Override
public List<Class> getBundlers() {
boolean buildCategories = false;
for (PublishQueueElement element : ((PushPublisherConfig) config).getAssets())
if (element.getType().equals("category")) buildCategories = true;
List<Class> list = new ArrayList<Class>();
// The order is important cause
// I need to add all containers associated with templates
/** ISSUE #2244: https://github.com/dotCMS/dotCMS/issues/2244 */
if (buildCategories) list.add(CategoryBundler.class);
else {
list.add(DependencyBundler.class);
list.add(HostBundler.class);
list.add(ContentBundler.class);
list.add(FolderBundler.class);
list.add(TemplateBundler.class);
list.add(ContainerBundler.class);
list.add(HTMLPageBundler.class);
list.add(LinkBundler.class);
if (Config.getBooleanProperty("PUSH_PUBLISHING_PUSH_STRUCTURES")) {
list.add(StructureBundler.class);
/** ISSUE #2222: https://github.com/dotCMS/dotCMS/issues/2222 */
list.add(RelationshipBundler.class);
}
list.add(LanguageBundler.class);
}
return list;
}
}
/**
* Allow the user to send or try to send again failed and successfully sent bundles, in order to
* do that<br>
* we send the bundle again to que publisher queue job which will try to remote publish again the
* bundle.
*
* @param request HttpRequest
* @param response HttpResponse
* @throws IOException If fails sending back to the user a proper response
* @throws DotPublisherException If fails retrieving the Bundle related information like elements
* on it and statuses
* @throws LanguageException If fails using i18 messages
*/
public void retry(HttpServletRequest request, HttpServletResponse response)
throws IOException, DotPublisherException, LanguageException {
PublisherAPI publisherAPI = PublisherAPI.getInstance();
PublishAuditAPI publishAuditAPI = PublishAuditAPI.getInstance();
// Read the parameters
String bundlesIds = request.getParameter("bundlesIds");
String[] ids = bundlesIds.split(",");
StringBuilder responseMessage = new StringBuilder();
for (String bundleId : ids) {
if (bundleId.trim().isEmpty()) {
continue;
}
PublisherConfig basicConfig = new PublisherConfig();
basicConfig.setId(bundleId);
File bundleRoot = BundlerUtil.getBundleRoot(basicConfig);
// Get the audit records related to this bundle
PublishAuditStatus status = PublishAuditAPI.getInstance().getPublishAuditStatus(bundleId);
String pojo_string = status.getStatusPojo().getSerialized();
PublishAuditHistory auditHistory = PublishAuditHistory.getObjectFromString(pojo_string);
// First we need to verify is this bundle is already in the queue job
List<PublishQueueElement> foundBundles = publisherAPI.getQueueElementsByBundleId(bundleId);
if (foundBundles != null && !foundBundles.isEmpty()) {
appendMessage(responseMessage, "publisher_retry.error.already.in.queue", bundleId, true);
continue;
}
// We will be able to retry failed and successfully bundles
if (!(status.getStatus().equals(Status.FAILED_TO_PUBLISH)
|| status.getStatus().equals(Status.SUCCESS))) {
appendMessage(responseMessage, "publisher_retry.error.only.failed.publish", bundleId, true);
continue;
}
/*
Verify if the bundle exist and was created correctly..., meaning, if there is not a .tar.gz file is because
something happened on the creation of the bundle.
*/
File bundleFile =
new File(
bundleRoot
+ File.separator
+ ".."
+ File.separator
+ basicConfig.getId()
+ ".tar.gz");
if (!bundleFile.exists()) {
Logger.error(this.getClass(), "No Bundle with id: " + bundleId + " found.");
appendMessage(responseMessage, "publisher_retry.error.not.found", bundleId, true);
continue;
}
if (!BundlerUtil.bundleExists(basicConfig)) {
Logger.error(
this.getClass(), "No Bundle Descriptor for bundle id: " + bundleId + " found.");
appendMessage(
responseMessage, "publisher_retry.error.not.descriptor.found", bundleId, true);
continue;
}
try {
// Read the bundle to see what kind of configuration we need to apply
String bundlePath = ConfigUtils.getBundlePath() + File.separator + basicConfig.getId();
File xml = new File(bundlePath + File.separator + "bundle.xml");
PushPublisherConfig config = (PushPublisherConfig) BundlerUtil.xmlToObject(xml);
// We can not retry Received Bundles, just bundles that we are trying to send
Boolean sending = sendingBundle(request, config, bundleId);
if (!sending) {
appendMessage(
responseMessage, "publisher_retry.error.cannot.retry.received", bundleId, true);
continue;
}
if (status.getStatus().equals(Status.SUCCESS)) {
// Get the bundle
Bundle bundle = APILocator.getBundleAPI().getBundleById(bundleId);
if (bundle == null) {
Logger.error(this.getClass(), "No Bundle with id: " + bundleId + " found.");
appendMessage(responseMessage, "publisher_retry.error.not.found", bundleId, true);
continue;
}
bundle.setForcePush(true);
APILocator.getBundleAPI().updateBundle(bundle);
}
// Clean the number of tries, we want to try it again
auditHistory.setNumTries(0);
publishAuditAPI.updatePublishAuditStatus(
config.getId(), status.getStatus(), auditHistory, true);
// Get the identifiers on this bundle
HashSet<String> identifiers = new HashSet<String>();
List<PublishQueueElement> assets = config.getAssets();
if (config.getLuceneQueries() != null && !config.getLuceneQueries().isEmpty()) {
identifiers.addAll(PublisherUtil.getContentIds(config.getLuceneQueries()));
}
if (assets != null && !assets.isEmpty()) {
for (PublishQueueElement asset : assets) {
identifiers.add(asset.getAsset());
}
}
// Now depending of the operation lets add it to the queue job
if (config.getOperation().equals(PushPublisherConfig.Operation.PUBLISH)) {
publisherAPI.addContentsToPublish(
new ArrayList<String>(identifiers), bundleId, new Date(), getUser());
} else {
publisherAPI.addContentsToUnpublish(
new ArrayList<String>(identifiers), bundleId, new Date(), getUser());
}
// Success...
appendMessage(responseMessage, "publisher_retry.success", bundleId, false);
} catch (Exception e) {
Logger.error(
this.getClass(),
"Error trying to add bundle id: " + bundleId + " to the Publishing Queue.",
e);
appendMessage(responseMessage, "publisher_retry.error.adding.to.queue", bundleId, true);
}
}
response.getWriter().println(responseMessage.toString());
}