/**
* Upload one SIP file from the input folder. First call the DESA API method
* asyncSubmitPackageStart, then copy the zip file to the mapped DESA target folder and finally
* call the DESA API method asyncSubmitPackageEnd. Add the corresponding entry into the results
* file JAXB representation.
*
* @param file
* @param sipType
* @param writeResults
*/
private void uploadFile(File file, SipType sipType, boolean writeResults) {
Holder<String> sipId = new Holder<String>(getSipId(file));
Holder<String> idSipVersion = new Holder<String>();
String checksum = getMD5Checksum(file);
log.info("Transporting file: " + file.getName());
if (useRest) {
idSipVersion.value =
desaClient.submitPackage(
file, operatorName, producerCode, sipId.value, FileHashAlg.MD_5, checksum, "cs");
log.info("Received idSipVersion:" + idSipVersion.value);
if (idSipVersion.value == null || "".equals(idSipVersion.value)) {
throw new RuntimeException(
"DESA REST call did not return idSipVersion for file " + file.getName());
}
} else {
try {
desaPort.asyncSubmitPackageStart(
0,
producerCode,
operatorName,
sipId,
(int) file.length(),
checksum,
FileHashAlg.MD_5,
idSipVersion);
} catch (SIPSubmissionFault sipSubmissionFault) {
throw new RuntimeException(sipSubmissionFault);
}
if (idSipVersion.value == null || "".equals(idSipVersion.value)) {
throw new RuntimeException(
"DESA SOAP call did not return idSipVersion for file " + file.getName());
}
File target = new File(getDesaFolder(), idSipVersion.value + ".sip");
try {
FileUtils.copyFile(file, target);
} catch (IOException e) {
throw new RuntimeException(e);
}
log.info("Received idSipVersion:" + idSipVersion.value);
try {
desaPort.asyncSubmitPackageEnd(0, producerCode, operatorName, idSipVersion.value);
} catch (SIPSubmissionFault sipSubmissionFault) {
throw new RuntimeException(sipSubmissionFault);
}
}
if (writeResults) {
XMLGregorianCalendar currentDate =
desaClient.getXmlTypes().newXMLGregorianCalendar(new GregorianCalendar());
PSPSIP.SIP entry = resultsFactory.createPSPSIPSIP();
entry.setIdentifier(sipId.value);
entry.setIdSIPVersion(idSipVersion.value);
entry.setResultTime(currentDate);
entry.setResultCode(ResultType.PROGRESS);
entry.setType(sipType);
results.getSIP().add(entry);
}
}
/**
* Main class for import of the SIP packages to DESA repository and for checking the progress of the
* import.
*/
public final class SIP2DESATransporter {
private static final Logger log = Logger.getLogger(SIP2DESATransporter.class.toString());
private static JAXBContext PSPSIP_JAXB;
private final SIPSubmission desaPort;
private final DesaClient desaClient;
private final String desaFolderPath;
private final String operatorName;
private final String producerCode;
private boolean checkMTDPSP = false;
private final boolean useRest;
private Marshaller marshaller = null;
private Unmarshaller unmarshaller = null;
private int filesIndex = -1;
private int mtdpspIndex = -1;
private String packageid = null;
private final ObjectFactory resultsFactory = new ObjectFactory();
private PSPSIP results = resultsFactory.createPSPSIP();
/**
* Returns the parsed result from the result file
*
* @return
*/
public PSPSIP getResults() {
return results;
}
/** used just in case non REST usage! */
private File desaFolder;
private String logRoot = "";
// /**
// * The transporter entry point
// *
// * @param args
// * command line arguments for transport mode : transport
// * <input-folder> <results-folder> <log-folder> command line
// * arguments for checkStatus mode : checkStatus <results-file>
// * <log-folder>
// */
// public static void main(String[] args) {
// try {
// if (args.length == 4 && "transport".equalsIgnoreCase(args[0])) {
// String importRoot = args[1];
// String resultsRoot = args[2];
// String logRoot = args[3];
// new SIP2DESATransporter().transport(importRoot, resultsRoot, logRoot);
// } else if (args.length == 3 && "checkStatus".equalsIgnoreCase(args[0])) {
// String resultsRoot = args[1];
// String logRoot = args[2];
// new SIP2DESATransporter().checkStatus(resultsRoot, logRoot);
// } else if (args.length == 2 && "adminUpload".equalsIgnoreCase(args[0])) {
// String importRoot = args[1];
// new SIP2DESATransporter().adminUpload(importRoot);
// } else {
// System.out.println("SIP to DESA transporter.\n");
// System.out.println("Usage for transport: sip2desa.SIP2DESATransporter transport
// <input-folder> <results-folder> <log-folder>");
// System.out.println("Usage for checkStatus: sip2desa.SIP2DESATransporter
// checkStatus <results-file> <log-folder>");
// System.out.println("Usage for adminUpload: sip2desa.SIP2DESATransporter
// adminUpload <input-folder>");
//
// System.exit(1);
// }
// } catch (Exception e) {
// log.log(Level.SEVERE, e.getMessage(), e);
// }
// }
/** Submits SIPs through REST interface. */
SIP2DESATransporter(DesaClient desaClient, String operator, String producerCode) {
this(desaClient, true, null, operator, producerCode);
}
/** Ignores REST interface. */
SIP2DESATransporter(
DesaClient desaClient, String desaFolderPath, String operator, String producerCode) {
this(desaClient, false, desaFolderPath, operator, producerCode);
}
private SIP2DESATransporter(
DesaClient desaClient,
boolean useRest,
String desaFolderPath,
String operator,
String producerCode) {
this.desaPort = desaClient.getSoapClient();
this.desaClient = desaClient;
this.useRest = useRest;
this.desaFolderPath = desaFolderPath;
this.operatorName = operator;
this.producerCode = producerCode;
}
/**
* Main execution method for the transport mode. Imports the SIP files in the sourceRoot to DESA
* repository (configured in config property file). The status of the import of each SIP is
* written in the results XML file named TRANSF_packageid.xml in the resultsRoot folder. The JDK
* logging output is mirrored in the packageid.log file in the logRoot folder.
*
* @param sourceRoot
* @param resultsRoot
* @param logRootIn
*/
public int[] transport(String sourceRoot, String resultsRoot, String logRootIn) {
long timeStart = System.currentTimeMillis();
logRoot = logRootIn;
Handler handler = null;
File[] sourceFiles;
File resultsFolder;
try {
try {
initJAXB();
File sourceFolder = new File(sourceRoot);
if (!sourceFolder.exists()) {
handler = setupLogHandler(sourceFolder.getName());
log.log(Level.SEVERE, "Source folder doesn't exist: " + sourceFolder.getAbsolutePath());
throw new IllegalStateException(
"Source folder doesn't exist: " + sourceFolder.getAbsolutePath());
}
sourceFiles =
sourceFolder.listFiles(
new FileFilter() {
@Override
public boolean accept(File pathname) {
return (pathname.getName().endsWith(".zip"));
}
});
if (sourceFiles == null || sourceFiles.length == 0) {
handler = setupLogHandler(sourceFolder.getName());
log.log(Level.SEVERE, "Empty source folder: " + sourceFolder.getAbsolutePath());
throw new IllegalStateException("Empty source folder: " + sourceFolder.getAbsolutePath());
}
preprocessFiles(sourceFolder, sourceFiles);
handler = setupLogHandler(packageid);
resultsFolder = checkDirectory(resultsRoot);
log.info("Loaded source folder: " + sourceFolder);
log.info("Results directory: " + resultsFolder);
} catch (Exception e) {
log.log(Level.SEVERE, "Error in transporter initialization.", e);
throw new IllegalStateException("Error in transporter initialization.", e);
}
try {
for (int i = 0; i < sourceFiles.length; i++) {
if (i != filesIndex && i != mtdpspIndex) {
uploadFile(sourceFiles[i], SipType.RECORD, true);
}
}
if (mtdpspIndex > -1) {
uploadFile(sourceFiles[mtdpspIndex], SipType.RECORD, true);
}
if (filesIndex >= 0) {
uploadFile(sourceFiles[filesIndex], SipType.FILE, true);
}
} catch (Throwable th) {
log.log(Level.SEVERE, "Error in file upload: ", th);
throw new IllegalStateException("Error in file upload: ", th);
} finally {
results.setPspResultCode("progress");
try {
marshaller.marshal(results, new File(resultsFolder, "TRANSF_" + packageid + ".xml"));
} catch (JAXBException e) {
log.log(Level.SEVERE, "Error writing results file: ", e);
throw new IllegalStateException(e);
}
}
long timeFinish = System.currentTimeMillis();
log.info(
"Elapsed time: "
+ ((timeFinish - timeStart) / 1000.0)
+ " seconds. "
+ sourceFiles.length
+ " SIP packages transported.");
log.info("RESULT: OK");
} finally {
if (handler != null) {
removeLogHandler(handler);
}
}
return countSip();
}
private Handler setupLogHandler(String fileName) {
Handler handler;
try {
checkDirectory(logRoot);
handler =
new FileHandler(
logRoot + System.getProperty("file.separator") + fileName + ".txt", 0, 1, true);
handler.setFormatter(new SimpleFormatter());
} catch (IOException e) {
throw new RuntimeException(e);
}
Logger.getLogger("").addHandler(handler);
return handler;
}
private static void removeLogHandler(Handler handler) {
Logger.getLogger("").removeHandler(handler);
handler.close();
}
public void adminUpload(String sourceRoot) {
long timeStart = System.currentTimeMillis();
File[] sourceFiles;
try {
initJAXB();
File sourceFolder = new File(sourceRoot);
if (!sourceFolder.exists()) {
log.log(Level.SEVERE, "Source folder doesn't exist: " + sourceFolder.getAbsolutePath());
throw new IllegalStateException(
"Source folder doesn't exist: " + sourceFolder.getAbsolutePath());
}
sourceFiles =
sourceFolder.listFiles(
new FileFilter() {
@Override
public boolean accept(File pathname) {
return (pathname.getName().endsWith(".zip"));
}
});
if (sourceFiles == null || sourceFiles.length == 0) {
log.log(Level.SEVERE, "Empty source folder: " + sourceFolder.getAbsolutePath());
throw new IllegalStateException("Empty source folder: " + sourceFolder.getAbsolutePath());
}
log.info("Loaded source folder: " + sourceFolder);
} catch (Exception e) {
log.log(Level.SEVERE, "Error in transporter initialization.", e);
throw new IllegalStateException("Error in transporter initialization.", e);
}
try {
for (int i = 0; i < sourceFiles.length; i++) {
uploadFile(sourceFiles[i], null, false);
}
} catch (Throwable th) {
log.log(Level.SEVERE, "Error in file upload: ", th);
throw new IllegalStateException("Error in file upload: ", th);
}
long timeFinish = System.currentTimeMillis();
log.info(
"Elapsed time: "
+ ((timeFinish - timeStart) / 1000.0)
+ " seconds. "
+ sourceFiles.length
+ " SIP packages transported.");
log.info("RESULT: OK");
}
/** Gets the cached thread safe JAXB context. */
private static JAXBContext getPspipJaxb() throws JAXBException {
if (PSPSIP_JAXB == null) {
PSPSIP_JAXB = JAXBContext.newInstance(PSPSIP.class);
}
return PSPSIP_JAXB;
}
/**
* Initialize JAXB transformers
*
* @throws JAXBException
*/
private void initJAXB() throws JAXBException {
if (marshaller != null) {
return;
}
JAXBContext jaxbContext = getPspipJaxb();
marshaller = jaxbContext.createMarshaller();
marshaller.setProperty(Marshaller.JAXB_ENCODING, "utf-8");
marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
unmarshaller = jaxbContext.createUnmarshaller();
}
/**
* Upload one SIP file from the input folder. First call the DESA API method
* asyncSubmitPackageStart, then copy the zip file to the mapped DESA target folder and finally
* call the DESA API method asyncSubmitPackageEnd. Add the corresponding entry into the results
* file JAXB representation.
*
* @param file
* @param sipType
* @param writeResults
*/
private void uploadFile(File file, SipType sipType, boolean writeResults) {
Holder<String> sipId = new Holder<String>(getSipId(file));
Holder<String> idSipVersion = new Holder<String>();
String checksum = getMD5Checksum(file);
log.info("Transporting file: " + file.getName());
if (useRest) {
idSipVersion.value =
desaClient.submitPackage(
file, operatorName, producerCode, sipId.value, FileHashAlg.MD_5, checksum, "cs");
log.info("Received idSipVersion:" + idSipVersion.value);
if (idSipVersion.value == null || "".equals(idSipVersion.value)) {
throw new RuntimeException(
"DESA REST call did not return idSipVersion for file " + file.getName());
}
} else {
try {
desaPort.asyncSubmitPackageStart(
0,
producerCode,
operatorName,
sipId,
(int) file.length(),
checksum,
FileHashAlg.MD_5,
idSipVersion);
} catch (SIPSubmissionFault sipSubmissionFault) {
throw new RuntimeException(sipSubmissionFault);
}
if (idSipVersion.value == null || "".equals(idSipVersion.value)) {
throw new RuntimeException(
"DESA SOAP call did not return idSipVersion for file " + file.getName());
}
File target = new File(getDesaFolder(), idSipVersion.value + ".sip");
try {
FileUtils.copyFile(file, target);
} catch (IOException e) {
throw new RuntimeException(e);
}
log.info("Received idSipVersion:" + idSipVersion.value);
try {
desaPort.asyncSubmitPackageEnd(0, producerCode, operatorName, idSipVersion.value);
} catch (SIPSubmissionFault sipSubmissionFault) {
throw new RuntimeException(sipSubmissionFault);
}
}
if (writeResults) {
XMLGregorianCalendar currentDate =
desaClient.getXmlTypes().newXMLGregorianCalendar(new GregorianCalendar());
PSPSIP.SIP entry = resultsFactory.createPSPSIPSIP();
entry.setIdentifier(sipId.value);
entry.setIdSIPVersion(idSipVersion.value);
entry.setResultTime(currentDate);
entry.setResultCode(ResultType.PROGRESS);
entry.setType(sipType);
results.getSIP().add(entry);
}
}
private String getSipId(File sipFile) {
return sipFile.getName().replace(".zip", "");
}
/**
* Check the contents of the input directory and find packageid and FILES and MTDPSP SIP packages
* (which will be imported last)
*/
private void preprocessFiles(File sourceFolder, File[] sourceFiles) {
for (int i = 0; i < sourceFiles.length; i++) {
if (sourceFiles[i].getName().endsWith("_FILE.zip")) {
packageid = sourceFiles[i].getName().replace("_FILE.zip", "");
filesIndex = i;
} else if (sourceFiles[i].getName().endsWith("_MTDPSP.zip")) {
mtdpspIndex = i;
}
}
if (packageid == null) {
for (int i = 0; i < sourceFiles.length; i++) {
if (sourceFiles[i].getName().endsWith(".zip")) {
if (sourceFiles[i].getName().indexOf("_") > 0) {
packageid =
sourceFiles[i].getName().substring(0, sourceFiles[i].getName().indexOf("_"));
} else {
packageid = sourceFiles[i].getName().replace(".zip", "");
}
break;
}
}
}
if (packageid == null) {
Handler handler = setupLogHandler(sourceFolder.getName());
log.log(Level.SEVERE, "Invalid source folder contents. Missing FILE.zip.");
removeLogHandler(handler);
throw new IllegalStateException("Invalid source folder contents. Missing FILE.zip.");
}
if (checkMTDPSP && mtdpspIndex == -1) {
Handler handler = setupLogHandler(sourceFolder.getName());
log.log(Level.SEVERE, "Invalid source folder contents. Missing MTDPSP.zip.");
removeLogHandler(handler);
throw new IllegalStateException("Invalid source folder contents. Missing MTDPSP.zip.");
}
results.setPackageID(packageid);
}
/**
* Main execution method for the checkStatus mode. The status of the import of each SIP in the
* results XML file named TRANSF_packageid.xml in the resultsRoot folder is checked by call to the
* DESA API getPackageStatus method. The results file is then updated accordingly. The JDK logging
* output is mirrored in the packageid.log file in the logRoot folder.
*
* @param resultsFileStr
* @param logRootIn
*/
public int[] checkStatus(String resultsFileStr, String logRootIn) {
long timeStart = System.currentTimeMillis();
logRoot = logRootIn;
Handler handler = null;
try {
File resultsFile = new File(resultsFileStr);
if (!resultsFile.exists()) {
handler = setupLogHandler(resultsFile.getName());
log.log(Level.SEVERE, "Results file does not exist.");
throw new IllegalStateException("Results file does not exist.");
}
try {
initJAXB();
log.info("Loaded results file: " + resultsFile);
parseResultsFile(resultsFile);
} catch (Exception e) {
handler = setupLogHandler(resultsFile.getName());
log.log(Level.SEVERE, "Error in transporter initialization.", e);
throw new IllegalStateException("Error in transporter initialization.", e);
}
handler = setupLogHandler(packageid);
int objectCounter = 0;
try {
boolean finishedAll = true;
for (PSPSIP.SIP sip : results.getSIP()) {
if (!checkSIP(sip)) {
finishedAll = false;
}
;
objectCounter++;
}
if (finishedAll) {
results.setPspResultCode("finished");
} else {
results.setPspResultCode("progress");
}
marshaller.marshal(results, resultsFile);
} catch (Exception e) {
log.log(Level.SEVERE, "Error checking results: ", e);
throw new IllegalStateException("Error checking results: ", e);
}
long timeFinish = System.currentTimeMillis();
log.info(
"Elapsed time: "
+ ((timeFinish - timeStart) / 1000.0)
+ " seconds. "
+ objectCounter
+ " SIP packages checked.");
log.info("RESULT: OK");
} finally {
if (handler != null) {
removeLogHandler(handler);
}
}
return countSip();
}
private int[] countSip() {
List<PSPSIP.SIP> sipList = results.getSIP();
int sipCount = sipList.size();
int sipFinishedCount = 0;
for (PSPSIP.SIP sip : sipList) {
if (ResultType.FINISHED.equals(sip.getResultCode())) {
sipFinishedCount++;
}
}
return new int[] {sipCount, sipFinishedCount};
}
/**
* Convert the existing results file to JAXB representation
*
* @param resultsFile
*/
private void parseResultsFile(File resultsFile) {
try {
results = (PSPSIP) unmarshaller.unmarshal(resultsFile);
packageid = results.getPackageID();
} catch (Exception e) {
Handler handler = setupLogHandler(resultsFile.getName());
log.log(Level.SEVERE, "Error in parsing results file.", e);
removeLogHandler(handler);
throw new IllegalStateException("Error in parsing results file.", e);
}
}
/**
* Check the status of the SIP of one entry in the results file and update the entry in the JAXB
* object.
*
* @param sip
* @return
*/
private boolean checkSIP(PSPSIP.SIP sip) {
Holder<String> sipId = new Holder<String>(sip.getIdentifier());
Holder<String> idSipVersion = new Holder<String>(sip.getIdSIPVersion());
Holder<String> packageStateCode = new Holder<String>();
Holder<String> packageStateText = new Holder<String>();
Holder<String> errorCode = new Holder<String>();
log.info("Checking file: " + sipId.value);
/*
* if(config.getBoolean("desa.rest")){ try { URLConnection connection =
* new
* URL(config.getString("desa.restapi")+"/packagestatus"+"?userName="******"desa.user")
* +"&producerCode="+config.getString("desa.producer"
* )+"&producerSipId="+
* sipId.value+"&idSIPVersion="+idSipVersion.value).openConnection();
* packageStateCode
* .value=connection.getHeaderField("X-DEA-packageStateCode");
* packageStateText
* .value=connection.getHeaderField("X-DEA-packageStateText");
* errorCode.value=connection.getHeaderField("X-DEA-errorCode"); } catch
* (Exception e) { throw new RuntimeException(e); } }else{
*/
try {
desaPort.getPackageStatus(
0,
producerCode,
operatorName,
idSipVersion,
sipId,
packageStateCode,
packageStateText,
errorCode);
} catch (SIPSubmissionFault sipSubmissionFault) {
throw new RuntimeException(sipSubmissionFault);
} catch (Exception e1) {
log.log(Level.SEVERE, "DESA exception", e1);
throw new IllegalStateException("DESA exception", e1);
}
/* } */
log.info(
"Status: "
+ packageStateCode.value
+ " ("
+ packageStateText.value
+ ")"
+ (errorCode.value != null ? (", errorCode:" + errorCode.value) : ""));
XMLGregorianCalendar currentDate =
desaClient.getXmlTypes().newXMLGregorianCalendar(new GregorianCalendar());
;
boolean retval = false;
sip.setResultTime(currentDate);
if ("AI_ACC_OK".equalsIgnoreCase(packageStateCode.value)) {
sip.setResultCode(ResultType.FINISHED);
retval = true;
} else if ("AI_ERROR".equalsIgnoreCase(packageStateCode.value)
|| "AI_INVALID".equalsIgnoreCase(packageStateCode.value)
|| "AI_REJECT".equalsIgnoreCase(packageStateCode.value)
|| "AI_INFECTED".equalsIgnoreCase(packageStateCode.value)
|| "AI_QA_ERR".equalsIgnoreCase(packageStateCode.value)) {
sip.setResultCode(ResultType.ERROR);
} else {
sip.setResultCode(ResultType.PROGRESS);
}
return retval;
}
/** Gets target folder for SOAP submit package. */
private File getDesaFolder() {
if (!useRest) {
if (desaFolder == null) {
desaFolder = checkDirectory(desaFolderPath);
}
}
return desaFolder;
}
/**
* Check if the directory with the given path exists and create it if necessary
*
* @param name The path of the requested directory
* @return The File representation of the requested directory
*/
private static File checkDirectory(String name) {
File directory = new File(name);
try {
FileUtils.forceMkdir(directory);
return directory;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Calculate the MD5 checksum of the given file
*
* @param file
* @return
*/
private static String getMD5Checksum(File file) {
try {
InputStream fis = new FileInputStream(file);
byte[] buffer = new byte[1024];
MessageDigest complete = MessageDigest.getInstance("MD5");
int numRead;
do {
numRead = fis.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
}
} while (numRead != -1);
fis.close();
byte[] bytes = complete.digest();
StringBuilder sb = new StringBuilder(2 * bytes.length);
for (byte b : bytes) {
sb.append(hexDigits[(b >> 4) & 0xf]).append(hexDigits[b & 0xf]);
}
return sb.toString();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static final char[] hexDigits = "0123456789abcdef".toCharArray();
}