private static Map<String, List<String>> generate_result(Directory directory) {
Map<String, List<String>> result_map = new HashMap<String, List<String>>();
try {
IndexReader reader = IndexReader.open(directory);
TermEnum termEnum = reader.terms();
while (termEnum.next()) {
String termEnumString = termEnum.term().toString();
if (termEnumString.startsWith("content:")) {
String term = termEnumString.substring(termEnumString.lastIndexOf(":") + 1);
TermDocs termDocs = reader.termDocs(termEnum.term());
while (termDocs.next()) {
Document doc = reader.document(termDocs.doc());
String relative_path = doc.get("relative_path");
if (result_map.containsKey(relative_path)) {
result_map.get(relative_path).add(term + termDocs.freq());
} else {
result_map.put(relative_path, new ArrayList<String>());
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
}
return result_map;
}
private Map<String, Integer> loadFixedClasses(String file) {
FileReader input;
try {
Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
File f = new File(file);
if (!f.exists()) {
System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
return invertedFixedClasses;
}
input = new FileReader(f);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
String parts[] = line.split(",");
int clazz = Integer.parseInt(parts[0]);
List<String> symbols = new ArrayList<String>();
symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
fixedClaszzes.put(clazz, symbols);
}
for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
for (String s : e.getValue()) {
invertedFixedClasses.put(s, e.getKey());
}
}
return invertedFixedClasses;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private static Directory index(Analyzer analyzer, String processingPath) {
RAMDirectory directory = null;
IndexWriter indexWriter = null;
try {
directory = new RAMDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_35, analyzer);
indexWriter = new IndexWriter(directory, iwc);
File file = new File(processingPath);
index_h("", file, indexWriter);
} catch (IOException e) {
e.printStackTrace();
} finally {
if (indexWriter != null) {
try {
indexWriter.close();
} catch (CorruptIndexException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
return directory;
}
private void run_poolAvail() throws ExecutionProblem {
AssociationEntry[] entries = null;
try {
if (this.nodePool != null) {
final String entriesJson =
this.remoteNodeManagement.getNetworkPool(this.nodePool, this.inUse);
if (entriesJson == null) {
System.err.println("No entries with pool name " + nodePool + " found");
return;
}
entries = gson.fromJson(entriesJson, AssociationEntry[].class);
} else {
final String entriesJson = this.remoteNodeManagement.getAllNetworkPools(this.inUse);
if (entriesJson == null) {
System.err.println("No pool entries found");
return;
}
entries = gson.fromJson(entriesJson, AssociationEntry[].class);
}
} catch (RemoteException e) {
System.err.println(e.getMessage());
}
try {
reporter.report(nodeAllocationToMaps(entries), this.outStream);
} catch (IOException e) {
throw new ExecutionProblem("Problem writing output: " + e.getMessage(), e);
}
}
private static void output_result(String outPath, Map<String, List<String>> result_map) {
for (String s : result_map.keySet()) {
try {
FileUtils.writeLines(new File(outPath + s), result_map.get(s));
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static String[] getStopWords(String path) {
try {
List<String> list = FileUtils.readLines(new File(path), "utf-8");
System.out.println("==============================================================");
System.out.println("getStopWords " + list.get(159));
System.out.println("==============================================================");
return list.toArray(new String[list.size()]);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
}
return null;
}
private void run_listNodes() throws ExecutionProblem {
VmmNode[] nodes = null;
try {
final String nodesJson = this.remoteNodeManagement.listNodes();
nodes = this.gson.fromJson(nodesJson, VmmNode[].class);
} catch (RemoteException e) {
handleRemoteException(e);
}
try {
reporter.report(nodesToMaps(nodes), this.outStream);
} catch (IOException e) {
throw new ExecutionProblem("Problem writing output: " + e.getMessage(), e);
}
}
private void run_removeNodes() throws ExecutionProblem {
NodeReport[] reports = null;
try {
final String[] hostnames = this.hosts.toArray(new String[this.hosts.size()]);
final String reportJson = this.remoteNodeManagement.removeNodes(hostnames);
reports = gson.fromJson(reportJson, NodeReport[].class);
} catch (RemoteException e) {
handleRemoteException(e);
}
try {
reporter.report(nodeReportsToMaps(reports), this.outStream);
} catch (IOException e) {
throw new ExecutionProblem("Problem writing output: " + e.getMessage(), e);
}
}
private void run_addNodes() throws ParameterProblem, ExecutionProblem {
if (!this.nodeMemoryConfigured) {
throw new ParameterProblem(
"Node max memory must be specified as an argument ("
+ Opts.MEMORY_LONG
+ ") or config value");
}
if (this.nodeNetworks == null) {
throw new ParameterProblem(
"Node network associations must be specified as an argument ("
+ Opts.NETWORKS_LONG
+ ") or config value");
}
if (this.nodePool == null) {
throw new ParameterProblem(
"Node pool name must be specified as an argument ("
+ Opts.POOL_LONG
+ ") or config value");
}
final List<VmmNode> nodes = new ArrayList<VmmNode>(this.hosts.size());
for (String hostname : this.hosts) {
nodes.add(
new VmmNode(
hostname, this.nodeActive, this.nodePool, this.nodeMemory, this.nodeNetworks, true));
}
final String nodesJson = gson.toJson(nodes);
NodeReport[] reports = null;
try {
final String reportJson = this.remoteNodeManagement.addNodes(nodesJson);
reports = gson.fromJson(reportJson, NodeReport[].class);
} catch (RemoteException e) {
handleRemoteException(e);
}
try {
reporter.report(nodeReportsToMaps(reports), this.outStream);
} catch (IOException e) {
throw new ExecutionProblem("Problem writing output: " + e.getMessage(), e);
}
}
private void run_updateNodes() throws ExecutionProblem {
final String[] hostnames = this.hosts.toArray(new String[this.hosts.size()]);
final Boolean active = this.nodeActiveConfigured ? this.nodeActive : null;
final String resourcepool = this.nodePool;
final Integer memory = this.nodeMemoryConfigured ? this.nodeMemory : null;
final String networks = this.nodeNetworks;
NodeReport[] reports = null;
try {
final String reportJson =
this.remoteNodeManagement.updateNodes(hostnames, active, resourcepool, memory, networks);
reports = gson.fromJson(reportJson, NodeReport[].class);
} catch (RemoteException e) {
super.handleRemoteException(e);
}
try {
reporter.report(nodeReportsToMaps(reports), this.outStream);
} catch (IOException e) {
throw new ExecutionProblem("Problem writing output: " + e.getMessage(), e);
}
}
public static void main(String[] args) {
try {
// IMPORTANT HINT REGARDING STRING ENCODING
// in Java all Strings have UTF-8 as default encoding
// therefore: there are only a few references to UTF-8 encoding in this demo code
// however, if values are retrieved from a database or another program language is used, then
// one needs to
// make sure that the UTF-8 encoding is correctly implemented
// create CMD line option object
Options options = new Options();
// add CMD line options
options.addOption(
"o",
"output-dir",
true,
"specify base output directory, if none is specified a new directory will be created in the current path");
options.addOption(
"n",
"number-of-generated-receipts",
true,
"specify number of receipts to be randomly generated, 50 is default");
options.addOption(
"g",
"signature-creation-device-cannot-fail",
false,
"deactivate glitches in signature-creation-device");
options.addOption(
"s",
"no-signature-certificate-switch",
false,
"deactivate switching of signature certificates after 5 receipts");
options.addOption(
"t", "no-training-receipts", false, "deactivate random generation of training-receipts");
/// parse CMD line options
CommandLineParser parser = new DefaultParser();
CommandLine cmd = parser.parse(options, args);
boolean signatureCreationDeviceAlwaysWorks = cmd.hasOption("g");
boolean deactivateSignatureCertificateSwitching = cmd.hasOption("s");
boolean deactivateTrainingReceipts = cmd.hasOption("t");
String outputParentDirectoryString = cmd.getOptionValue("o");
if (outputParentDirectoryString == null) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
outputParentDirectoryString = "./CashBoxDemoOutput" + df.format(new Date());
}
File OUTPUT_PARENT_DIRECTORY = new File(outputParentDirectoryString);
OUTPUT_PARENT_DIRECTORY.mkdirs();
System.out.println("Setting workdir to " + OUTPUT_PARENT_DIRECTORY.getAbsolutePath());
String numberOfReceiptsString = cmd.getOptionValue("n");
int NUMBER_OF_RECEIPTS = DEFAULT_NUMBER_OF_GENERATED_RECEIPTS;
if (numberOfReceiptsString != null) {
NUMBER_OF_RECEIPTS = new Integer(numberOfReceiptsString);
}
// TODO add provider independent functionality
// initialise cryptographic providers
Security.addProvider(new BouncyCastleProvider());
// prepare cashbox init parameters
CashBoxParameters cashBoxParameters = new CashBoxParameters();
// set parameter for signature certificate switching
// if > 0 then switch signature certificate after so many signatures
// this is important for demonstrating the handling of the DEP export format
// when receipts where signed with multiple signature certificates
if (deactivateSignatureCertificateSwitching) {
cashBoxParameters.setChangeSignatureCertificateAfterSoManyReceipts(-1);
} else {
cashBoxParameters.setChangeSignatureCertificateAfterSoManyReceipts(10);
}
// generate and set random cash box ID ("Kassen-ID")
// REF TO SPECIFICATION: Detailspezifikation/Abs 4
String CASH_BOX_ID = "DEMO-CASH-BOX" + Math.round(Math.random() * 1000);
cashBoxParameters.setCashBoxID(CASH_BOX_ID);
// set cashbox suite
// REF TO SPECIFICATION: Detailspezifikation/Abs 2
// AT0 is used here for demonstration purposes, see Abs 2 for details on AT0
cashBoxParameters.setRkSuite(RKSuite.R1_AT0);
// set initial receipt identifier
// in this demo cashbox integer values are used as receipt identifiers ("Belegnummer"),
// however the specification does not
// impose that limit. An arbitrary UTF-8 String could be used, the only requirement is that
// the same combination of
// the cashBox ID ("Kassen-ID") and the receipt identifier ("Belegnummer") is NEVER used for
// more than one receipt
// using the same multiple times compromises the security of the encrypted turnover value,
// which might lead
// to leaked turnover data.
// REF TO SPECIFICATION: Detailspezifikation/Abs 4, Abs 8, Abs 9, Abs 10
long initialReceiptIdentifier = Math.round(Math.random() * 1000000);
cashBoxParameters.setInitialReceiptIdentifier(initialReceiptIdentifier);
// set DEP module for storing and exporting receipts
// REF TO SPECIFICATION: Detailspezifikation/Abs 3, 11
cashBoxParameters.setDepModul(new SimpleMemoryDEPModule());
// create random AES key for turnover encryption
// REF TO SPECIFICATION: Detailspezifikation/Abs 4, Abs 8, Abs 9, Abs 10
cashBoxParameters.setTurnoverKeyAESkey(CashBoxUtils.createAESKey());
// set up signature module
// the signature module is composed of an JWS module that create the JSON Web Signature (JWS)
// and
// a low level signature module for signing the hash values.
// REF TO SPECIFICATION: Detailspezifikation/Abs 2, Abs 4, Abs 5, Abs 6
// JWSModule jwsModule = new OrgBitbucketBcJwsModule(); //requires bouncycastle provider
JWSModule jwsModule = new ManualJWSModule(); // allows for provider independent use cases
// set damage flag, which simulates the failure of the signature creation device and the
// correct handling
// of this case, obviously this is only suitable for demonstration purposes
jwsModule.setDamageIsPossible(!signatureCreationDeviceAlwaysWorks);
jwsModule.setProbabilityOfDamagedSignatureDevice(PROPABILITY_DAMAGED_SIGNATURE_DEVICE);
jwsModule.setSignatureModule(new DO_NOT_USE_IN_REAL_CASHBOX_DemoSoftwareSignatureModule());
// jwsModule.setSignatureModule(new PKCS11SignatureModule());
cashBoxParameters.setJwsModule(jwsModule);
// set printer module
// REF TO SPECIFICATION: Detailspezifikation/Abs 12, Abs 13, Abs 14, Abs 15
PrinterModule printerModule = new SimplePDFPrinterModule();
cashBoxParameters.setPrinterModule(printerModule);
// init the cash box with the parameters
DemoCashBox demoCashBox = new DemoCashBox(cashBoxParameters);
// init done, start interaction with cashbox
// create random receipt data that will be handled by the cashbox
List<RawReceiptData> receipts =
RandomReceiptGenerator.generateRandomReceipts(NUMBER_OF_RECEIPTS);
// store first receipt (Startbeleg) in cashbox
// all taxtype values are set to zero (per default in this demo)
RawReceiptData firstReceipt = new RawReceiptData();
demoCashBox.storeReceipt(firstReceipt, false, false);
// now store the other receipts
for (RawReceiptData rawReceiptData : receipts) {
// store receipt within cashbox: (prepare data-to-be-signed, sign with JWS, store signed
// receipt in DEP)
// pre-defined chance for a training receipt (just for demo purposes)
boolean isTrainingReceipt = false;
if (Math.random() < PROPABILITY_TRAINING_RECEIPT && !deactivateTrainingReceipts) {
isTrainingReceipt = true;
}
// pre-defined chance for a storno receipt
boolean isStornoReceipt = false;
if (Math.random() < PROPABILITY_OF_STORNO_RECEIPT) {
isStornoReceipt = true;
}
demoCashBox.storeReceipt(rawReceiptData, isTrainingReceipt, isStornoReceipt);
}
// dump machine readable code of receipts (this "code" is used for the QR-codes)
// REF TO SPECIFICATION: Detailspezifikation/Abs 12
// dump to File
File qrCoreRepExportFile = new File(OUTPUT_PARENT_DIRECTORY, "qr-code-rep.txt");
List<ReceiptPackage> receiptPackages = demoCashBox.getStoredReceipts();
PrintWriter writer = new PrintWriter(new FileWriter(qrCoreRepExportFile));
System.out.println("------------QR-CODE-REP------------");
for (ReceiptPackage receiptPackage : receiptPackages) {
System.out.println(receiptPackage.getQRCodeRepresentation());
writer.println(receiptPackage.getQRCodeRepresentation());
}
System.out.println("");
writer.close();
// dump OCR code of receipts
// REF TO SPECIFICATION: Detailspezifikation/Abs 14
// dump to File
File ocrCoreRepExportFile = new File(OUTPUT_PARENT_DIRECTORY, "ocr-code-rep.txt");
writer = new PrintWriter(new FileWriter(ocrCoreRepExportFile));
System.out.println("------------OCR-CODE-REP------------");
for (ReceiptPackage receiptPackage : receiptPackages) {
System.out.println(receiptPackage.getOcrCodeRepresentation());
writer.println(receiptPackage.getOcrCodeRepresentation());
}
System.out.println("");
writer.close();
// export DEP from cashbox
// REF TO SPECIFICATION: Detailspezifikation/Abs 3
DEPExportFormat depExportFormat = demoCashBox.exportDEP();
// get JSON rep and dump export format to file/std output
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String exportFormatJSONString = gson.toJson(depExportFormat);
System.out.println("------------DEP-EXPORT-FORMAT------------");
System.out.println(exportFormatJSONString);
System.out.println("");
// dump DEP export to file
File depExportFile = new File(OUTPUT_PARENT_DIRECTORY, "dep-export.txt");
FileOutputStream outputStream = new FileOutputStream(depExportFile);
outputStream.write(exportFormatJSONString.getBytes());
outputStream.close();
// export receipts as PDF (QR-CODE)
// REF TO SPECIFICATION: Detailspezifikation/Abs 12, Abs 13
File qrCodeDumpDirectory = new File(OUTPUT_PARENT_DIRECTORY, "qr-code-dir-pdf");
qrCodeDumpDirectory.mkdirs();
List<byte[]> printedQRCodeReceipts =
demoCashBox.printReceipt(receiptPackages, ReceiptPrintType.QR_CODE);
CashBoxUtils.writeReceiptsToFiles(printedQRCodeReceipts, "QR-", qrCodeDumpDirectory);
// export receipts as PDF (OCR)
// REF TO SPECIFICATION: Detailspezifikation/Abs 14, Abs 15
File ocrCodeDumpDirectory = new File(OUTPUT_PARENT_DIRECTORY, "ocr-code-dir-pdf");
ocrCodeDumpDirectory.mkdirs();
List<byte[]> printedOCRCodeReceipts =
demoCashBox.printReceipt(receiptPackages, ReceiptPrintType.OCR);
CashBoxUtils.writeReceiptsToFiles(printedOCRCodeReceipts, "OCR-", ocrCodeDumpDirectory);
// store signature certificates (so that they can be used for verification purposes)
// only for demonstration purposes
List<String> signatureCertificates = new ArrayList<>();
List<List<String>> certificateChains = new ArrayList<>();
DEPBelegDump[] belegDumps = depExportFormat.getBelegPackage();
for (DEPBelegDump depBelegDump : belegDumps) {
signatureCertificates.add(depBelegDump.getSignatureCertificate());
certificateChains.add(Arrays.asList(depBelegDump.getCertificateChain()));
}
File signatureCertificatesOutputFile =
new File(OUTPUT_PARENT_DIRECTORY, "signatureCertificates.txt");
String signatureCertificatesJSON = gson.toJson(signatureCertificates);
BufferedOutputStream bufferedOutputStream =
new BufferedOutputStream(new FileOutputStream(signatureCertificatesOutputFile));
ByteArrayInputStream bIn = new ByteArrayInputStream(signatureCertificatesJSON.getBytes());
IOUtils.copy(bIn, bufferedOutputStream);
bufferedOutputStream.close();
// store certificate chains (so that they can be used for verification purposes)
// only for demonstration purposes
File signatureCertificateChainsOutputFile =
new File(OUTPUT_PARENT_DIRECTORY, "signatureCertificateChains.txt");
String signatureCertificateChainsJSON = gson.toJson(certificateChains);
bufferedOutputStream =
new BufferedOutputStream(new FileOutputStream(signatureCertificateChainsOutputFile));
bIn = new ByteArrayInputStream(signatureCertificateChainsJSON.getBytes());
IOUtils.copy(bIn, bufferedOutputStream);
bufferedOutputStream.close();
// store AES key as BASE64 String (for demonstration purposes: to allow decryption of turnover
// value)
// ATTENTION, this is only for demonstration purposes, the AES key must be stored in a secure
// area
byte[] aesKey = cashBoxParameters.getTurnoverKeyAESkey().getEncoded();
String aesKeyBase64 = CashBoxUtils.base64Encode(aesKey, false);
writer = new PrintWriter(new File(OUTPUT_PARENT_DIRECTORY, "aesKeyBase64.txt"));
writer.print(aesKeyBase64);
writer.close();
} catch (IOException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
}
}
/**
* constructs the folders required by the application. These are, typically: <br>
*
* <ul>
* <li>/input and subfolders
* <ul>
* <li>/input/<i>sourceLang</i>, /input/<i>targetLang</i> (for storing the results of
* processing the input files with various tools, such as pos tagger, transliterator,
* morphological analyser),<br>
* <li>/input/systems/<i>systemName</i> (for storing system specific resources - for
* example, the compiled and processed word lattices in the case of the IBM system
* </ul>
* <li>/output (for storing the resulting feature files),
* </ul>
*/
public void constructFolders() {
File f = new File(input);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + sourceLang);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + targetLang);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + targetLang + File.separator + "temp");
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
/*
f = new File(input + File.separator + "systems");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + "systems" + File.separator
+ "IBM");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + "systems" + File.separator
+ "MOSES");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
*/
String output = resourceManager.getString("output");
f = new File(output);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
if (featureManager.hasFeature("1700")) {
String lang_resources = workDir + File.separator + "lang_resources";
f = new File(lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, lang_resources are needed.");
System.exit(0);
}
String source_lang_resources = lang_resources + File.separator + sourceLang;
f = new File(source_lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, source lang_resources are needed.");
System.exit(0);
}
String source_lucene_path =
lang_resources + File.separator + sourceLang + File.separator + "luceneIndex";
f = new File(source_lucene_path);
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
String source_lucene_corpus = source_lucene_path + File.separator + sourceLang + ".corpus";
try {
Runtime.getRuntime()
.exec(
"ln -s "
+ workDir
+ File.separator
+ resourceManager.getString(sourceLang + ".corpus")
+ " "
+ source_lucene_corpus);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// Indexing the target
String target_lang_resources = lang_resources + File.separator + targetLang;
f = new File(target_lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, target lang_resources are needed.");
System.exit(0);
}
String target_lucene_path =
lang_resources + File.separator + targetLang + File.separator + "luceneIndex";
f = new File(target_lucene_path);
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
String target_lucene_corpus = target_lucene_path + File.separator + targetLang + ".corpus";
try {
Runtime.getRuntime()
.exec(
"ln -s "
+ workDir
+ File.separator
+ resourceManager.getString(targetLang + ".corpus")
+ " "
+ target_lucene_corpus);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
