static Configuration parseConfiguration(String[] args) {
Configuration configuration = new Configuration();
JCommander jCommander;
List<String> crateArgs = new ArrayList<>();
List<String> safeArgs = new ArrayList<>(args.length);
for (String arg : args) {
if (HELP_OPTIONS.contains(arg)) {
jCommander = new JCommander(configuration);
jCommander.usage();
System.exit(1);
}
if (arg.startsWith("-Des.")) {
String argKey = arg.split("\\=")[0];
if (PROTECTED_CRATE_ARGS.contains(argKey)) {
throw new IllegalArgumentException(
String.format(
"Argument \"%s\" is protected and managed by the framework. "
+ "It cannot be set by the user",
argKey));
} else {
crateArgs.add(arg);
}
} else {
safeArgs.add(arg);
}
}
jCommander = new JCommander(configuration, safeArgs.toArray(new String[safeArgs.size()]));
configuration.crateArgs(crateArgs);
LOGGER.debug("args: {}", configuration);
return configuration;
}
public static void main(String[] args) throws Throwable {
StreamFile2RDF streamfile2rdf = new StreamFile2RDF();
JCommander com = new JCommander(streamfile2rdf, args);
com.setProgramName("stream2file");
if (streamfile2rdf.parameters.size() == 1) {
System.err.println("No output file specified, writing to standard output.");
streamfile2rdf.rdfOutput = "stdout";
streamfile2rdf.InputStream = streamfile2rdf.parameters.get(0);
} else if (streamfile2rdf.parameters.size() == 2) {
streamfile2rdf.InputStream = streamfile2rdf.parameters.get(0);
streamfile2rdf.rdfOutput = streamfile2rdf.parameters.get(1);
} else {
com.usage();
System.exit(1);
}
System.out.println(
"Converting '"
+ streamfile2rdf.InputStream
+ "' to Stream File'"
+ streamfile2rdf.rdfOutput
+ "'");
streamfile2rdf.execute();
System.out.println("Bye!");
System.exit(0);
}
public AnnotationsToHTML(String[] args) {
jc = new JCommander(this);
jc.addCommand(annotationEndpoints);
try {
jc.parse(args);
} catch (Exception e) {
System.out.println(e.getMessage());
jc.usage();
System.exit(1);
}
if (help || jc.getParsedCommand() == null) {
jc.usage();
System.exit(0);
}
}
public static Parameters parseArgs(String[] args) {
Parameters parameters = new Parameters();
JCommander jc = new JCommander(parameters, args);
jc.setProgramName(name);
if (parameters.help) {
jc.usage();
}
return parameters;
}
/**
* Parses command line arguments and populates this command line instance.
*
* <p>If the command line arguments include the "help" argument, or if the arguments have
* incorrect values or order, then usage information is printed to {@link System#out} and the
* program terminates.
*
* @param args the command line arguments
* @return an instance of the parsed arguments object
*/
public Arguments parse(String[] args) {
JCommander jCommander = new JCommander(this);
jCommander.setProgramName("jsonschema2pojo");
try {
jCommander.parse(args);
if (this.showHelp) {
jCommander.usage();
exit(EXIT_OKAY);
}
} catch (ParameterException e) {
System.err.println(e.getMessage());
jCommander.usage();
exit(EXIT_ERROR);
}
return this;
}
public static void main(String[] args) {
/* Parse and validate command line parameters */
CommandLineParameters params = new CommandLineParameters();
try {
JCommander jc = new JCommander(params, args);
if (params.help) {
jc.setProgramName("java -Xmx[several]G -jar otp.jar");
jc.usage();
System.exit(0);
}
params.infer();
} catch (ParameterException pex) {
LOG.error("Parameter error: {}", pex.getMessage());
System.exit(1);
}
OTPConfigurator configurator = new OTPConfigurator(params);
// start graph builder, if asked for
GraphBuilderTask graphBuilder = configurator.builderFromParameters();
if (graphBuilder != null) {
graphBuilder.run();
// Inform configurator which graph is to be used for in-memory handoff.
if (params.inMemory) configurator.makeGraphService(graphBuilder.getGraph());
}
// start visualizer, if asked for
GraphVisualizer graphVisualizer = configurator.visualizerFromParameters();
if (graphVisualizer != null) {
graphVisualizer.run();
}
// start web server, if asked for
GrizzlyServer grizzlyServer = configurator.serverFromParameters();
if (grizzlyServer != null) {
while (true) { // Loop to restart server on uncaught fatal exceptions.
try {
grizzlyServer.run();
return;
} catch (Throwable throwable) {
throwable.printStackTrace();
LOG.error(
"An uncaught {} occurred inside OTP. Restarting server.",
throwable.getClass().getSimpleName());
}
}
}
if (graphBuilder == null && graphVisualizer == null && grizzlyServer == null) {
LOG.info("Nothing to do. Use --help to see available tasks.");
;
}
}
TrackAnalyzer(String[] args) throws Exception {
JCommander jcommander = new JCommander(c, args);
jcommander.setProgramName("TrackAnalyzer");
if ((c.filenames.size() == 0 && Utils.isEmpty(c.filelist)) || c.help) {
jcommander.usage();
System.exit(-1);
}
if (c.debug) {
Logger.getLogger(TrackAnalyzer.class.getName()).setLevel(Level.ALL);
} else {
Logger.getLogger(TrackAnalyzer.class.getName()).setLevel(Level.WARNING);
}
// we have a list, read all the filenames in the list and
// collect them in 'filenames'
if (!Utils.isEmpty(c.filelist)) {
try {
// use buffering, reading one line at a time
// FileReader always assumes default encoding is OK!
BufferedReader input = new BufferedReader(new FileReader(new File(c.filelist)));
try {
String line = null; // not declared within while loop
/*
* readLine is a bit quirky :
* it returns the content of a line MINUS the newline.
* it returns null only for the END of the stream.
* it returns an empty String if two newlines appear in a row.
*/
while ((line = input.readLine()) != null) {
filenames.add(line);
}
} finally {
input.close();
}
} catch (IOException ex) {
ex.printStackTrace();
System.exit(-1);
}
}
// add filenames from command line
filenames.addAll(c.filenames);
if (!Utils.isEmpty(c.writeList)) {
try {
writeListWriter = new BufferedWriter(new FileWriter(c.writeList));
} catch (IOException ex) {
Logger.getLogger(TrackAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
}
}
k = new KeyFinder();
p = new Parameters();
p.setHopSize(8192);
}
public static void main(String... args) throws Exception {
Main main = new Main();
JCommander commander = new JCommander(main);
commander.setProgramName(Main.class.getName());
try {
commander.parse(args);
} catch (ParameterException pe) {
commander.usage();
System.exit(-1);
}
if (main.help) {
commander.usage();
System.exit(0);
}
main.run();
}
public static void main(String... args) {
CommandLineArgs parsedArgs = new CommandLineArgs();
JCommander jc = new JCommander(parsedArgs);
try {
jc.parse(args);
} catch (ParameterException e) {
StringBuilder out = new StringBuilder(e.getLocalizedMessage()).append("\n\n");
jc.usage(out);
System.err.println(out.toString());
System.exit(1);
return;
}
if (parsedArgs.help) {
jc.usage();
return;
}
SpoonRunner spoonRunner =
new SpoonRunner.Builder() //
.setTitle(parsedArgs.title)
.setApplicationApk(parsedArgs.apk)
.setInstrumentationApk(parsedArgs.testApk)
.setOutputDirectory(parsedArgs.output)
.setDebug(parsedArgs.debug)
.setAndroidSdk(parsedArgs.sdk)
.setNoAnimations(parsedArgs.noAnimations)
.setTestSize(parsedArgs.size)
.setAdbTimeout(parsedArgs.adbTimeoutSeconds * 1000)
.setClassName(parsedArgs.className)
.setMethodName(parsedArgs.methodName)
.useAllAttachedDevices()
.build();
if (!spoonRunner.run() && parsedArgs.failOnFailure) {
System.exit(1);
}
}
private void showHelpForOneCommand(List<ICommand> commands, PrintWriter pw) {
String commandName = commandNames.get(0);
for (ICommand command : commands) {
if (commandName.equals(command.getName())) {
JCommander commander = new JCommander();
commander.setProgramName(command.getName());
commander.addObject(command);
StringBuilder sb = new StringBuilder();
sb.append("\n");
commander.usage(sb);
pw.println(sb.toString());
}
}
}
public static void main(String[] args) throws Exception {
System.setProperty("javax.net.debug", "none");
App app = new App();
AppParameters params = new AppParameters();
JCommander jcmd = new JCommander(params);
try {
jcmd.parse(args);
app.setParams(params);
app.scepCLI();
} catch (ParameterException e) {
jcmd.usage();
}
}
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws RuntimeException, Exception {
is = new ImageSimilarity();
jc = new JCommander(is);
jc.setProgramName("ImageSimilarity");
try {
jc.parse(args);
} catch (ParameterException pe) {
System.out.println("Wrong console parameters. See usage of ImageSimilarity below:");
jc.usage();
}
is.setup();
}
public static void main(String[] args) {
PublishNanopub obj = new PublishNanopub();
JCommander jc = new JCommander(obj);
try {
jc.parse(args);
} catch (ParameterException ex) {
jc.usage();
System.exit(1);
}
try {
obj.run();
} catch (Exception ex) {
ex.printStackTrace();
System.exit(1);
}
}
public static void main(String[] args) throws Exception {
Options opts = new Options();
JCommander jcmd = new JCommander(opts);
jcmd.setProgramName("itunesclone");
jcmd.parse(args);
if (opts.help) {
jcmd.usage();
return;
}
RunType runType = opts.dryRun ? RunType.DRY_RUN : RunType.THE_REAL_DEAL;
CloneProgressReporter progressReporter = null;
CountingProgressReporter counter = null;
switch (opts.logLevel) {
case 0:
progressReporter = new SilentProgressReporter();
break;
case 1:
counter = new CountingProgressReporter();
progressReporter = counter;
break;
case 2:
TextProgressReporter tpr = new TextProgressReporter(System.out, false);
counter = new CountingProgressReporter();
progressReporter = new ChainedProgressReporter(tpr, counter);
break;
}
assert (progressReporter != null);
FFMPEGMetadataCache.getInstance().setCacheDirectory(opts.cache.toPath());
(new DirCloner(runType, opts.codec))
.clone(opts.src.toPath(), opts.dest.toPath(), progressReporter);
if (opts.logLevel != 0) {
counter.printStatistics(System.out);
FFMPEGMetadataCache.getInstance().printStatistics(System.out);
}
}
private static void initParameters(String[] args) {
Parameters parameters = new Parameters();
JCommander jCommander = new JCommander(parameters);
try {
jCommander.parse(args);
precision = parameters.getPrecision() + 1;
tasks = parameters.getTasks();
quiet = parameters.getQuiet();
fileName = parameters.getFileName();
} catch (ParameterException ex) {
System.out.println(MESSAGE_COULD_NOT_GET_PARAMETERS);
jCommander.usage();
}
}
private void go(String[] args) throws IOException, JSONException, TemplateException {
Arguments arguments = new Arguments();
JCommander jCommander = new JCommander(arguments, args);
jCommander.setProgramName("GenerateAndroidProvider");
if (arguments.help) {
jCommander.usage();
return;
}
getConfig(arguments.inputDir);
loadModel(arguments.inputDir);
generateColumns(arguments);
generateWrappers(arguments);
generateContentProvider(arguments);
generateSqliteHelper(arguments);
}
public static void main(String[] argv) {
CopyToHdfs obj = new CopyToHdfs();
JCommander cmdr = new JCommander(obj, argv);
// Dump the help screen if we have to
if (obj.help || argv.length == 0) {
cmdr.usage();
return;
}
try {
obj.copy_jsonlog();
} catch (RuntimeException re_ex) {
Writer result = new StringWriter();
PrintWriter printWriter = new PrintWriter(result);
re_ex.printStackTrace(printWriter);
new Syslog().error(result.toString());
}
}
/** @param args */
public static void main(String... args) {
Thread.setDefaultUncaughtExceptionHandler(new LoggedUncaughtExceptionHandler());
GigawordIngesterRunner run = new GigawordIngesterRunner();
JCommander jc = new JCommander(run, args);
jc.setProgramName(GigawordIngesterRunner.class.getSimpleName());
if (run.delegate.help) {
jc.usage();
}
try {
Path outpath = Paths.get(run.delegate.outputPath);
IngesterParameterDelegate.prepare(outpath);
GigawordDocumentConverter conv = new GigawordDocumentConverter();
for (String pstr : run.delegate.paths) {
LOGGER.debug("Running on file: {}", pstr);
Path p = Paths.get(pstr);
new ExistingNonDirectoryFile(p);
Path outWithExt = outpath.resolve(p.getFileName().toString().split("\\.")[0] + ".tar.gz");
if (Files.exists(outWithExt)) {
if (!run.delegate.overwrite) {
LOGGER.info(
"File: {} exists and overwrite disabled. Not running.", outWithExt.toString());
continue;
} else {
Files.delete(outWithExt);
}
}
try (OutputStream os = Files.newOutputStream(outWithExt);
GzipCompressorOutputStream gout = new GzipCompressorOutputStream(os);
TarArchiver arch = new TarArchiver(gout)) {
Iterator<Communication> iter = conv.gzToStringIterator(p);
while (iter.hasNext()) {
arch.addEntry(new ArchivableCommunication(iter.next()));
}
}
}
} catch (NotFileException | IOException e) {
LOGGER.error("Caught exception processing.", e);
}
}
public static void main(String[] args) {
final Options options = new Options();
JCommander jCommander = new JCommander(options);
jCommander.setProgramName("java -jar ColorBleeding.jar");
try {
String inPath = "/tmp/images";
String[] argv = {
/* "-overwrite", */
"-dir",
inPath,
"-debug",
"-maxiterations",
"2",
"-outdir",
"/tmp/result",
"-numthreads",
"8"
};
if (System.getProperty("overrideArgs", "false").equals("true")) {
args = argv;
}
jCommander.parse(args);
execute(options);
} catch (ParameterException e) {
logger.error(e.getMessage());
if (jCommander != null) {
jCommander.usage();
}
} catch (IOException e) {
logger.error("Error while processing images", e);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* Setups all necessary things for this application
*
* @author Grzegorz Polek <*****@*****.**>
* @author Lukasz Pycia <*****@*****.**>
*/
private void setup() {
// Show help
if (help) {
jc.usage();
}
// Disable MediaLib. lol.
System.setProperty("com.sun.media.jai.disableMediaLib", "true");
// GUI enabled
if (gui) {
try {
UserInterface ui = UserInterface.getInstance();
ui.setVisible(true);
} catch (Exception e) {
e.printStackTrace();
}
}
if (threads == 0) {
threads = 1;
}
// Set up the logger
try {
ISLogger.setup();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("Problems with creating the log files!");
}
// Run if not gui and all arguments are set up
if (img != null && dir != null) {
if (!gui) {
check();
run();
}
}
}
public static void main(String[] args) {
JCommanderTwitterStream jcts = new JCommanderTwitterStream();
new JCommander(jcts, args);
if (jcts.isHelp()) {
JCommander jc = new JCommander(jcts, args);
jc.usage();
return;
}
String stringKeywords = jcts.getKeywords().stream().collect(joining(" "));
if (jcts.isStream()) {
try {
if (jcts.getLimit() < Integer.MAX_VALUE) {
throw new ParameterException("You can't have stream with limit.");
}
streamTweets(stringKeywords, jcts.getLocation(), jcts.isHideRetweets());
} catch (URLs.ConnectionException
| URLs.HTTPQueryException
| MalformedURLException
| ParameterException
| GeolocationSearch.NoKeyException
| GeolocationSearch.SearchLocationException e) {
System.err.println(e.getMessage());
System.exit(1);
}
} else {
try {
printTweets(stringKeywords, jcts.getLocation(), jcts.getLimit(), jcts.isHideRetweets());
} catch (TwitterException
| URLs.ConnectionException
| GeolocationSearch.SearchLocationException
| MalformedURLException
| GeolocationSearch.NoKeyException
| URLs.HTTPQueryException e) {
System.err.println(e.getMessage());
System.exit(1);
}
}
}
public static void main(String[] args) throws Exception {
EncoderConfig config = new EncoderConfig();
JCommander jCommander = new JCommander(config, args);
jCommander.setProgramName(CommandLineEncoder.class.getSimpleName());
if (config.help) {
jCommander.usage();
return;
}
String outFileString = config.outputFileBase;
if (EncoderConfig.DEFAULT_OUTPUT_FILE_BASE.equals(outFileString)) {
outFileString += '.' + config.imageFormat.toLowerCase(Locale.ENGLISH);
}
Map<EncodeHintType, Object> hints = new EnumMap<>(EncodeHintType.class);
if (config.errorCorrectionLevel != null) {
hints.put(EncodeHintType.ERROR_CORRECTION, config.errorCorrectionLevel);
}
BitMatrix matrix =
new MultiFormatWriter()
.encode(
config.contents.get(0), config.barcodeFormat, config.width, config.height, hints);
MatrixToImageWriter.writeToPath(matrix, config.imageFormat, Paths.get(outFileString));
}
/**
* @param args
* @throws IOException
* @throws MarshallingException
*/
public static void main(final String[] args) throws Exception {
final Arguments arguments = new Arguments();
final JCommander commander = new JCommander(arguments);
try {
commander.parse(args);
} catch (final ParameterException e) {
System.err.println(e.getMessage());
final StringBuilder builder = new StringBuilder();
commander.usage(builder);
System.err.println(builder);
return;
}
final KeyStore keystore = KeyStore.getInstance("jks");
InputStream keystoreStream = null;
try {
keystoreStream = new FileInputStream(arguments.getKeystore());
keystore.load(keystoreStream, arguments.getKeystorePassword().toCharArray());
} finally {
if (keystoreStream != null) keystoreStream.close();
}
final Map<String, String> keyPasswords = new HashMap<>();
if (arguments.getEncryptionKey() != null)
keyPasswords.put(arguments.getEncryptionKey(), arguments.getEncryptionKeyPassword());
if (arguments.getSigningKey() != null)
keyPasswords.put(arguments.getSigningKey(), arguments.getSigningKeyPassword());
if (arguments.getTlsKey() != null)
keyPasswords.put(arguments.getTlsKey(), arguments.getTlsKeyPassword());
bootstrap();
final JKSKeyManager keyManager = new JKSKeyManager(keystore, keyPasswords, null);
final MetadataGenerator generator = new MetadataGenerator();
generator.setKeyManager(keyManager);
generator.setEntityId(arguments.getEntityId());
generator.setEntityAlias(arguments.getAlias());
generator.setEntityBaseURL(arguments.getBaseURL());
generator.setSignMetadata(arguments.isSignMetadata());
generator.setRequestSigned(arguments.isRequestSigned());
generator.setWantAssertionSigned(arguments.isWantAssertionSigned());
generator.setSigningKey(arguments.getSigningKey());
generator.setEncryptionKey(arguments.getEncryptionKey());
if (arguments.getTlsKey() != null && arguments.getTlsKey().length() > 0) {
generator.setTlsKey(arguments.getTlsKey());
}
final Collection<String> bindingsSSO = new LinkedList<String>();
final Collection<String> bindingsHoKSSO = new LinkedList<String>();
final AllowedSSOBindings defaultBinding = arguments.getSsoDefaultBinding();
int assertionConsumerIndex = 0;
for (final AllowedSSOBindings binding : arguments.getSsoBindings()) {
// Set default binding
if (binding.equals(defaultBinding)) {
assertionConsumerIndex = bindingsSSO.size() + bindingsHoKSSO.size();
}
// Set included bindings
if (AllowedSSOBindings.SSO_POST.equals(binding)) {
bindingsSSO.add(SAMLConstants.SAML2_POST_BINDING_URI);
} else if (AllowedSSOBindings.SSO_ARTIFACT.equals(binding)) {
bindingsSSO.add(SAMLConstants.SAML2_ARTIFACT_BINDING_URI);
} else if (AllowedSSOBindings.SSO_PAOS.equals(binding)) {
bindingsSSO.add(SAMLConstants.SAML2_PAOS_BINDING_URI);
} else if (AllowedSSOBindings.HOKSSO_POST.equals(binding)) {
bindingsHoKSSO.add(SAMLConstants.SAML2_POST_BINDING_URI);
} else if (AllowedSSOBindings.HOKSSO_ARTIFACT.equals(binding)) {
bindingsHoKSSO.add(SAMLConstants.SAML2_ARTIFACT_BINDING_URI);
}
}
// Set bindings
generator.setBindingsSSO(bindingsSSO);
generator.setBindingsHoKSSO(bindingsHoKSSO);
generator.setAssertionConsumerIndex(assertionConsumerIndex);
// Discovery
if (arguments.isIncludeDiscovery()) {
generator.setIncludeDiscovery(true);
if (arguments.getCustomDiscoveryURL() != null
&& arguments.getCustomDiscoveryURL().length() > 0) {
generator.setCustomDiscoveryURL(arguments.getCustomDiscoveryURL());
}
} else {
generator.setIncludeDiscovery(false);
}
generator.setNameID(arguments.getNameID());
final EntityDescriptor descriptor = generator.generateMetadata();
final ExtendedMetadata extendedMetadata = generator.generateExtendedMetadata();
extendedMetadata.setSecurityProfile(arguments.getSecurityProfile());
extendedMetadata.setSslSecurityProfile(arguments.getSslSecurityProfile());
extendedMetadata.setRequireLogoutRequestSigned(arguments.isRequireLogoutRequestSigned());
extendedMetadata.setRequireLogoutResponseSigned(arguments.isRequireLogoutResponseSigned());
extendedMetadata.setRequireArtifactResolveSigned(arguments.isRequireArtifactResolveSigned());
Writer metadataWriter = null;
try {
metadataWriter = new FileWriter(arguments.getMetadataOutput());
metadataWriter.write(getMetadataAsString(descriptor));
} finally {
if (metadataWriter != null) {
metadataWriter.flush();
metadataWriter.close();
}
}
if (arguments.getExtendedMetadataOutput() != null) {
Writer extendedMetadataWriter = null;
try {
extendedMetadataWriter = new FileWriter(arguments.getExtendedMetadataOutput());
extendedMetadataWriter.write(
getConfiguration(arguments.getExtendedMetadataOutput().getName(), extendedMetadata));
} finally {
if (extendedMetadataWriter != null) {
extendedMetadataWriter.flush();
extendedMetadataWriter.close();
}
}
}
}
@Override
public int run(String[] args) throws Exception {
// Validate params etc
JCommander jComm = new JCommander(this);
jComm.setProgramName("Splout Page Counts example");
try {
jComm.parse(args);
} catch (ParameterException e) {
System.err.println(e.getMessage());
jComm.usage();
System.exit(-1);
}
boolean generate = !noGenerate; // just for clarifying
if (generateTupleFiles && deploy) {
System.err.println("Can't run a 'dry' TupleFile generation and deploy it.");
jComm.usage();
System.exit(-1);
}
Path outPath = new Path(outputPath);
FileSystem outFs = outPath.getFileSystem(getConf());
if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) {
File nativeLibs = new File("native");
if (nativeLibs.exists()) {
SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf);
}
}
if (generate) {
Path inputPath = new Path(this.inputPath);
FileSystem inputFileSystem = inputPath.getFileSystem(conf);
FileStatus[] fileStatuses = inputFileSystem.listStatus(inputPath);
// define the schema that the resultant table will have: date, hour, pagename, pageviews
final Schema tableSchema =
new Schema(
"pagecounts",
Fields.parse("date:string, hour:string, pagename:string, pageviews:int"));
// define the schema of the input files: projectcode, pagename, pageviews, bytes
Schema fileSchema =
new Schema(
"pagecountsfile",
Fields.parse("projectcode:string, pagename:string, pageviews:int, bytes:long"));
// instantiate a TableBuilder
TableBuilder tableBuilder = new TableBuilder(tableSchema);
// for every input file...
for (FileStatus fileStatus : fileStatuses) {
String fileName = fileStatus.getPath().getName().toString();
// strip the date and the hour from the file name
String fileDate = fileName.split("-")[1];
String fileHour = fileName.split("-")[2].substring(0, 2);
// instantiate a custom RecordProcessor to process the records of this file
PageCountsRecordProcessor recordProcessor =
new PageCountsRecordProcessor(tableSchema, fileDate, fileHour);
// use the tableBuilder method for adding each of the files to the mix
tableBuilder.addCSVTextFile(
fileStatus.getPath(),
' ',
TupleTextInputFormat.NO_QUOTE_CHARACTER,
TupleTextInputFormat.NO_ESCAPE_CHARACTER,
false,
false,
TupleTextInputFormat.NO_NULL_STRING,
fileSchema,
recordProcessor);
}
// partition the dataset by pagename - which should give a fair even distribution.
tableBuilder.partitionBy("pagename");
// create a compound index on pagename, date so that typical queries for the dataset will be
// fast
tableBuilder.createIndex("pagename", "date");
long nonExactPageSize = memoryForIndexing / 32000; // number of pages
int pageSize = (int) Math.pow(2, (int) Math.round(Math.log(nonExactPageSize) / Math.log(2)));
Log.info(
"Pagesize = "
+ pageSize
+ " as memory for indexing was ["
+ memoryForIndexing
+ "] and there are 32000 pages.");
tableBuilder.initialSQL("pragma page_size=" + pageSize);
// insertion order is very important for optimizing query speed because it makes data be
// co-located in disk
tableBuilder.insertionSortOrder(OrderBy.parse("pagename:asc, date:asc"));
// instantiate a TablespaceBuilder
TablespaceBuilder tablespaceBuilder = new TablespaceBuilder();
// we will partition this dataset in as many partitions as:
tablespaceBuilder.setNPartitions(nPartitions);
tablespaceBuilder.add(tableBuilder.build());
// we turn a specific SQLite pragma on for making autocomplete queries fast
tablespaceBuilder.initStatements("pragma case_sensitive_like=true;");
HadoopUtils.deleteIfExists(outFs, outPath);
// finally, instantiate a TablespaceGenerator and execute it
TablespaceGenerator tablespaceViewBuilder;
if (generateTupleFiles) {
// we subclass TablespaceGenerator to be able to run the generation without outputting the
// SQLite stores, for
// benchmark comparisons.
// In the future this feature may be useful in general for debugging store creation.
tablespaceViewBuilder =
new TablespaceGenerator(tablespaceBuilder.build(), outPath) {
@Override
public void generateView(
Configuration conf, SamplingType samplingType, SamplingOptions samplingOptions)
throws Exception {
prepareOutput(conf);
final int nPartitions = tablespace.getnPartitions();
if (nPartitions > 1) {
partitionMap = sample(nPartitions, conf, samplingType, samplingOptions);
} else {
partitionMap = PartitionMap.oneShardOpenedMap();
}
writeOutputMetadata(conf);
TupleMRBuilder builder = createMRBuilder(nPartitions, conf);
// Set a TupleOutput here instead of SQLiteOutput
builder.setOutput(
new Path(outputPath, OUT_STORE),
new TupleOutputFormat(tableSchema),
ITuple.class,
NullWritable.class);
Job job = builder.createJob();
executeViewGeneration(job);
}
};
} else {
// ... otherwise a standard TablespaceGenerator is used.
tablespaceViewBuilder = new TablespaceGenerator(tablespaceBuilder.build(), outPath);
}
tablespaceViewBuilder.generateView(
getConf(), SamplingType.RESERVOIR, new TupleSampler.DefaultSamplingOptions());
}
if (deploy) {
// use StoreDeployerTool for deploying the already generated dataset
StoreDeployerTool deployer = new StoreDeployerTool(qnode, getConf());
ArrayList<TablespaceDepSpec> deployments = new ArrayList<TablespaceDepSpec>();
deployments.add(new TablespaceDepSpec("pagecounts", outPath.toString(), repFactor, null));
deployer.deploy(deployments);
}
return 1;
}
