public static CompilerConfiguration generateCompilerConfigurationFromOptions(CommandLine cli)
throws IOException {
//
// Setup the configuration data
CompilerConfiguration configuration = new CompilerConfiguration();
if (cli.hasOption("classpath")) {
configuration.setClasspath(cli.getOptionValue("classpath"));
}
if (cli.hasOption('d')) {
configuration.setTargetDirectory(cli.getOptionValue('d'));
}
if (cli.hasOption("encoding")) {
configuration.setSourceEncoding(cli.getOptionValue("encoding"));
}
if (cli.hasOption("basescript")) {
configuration.setScriptBaseClass(cli.getOptionValue("basescript"));
}
// joint compilation parameters
if (cli.hasOption('j')) {
Map<String, Object> compilerOptions = new HashMap<String, Object>();
String[] opts = cli.getOptionValues("J");
compilerOptions.put("namedValues", opts);
opts = cli.getOptionValues("F");
compilerOptions.put("flags", opts);
configuration.setJointCompilationOptions(compilerOptions);
}
if (cli.hasOption("indy")) {
configuration.getOptimizationOptions().put("int", false);
configuration.getOptimizationOptions().put("indy", true);
}
if (cli.hasOption("configscript")) {
String path = cli.getOptionValue("configscript");
File groovyConfigurator = new File(path);
Binding binding = new Binding();
binding.setVariable("configuration", configuration);
CompilerConfiguration configuratorConfig = new CompilerConfiguration();
ImportCustomizer customizer = new ImportCustomizer();
customizer.addStaticStars(
"org.codehaus.groovy.control.customizers.builder.CompilerCustomizationBuilder");
configuratorConfig.addCompilationCustomizers(customizer);
GroovyShell shell = new GroovyShell(binding, configuratorConfig);
shell.evaluate(groovyConfigurator);
}
return configuration;
}
private static Cassandra.Client createThriftClient(
String host, int port, String user, String passwd, ITransportFactory transportFactory)
throws Exception {
TTransport trans = transportFactory.openTransport(host, port);
TProtocol protocol = new TBinaryProtocol(trans);
Cassandra.Client client = new Cassandra.Client(protocol);
if (user != null && passwd != null) {
Map<String, String> credentials = new HashMap<>();
credentials.put(IAuthenticator.USERNAME_KEY, user);
credentials.put(IAuthenticator.PASSWORD_KEY, passwd);
AuthenticationRequest authenticationRequest = new AuthenticationRequest(credentials);
client.login(authenticationRequest);
}
return client;
}
private static Map<String, String> createAuthorizationAttributeMap(
String snaaName, Properties props) {
Map<String, String> attributes = new HashMap<String, String>();
List<String> keys = new LinkedList<String>();
// getting keys from properties
for (Object o : props.keySet()) {
if (((String) o).startsWith(snaaName + authorizationAtt)
&& ((String) o).endsWith(authorizationKeyAtt)) {
keys.add((String) o);
}
}
for (String k : keys) {
String key = props.getProperty(k);
// getting plain key-number from properties
String plainKeyProperty = k.replaceAll(snaaName + authorizationAtt + ".", "");
plainKeyProperty = plainKeyProperty.replaceAll(authorizationKeyAtt, "");
String keyPrefix = snaaName + authorizationAtt + "." + plainKeyProperty;
// building value-property-string
String value = props.getProperty(keyPrefix + authorizationValAtt);
// finally put key and values
attributes.put(key, value);
}
return attributes;
}
private Map<String, List<String>> loadHistoSectors(String sectorFile) {
FileReader input;
Map<String, List<String>> sectors = new HashMap<String, List<String>>();
try {
input = new FileReader(sectorFile);
BufferedReader bufRead = new BufferedReader(input);
String line;
int i = 1;
while ((line = bufRead.readLine()) != null) {
Bin sectorRecord = Utils.readBin(line);
List<String> stockList = sectorRecord.symbols;
String startEnd = formatter.format(sectorRecord.end);
String key = intgerFormaterr.format(i) + ":" + startEnd;
sectors.put(key, stockList);
for (String s : stockList) {
invertedSectors.put(s, key);
}
i++;
}
} catch (IOException e) {
throw new RuntimeException("Failed to load sector file", e);
}
return sectors;
}
private Map<String, Integer> loadFixedClasses(String file) {
FileReader input;
try {
Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
File f = new File(file);
if (!f.exists()) {
System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
return invertedFixedClasses;
}
input = new FileReader(f);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
String parts[] = line.split(",");
int clazz = Integer.parseInt(parts[0]);
List<String> symbols = new ArrayList<String>();
symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
fixedClaszzes.put(clazz, symbols);
}
for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
for (String s : e.getValue()) {
invertedFixedClasses.put(s, e.getKey());
}
}
return invertedFixedClasses;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public void changeClassLabels() {
System.out.println("Reading cluster file: " + cluserInputFile);
Clusters clusters = loadClusters(cluserInputFile);
if (clusters == null) {
System.out.println("Not clusters found to change");
return;
}
Map<Integer, String> classToSector = new HashMap<Integer, String>();
for (Map.Entry<String, Integer> e : sectorToClazz.entrySet()) {
classToSector.put(e.getValue(), e.getKey());
}
for (Cluster c : clusters.getCluster()) {
// find the cluster label
// this is the label
String key = classToSector.get(c.getKey());
if (key != null) {
System.out.println("Setting label: " + key + " to cluster: " + c.getKey());
c.setLabel(key);
}
}
try {
System.out.println("Writing cluster file: " + clusterOutputFile);
saveClusters(clusterOutputFile, clusters);
} catch (FileNotFoundException | JAXBException e) {
throw new RuntimeException("Failed to write clusters", e);
}
}
private void init() {
permNoToSymbol = Utils.loadMapping(originalStockFile);
Map<String, Integer> symbolToPerm = new HashMap<String, Integer>();
for (Map.Entry<Integer, String> entry : permNoToSymbol.entrySet()) {
symbolToPerm.put(entry.getValue(), entry.getKey());
}
}
private static void configureTransportFactory(
ITransportFactory transportFactory, LoaderOptions opts) {
Map<String, String> options = new HashMap<>();
// If the supplied factory supports the same set of options as our SSL impl, set those
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
options.put(
SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);
// Now check if any of the factory's supported options are set as system properties
for (String optionKey : transportFactory.supportedOptions())
if (System.getProperty(optionKey) != null)
options.put(optionKey, System.getProperty(optionKey));
transportFactory.setOptions(options);
}
private Map<String, Integer> convertSectorsToClazz(Map<String, List<String>> sectors) {
List<String> sectorNames = new ArrayList<String>(sectors.keySet());
Collections.sort(sectorNames);
Map<String, Integer> sectorsToClazz = new HashMap<String, Integer>();
for (int i = 0; i < sectorNames.size(); i++) {
sectorsToClazz.put(sectorNames.get(i), i + 1);
System.out.println(sectorNames.get(i) + ": " + (i + 1));
}
return sectorsToClazz;
}
public void handleStreamEvent(StreamEvent event) {
if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
sessionsByHost.put(session.peer, session);
} else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;
// update progress
Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
if (progresses == null) {
progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
progressByHost.put(progressInfo.peer, progresses);
}
if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
progresses.add(progressInfo);
StringBuilder sb = new StringBuilder();
sb.append("\rprogress: ");
long totalProgress = 0;
long totalSize = 0;
for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
SessionInfo session = sessionsByHost.get(entry.getKey());
long size = session.getTotalSizeToSend();
long current = 0;
int completed = 0;
for (ProgressInfo progress : entry.getValue()) {
if (progress.currentBytes == progress.totalBytes) completed++;
current += progress.currentBytes;
}
totalProgress += current;
totalSize += size;
sb.append("[").append(entry.getKey());
sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
}
long time = System.nanoTime();
long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
lastTime = time;
long deltaProgress = totalProgress - lastProgress;
lastProgress = totalProgress;
sb.append("[total: ")
.append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
.append("% - ");
sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
sb.append(" (avg: ")
.append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
.append("MB/s)]");
System.out.print(sb.toString());
}
}
private boolean register(File f) {
boolean rc = false;
if (f.isDirectory()) {
File file[] = f.listFiles(this);
for (int i = 0; i < file.length; i++) {
if (register(file[i])) rc = true;
}
} else if (dirMap.get(f) == null) {
dirMap.put(f, new QEntry());
rc = true;
}
return rc;
}
private Map<Integer, String> readTraining(File f) throws IOException {
Map<Integer, String> map = new TreeMap<Integer, String>();
LineNumberReader lr =
new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
String line = null;
StringBuilder sb = new StringBuilder();
int id = -1;
while ((line = lr.readLine()) != null) {
String[] s = line.split("\t");
if (s.length > 0 && s[0].length() > 0 && !s[0].equals("sid")) {
sb.append(line + "\n");
id = Integer.parseInt(s[0]);
} else {
if (sb.length() > 0) {
map.put(id, sb.toString());
}
sb = new StringBuilder();
}
}
map.put(id, sb.toString());
return map;
}
private Map<String, String[]> readRoles(File f) throws IOException {
Map<String, String[]> map = new TreeMap<String, String[]>();
LineNumberReader lr =
new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
String line = null;
StringBuilder sb = new StringBuilder();
while ((line = lr.readLine()) != null) {
String[] s = line.toLowerCase().split("\t");
if (s.length > 2) {
logger.trace(Arrays.toString(s));
map.put(s[1], s);
}
}
return map;
}
@Override
public void init(String keyspace) {
Iterator<InetAddress> hostiter = hosts.iterator();
while (hostiter.hasNext()) {
try {
// Query endpoint to ranges map and schemas from thrift
InetAddress host = hostiter.next();
Cassandra.Client client =
createThriftClient(
host.getHostAddress(), rpcPort, this.user, this.passwd, this.transportFactory);
setPartitioner(client.describe_partitioner());
Token.TokenFactory tkFactory = getPartitioner().getTokenFactory();
for (TokenRange tr : client.describe_ring(keyspace)) {
Range<Token> range =
new Range<>(
tkFactory.fromString(tr.start_token),
tkFactory.fromString(tr.end_token),
getPartitioner());
for (String ep : tr.endpoints) {
addRangeForEndpoint(range, InetAddress.getByName(ep));
}
}
String query =
String.format(
"SELECT * FROM %s.%s WHERE keyspace_name = '%s'",
Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, keyspace);
CqlResult result =
client.execute_cql3_query(
ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
for (CqlRow row : result.rows) {
CFMetaData metadata = CFMetaData.fromThriftCqlRow(row);
knownCfs.put(metadata.cfName, metadata);
}
break;
} catch (Exception e) {
if (!hostiter.hasNext())
throw new RuntimeException("Could not retrieve endpoint ranges: ", e);
}
}
}
private Map<String, List<String>> loadStockSectors(String sectorFile) {
FileReader input;
Map<String, List<String>> sectors = new HashMap<String, List<String>>();
try {
input = new FileReader(sectorFile);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
SectorRecord sectorRecord = Utils.readSectorRecord(line);
List<String> stockList = sectors.get(sectorRecord.getSector());
if (stockList == null) {
stockList = new ArrayList<String>();
sectors.put(sectorRecord.getSector(), stockList);
}
stockList.add(sectorRecord.getSymbol());
invertedSectors.put(sectorRecord.getSymbol(), sectorRecord.getSector());
}
} catch (IOException e) {
throw new RuntimeException("Failed to load sector file", e);
}
return sectors;
}
private static long getTermCount(
SolrServer server,
Map<String, Integer> singleCountMap,
SearchTermAndList searchTermAndList,
Map<String, String> filterGrayList,
Map<String, String> keepGrayList) {
List<String[]> searchTerms = searchTermAndList.asListArray();
Integer searchTermCountObject =
searchTerms.size() != 1 ? null : singleCountMap.get(join(searchTerms.get(0), ","));
long searchTermCount = 0;
if (searchTermCountObject == null) {
// didn't find it in map, so need to go get count
SolrQuery query = new SolrQuery();
query.setQuery("+text:(*:*)");
query.addFilterQuery("+pub_date_year:[1993 TO 2013]");
query.setParam("fl", "pmid");
for (int i = 0; i < searchTerms.size(); i++) {
String term1 = "";
for (String aTermArray : searchTerms.get(i)) {
if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
String[] splitFilterTerms = filterTerms.split(",");
term1 = term1 + "(+\"" + aTermArray + "\" -(";
for (String splitFilterTerm : splitFilterTerms) {
term1 = term1 + "\"" + splitFilterTerm + "\" ";
}
term1 = term1 + ")) ";
} else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
String[] splitKeepTerms = keepTerms.split(",");
term1 = term1 + "(+\"" + aTermArray + "\" +(";
for (String splitKeepTerm : splitKeepTerms) {
term1 = term1 + "\"" + splitKeepTerm + "\" ";
}
term1 = term1 + ")) ";
} else {
term1 = term1 + "\"" + aTermArray + "\" ";
}
}
query.addFilterQuery("+text:(" + term1 + ")");
}
try {
QueryResponse rsp = server.query(query);
searchTermCount = rsp.getResults().getNumFound();
singleCountMap.put(
join(searchTerms.get(0), ","), Integer.parseInt(Long.toString(searchTermCount)));
} catch (SolrServerException e) {
// exit out if there is an error
log.warning(e.getMessage());
System.exit(1);
}
} else {
searchTermCount = searchTermCountObject;
}
return searchTermCount;
}
