private Map<String, Integer> loadFixedClasses(String file) {
FileReader input;
try {
Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
File f = new File(file);
if (!f.exists()) {
System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
return invertedFixedClasses;
}
input = new FileReader(f);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
String parts[] = line.split(",");
int clazz = Integer.parseInt(parts[0]);
List<String> symbols = new ArrayList<String>();
symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
fixedClaszzes.put(clazz, symbols);
}
for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
for (String s : e.getValue()) {
invertedFixedClasses.put(s, e.getKey());
}
}
return invertedFixedClasses;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public void changeClassLabels() {
System.out.println("Reading cluster file: " + cluserInputFile);
Clusters clusters = loadClusters(cluserInputFile);
if (clusters == null) {
System.out.println("Not clusters found to change");
return;
}
Map<Integer, String> classToSector = new HashMap<Integer, String>();
for (Map.Entry<String, Integer> e : sectorToClazz.entrySet()) {
classToSector.put(e.getValue(), e.getKey());
}
for (Cluster c : clusters.getCluster()) {
// find the cluster label
// this is the label
String key = classToSector.get(c.getKey());
if (key != null) {
System.out.println("Setting label: " + key + " to cluster: " + c.getKey());
c.setLabel(key);
}
}
try {
System.out.println("Writing cluster file: " + clusterOutputFile);
saveClusters(clusterOutputFile, clusters);
} catch (FileNotFoundException | JAXBException e) {
throw new RuntimeException("Failed to write clusters", e);
}
}
private static Map<String, String> createAuthorizationAttributeMap(
String snaaName, Properties props) {
Map<String, String> attributes = new HashMap<String, String>();
List<String> keys = new LinkedList<String>();
// getting keys from properties
for (Object o : props.keySet()) {
if (((String) o).startsWith(snaaName + authorizationAtt)
&& ((String) o).endsWith(authorizationKeyAtt)) {
keys.add((String) o);
}
}
for (String k : keys) {
String key = props.getProperty(k);
// getting plain key-number from properties
String plainKeyProperty = k.replaceAll(snaaName + authorizationAtt + ".", "");
plainKeyProperty = plainKeyProperty.replaceAll(authorizationKeyAtt, "");
String keyPrefix = snaaName + authorizationAtt + "." + plainKeyProperty;
// building value-property-string
String value = props.getProperty(keyPrefix + authorizationValAtt);
// finally put key and values
attributes.put(key, value);
}
return attributes;
}
public String createCombinedTerm(String[] termArray) {
String term1Combined = "";
for (String aTermArray : termArray) {
if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
String[] splitFilterTerms = filterTerms.split(",");
term1Combined = term1Combined + "(+\"" + aTermArray + "\" -(";
for (String splitFilterTerm : splitFilterTerms) {
term1Combined = term1Combined + "\"" + splitFilterTerm + "\" ";
}
term1Combined = term1Combined + ")) ";
} else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
String[] splitKeepTerms = keepTerms.split(",");
term1Combined = term1Combined + "(+\"" + aTermArray + "\" +(";
for (String splitKeepTerm : splitKeepTerms) {
term1Combined = term1Combined + "\"" + splitKeepTerm + "\" ";
}
term1Combined = term1Combined + ")) ";
} else {
term1Combined = term1Combined + "\"" + aTermArray + "\" ";
}
}
return term1Combined;
}
private void init() {
permNoToSymbol = Utils.loadMapping(originalStockFile);
Map<String, Integer> symbolToPerm = new HashMap<String, Integer>();
for (Map.Entry<Integer, String> entry : permNoToSymbol.entrySet()) {
symbolToPerm.put(entry.getValue(), entry.getKey());
}
}
public static CompilerConfiguration generateCompilerConfigurationFromOptions(CommandLine cli)
throws IOException {
//
// Setup the configuration data
CompilerConfiguration configuration = new CompilerConfiguration();
if (cli.hasOption("classpath")) {
configuration.setClasspath(cli.getOptionValue("classpath"));
}
if (cli.hasOption('d')) {
configuration.setTargetDirectory(cli.getOptionValue('d'));
}
if (cli.hasOption("encoding")) {
configuration.setSourceEncoding(cli.getOptionValue("encoding"));
}
if (cli.hasOption("basescript")) {
configuration.setScriptBaseClass(cli.getOptionValue("basescript"));
}
// joint compilation parameters
if (cli.hasOption('j')) {
Map<String, Object> compilerOptions = new HashMap<String, Object>();
String[] opts = cli.getOptionValues("J");
compilerOptions.put("namedValues", opts);
opts = cli.getOptionValues("F");
compilerOptions.put("flags", opts);
configuration.setJointCompilationOptions(compilerOptions);
}
if (cli.hasOption("indy")) {
configuration.getOptimizationOptions().put("int", false);
configuration.getOptimizationOptions().put("indy", true);
}
if (cli.hasOption("configscript")) {
String path = cli.getOptionValue("configscript");
File groovyConfigurator = new File(path);
Binding binding = new Binding();
binding.setVariable("configuration", configuration);
CompilerConfiguration configuratorConfig = new CompilerConfiguration();
ImportCustomizer customizer = new ImportCustomizer();
customizer.addStaticStars(
"org.codehaus.groovy.control.customizers.builder.CompilerCustomizationBuilder");
configuratorConfig.addCompilationCustomizers(customizer);
GroovyShell shell = new GroovyShell(binding, configuratorConfig);
shell.evaluate(groovyConfigurator);
}
return configuration;
}
private Map<String, Integer> convertSectorsToClazz(Map<String, List<String>> sectors) {
List<String> sectorNames = new ArrayList<String>(sectors.keySet());
Collections.sort(sectorNames);
Map<String, Integer> sectorsToClazz = new HashMap<String, Integer>();
for (int i = 0; i < sectorNames.size(); i++) {
sectorsToClazz.put(sectorNames.get(i), i + 1);
System.out.println(sectorNames.get(i) + ": " + (i + 1));
}
return sectorsToClazz;
}
public void handleStreamEvent(StreamEvent event) {
if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
sessionsByHost.put(session.peer, session);
} else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;
// update progress
Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
if (progresses == null) {
progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
progressByHost.put(progressInfo.peer, progresses);
}
if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
progresses.add(progressInfo);
StringBuilder sb = new StringBuilder();
sb.append("\rprogress: ");
long totalProgress = 0;
long totalSize = 0;
for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
SessionInfo session = sessionsByHost.get(entry.getKey());
long size = session.getTotalSizeToSend();
long current = 0;
int completed = 0;
for (ProgressInfo progress : entry.getValue()) {
if (progress.currentBytes == progress.totalBytes) completed++;
current += progress.currentBytes;
}
totalProgress += current;
totalSize += size;
sb.append("[").append(entry.getKey());
sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
}
long time = System.nanoTime();
long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
lastTime = time;
long deltaProgress = totalProgress - lastProgress;
lastProgress = totalProgress;
sb.append("[total: ")
.append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
.append("% - ");
sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
sb.append(" (avg: ")
.append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
.append("MB/s)]");
System.out.print(sb.toString());
}
}
private boolean register(File f) {
boolean rc = false;
if (f.isDirectory()) {
File file[] = f.listFiles(this);
for (int i = 0; i < file.length; i++) {
if (register(file[i])) rc = true;
}
} else if (dirMap.get(f) == null) {
dirMap.put(f, new QEntry());
rc = true;
}
return rc;
}
private void applyLabel(String inPointsFile, String outPointsFile, List<String> symbols) {
System.out.println("Applying labels for points file: " + inPointsFile);
FileReader input;
BufferedWriter bufWriter = null;
try {
FileOutputStream fos = new FileOutputStream(outPointsFile);
bufWriter = new BufferedWriter(new OutputStreamWriter(fos));
File inFile = new File(inPointsFile);
if (!inFile.exists()) {
System.out.println("ERROR: In file doens't exist");
return;
}
input = new FileReader(inPointsFile);
BufferedReader bufRead = new BufferedReader(input);
String inputLine;
int index = 0;
while ((inputLine = bufRead.readLine()) != null && index < symbols.size()) {
Point p = Utils.readPoint(inputLine);
String symbol = symbols.get(index);
int clazz = 0;
if (this.invertedFixedClases.containsKey(symbol)) {
clazz = this.invertedFixedClases.get(symbol);
} else {
// get the corresponding symbol
// get the class for this one
String sector = invertedSectors.get(symbol);
if (sector != null) {
clazz = sectorToClazz.get(sector);
} else {
// System.out.println("No sector: " + symbol);
}
}
p.setClazz(clazz);
String s = p.serialize();
bufWriter.write(s);
bufWriter.newLine();
index++;
}
System.out.println("Read lines: " + index);
} catch (Exception e) {
throw new RuntimeException("Failed to read/write file", e);
} finally {
if (bufWriter != null) {
try {
bufWriter.close();
} catch (IOException ignore) {
}
}
}
}
private static void configureTransportFactory(
ITransportFactory transportFactory, LoaderOptions opts) {
Map<String, String> options = new HashMap<>();
// If the supplied factory supports the same set of options as our SSL impl, set those
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
options.put(
SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);
// Now check if any of the factory's supported options are set as system properties
for (String optionKey : transportFactory.supportedOptions())
if (System.getProperty(optionKey) != null)
options.put(optionKey, System.getProperty(optionKey));
transportFactory.setOptions(options);
}
private static Cassandra.Client createThriftClient(
String host, int port, String user, String passwd, ITransportFactory transportFactory)
throws Exception {
TTransport trans = transportFactory.openTransport(host, port);
TProtocol protocol = new TBinaryProtocol(trans);
Cassandra.Client client = new Cassandra.Client(protocol);
if (user != null && passwd != null) {
Map<String, String> credentials = new HashMap<>();
credentials.put(IAuthenticator.USERNAME_KEY, user);
credentials.put(IAuthenticator.PASSWORD_KEY, passwd);
AuthenticationRequest authenticationRequest = new AuthenticationRequest(credentials);
client.login(authenticationRequest);
}
return client;
}
private Map<String, List<String>> loadHistoSectors(String sectorFile) {
FileReader input;
Map<String, List<String>> sectors = new HashMap<String, List<String>>();
try {
input = new FileReader(sectorFile);
BufferedReader bufRead = new BufferedReader(input);
String line;
int i = 1;
while ((line = bufRead.readLine()) != null) {
Bin sectorRecord = Utils.readBin(line);
List<String> stockList = sectorRecord.symbols;
String startEnd = formatter.format(sectorRecord.end);
String key = intgerFormaterr.format(i) + ":" + startEnd;
sectors.put(key, stockList);
for (String s : stockList) {
invertedSectors.put(s, key);
}
i++;
}
} catch (IOException e) {
throw new RuntimeException("Failed to load sector file", e);
}
return sectors;
}
private void deploy() {
List startList = new ArrayList();
Iterator iter = dirMap.entrySet().iterator();
try {
while (iter.hasNext() && !shutdown) {
Map.Entry entry = (Map.Entry) iter.next();
File f = (File) entry.getKey();
QEntry qentry = (QEntry) entry.getValue();
long deployed = qentry.getDeployed();
if (deployed == 0) {
if (deploy(f)) {
if (qentry.isQBean()) startList.add(qentry.getInstance());
qentry.setDeployed(f.lastModified());
} else {
// deploy failed, clean up.
iter.remove();
}
} else if (deployed != f.lastModified()) {
undeploy(f);
iter.remove();
loader.forceNewClassLoaderOnNextScan();
}
}
iter = startList.iterator();
while (iter.hasNext()) {
start((ObjectInstance) iter.next());
}
} catch (Exception e) {
log.error("deploy", e);
}
}
/**
* Construct table schema from info stored in SSTable's Stats.db
*
* @param desc SSTable's descriptor
* @return Restored CFMetaData
* @throws IOException when Stats.db cannot be read
*/
public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException {
if (!desc.version.storeRows()) throw new IOException("pre-3.0 SSTable is not supported.");
EnumSet<MetadataType> types =
EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
Map<MetadataType, MetadataComponent> sstableMetadata =
desc.getMetadataSerializer().deserialize(desc, types);
ValidationMetadata validationMetadata =
(ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
SerializationHeader.Component header =
(SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
IPartitioner partitioner =
SecondaryIndexManager.isIndexColumnFamily(desc.cfname)
? new LocalPartitioner(header.getKeyType())
: FBUtilities.newPartitioner(validationMetadata.partitioner);
CFMetaData.Builder builder =
CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
header
.getStaticColumns()
.entrySet()
.stream()
.forEach(
entry -> {
ColumnIdentifier ident =
ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addStaticColumn(ident, entry.getValue());
});
header
.getRegularColumns()
.entrySet()
.stream()
.forEach(
entry -> {
ColumnIdentifier ident =
ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addRegularColumn(ident, entry.getValue());
});
builder.addPartitionKey("PartitionKey", header.getKeyType());
for (int i = 0; i < header.getClusteringTypes().size(); i++) {
builder.addClusteringColumn(
"clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
}
return builder.build();
}
public String getPath() {
Map<String, String> environment = new ProcessBuilder().environment();
String path = environment.get("Path");
if (path == null) {
path = environment.get("PATH");
}
if (path == null) {
path = environment.get("path");
}
if (path == null) {
throw new UnexpectedLiquibaseException(
"Cannot find path variable in environment. Possible variables are "
+ StringUtils.join(environment.keySet(), ","));
}
return path;
}
private Map<String, String[]> readRoles(File f) throws IOException {
Map<String, String[]> map = new TreeMap<String, String[]>();
LineNumberReader lr =
new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
String line = null;
StringBuilder sb = new StringBuilder();
while ((line = lr.readLine()) != null) {
String[] s = line.toLowerCase().split("\t");
if (s.length > 2) {
logger.trace(Arrays.toString(s));
map.put(s[1], s);
}
}
return map;
}
private void undeploy() {
Object[] set = dirMap.entrySet().toArray();
int l = set.length;
while (l-- > 0) {
Map.Entry entry = (Map.Entry) set[l];
File f = (File) entry.getKey();
undeploy(f);
}
}
public void process() {
File inFolder = new File(vectorFolder);
if (!inFolder.isDirectory()) {
System.out.println("In should be a folder: " + vectorFolder);
return;
}
boolean clusterSaved = false;
this.invertedFixedClases = loadFixedClasses(fixedClassesFile);
if (!histogram) {
Map<String, List<String>> sectors = loadStockSectors(sectorFile);
sectorToClazz = convertSectorsToClazz(sectors);
if (!clusterSaved) {
changeClassLabels();
clusterSaved = true;
}
for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
for (File inFile : inFolder.listFiles()) {
String fileName = inFile.getName();
String fileNameWithOutExt = FilenameUtils.removeExtension(fileName);
if (histogram) {
sectorToClazz.clear();
invertedSectors.clear();
Map<String, List<String>> sectors =
loadHistoSectors(sectorFile + "/" + fileNameWithOutExt + ".csv");
sectorToClazz = convertSectorsToClazz(sectors);
if (!clusterSaved) {
changeClassLabels();
clusterSaved = true;
}
for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
processFile(fileNameWithOutExt);
}
}
private void undeploy(File f) {
QEntry qentry = (QEntry) dirMap.get(f);
try {
if (log != null) log.trace("undeploying:" + f.getCanonicalPath());
Object obj = qentry.getObject();
ObjectName name = qentry.getObjectName();
factory.destroyQBean(this, name, obj);
if (log != null) log.info("undeployed:" + f.getCanonicalPath());
} catch (Exception e) {
getLog().warn("undeploy", e);
}
}
// load symbols for each point in file
private List<String> loadSymbols(String vectorFile) {
System.out.println("Loading symbols from vector file: " + vectorFile);
File vf = new File(vectorFile);
List<VectorPoint> vectorPoints = Utils.readVectors(vf, 0, 7000);
List<String> symbols = new ArrayList<String>();
for (int i = 0; i < vectorPoints.size(); i++) {
VectorPoint v = vectorPoints.get(i);
String symbol = permNoToSymbol.get(v.getKey());
symbols.add(symbol);
}
System.out.println("No of symbols for point: " + symbols.size());
return symbols;
}
private boolean deploy(File f) {
try {
if (log != null) log.info("deploy:" + f.getCanonicalPath());
QEntry qentry = (QEntry) dirMap.get(f);
SAXBuilder builder = createSAXBuilder();
Document doc;
if (decorator != null && !f.getName().equals(LOGGER_CONFIG)) {
doc = decrypt(builder.build(new StringReader(decorator.decorateFile(f))));
} else {
doc = decrypt(builder.build(f));
}
Element rootElement = doc.getRootElement();
String iuuid = rootElement.getAttributeValue("instance");
if (iuuid != null) {
UUID uuid = UUID.fromString(iuuid);
if (!uuid.equals(getInstanceId())) {
deleteFile(f, iuuid);
return false;
}
}
Object obj = factory.instantiate(this, rootElement);
qentry.setObject(obj);
ObjectInstance instance = factory.createQBean(this, doc.getRootElement(), obj);
qentry.setInstance(instance);
} catch (InstanceAlreadyExistsException e) {
/*
* Ok, the file we tried to deploy, holds an object
* that already has been deployed.
*
* Rename it out of the way.
*
*/
tidyFileAway(f, DUPLICATE_EXTENSION);
getLog().warn("deploy", e);
return false;
} catch (Exception e) {
getLog().warn("deploy", e);
tidyFileAway(f, ERROR_EXTENSION);
// This will also save deploy error repeats...
return false;
} catch (Error e) {
getLog().warn("deploy", e);
tidyFileAway(f, ENV_EXTENSION);
// This will also save deploy error repeats...
return false;
}
return true;
}
private void checkModified() {
Iterator iter = dirMap.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry entry = (Map.Entry) iter.next();
File f = (File) entry.getKey();
QEntry qentry = (QEntry) entry.getValue();
if (qentry.isQBean() && qentry.isQPersist()) {
ObjectName name = qentry.getObjectName();
if (getState(name) == QBean.STARTED && isModified(name)) {
qentry.setDeployed(persist(f, name));
}
}
}
}
private Map<Integer, String> readTraining(File f) throws IOException {
Map<Integer, String> map = new TreeMap<Integer, String>();
LineNumberReader lr =
new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
String line = null;
StringBuilder sb = new StringBuilder();
int id = -1;
while ((line = lr.readLine()) != null) {
String[] s = line.split("\t");
if (s.length > 0 && s[0].length() > 0 && !s[0].equals("sid")) {
sb.append(line + "\n");
id = Integer.parseInt(s[0]);
} else {
if (sb.length() > 0) {
map.put(id, sb.toString());
}
sb = new StringBuilder();
}
}
map.put(id, sb.toString());
return map;
}
@Override
public void init(String keyspace) {
Iterator<InetAddress> hostiter = hosts.iterator();
while (hostiter.hasNext()) {
try {
// Query endpoint to ranges map and schemas from thrift
InetAddress host = hostiter.next();
Cassandra.Client client =
createThriftClient(
host.getHostAddress(), rpcPort, this.user, this.passwd, this.transportFactory);
setPartitioner(client.describe_partitioner());
Token.TokenFactory tkFactory = getPartitioner().getTokenFactory();
for (TokenRange tr : client.describe_ring(keyspace)) {
Range<Token> range =
new Range<>(
tkFactory.fromString(tr.start_token),
tkFactory.fromString(tr.end_token),
getPartitioner());
for (String ep : tr.endpoints) {
addRangeForEndpoint(range, InetAddress.getByName(ep));
}
}
String query =
String.format(
"SELECT * FROM %s.%s WHERE keyspace_name = '%s'",
Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, keyspace);
CqlResult result =
client.execute_cql3_query(
ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
for (CqlRow row : result.rows) {
CFMetaData metadata = CFMetaData.fromThriftCqlRow(row);
knownCfs.put(metadata.cfName, metadata);
}
break;
} catch (Exception e) {
if (!hostiter.hasNext())
throw new RuntimeException("Could not retrieve endpoint ranges: ", e);
}
}
}
private Map<String, List<String>> loadStockSectors(String sectorFile) {
FileReader input;
Map<String, List<String>> sectors = new HashMap<String, List<String>>();
try {
input = new FileReader(sectorFile);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
SectorRecord sectorRecord = Utils.readSectorRecord(line);
List<String> stockList = sectors.get(sectorRecord.getSector());
if (stockList == null) {
stockList = new ArrayList<String>();
sectors.put(sectorRecord.getSector(), stockList);
}
stockList.add(sectorRecord.getSymbol());
invertedSectors.put(sectorRecord.getSymbol(), sectorRecord.getSector());
}
} catch (IOException e) {
throw new RuntimeException("Failed to load sector file", e);
}
return sectors;
}
public AnnotationMigration(File labelled, File unlabelled, File roles, File output, int start)
throws IOException {
pw =
new PrintWriter(
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8")));
tokenier = new HardTokenizer();
Map<Integer, String> labelledMap = readTraining(labelled);
Map<Integer, String> unlabelledMap = readTraining(unlabelled);
roleMap = readRoles(roles);
// logger.info(labelledMap);
if (labelledMap.size() != unlabelledMap.size()) {
logger.warn(labelledMap.size() + " != " + unlabelledMap.size());
}
for (int i = start; i < labelledMap.size(); i++) {
logger.info(i);
logger.info("\n" + labelledMap.get(i));
logger.info("\n" + unlabelledMap.get(i));
process(labelledMap.get(i), unlabelledMap.get(i));
logger.info("***\n");
}
/*Iterator<Integer> it = unlabelledMap.keySet().iterator();
while(it.hasNext()) {
int id = it.next();
if (labelledMap.get(id)==null){
logger.warn(id);
logger.warn(unlabelledMap.get(id));
}
}*/
pw.close();
}
@Override
public CFMetaData getCFMetaData(String keyspace, String cfName) {
return knownCfs.get(cfName);
}
private static long getTermCount(
SolrServer server,
Map<String, Integer> singleCountMap,
SearchTermAndList searchTermAndList,
Map<String, String> filterGrayList,
Map<String, String> keepGrayList) {
List<String[]> searchTerms = searchTermAndList.asListArray();
Integer searchTermCountObject =
searchTerms.size() != 1 ? null : singleCountMap.get(join(searchTerms.get(0), ","));
long searchTermCount = 0;
if (searchTermCountObject == null) {
// didn't find it in map, so need to go get count
SolrQuery query = new SolrQuery();
query.setQuery("+text:(*:*)");
query.addFilterQuery("+pub_date_year:[1993 TO 2013]");
query.setParam("fl", "pmid");
for (int i = 0; i < searchTerms.size(); i++) {
String term1 = "";
for (String aTermArray : searchTerms.get(i)) {
if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
String[] splitFilterTerms = filterTerms.split(",");
term1 = term1 + "(+\"" + aTermArray + "\" -(";
for (String splitFilterTerm : splitFilterTerms) {
term1 = term1 + "\"" + splitFilterTerm + "\" ";
}
term1 = term1 + ")) ";
} else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
String[] splitKeepTerms = keepTerms.split(",");
term1 = term1 + "(+\"" + aTermArray + "\" +(";
for (String splitKeepTerm : splitKeepTerms) {
term1 = term1 + "\"" + splitKeepTerm + "\" ";
}
term1 = term1 + ")) ";
} else {
term1 = term1 + "\"" + aTermArray + "\" ";
}
}
query.addFilterQuery("+text:(" + term1 + ")");
}
try {
QueryResponse rsp = server.query(query);
searchTermCount = rsp.getResults().getNumFound();
singleCountMap.put(
join(searchTerms.get(0), ","), Integer.parseInt(Long.toString(searchTermCount)));
} catch (SolrServerException e) {
// exit out if there is an error
log.warning(e.getMessage());
System.exit(1);
}
} else {
searchTermCount = searchTermCountObject;
}
return searchTermCount;
}
private void process(String labelled, String unlabelled) throws IOException {
if (labelled == null || unlabelled == null) {
return;
}
List<String[]> examples = extractExamples(unlabelled);
List<String> roles = extractRoles(labelled);
boolean[] found = new boolean[roles.size()];
logger.debug("roles: " + roles);
for (int i = 0; i < roles.size(); i++) {
String[] t = roles.get(i).split("\t");
String[] translatedRole = roleMap.get(t[0]);
if (translatedRole != null) {
for (int k = 2; k < translatedRole.length; k++) {
logger.debug(i + "\t" + roles.get(i) + "\t" + translatedRole[k]);
if (translatedRole[k] != null) {
Boundary boundary = matcher(translatedRole[k], examples);
if (boundary != null) {
logger.info("///");
logger.debug(boundary + "\t" + translatedRole[k] + ":");
logger.debug(boundary.begin + "\t" + Arrays.toString(examples.get(boundary.begin)));
examples.get(boundary.begin)[5] = t[1];
examples.get(boundary.begin)[6] = t[2];
for (int j = boundary.begin + 1; j <= boundary.end; j++) {
logger.debug(j + "\t" + Arrays.toString(examples.get(j)));
examples.get(j)[5] = t[1];
examples.get(j)[6] = t[2].replaceAll("B-", "I-");
}
found[i] = true;
logger.info("\\\\\\ break at " + k + "/" + translatedRole.length);
break;
}
}
}
}
}
logger.info("===");
boolean b = false;
for (int i = 0; i < found.length; i++) {
if (!found[i]) {
String[] t = roles.get(i).split("\t");
logger.info("# Missing " + t[0] + "\t" + t[1] + "\t" + t[2]);
pw.println("# Missing " + t[0] + "\t" + t[1] + "\t" + t[2]);
b = true;
}
}
if (b) {
StringBuilder sb = new StringBuilder();
for (int j = 0; j < examples.size(); j++) {
if (j > 0) {
sb.append(" ");
}
sb.append(examples.get(j)[2]);
}
logger.info("# " + sb.toString());
}
for (int i = 0; i < examples.size(); i++) {
String[] example = examples.get(i);
logger.info(
example[0]
+ "\t"
+ example[1]
+ "\t"
+ example[2]
+ "\t"
+ example[3]
+ "\t"
+ example[4]
+ "\t"
+ example[5]
+ "\t"
+ example[6]);
pw.println(
example[0]
+ "\t"
+ example[1]
+ "\t"
+ example[2]
+ "\t"
+ example[3]
+ "\t"
+ example[4]
+ "\t"
+ example[5]
+ "\t"
+ example[6]);
}
pw.print("\n");
logger.info("***");
}
