@Override public void write(DataOutput output) throws IOException { Bytes.writeByteArray(output, name.getBytes()); WritableUtils.writeVInt(output, type.ordinal()); WritableUtils.writeVLong(output, sequenceNumber); output.writeLong(timeStamp); Bytes.writeByteArray( output, pkName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(pkName)); WritableUtils.writeVInt(output, allColumns.size()); for (int i = 0; i < allColumns.size(); i++) { PColumn column = allColumns.get(i); column.write(output); } stats.write(output); }
@Override public PColumn getPKColumn(String name) throws ColumnNotFoundException { List<PColumn> columns = columnsByName.get(name); int size = columns.size(); if (size == 0) { throw new ColumnNotFoundException(name); } if (size > 1) { do { PColumn column = columns.get(--size); if (column.getFamilyName() == null) { return column; } } while (size > 0); throw new ColumnNotFoundException(name); } return columns.get(0); }
@Override public PColumn getColumn(String name) throws ColumnNotFoundException, AmbiguousColumnException { List<PColumn> columns = columnsByName.get(name); int size = columns.size(); if (size == 0) { throw new ColumnNotFoundException(name); } if (size > 1) { for (PColumn column : columns) { if (QueryConstants.DEFAULT_COLUMN_FAMILY.equals(column.getFamilyName().getString())) { // Allow ambiguity with default column, since a user would not know how to prefix it. return column; } } throw new AmbiguousColumnException(name); } return columns.get(0); }
@Override public QueryResult<Variant> getAllVariantsByRegionAndStudy( Region region, String sourceId, QueryOptions options) { Long start, end, dbstart, dbend; start = System.currentTimeMillis(); QueryResult<Variant> queryResult = new QueryResult<>( String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd())); List<Variant> results = new LinkedList<>(); boolean includeSamples; boolean includeStats; boolean includeEffects; if (!options.containsKey("samples") && !options.containsKey("stats") && !options.containsKey("effects")) { includeSamples = true; includeStats = true; includeEffects = true; } else { includeSamples = options.containsKey("samples") && options.getBoolean("samples"); includeStats = options.containsKey("stats") && options.getBoolean("stats"); includeEffects = options.containsKey("effects") && options.getBoolean("effects"); } try { String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart())); String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd())); HTable table = new HTable(admin.getConfiguration(), tableName); dbstart = System.currentTimeMillis(); Scan regionScan = new Scan(startRow.getBytes(), stopRow.getBytes()); ResultScanner scanres = table.getScanner(regionScan); dbend = System.currentTimeMillis(); queryResult.setDbTime(dbend - dbstart); // Iterate over results and, optionally, their samples and statistics for (Result result : scanres) { String[] rowkeyParts = new String(result.getRow(), CHARSET_UTF_8).split("_"); String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", ""); int position = Integer.parseInt(rowkeyParts[1]); // Get basic result fields from Protocol Buffers message NavigableMap<byte[], byte[]> infoMap = result.getFamilyMap("i".getBytes()); byte[] byteInfo = infoMap.get((sourceId + "_data").getBytes()); VariantFieldsProtos.VariantInfo protoInfo = VariantFieldsProtos.VariantInfo.parseFrom(byteInfo); String reference = protoInfo.getReference(); String alternate = StringUtils.join(protoInfo.getAlternateList(), ","); String format = StringUtils.join(protoInfo.getFormatList(), ":"); Variant variant = new Variant(chromosome, position, position, reference, alternate); // Set samples if requested if (includeSamples) { NavigableMap<byte[], byte[]> sampleMap = result.getFamilyMap("d".getBytes()); Map<String, Map<String, String>> resultSampleMap = new HashMap<>(); // Set samples for (byte[] s : sampleMap.keySet()) { String sampleName = (new String(s, CHARSET_UTF_8)).replaceAll(sourceId + "_", ""); VariantFieldsProtos.VariantSample sample = VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s)); String sample1 = sample.getSample(); String[] values = sample1.split(":"); String[] fields = format.split(":"); Map<String, String> singleSampleMap = new HashMap<>(); for (int i = 0; i < fields.length; i++) { singleSampleMap.put(fields[i], values[i]); } // TODO // variant.addSampleData(sampleName, singleSampleMap); } } // Set stats if requested if (includeStats) { byte[] byteStats = infoMap.get((sourceId + "_stats").getBytes()); VariantFieldsProtos.VariantStats protoStats = VariantFieldsProtos.VariantStats.parseFrom(byteStats); VariantStats variantStats = new VariantStats( chromosome, position, reference, alternate, protoStats.getMaf(), protoStats.getMgf(), protoStats.getMafAllele(), protoStats.getMgfGenotype(), protoStats.getMissingAlleles(), protoStats.getMissingGenotypes(), protoStats.getMendelianErrors(), protoStats.getIsIndel(), protoStats.getCasesPercentDominant(), protoStats.getControlsPercentDominant(), protoStats.getCasesPercentRecessive(), protoStats.getControlsPercentRecessive()); variant.setStats(variantStats); } if (includeEffects) { QueryResult<VariantEffect> queryEffects = getEffectsByVariant(variant, options); variant.setEffect(queryEffects.getResult()); } results.add(variant); } } catch (IOException e) { System.err.println(e.getClass().getName() + ": " + e.getMessage()); } queryResult.setResult(results); queryResult.setNumResults(results.size()); end = System.currentTimeMillis(); queryResult.setTime(end - start); return queryResult; }
public QueryResult getSimpleVariantsByRegion( Region region, String sourceId, QueryOptions options) { Long start, end, dbstart, dbend; start = System.currentTimeMillis(); boolean includeStats; boolean includeEffects; if (!options.containsKey("stats") && !options.containsKey("effects")) { includeStats = true; includeEffects = true; } else { includeStats = options.containsKey("stats") && options.getBoolean("stats"); includeEffects = options.containsKey("effects") && options.getBoolean("effects"); } QueryResult<Variant> queryResult = new QueryResult<>( String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd())); List<Variant> results = new ArrayList<>(); String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart())); String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd())); BasicDBObject query = new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow)) .append("sources.sourceId", sourceId); DBCollection collection = db.getCollection("variants"); dbstart = System.currentTimeMillis(); DBCursor variantInStudies = collection.find(query); dbend = System.currentTimeMillis(); queryResult.setDbTime(dbend - dbstart); for (DBObject result : variantInStudies) { String[] rowkeyParts = result.get("position").toString().split("_"); String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", ""); int position = Integer.parseInt(rowkeyParts[1]); BasicDBList studies = (BasicDBList) result.get("sources"); BasicDBObject st = (BasicDBObject) studies.get(0); String ref = (String) st.get("ref"); String alt = StringUtils.join((ArrayList<String>) st.get("alt"), ","); // TODO Needs rework Variant variant = new Variant(chromosome, position, position, ref, alt); // Set stats informations if (includeStats) { VariantStats stats = new VariantStats(); BasicDBObject mongoStats = (BasicDBObject) st.get("stats"); stats.setMaf((float) (double) mongoStats.get("maf")); stats.setMafAllele((String) mongoStats.get("alleleMaf")); stats.setMissingGenotypes((int) mongoStats.get("missing")); List<Genotype> genotypeCount = new ArrayList<>(); for (BasicDBObject s : (List<BasicDBObject>) mongoStats.get("genotypeCount")) { for (Map.Entry<String, Object> entry : s.entrySet()) { Genotype genotype = new Genotype(entry.getKey()); genotype.setCount((Integer) entry.getValue()); genotypeCount.add(genotype); } } stats.setGenotypes(genotypeCount); variant.setStats(stats); } // TODO Set consequence type names if (includeEffects) { BasicDBList mongoEffects = (BasicDBList) st.get("effects"); if (mongoEffects != null) { for (Object e : mongoEffects) { String effectObo = e.toString(); VariantEffect effect = new VariantEffect(); effect.setConsequenceTypeObo(effectObo); variant.addEffect(effect); } } } results.add(variant); } queryResult.setResult(results); queryResult.setNumResults(results.size()); end = System.currentTimeMillis(); queryResult.setTime(end - start); return queryResult; }
@Override public QueryResult getVariantsHistogramByRegion( Region region, String sourceId, boolean histogramLogarithm, int histogramMax) { QueryResult<ObjectMap> queryResult = new QueryResult<>( String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd())); List<ObjectMap> data = new ArrayList<>(); String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart())); String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd())); long startTime = System.currentTimeMillis(); long startDbTime = System.currentTimeMillis(); BasicDBObject query = new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow)) .append("studies.studyId", sourceId); DBCollection collection = db.getCollection("variants"); DBCursor queryResults = collection.find(query); queryResult.setDbTime(System.currentTimeMillis() - startDbTime); int resultSize = queryResults.size(); if (resultSize > histogramMax) { // Need to group results to fit maximum size of the histogram int sumChunkSize = resultSize / histogramMax; int i = 0, j = 0; int featuresCount = 0; ObjectMap item = null; for (DBObject result : queryResults) { // featuresCount += result.getInt("features_count"); // if (i == 0) { // item = new ObjectMap("chromosome", result.getString("chromosome")); // item.put("chunkId", result.getInt("chunk_id")); // item.put("start", result.getInt("start")); // } else if (i == sumChunkSize - 1 || j == resultSize - 1) { // if (histogramLogarithm) { // item.put("featuresCount", (featuresCount > 0) ? // Math.log(featuresCount) : 0); // } else { // item.put("featuresCount", featuresCount); // } // item.put("end", result.getInt("end")); // data.add(item); // i = -1; // featuresCount = 0; // } // j++; // i++; } } else { for (DBObject result : queryResults) { // ObjectMap item = new ObjectMap("chromosome", // result.getString("chromosome")); // item.put("chunkId", result.getInt("chunk_id")); // item.put("start", result.getInt("start")); // if (histogramLogarithm) { // int features_count = result.getInt("features_count"); // result.put("featuresCount", (features_count > 0) ? // Math.log(features_count) : 0); // } else { // item.put("featuresCount", result.getInt("features_count")); // } // item.put("end", result.getInt("end")); // data.add(item); } } queryResult.setResult(data); queryResult.setNumResults(data.size()); queryResult.setTime(System.currentTimeMillis() - startTime); return queryResult; }
public int run(String[] args) throws Exception { // printUsage(); /* * SETUP */ Configuration argConf = getConf(); Hashtable<String, String> confArg = new Hashtable<String, String>(); setup(confArg, argConf); Date currentTime = new Date(); Date endDate = new Date(new Long(confArg.get("timestamp_stop"))); Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*"); Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*"); logger.info("Running GeStore"); // ZooKeeper setup Configuration config = HBaseConfiguration.create(); zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config)); zkInstance = new ZooKeeper( ZKConfig.getZKQuorumServersString(config), config.getInt("zookeeper.session.timeout", -1), zkWatcher); if (!confArg.get("task_id").isEmpty()) { confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id")); } String lockRequest = confArg.get("file_id"); if (!confArg.get("run_id").isEmpty()) lockRequest = lockRequest + "_" + confArg.get("run_id") + "_"; if (!confArg.get("task_id").isEmpty()) lockRequest = lockRequest + "_" + confArg.get("task_id") + "_"; // Get type of movement toFrom type_move = checkArgs(confArg); if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) { List<String> arguments = new ArrayList<String>(); arguments.add("-Dinput=" + confArg.get("local_path")); arguments.add("-Dtable=" + confArg.get("file_id")); arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop")); arguments.add("-Dtype=" + confArg.get("format")); arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id")); arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path")); arguments.add("-Drun_id=" + confArg.get("run_id")); if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id")); if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id")); if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add")); String lockName = lock(lockRequest); String[] argumentString = arguments.toArray(new String[arguments.size()]); adddb.main(argumentString); unlock(lockName); System.exit(0); } // Database registration dbutil db_util = new dbutil(config); db_util.register_database(confArg.get("db_name_files"), true); db_util.register_database(confArg.get("db_name_runs"), true); db_util.register_database(confArg.get("db_name_updates"), true); FileSystem hdfs = FileSystem.get(config); FileSystem localFS = FileSystem.getLocal(config); // Get source type confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id"))); confArg.put( "database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id"))); if (!confArg.get("source").equals("local") && type_move == toFrom.REMOTE2LOCAL && !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) { confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util))); } /* * Get previous timestamp */ Get run_id_get = new Get(confArg.get("run_id").getBytes()); Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get); KeyValue run_file_prev = run_get.getColumnLatest( "d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes()); String last_timestamp = new String("0"); if (null != run_file_prev && !confArg.get("source").equals("local")) { long last_timestamp_real = run_file_prev.getTimestamp(); Long current_timestamp = new Long(confArg.get("timestamp_real")); if ((current_timestamp - last_timestamp_real) > 36000) { last_timestamp = new String(run_file_prev.getValue()); Integer lastTimestamp = new Integer(last_timestamp); lastTimestamp += 1; last_timestamp = lastTimestamp.toString(); logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate); Date last_run = new Date(run_file_prev.getTimestamp()); if (last_run.before(endDate) && !full_run) { confArg.put("timestamp_start", last_timestamp); } } } Integer tse = new Integer(confArg.get("timestamp_stop")); Integer tss = new Integer(confArg.get("timestamp_start")); if (tss > tse) { logger.info("No new version of requested file."); return 0; } /* * Generate file */ String lockName = lock(lockRequest); Get file_id_get = new Get(confArg.get("file_id").getBytes()); Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get); if (!file_get.isEmpty()) { boolean found = hasFile( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg)); if (confArg.get("source").equals("fullfile")) { found = false; } String filenames_put = getFileNames( db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg)); // Filename not found in file database if (!found && type_move == toFrom.REMOTE2LOCAL) { if (!confArg.get("source").equals("local")) { // Generate intermediate file if (getFile(hdfs, confArg, db_util) == null) { unlock(lockName); return 1; } // Put generated file into file database if (!confArg.get("format").equals("fullfile")) { putFileEntry( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), confArg.get("full_file_name"), confArg.get("source")); } } else { logger.warn("Remote file not found, and cannot be generated! File: " + confArg); unlock(lockName); return 1; } } } else { if (type_move == toFrom.REMOTE2LOCAL) { logger.warn("Remote file not found, and cannot be generated."); unlock(lockName); return 1; } } /* * Copy file * Update tables */ if (type_move == toFrom.LOCAL2REMOTE) { if (!confArg.get("format").equals("fullfile")) { putFileEntry( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg), confArg.get("source")); } putRunEntry( db_util, confArg.get("db_name_runs"), confArg.get("run_id"), confArg.get("file_id"), confArg.get("type"), confArg.get("timestamp_real"), confArg.get("timestamp_stop"), getFullPath(confArg), confArg.get("delimiter")); hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg))); } else if (type_move == toFrom.REMOTE2LOCAL) { FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*")); putRunEntry( db_util, confArg.get("db_name_runs"), confArg.get("run_id"), confArg.get("file_id"), confArg.get("type"), confArg.get("timestamp_real"), confArg.get("timestamp_stop"), getFullPath(confArg), confArg.get("delimiter")); unlock(lockName); for (FileStatus file : files) { Path cur_file = file.getPath(); Path cur_local_path = new Path(new String(confArg.get("local_path") + confArg.get("file_id"))); String suffix = getSuffix(getFileName(confArg), cur_file.getName()); if (suffix.length() > 0) { cur_local_path = cur_local_path.suffix(new String("." + suffix)); } if (confArg.get("copy").equals("true")) { String crc = hdfs.getFileChecksum(cur_file).toString(); if (checksumLocalTest(cur_local_path, crc)) { continue; } else { hdfs.copyToLocalFile(cur_file, cur_local_path); writeChecksum(cur_local_path, crc); } } else { System.out.println(cur_local_path + "\t" + cur_file); } } } unlock(lockName); return 0; }
private void init( PName name, PTableType type, long timeStamp, long sequenceNumber, String pkName, List<PColumn> columns, PTableStats stats) { this.name = name; this.type = type; this.timeStamp = timeStamp; this.sequenceNumber = sequenceNumber; this.columnsByName = ArrayListMultimap.create(columns.size(), 1); this.pkName = pkName; List<PColumn> pkColumns = Lists.newArrayListWithExpectedSize(columns.size() - 1); PColumn[] allColumns = new PColumn[columns.size()]; RowKeySchemaBuilder builder = new RowKeySchemaBuilder(); for (int i = 0; i < allColumns.length; i++) { PColumn column = columns.get(i); allColumns[column.getPosition()] = column; PName familyName = column.getFamilyName(); if (familyName == null) { pkColumns.add(column); builder.addField(column); } columnsByName.put(column.getName().getString(), column); } this.pkColumns = ImmutableList.copyOf(pkColumns); this.rowKeySchema = builder.setMinNullable(pkColumns.size()).build(); this.allColumns = ImmutableList.copyOf(allColumns); // Two pass so that column order in column families matches overall column order // and to ensure that column family order is constant int maxExpectedSize = allColumns.length - pkColumns.size(); // Maintain iteration order so that column families are ordered as they are listed Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap(); for (PColumn column : allColumns) { PName familyName = column.getFamilyName(); if (familyName != null) { List<PColumn> columnsInFamily = familyMap.get(familyName); if (columnsInFamily == null) { columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize); familyMap.put(familyName, columnsInFamily); } columnsInFamily.add(column); } } Iterator<Map.Entry<PName, List<PColumn>>> iterator = familyMap.entrySet().iterator(); PColumnFamily[] families = new PColumnFamily[familyMap.size()]; ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder(); ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR); for (int i = 0; i < families.length; i++) { Map.Entry<PName, List<PColumn>> entry = iterator.next(); PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue()); families[i] = family; familyByString.put(family.getName().getString(), family); familyByBytes.put(family.getName().getBytes(), family); } this.families = ImmutableList.copyOf(families); this.familyByBytes = familyByBytes.build(); this.familyByString = familyByString.build(); this.stats = stats; }
@Override public int newKey(ImmutableBytesWritable key, byte[][] values) { int i = 0; TrustedByteArrayOutputStream os = new TrustedByteArrayOutputStream(SchemaUtil.estimateKeyLength(this)); try { List<PColumn> columns = getPKColumns(); int nColumns = columns.size(); PColumn lastPKColumn = columns.get(nColumns - 1); while (i < values.length && i < nColumns) { PColumn column = columns.get(i); PDataType type = column.getDataType(); // This will throw if the value is null and the type doesn't allow null byte[] byteValue = values[i++]; if (byteValue == null) { byteValue = ByteUtil.EMPTY_BYTE_ARRAY; } // An empty byte array return value means null. Do this, // since a type may have muliple representations of null. // For example, VARCHAR treats both null and an empty string // as null. This way we don't need to leak that part of the // implementation outside of PDataType by checking the value // here. if (byteValue.length == 0 && !column.isNullable()) { throw new ConstraintViolationException( name.getString() + "." + column.getName().getString() + " may not be null"); } Integer byteSize = column.getByteSize(); if (type.isFixedWidth()) { // TODO: handle multi-byte characters if (byteValue.length != byteSize) { throw new ConstraintViolationException( name.getString() + "." + column.getName().getString() + " must be " + byteSize + " bytes (" + SchemaUtil.toString(type, byteValue) + ")"); } } else if (byteSize != null && byteValue.length > byteSize) { throw new ConstraintViolationException( name.getString() + "." + column.getName().getString() + " may not exceed " + byteSize + " bytes (" + SchemaUtil.toString(type, byteValue) + ")"); } os.write(byteValue, 0, byteValue.length); // Separate variable length column values in key with zero byte if (!type.isFixedWidth() && column != lastPKColumn) { os.write(SEPARATOR_BYTE); } } // If some non null pk values aren't set, then throw if (i < nColumns) { PColumn column = columns.get(i); PDataType type = column.getDataType(); if (type.isFixedWidth() || !column.isNullable()) { throw new ConstraintViolationException( name.getString() + "." + column.getName().getString() + " may not be null"); } // Separate variable length column values in key with zero byte if (column != lastPKColumn) { os.write(SEPARATOR_BYTE); } } key.set(os.getBuffer(), 0, os.size()); return i; } finally { try { os.close(); } catch (IOException e) { throw new RuntimeException(e); // Impossible } } }
/** * init station code map * * @param file station code file path */ private static void initStationCode(String file) { List<String> lstStation = new Vector<>(); strConfig = file + ".process"; File fsP = new File(strConfig); if (!fsP.exists()) { try { fsP.createNewFile(); } catch (IOException e) { e.printStackTrace(); } } else { BufferedReader readp = null; try { String line = null; readp = new BufferedReader(new FileReader(fsP)); while ((line = readp.readLine()) != null) { processed.add(line); } } catch (IOException e) { e.printStackTrace(); } finally { if (readp != null) { try { readp.close(); } catch (IOException e) { e.printStackTrace(); } } } } File fs = new File(file); BufferedReader reader = null; try { String stationName = null; String stationCode = null; String line = null; String[] splitLine = null; reader = new BufferedReader(new FileReader(fs)); while ((line = reader.readLine()) != null) { splitLine = line.split("\t"); if (splitLine.length != 2) { continue; } stationName = splitLine[0]; stationCode = splitLine[1]; mapStationCode.put(stationName, stationCode); lstStation.add(stationName); } } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { if (reader == null) { try { reader.close(); } catch (IOException e1) { e1.printStackTrace(); } } } int len = lstStation.size(); String start = null; for (int i = 0; i < len; i++) { start = lstStation.get(i); for (String end : lstStation) { if (start.equals(end) || processed.contains(start + ":" + end)) { continue; } lstAllProcessStation.add(new DefaultKeyValue(start, end)); } } }