public void createDDL() throws SQLException, IOException { TableReader sourceReader = new TableReader(sourceConnection); TableReader destionationReader = new TableReader(destinationConnection); List<Index> sourceIndexes = sourceReader.getIndexes(schema, table); List<Index> destinationIndexes = destionationReader.getIndexes(schema, table); for (Index destIndex : destinationIndexes) { boolean found = false; boolean drop = false; if (destIndex.getName().startsWith("SYS")) { continue; } for (Index sourceIndex : sourceIndexes) { if (destIndex.getName().equalsIgnoreCase(sourceIndex.getName())) { if (!destIndex.getColumns().containsAll(sourceIndex.getColumns())) { drop = true; } found = true; break; } else if (destIndex.getColumns().size() == sourceIndex.getColumns().size() && destIndex.getColumns().containsAll(sourceIndex.getColumns())) { found = true; break; } } if (drop) { dropDDL(destIndex.getName()); createIndexDDL(destIndex); } else if (!found) { createIndexDDL(destIndex); } } }
public void takeSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { this.signature = table.getSignature(); this.columnDelimeter = table.getColumnDelimeter(); int maxIndex = tableDesc.getMaxColumnIndex(); TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); TableReader reader = table.getReader(); while (reader.next()) { String[] row = reader.getRow(); if (row.length <= maxIndex) { throw new IllegalStateException( "Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row)); } for (String cell : row) { if (cell != null) b.addValue(cell); } } this.dict = b.build(0); reader = table.getReader(); ArrayList<int[]> allRowIndices = new ArrayList<int[]>(); while (reader.next()) { String[] row = reader.getRow(); int[] rowIndex = new int[row.length]; for (int i = 0; i < row.length; i++) { rowIndex[i] = dict.getIdFromValue(row[i]); } allRowIndices.add(rowIndex); } this.rowIndices = allRowIndices; }
// validity is to be checked in read() method of the corresponding provider public static <V, T extends GenericTable<V>> void readMultiTable( File root, T table, GenericTableProvider<V, T> provider) throws IOException, ValidityCheckFailedException { ArrayList<ArrayList<T>> tableParts = new ArrayList<ArrayList<T>>(DIRECTORIES_QUANTITY); for (int i = 0; i < DIRECTORIES_QUANTITY; ++i) { tableParts.add(i, new ArrayList<T>(FILES_QUANTITY)); for (int j = 0; j < FILES_QUANTITY; ++j) { tableParts.get(i).add(j, (T) table.clone()); } } for (File directory : root.listFiles()) { // validity is already checked, therefore, the file is there for a reason; // or I could delete it. Both options - not that nice. if (directory.isFile()) { continue; } ValidityChecker.checkMultiFileStorageDir(directory, DIRECTORIES_QUANTITY); for (File file : directory.listFiles()) { ValidityChecker.checkMultiFileStorageFile(file, FILES_QUANTITY); TableReader.readTable( directory, file, tableParts.get(getNum(directory)).get(getNum(file)), provider); // iterating is only possible over commited entries tableParts.get(getNum(directory)).get(getNum(file)).commit(); checkKeys(tableParts.get(getNum(directory)).get(getNum(file)), file, directory); } } for (int i = 0; i < tableParts.size(); ++i) { for (int j = 0; j < tableParts.get(i).size(); ++j) { addTablePart(tableParts.get(i).get(j), table); } } }