private static void checkAttributesSanity(
final int attributeRecordId,
final IntArrayList usedAttributeRecordIds,
final IntArrayList validAttributeIds)
throws IOException {
assert !usedAttributeRecordIds.contains(attributeRecordId);
usedAttributeRecordIds.add(attributeRecordId);
final DataInputStream dataInputStream = getAttributesStorage().readStream(attributeRecordId);
try {
while (dataInputStream.available() > 0) {
int attId = DataInputOutputUtil.readINT(dataInputStream);
int attDataRecordId = DataInputOutputUtil.readINT(dataInputStream);
assert !usedAttributeRecordIds.contains(attDataRecordId);
usedAttributeRecordIds.add(attDataRecordId);
if (!validAttributeIds.contains(attId)) {
assert getNames().valueOf(attId).length() > 0;
validAttributeIds.add(attId);
}
getAttributesStorage().checkSanity(attDataRecordId);
}
} finally {
dataInputStream.close();
}
}
public void readFrom(DataInputStream stream, DataExternalizer<Value> externalizer)
throws IOException {
while (stream.available() > 0) {
final int valueCount = DataInputOutputUtil.readINT(stream);
if (valueCount < 0) {
removeAssociatedValue(
-valueCount); // ChangeTrackingValueContainer marked inputId as invalidated, see
// ChangeTrackingValueContainer.saveTo
setNeedsCompacting(true);
} else {
for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
final Value value = externalizer.read(stream);
int idCountOrSingleValue = DataInputOutputUtil.readINT(stream);
if (idCountOrSingleValue > 0) {
addValue(idCountOrSingleValue, value);
} else {
idCountOrSingleValue = -idCountOrSingleValue;
ensureFileSetCapacityForValue(value, idCountOrSingleValue);
int prev = 0;
for (int i = 0; i < idCountOrSingleValue; i++) {
final int id = DataInputOutputUtil.readINT(stream);
addValue(prev + id, value);
prev += id;
}
}
}
}
}
}
private void writeToStream(final DataOutputStream stream) throws IOException {
if (myIndexStamps != null && !myIndexStamps.isEmpty()) {
final long[] dominatingIndexStamp = new long[1];
myIndexStamps.forEachEntry(
new TObjectLongProcedure<ID<?, ?>>() {
@Override
public boolean execute(ID<?, ?> a, long b) {
dominatingIndexStamp[0] = Math.max(dominatingIndexStamp[0], b);
return true;
}
});
DataInputOutputUtil.writeTIME(stream, dominatingIndexStamp[0]);
myIndexStamps.forEachEntry(
new TObjectLongProcedure<ID<?, ?>>() {
@Override
public boolean execute(final ID<?, ?> id, final long timestamp) {
try {
DataInputOutputUtil.writeINT(stream, id.getUniqueId());
return true;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
} else {
DataInputOutputUtil.writeTIME(stream, DataInputOutputUtil.timeBase);
}
}
public static Pair<String[], int[]> listAll(int parentId) {
try {
r.lock();
try {
final DataInputStream input = readAttribute(parentId, CHILDREN_ATT);
if (input == null)
return Pair.create(ArrayUtil.EMPTY_STRING_ARRAY, ArrayUtil.EMPTY_INT_ARRAY);
final int count = DataInputOutputUtil.readINT(input);
final int[] ids = ArrayUtil.newIntArray(count);
final String[] names = ArrayUtil.newStringArray(count);
for (int i = 0; i < count; i++) {
int id = DataInputOutputUtil.readINT(input);
id = id >= 0 ? id + parentId : -id;
ids[i] = id;
names[i] = getName(id);
}
input.close();
return Pair.create(names, ids);
} finally {
r.unlock();
}
} catch (Throwable e) {
throw DbConnection.handleError(e);
}
}
public ValueContainerImpl<T> read(final DataInput in) throws IOException {
DataInputStream stream = (DataInputStream) in;
final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
while (stream.available() > 0) {
final int valueCount = DataInputOutputUtil.readSINT(in);
if (valueCount < 0) {
valueContainer.removeAllValues(-valueCount);
valueContainer.setNeedsCompacting(true);
} else {
for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
final T value = myExternalizer.read(in);
final int idCount = DataInputOutputUtil.readSINT(in);
for (int i = 0; i < idCount; i++) {
final int id = DataInputOutputUtil.readSINT(in);
if (id < 0) {
valueContainer.removeValue(-id, value);
valueContainer.setNeedsCompacting(true);
} else {
valueContainer.addValue(id, value);
}
}
}
}
}
return valueContainer;
}
@Override
public void save(final DataOutput out) {
try {
out.writeByte(CLASS_TYPE);
DataInputOutputUtil.writeINT(out, className);
DataInputOutputUtil.writeINT(out, typeArgs.length);
for (AbstractType t : typeArgs) {
t.save(out);
}
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
}
@Override
public void save(final DataOutput out, final TIntArrayList value) throws IOException {
int size = value.size();
if (size == 0) {
DataInputOutputUtil.writeSINT(out, Integer.MAX_VALUE);
} else if (size == 1) {
DataInputOutputUtil.writeSINT(out, -value.get(0));
} else {
DataInputOutputUtil.writeSINT(out, size);
for (int i = 0; i < size; i++) {
DataInputOutputUtil.writeINT(out, value.get(i));
}
}
}
public void save(final DataOutput out) {
super.save(out);
myType.save(out);
try {
if (myValue instanceof String) {
out.writeByte(STRING);
String value = (String) myValue;
RW.writeUTF(out, value);
} else if (myValue instanceof Integer) {
out.writeByte(INTEGER);
DataInputOutputUtil.writeINT(out, ((Integer) myValue).intValue());
} else if (myValue instanceof Long) {
out.writeByte(LONG);
out.writeLong(((Long) myValue).longValue());
} else if (myValue instanceof Float) {
out.writeByte(FLOAT);
out.writeFloat(((Float) myValue).floatValue());
} else if (myValue instanceof Double) {
out.writeByte(DOUBLE);
out.writeDouble(((Double) myValue).doubleValue());
} else if (myValue instanceof Type) {
out.writeByte(TYPE);
RW.writeUTF(out, ((Type) myValue).getDescriptor());
} else {
out.writeByte(NONE);
}
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
}
private static Object loadTyped(final DataInput in) {
try {
switch (in.readByte()) {
case STRING:
return RW.readUTF(in);
case NONE:
return null;
case INTEGER:
return DataInputOutputUtil.readINT(in);
case LONG:
return in.readLong();
case FLOAT:
return in.readFloat();
case DOUBLE:
return in.readDouble();
case TYPE:
return Type.getType(RW.readUTF(in));
}
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
assert (false);
return null;
}
PrimitiveType(final DataInput in) {
try {
type = DataInputOutputUtil.readINT(in);
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
}
@Override
public void save(@NotNull DataOutput out, TIntArrayList list) throws IOException {
if (list.size() == 2) {
DataInputOutputUtil.writeINT(out, list.getQuick(0));
DataInputOutputUtil.writeINT(out, list.getQuick(1));
} else {
DataInputOutputUtil.writeINT(out, -list.size());
int prev = 0;
for (int i = 0, len = list.size(); i < len; i += 2) {
int value = list.getQuick(i);
DataInputOutputUtil.writeINT(out, value - prev);
prev = value;
DataInputOutputUtil.writeINT(out, list.getQuick(i + 1));
}
}
}
ClassType(final DependencyContext context, final DataInput in) {
try {
className = DataInputOutputUtil.readINT(in);
final int size = DataInputOutputUtil.readINT(in);
if (size == 0) {
typeArgs = EMPTY_TYPE_ARRAY;
} else {
typeArgs = new AbstractType[size];
final DataExternalizer<AbstractType> externalizer = externalizer(context);
for (int i = 0; i < size; i++) {
typeArgs[i] = externalizer.read(in);
}
}
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
}
@Override
public TIntArrayList read(final DataInput in) throws IOException {
int size = DataInputOutputUtil.readSINT(in);
if (size == Integer.MAX_VALUE) {
return new TIntArrayList();
} else if (size <= 0) {
TIntArrayList result = new TIntArrayList(1);
result.add(-size);
return result;
} else {
TIntArrayList result = new TIntArrayList(size);
for (int i = 0; i < size; i++) {
result.add(DataInputOutputUtil.readINT(in));
}
return result;
}
}
private Timestamps(@Nullable DataInputStream stream) throws IOException {
if (stream != null) {
try {
long dominatingIndexStamp = DataInputOutputUtil.readTIME(stream);
while (stream.available() > 0) {
ID<?, ?> id = ID.findById(DataInputOutputUtil.readINT(stream));
if (id != null) {
long stamp = IndexInfrastructure.getIndexCreationStamp(id);
if (myIndexStamps == null) myIndexStamps = new TObjectLongHashMap<ID<?, ?>>(5, 0.98f);
if (stamp <= dominatingIndexStamp) myIndexStamps.put(id, stamp);
}
}
} finally {
stream.close();
}
}
}
@Override
public void save(final DataOutput out) {
try {
out.writeByte(PRIMITIVE_TYPE);
DataInputOutputUtil.writeINT(out, type);
} catch (IOException e) {
throw new BuildDataCorruptedException(e);
}
}
private static void deleteContentAndAttributes(int id) throws IOException {
int content_page = getContentRecordId(id);
if (content_page != 0) {
getContentStorage().releaseRecord(content_page);
}
int att_page = getAttributeRecordId(id);
if (att_page != 0) {
final DataInputStream attStream = getAttributesStorage().readStream(att_page);
while (attStream.available() > 0) {
DataInputOutputUtil.readINT(attStream); // Attribute ID;
int attAddress = DataInputOutputUtil.readINT(attStream);
getAttributesStorage().deleteRecord(attAddress);
}
attStream.close();
getAttributesStorage().deleteRecord(att_page);
}
}
private static int findAttributePage(int fileId, String attrId, boolean toWrite)
throws IOException {
checkFileIsValid(fileId);
Storage storage = getAttributesStorage();
int encodedAttrId = DbConnection.getAttributeId(attrId);
int recordId = getAttributeRecordId(fileId);
if (recordId == 0) {
if (!toWrite) return 0;
recordId = storage.createNewRecord();
setAttributeRecordId(fileId, recordId);
} else {
DataInputStream attrRefs = storage.readStream(recordId);
try {
while (attrRefs.available() > 0) {
final int attIdOnPage = DataInputOutputUtil.readINT(attrRefs);
final int attrAddress = DataInputOutputUtil.readINT(attrRefs);
if (attIdOnPage == encodedAttrId) return attrAddress;
}
} finally {
attrRefs.close();
}
}
if (toWrite) {
Storage.AppenderStream appender = storage.appendStream(recordId);
DataInputOutputUtil.writeINT(appender, encodedAttrId);
int attrAddress = storage.createNewRecord();
DataInputOutputUtil.writeINT(appender, attrAddress);
DbConnection.REASONABLY_SMALL.myAttrPageRequested = true;
try {
appender.close();
} finally {
DbConnection.REASONABLY_SMALL.myAttrPageRequested = false;
}
return attrAddress;
}
return 0;
}
public Timestamps(@Nullable DataInputStream stream) throws IOException {
if (stream != null) {
try {
int count = DataInputOutputUtil.readINT(stream);
if (count > 0) {
myIndexStamps = new TObjectLongHashMap<ID<?, ?>>(20);
for (int i = 0; i < count; i++) {
ID<?, ?> id = ID.findById(DataInputOutputUtil.readINT(stream));
long timestamp = DataInputOutputUtil.readTIME(stream);
if (id != null) {
myIndexStamps.put(id, timestamp);
}
}
}
} finally {
stream.close();
}
}
}
@Override
public TIntArrayList read(@NotNull DataInput in) throws IOException {
int capacityOrValue = DataInputOutputUtil.readINT(in);
if (capacityOrValue >= 0) {
TIntArrayList list = new TIntArrayList(2);
list.add(capacityOrValue);
list.add(DataInputOutputUtil.readINT(in));
return list;
}
capacityOrValue = -capacityOrValue;
TIntArrayList list = new TIntArrayList(capacityOrValue);
int prev = 0;
while (capacityOrValue > 0) {
int value = DataInputOutputUtil.readINT(in) + prev;
list.add(value);
prev = value;
list.add(DataInputOutputUtil.readINT(in));
capacityOrValue -= 2;
}
return list;
}
private void saveImpl(
final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData)
throws IOException {
DataInputOutputUtil.writeSINT(out, container.size());
for (final Iterator<T> valueIterator = container.getValueIterator();
valueIterator.hasNext(); ) {
final T value = valueIterator.next();
myExternalizer.save(out, value);
final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
if (ids != null) {
DataInputOutputUtil.writeSINT(out, ids.size());
while (ids.hasNext()) {
final int id = ids.next();
DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
}
} else {
DataInputOutputUtil.writeSINT(out, 0);
}
}
}
public static void updateList(int id, @NotNull int[] children) {
try {
w.lock();
DbConnection.markDirty();
final DataOutputStream record = writeAttribute(id, CHILDREN_ATT, false);
DataInputOutputUtil.writeINT(record, children.length);
for (int child : children) {
if (child == id) {
LOG.error("Cyclic parent child relations");
} else {
child = child > id ? child - id : -child;
DataInputOutputUtil.writeINT(record, child);
}
}
record.close();
} catch (Throwable e) {
throw DbConnection.handleError(e);
} finally {
w.unlock();
}
}
public static int[] listRoots() throws IOException {
try {
r.lock();
final DataInputStream input = readAttribute(1, CHILDREN_ATT);
if (input == null) return ArrayUtil.EMPTY_INT_ARRAY;
try {
final int count = DataInputOutputUtil.readINT(input);
int[] result = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
DataInputOutputUtil.readINT(input); // Name
result[i] = DataInputOutputUtil.readINT(input); // Id
}
return result;
} finally {
input.close();
}
} finally {
r.unlock();
}
}
private static void saveVersion(File versionFile) {
try {
DataOutputStream versionOutputStream =
new DataOutputStream(new BufferedOutputStream(new FileOutputStream(versionFile)));
try {
DataInputOutputUtil.writeINT(versionOutputStream, VERSION);
} finally {
versionOutputStream.close();
}
} catch (IOException ignore) {
}
}
public static int findRootRecord(String rootUrl) throws IOException {
try {
w.lock();
DbConnection.markDirty();
final int root = getNames().enumerate(rootUrl);
final DataInputStream input = readAttribute(1, CHILDREN_ATT);
int[] names = ArrayUtil.EMPTY_INT_ARRAY;
int[] ids = ArrayUtil.EMPTY_INT_ARRAY;
if (input != null) {
try {
final int count = DataInputOutputUtil.readINT(input);
names = ArrayUtil.newIntArray(count);
ids = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
final int name = DataInputOutputUtil.readINT(input);
final int id = DataInputOutputUtil.readINT(input);
if (name == root) {
return id;
}
names[i] = name;
ids[i] = id;
}
} finally {
input.close();
}
}
final DataOutputStream output = writeAttribute(1, CHILDREN_ATT, false);
int id;
try {
id = createRecord();
DataInputOutputUtil.writeINT(output, names.length + 1);
for (int i = 0; i < names.length; i++) {
DataInputOutputUtil.writeINT(output, names[i]);
DataInputOutputUtil.writeINT(output, ids[i]);
}
DataInputOutputUtil.writeINT(output, root);
DataInputOutputUtil.writeINT(output, id);
} finally {
output.close();
}
return id;
} finally {
w.unlock();
}
}
public static int[] list(int id) {
try {
r.lock();
try {
final DataInputStream input = readAttribute(id, CHILDREN_ATT);
if (input == null) return ArrayUtil.EMPTY_INT_ARRAY;
final int count = DataInputOutputUtil.readINT(input);
final int[] result = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
int childId = DataInputOutputUtil.readINT(input);
childId = childId >= 0 ? childId + id : -childId;
result[i] = childId;
}
input.close();
return result;
} finally {
r.unlock();
}
} catch (Throwable e) {
throw DbConnection.handleError(e);
}
}
public void writeToStream(final DataOutputStream stream) throws IOException {
if (myIndexStamps != null) {
final int size = myIndexStamps.size();
final int[] count = new int[] {0};
DataInputOutputUtil.writeINT(stream, size);
myIndexStamps.forEachEntry(
new TObjectLongProcedure<ID<?, ?>>() {
public boolean execute(final ID<?, ?> id, final long timestamp) {
try {
DataInputOutputUtil.writeINT(stream, (int) id.getUniqueId());
DataInputOutputUtil.writeTIME(stream, timestamp);
count[0]++;
return true;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
assert count[0] == size;
}
}
public static void deleteRootRecord(int id) throws IOException {
try {
w.lock();
DbConnection.markDirty();
final DataInputStream input = readAttribute(1, CHILDREN_ATT);
assert input != null;
int count;
int[] names;
int[] ids;
try {
count = DataInputOutputUtil.readINT(input);
names = ArrayUtil.newIntArray(count);
ids = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
names[i] = DataInputOutputUtil.readINT(input);
ids[i] = DataInputOutputUtil.readINT(input);
}
} finally {
input.close();
}
final int index = ArrayUtil.find(ids, id);
assert index >= 0;
names = ArrayUtil.remove(names, index);
ids = ArrayUtil.remove(ids, index);
final DataOutputStream output = writeAttribute(1, CHILDREN_ATT, false);
try {
DataInputOutputUtil.writeINT(output, count - 1);
for (int i = 0; i < names.length; i++) {
DataInputOutputUtil.writeINT(output, names[i]);
DataInputOutputUtil.writeINT(output, ids[i]);
}
} finally {
output.close();
}
} finally {
w.unlock();
}
}
static {
File snapshotInfoFile = new File(new File(getJarsDir()), "snapshots_info");
int currentVersion = -1;
File versionFile = getVersionFile(snapshotInfoFile);
if (versionFile.exists()) {
try {
DataInputStream versionStream =
new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)));
try {
currentVersion = DataInputOutputUtil.readINT(versionStream);
} finally {
versionStream.close();
}
} catch (IOException ignore) {
}
}
if (currentVersion != VERSION) {
PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
saveVersion(versionFile);
}
PersistentHashMap<String, CacheLibraryInfo> info = null;
for (int i = 0; i < 2; ++i) {
try {
info =
new PersistentHashMap<String, CacheLibraryInfo>(
snapshotInfoFile,
new EnumeratorStringDescriptor(),
new DataExternalizer<CacheLibraryInfo>() {
@Override
public void save(@NotNull DataOutput out, CacheLibraryInfo value)
throws IOException {
IOUtil.writeUTF(out, value.mySnapshotPath);
out.writeLong(value.myModificationTime);
out.writeLong(value.myFileLength);
}
@Override
public CacheLibraryInfo read(@NotNull DataInput in) throws IOException {
return new CacheLibraryInfo(IOUtil.readUTF(in), in.readLong(), in.readLong());
}
});
if (i == 0) removeStaleJarFilesIfNeeded(snapshotInfoFile, info);
break;
} catch (IOException ex) {
PersistentHashMap.deleteFilesStartingWith(snapshotInfoFile);
saveVersion(versionFile);
}
}
assert info != null;
ourCachedLibraryInfo = info;
FlushingDaemon.everyFiveSeconds(
new Runnable() {
@Override
public void run() {
flushCachedLibraryInfos();
}
});
ShutDownTracker.getInstance()
.registerShutdownTask(
new Runnable() {
@Override
public void run() {
flushCachedLibraryInfos();
}
});
}
@Override
public void saveTo(DataOutput out, DataExternalizer<Value> externalizer) throws IOException {
DataInputOutputUtil.writeINT(out, size());
for (final Iterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext(); ) {
final Value value = valueIterator.next();
externalizer.save(out, value);
Object input = getInput(value);
if (input instanceof Integer) {
DataInputOutputUtil.writeINT(
out, (Integer) input); // most common 90% case during index building
} else {
// serialize positive file ids with delta encoding
ChangeBufferingList originalInput = (ChangeBufferingList) input;
IntIterator intIterator = originalInput.intIterator();
DataInputOutputUtil.writeINT(out, -intIterator.size());
if (intIterator.hasAscendingOrder()) {
IdSet checkSet = originalInput.getCheckSet();
if (checkSet != null && checkSet.size() != intIterator.size()) { // debug code
int a = 1;
assert false;
}
int prev = 0;
while (intIterator.hasNext()) {
int fileId = intIterator.next();
if (checkSet != null && !checkSet.contains(fileId)) { // debug code
int a = 1;
assert false;
}
DataInputOutputUtil.writeINT(out, fileId - prev);
prev = fileId;
}
} else {
// sorting numbers via bitset before writing deltas
int max = 0, min = Integer.MAX_VALUE;
while (intIterator.hasNext()) {
int nextInt = intIterator.next();
max = Math.max(max, nextInt);
min = Math.min(min, nextInt);
}
assert min > 0;
final int offset = (min >> INT_BITS_SHIFT) << INT_BITS_SHIFT;
final int bitsLength = ((max - offset) >> INT_BITS_SHIFT) + 1;
final int[] bits = ourSpareBuffer.getBuffer(bitsLength);
for (int i = 0; i < bitsLength; ++i) bits[i] = 0;
intIterator = originalInput.intIterator();
while (intIterator.hasNext()) {
final int id = intIterator.next() - offset;
bits[id >> INT_BITS_SHIFT] |= (1 << (id));
}
int pos = nextSetBit(0, bits, bitsLength);
int prev = 0;
while (pos != -1) {
DataInputOutputUtil.writeINT(out, pos + offset - prev);
prev = pos + offset;
pos = nextSetBit(pos + 1, bits, bitsLength);
}
}
}
}
}
public static void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
DataInputOutputUtil.writeSINT(out, -inputId);
}
