public void testUpdateOneBigRecord() throws IOException { byte[] bigRecord = new byte[2 * 65536 + 100]; MersenneTwisterFast mersenneTwisterFast = new MersenneTwisterFast(); mersenneTwisterFast.nextBytes(bigRecord); ORecordVersion recordVersion = OVersionFactory.instance().createVersion(); recordVersion.increment(); recordVersion.increment(); OPhysicalPosition physicalPosition = paginatedCluster.createRecord(bigRecord, recordVersion, (byte) 1); Assert.assertEquals(physicalPosition.clusterPosition, OClusterPositionFactory.INSTANCE.valueOf(0)); recordVersion.increment(); bigRecord = new byte[2 * 65536 + 20]; mersenneTwisterFast.nextBytes(bigRecord); paginatedCluster.updateRecord(physicalPosition.clusterPosition, bigRecord, recordVersion, (byte) 2); ORawBuffer rawBuffer = paginatedCluster.readRecord(physicalPosition.clusterPosition); Assert.assertNotNull(rawBuffer); Assert.assertEquals(rawBuffer.version, recordVersion); Assert.assertEquals(rawBuffer.buffer, bigRecord); Assert.assertEquals(rawBuffer.recordType, 2); }
public void testDeleteRecordAndAddNewOnItsPlace() throws IOException { byte[] smallRecord = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; ORecordVersion recordVersion = OVersionFactory.instance().createVersion(); recordVersion.increment(); recordVersion.increment(); OPhysicalPosition physicalPosition = paginatedCluster.createRecord(smallRecord, recordVersion, (byte) 1); Assert.assertEquals(physicalPosition.clusterPosition, OClusterPositionFactory.INSTANCE.valueOf(0)); paginatedCluster.deleteRecord(physicalPosition.clusterPosition); physicalPosition = paginatedCluster.createRecord(smallRecord, recordVersion, (byte) 1); Assert.assertEquals(physicalPosition.clusterPosition, OClusterPositionFactory.INSTANCE.valueOf(1)); recordVersion.increment(); Assert.assertEquals(physicalPosition.recordVersion, recordVersion); }
public void testBackwardIteration() throws IOException { final int records = 10000; long seed = System.currentTimeMillis(); MersenneTwisterFast mersenneTwisterFast = new MersenneTwisterFast(1381162033616L); System.out.println("testBackwardIteration seed : " + seed); NavigableMap<OClusterPosition, byte[]> positionRecordMap = new TreeMap<OClusterPosition, byte[]>(); ORecordVersion recordVersion = OVersionFactory.instance().createVersion(); recordVersion.increment(); recordVersion.increment(); for (int i = 0; i < records; i++) { int recordSize = mersenneTwisterFast.nextInt(2 * OClusterPage.MAX_RECORD_SIZE) + 1; byte[] record = new byte[recordSize]; mersenneTwisterFast.nextBytes(record); final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(record, recordVersion, (byte) 2); positionRecordMap.put(physicalPosition.clusterPosition, record); } Iterator<OClusterPosition> positionIterator = positionRecordMap.keySet().iterator(); while (positionIterator.hasNext()) { OClusterPosition clusterPosition = positionIterator.next(); if (mersenneTwisterFast.nextBoolean()) { Assert.assertTrue(paginatedCluster.deleteRecord(clusterPosition)); positionIterator.remove(); } } OPhysicalPosition physicalPosition = new OPhysicalPosition(); physicalPosition.clusterPosition = OClusterPositionFactory.INSTANCE.valueOf(Long.MAX_VALUE); OPhysicalPosition[] positions = paginatedCluster.floorPositions(physicalPosition); Assert.assertTrue(positions.length > 0); positionIterator = positionRecordMap.descendingKeySet().iterator(); int counter = 0; while (positionIterator.hasNext()) { Assert.assertTrue(positions.length > 0); OClusterPosition testedPosition = positionIterator.next(); Assert.assertEquals(positions[positions.length - 1].clusterPosition, testedPosition); OPhysicalPosition positionToFind = positions[positions.length - 1]; positions = paginatedCluster.lowerPositions(positionToFind); counter++; } Assert.assertEquals(paginatedCluster.getEntries(), counter); Assert.assertEquals(paginatedCluster.getFirstPosition(), positionRecordMap.firstKey()); Assert.assertEquals(paginatedCluster.getLastPosition(), positionRecordMap.lastKey()); }
public void testAddOneSmallRecord() throws IOException { byte[] smallRecord = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; ORecordVersion recordVersion = OVersionFactory.instance().createVersion(); recordVersion.increment(); recordVersion.increment(); OPhysicalPosition physicalPosition = paginatedCluster.createRecord(smallRecord, recordVersion, (byte) 1); Assert.assertEquals(physicalPosition.clusterPosition, OClusterPositionFactory.INSTANCE.valueOf(0)); ORawBuffer rawBuffer = paginatedCluster.readRecord(physicalPosition.clusterPosition); Assert.assertNotNull(rawBuffer); Assert.assertEquals(rawBuffer.version, recordVersion); Assert.assertEquals(rawBuffer.buffer, smallRecord); Assert.assertEquals(rawBuffer.recordType, 1); }
/** * Versions: * <li> * * <ul> * 3 = introduced file directory in physical segments and data-segment id in clusters * </ul> * * @author Luca */ @SuppressWarnings("serial") public class OStorageConfiguration implements OSerializableStream { public static final ORecordId CONFIG_RID = new OImmutableRecordId(0, OClusterPositionFactory.INSTANCE.valueOf(0)); public static final String DEFAULT_TIMEZONE = "UTC"; public static final String DEFAULT_CHARSET = "UTF-8"; public static final int CURRENT_VERSION = 4; public int version = -1; public String name; public String schemaRecordId; public String dictionaryRecordId; public String indexMgrRecordId; private String localeLanguage = Locale.getDefault().getLanguage(); private String localeCountry = Locale.getDefault().getCountry(); public String dateFormat = "yyyy-MM-dd"; public String dateTimeFormat = "yyyy-MM-dd HH:mm:ss"; private TimeZone timeZone = TimeZone.getTimeZone(DEFAULT_TIMEZONE); private String charset = DEFAULT_CHARSET; public final OStorageSegmentConfiguration fileTemplate; public List<OStorageClusterConfiguration> clusters = new ArrayList<OStorageClusterConfiguration>(); public List<OStorageDataConfiguration> dataSegments = new ArrayList<OStorageDataConfiguration>(); public OStorageTxConfiguration txSegment = new OStorageTxConfiguration(); public List<OStorageEntryConfiguration> properties = new ArrayList<OStorageEntryConfiguration>(); private transient Locale localeInstance; private transient DecimalFormatSymbols unusualSymbols; protected transient OStorage storage; public OStorageConfiguration(final OStorage iStorage) { storage = iStorage; fileTemplate = new OStorageSegmentConfiguration(); } /** * This method load the record information by the internal cluster segment. It's for compatibility * with older database than 0.9.25. * * @compatibility 0.9.25 * @return * @throws OSerializationException */ public OStorageConfiguration load() throws OSerializationException { final byte[] record = storage.readRecord(CONFIG_RID, null, false, null, false).getResult().buffer; if (record == null) throw new OStorageException( "Cannot load database's configuration. The database seems to be corrupted."); fromStream(record); return this; } public void update() throws OSerializationException { final byte[] record = toStream(); storage.updateRecord( CONFIG_RID, record, OVersionFactory.instance().createUntrackedVersion(), ORecordBytes.RECORD_TYPE, 0, null); } public boolean isEmpty() { return clusters.isEmpty(); } public String getDirectory() { return fileTemplate.location != null ? fileTemplate.getLocation() : ((OStorageLocal) storage).getStoragePath(); } public Locale getLocaleInstance() { if (localeInstance == null) localeInstance = new Locale(localeLanguage, localeCountry); return localeInstance; } public void resetLocaleInstance() { localeInstance = null; } public SimpleDateFormat getDateFormatInstance() { final SimpleDateFormat dateFormatInstance = new SimpleDateFormat(dateFormat); dateFormatInstance.setLenient(false); dateFormatInstance.setTimeZone(timeZone); return dateFormatInstance; } public SimpleDateFormat getDateTimeFormatInstance() { final SimpleDateFormat dateTimeFormatInstance = new SimpleDateFormat(dateTimeFormat); dateTimeFormatInstance.setLenient(false); dateTimeFormatInstance.setTimeZone(timeZone); return dateTimeFormatInstance; } public DecimalFormatSymbols getUnusualSymbols() { if (unusualSymbols == null) unusualSymbols = new DecimalFormatSymbols(getLocaleInstance()); return unusualSymbols; } public OSerializableStream fromStream(final byte[] iStream) throws OSerializationException { final String[] values = new String(iStream).split("\\|"); int index = 0; version = Integer.parseInt(read(values[index++])); name = read(values[index++]); schemaRecordId = read(values[index++]); dictionaryRecordId = read(values[index++]); if (version > 0) indexMgrRecordId = read(values[index++]); else // @COMPATIBILTY indexMgrRecordId = null; localeLanguage = read(values[index++]); localeCountry = read(values[index++]); dateFormat = read(values[index++]); dateTimeFormat = read(values[index++]); // @COMPATIBILTY 1.2.0 if (version >= 4) { timeZone = TimeZone.getTimeZone(read(values[index++])); charset = read(values[index++]); } // @COMPATIBILTY if (version > 1) index = phySegmentFromStream(values, index, fileTemplate); int size = Integer.parseInt(read(values[index++])); // PREPARE THE LIST OF CLUSTERS clusters = new ArrayList<OStorageClusterConfiguration>(size); for (int i = 0; i < size; ++i) { final int clusterId = Integer.parseInt(read(values[index++])); if (clusterId == -1) continue; final String clusterName = read(values[index++]); final int targetDataSegmentId = version >= 3 ? Integer.parseInt(read(values[index++])) : 0; final String clusterType = read(values[index++]); final OStorageClusterConfiguration currentCluster; if (clusterType.equals("p")) { // PHYSICAL CLUSTER final OStoragePhysicalClusterConfigurationLocal phyClusterLocal = new OStoragePhysicalClusterConfigurationLocal(this, clusterId, targetDataSegmentId); phyClusterLocal.name = clusterName; index = phySegmentFromStream(values, index, phyClusterLocal); phyClusterLocal.setHoleFile( new OStorageClusterHoleConfiguration( phyClusterLocal, read(values[index++]), read(values[index++]), read(values[index++]))); currentCluster = phyClusterLocal; } else if (clusterType.equals("m")) // MEMORY CLUSTER currentCluster = new OStorageMemoryClusterConfiguration(clusterName, clusterId, targetDataSegmentId); else if (clusterType.equals("h")) { final OStoragePhysicalClusterLHPEPSConfiguration phyClusterLocal = new OStoragePhysicalClusterLHPEPSConfiguration(this, clusterId, targetDataSegmentId); phyClusterLocal.name = clusterName; index = phySegmentFromStream(values, index, phyClusterLocal); phyClusterLocal.setOverflowFile( new OStorageClusterLocalLHPEOverflowConfiguration( this, phyClusterLocal.name, clusterId)); phyClusterLocal.setOverflowStatisticsFile( new OStorageClusterLocalLHPEStatisticConfiguration( phyClusterLocal, read(values[index++]), read(values[index++]), read(values[index++]))); currentCluster = phyClusterLocal; } else throw new IllegalArgumentException("Unsupported cluster type: " + clusterType); // MAKE ROOMS, EVENTUALLY FILLING EMPTIES ENTRIES for (int c = clusters.size(); c <= clusterId; ++c) clusters.add(null); clusters.set(clusterId, currentCluster); } // PREPARE THE LIST OF DATA SEGS size = Integer.parseInt(read(values[index++])); dataSegments = new ArrayList<OStorageDataConfiguration>(size); for (int i = 0; i < size; ++i) dataSegments.add(null); int dataId; String dataName; OStorageDataConfiguration data; for (int i = 0; i < size; ++i) { dataId = Integer.parseInt(read(values[index++])); if (dataId == -1) continue; dataName = read(values[index++]); data = new OStorageDataConfiguration(this, dataName, dataId); index = phySegmentFromStream(values, index, data); data.holeFile = new OStorageDataHoleConfiguration( data, read(values[index++]), read(values[index++]), read(values[index++])); dataSegments.set(dataId, data); } txSegment = new OStorageTxConfiguration( read(values[index++]), read(values[index++]), read(values[index++]), read(values[index++]), read(values[index++])); size = Integer.parseInt(read(values[index++])); properties = new ArrayList<OStorageEntryConfiguration>(size); for (int i = 0; i < size; ++i) { properties.add(new OStorageEntryConfiguration(read(values[index++]), read(values[index++]))); } return this; } public byte[] toStream() throws OSerializationException { final StringBuilder buffer = new StringBuilder(); write(buffer, CURRENT_VERSION); write(buffer, name); write(buffer, schemaRecordId); write(buffer, dictionaryRecordId); write(buffer, indexMgrRecordId); write(buffer, localeLanguage); write(buffer, localeCountry); write(buffer, dateFormat); write(buffer, dateTimeFormat); write(buffer, timeZone.getID()); write(buffer, charset); phySegmentToStream(buffer, fileTemplate); write(buffer, clusters.size()); for (OStorageClusterConfiguration c : clusters) { if (c == null) { write(buffer, -1); continue; } write(buffer, c.getId()); write(buffer, c.getName()); write(buffer, c.getDataSegmentId()); if (c instanceof OStoragePhysicalClusterConfigurationLocal) { // PHYSICAL write(buffer, "p"); phySegmentToStream(buffer, (OStoragePhysicalClusterConfigurationLocal) c); fileToStream(buffer, ((OStoragePhysicalClusterConfigurationLocal) c).getHoleFile()); } else if (c instanceof OStorageMemoryClusterConfiguration) { // MEMORY write(buffer, "m"); } else if (c instanceof OStoragePhysicalClusterLHPEPSConfiguration) { write(buffer, "h"); phySegmentToStream(buffer, (OStoragePhysicalClusterLHPEPSConfiguration) c); // TODO in this place we need serialize info about overflow bucket but it seems that there // is no such info // fileSegmentConfigurationToStream(buffer, ((OStoragePhysicalClusterLHPEPSConfiguration) // c).getOverflowSegment()); fileToStream( buffer, ((OStoragePhysicalClusterLHPEPSConfiguration) c).getOverflowStatisticsFile()); } } write(buffer, dataSegments.size()); for (OStorageDataConfiguration d : dataSegments) { if (d == null) { write(buffer, -1); continue; } write(buffer, d.id); write(buffer, d.name); phySegmentToStream(buffer, d); fileToStream(buffer, d.holeFile); } fileToStream(buffer, txSegment); write(buffer, txSegment.isSynchRecord()); write(buffer, txSegment.isSynchTx()); write(buffer, properties.size()); for (OStorageEntryConfiguration e : properties) entryToStream(buffer, e); // PLAIN: ALLOCATE ENOUGHT SPACE TO REUSE IT EVERY TIME buffer.append("|"); return buffer.toString().getBytes(); } private int phySegmentFromStream( final String[] values, int index, final OStorageSegmentConfiguration iSegment) { iSegment.location = version > 2 ? read(values[index++]) : null; iSegment.maxSize = read(values[index++]); iSegment.fileType = read(values[index++]); iSegment.fileStartSize = read(values[index++]); iSegment.fileMaxSize = read(values[index++]); iSegment.fileIncrementSize = read(values[index++]); iSegment.defrag = read(values[index++]); final int size = Integer.parseInt(read(values[index++])); iSegment.infoFiles = new OStorageFileConfiguration[size]; String fileName; for (int i = 0; i < size; ++i) { fileName = read(values[index++]); if (!fileName.contains("$")) { // @COMPATIBILITY 0.9.25 int pos = fileName.indexOf("/databases"); if (pos > -1) { fileName = "${" + Orient.ORIENTDB_HOME + "}" + fileName.substring(pos); } } iSegment.infoFiles[i] = new OStorageFileConfiguration( iSegment, fileName, read(values[index++]), read(values[index++]), iSegment.fileIncrementSize); } return index; } private void phySegmentToStream( final StringBuilder iBuffer, final OStorageSegmentConfiguration iSegment) { write(iBuffer, iSegment.location); write(iBuffer, iSegment.maxSize); write(iBuffer, iSegment.fileType); write(iBuffer, iSegment.fileStartSize); write(iBuffer, iSegment.fileMaxSize); write(iBuffer, iSegment.fileIncrementSize); write(iBuffer, iSegment.defrag); write(iBuffer, iSegment.infoFiles.length); for (OStorageFileConfiguration f : iSegment.infoFiles) fileToStream(iBuffer, f); } private void fileToStream(final StringBuilder iBuffer, final OStorageFileConfiguration iFile) { write(iBuffer, iFile.path); write(iBuffer, iFile.type); write(iBuffer, iFile.maxSize); } private void entryToStream(final StringBuilder iBuffer, final OStorageEntryConfiguration iEntry) { write(iBuffer, iEntry.name); write(iBuffer, iEntry.value); } private String read(final String iValue) { if (iValue.equals(" ")) return null; return iValue; } private void write(final StringBuilder iBuffer, final Object iValue) { if (iBuffer.length() > 0) iBuffer.append('|'); iBuffer.append(iValue != null ? iValue.toString() : ' '); } public void create() throws IOException { storage.createRecord( 0, CONFIG_RID, new byte[] {0, 0, 0, 0}, OVersionFactory.instance().createVersion(), ORecordBytes.RECORD_TYPE, (byte) 0, null); } public void synch() throws IOException {} public void setSoftlyClosed(boolean softlyClosed) throws IOException {} public void close() throws IOException {} public void setCluster(final OStorageClusterConfiguration config) { while (config.getId() >= clusters.size()) clusters.add(null); clusters.set(config.getId(), config); } public void dropCluster(final int iClusterId) { if (iClusterId < clusters.size()) { clusters.set(iClusterId, null); update(); } } public void dropDataSegment(final int iId) { if (iId < dataSegments.size()) { dataSegments.set(iId, null); update(); } } public TimeZone getTimeZone() { return timeZone; } public void setTimeZone(final TimeZone timeZone) { this.timeZone = timeZone; } public String getLocaleLanguage() { return localeLanguage; } public String getLocaleCountry() { return localeCountry; } public String getCharset() { return charset; } public void setCharset(String charset) { this.charset = charset; } public void setLocaleLanguage(final String iValue) { localeLanguage = iValue; localeInstance = null; } public void setLocaleCountry(final String iValue) { localeCountry = iValue; localeInstance = null; } }
protected void addRecord( final ORecordInternal<?> iRecord, final byte iStatus, final String iClusterName) { checkTransaction(); switch (iStatus) { case ORecordOperation.CREATED: database.checkSecurity( ODatabaseSecurityResources.CLUSTER, ORole.PERMISSION_CREATE, iClusterName); database.callbackHooks(TYPE.BEFORE_CREATE, iRecord); break; case ORecordOperation.LOADED: /** * Read hooks already invoked in {@link * com.orientechnologies.orient.core.db.record.ODatabaseRecordAbstract#executeReadRecord} . */ break; case ORecordOperation.UPDATED: database.checkSecurity( ODatabaseSecurityResources.CLUSTER, ORole.PERMISSION_UPDATE, iClusterName); database.callbackHooks(TYPE.BEFORE_UPDATE, iRecord); break; case ORecordOperation.DELETED: database.checkSecurity( ODatabaseSecurityResources.CLUSTER, ORole.PERMISSION_DELETE, iClusterName); database.callbackHooks(TYPE.BEFORE_DELETE, iRecord); break; } try { if (iRecord.getIdentity().isTemporary()) temp2persistent.put(iRecord.getIdentity().copy(), iRecord); if ((status == OTransaction.TXSTATUS.COMMITTING) && database.getStorage() instanceof OStorageEmbedded) { // I'M COMMITTING: BYPASS LOCAL BUFFER switch (iStatus) { case ORecordOperation.CREATED: case ORecordOperation.UPDATED: final ORID oldRid = iRecord.getIdentity().copy(); database.executeSaveRecord( iRecord, iClusterName, iRecord.getRecordVersion(), iRecord.getRecordType(), false, OPERATION_MODE.SYNCHRONOUS, false, null, null); updateIdentityAfterCommit(oldRid, iRecord.getIdentity()); break; case ORecordOperation.DELETED: database.executeDeleteRecord( iRecord, iRecord.getRecordVersion(), false, false, OPERATION_MODE.SYNCHRONOUS, false); break; } final ORecordOperation txRecord = getRecordEntry(iRecord.getIdentity()); if (txRecord == null) { // NOT IN TX, SAVE IT ANYWAY allEntries.put(iRecord.getIdentity(), new ORecordOperation(iRecord, iStatus)); } else if (txRecord.record != iRecord) { // UPDATE LOCAL RECORDS TO AVOID MISMATCH OF VERSION/CONTENT final String clusterName = getDatabase().getClusterNameById(iRecord.getIdentity().getClusterId()); if (!clusterName.equals(OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME) && !clusterName.equals(OMetadataDefault.CLUSTER_INDEX_NAME)) OLogManager.instance() .warn( this, "Found record in transaction with the same RID %s but different instance. Probably the record has been loaded from another transaction and reused on the current one: reload it from current transaction before to update or delete it", iRecord.getIdentity()); txRecord.record = iRecord; txRecord.type = iStatus; } } else { final ORecordId rid = (ORecordId) iRecord.getIdentity(); if (!rid.isValid()) { iRecord.onBeforeIdentityChanged(rid); // ASSIGN A UNIQUE SERIAL TEMPORARY ID if (rid.clusterId == ORID.CLUSTER_ID_INVALID) rid.clusterId = iClusterName != null ? database.getClusterIdByName(iClusterName) : database.getDefaultClusterId(); rid.clusterPosition = OClusterPositionFactory.INSTANCE.valueOf(newObjectCounter--); iRecord.onAfterIdentityChanged(iRecord); } else // REMOVE FROM THE DB'S CACHE database.getLevel1Cache().freeRecord(rid); ORecordOperation txEntry = getRecordEntry(rid); if (txEntry == null) { if (!(rid.isTemporary() && iStatus != ORecordOperation.CREATED)) { // NEW ENTRY: JUST REGISTER IT txEntry = new ORecordOperation(iRecord, iStatus); recordEntries.put(rid, txEntry); } } else { // UPDATE PREVIOUS STATUS txEntry.record = iRecord; switch (txEntry.type) { case ORecordOperation.LOADED: switch (iStatus) { case ORecordOperation.UPDATED: txEntry.type = ORecordOperation.UPDATED; break; case ORecordOperation.DELETED: txEntry.type = ORecordOperation.DELETED; break; } break; case ORecordOperation.UPDATED: switch (iStatus) { case ORecordOperation.DELETED: txEntry.type = ORecordOperation.DELETED; break; } break; case ORecordOperation.DELETED: break; case ORecordOperation.CREATED: switch (iStatus) { case ORecordOperation.DELETED: recordEntries.remove(rid); break; } break; } } } switch (iStatus) { case ORecordOperation.CREATED: database.callbackHooks(TYPE.AFTER_CREATE, iRecord); break; case ORecordOperation.LOADED: /** * Read hooks already invoked in {@link * com.orientechnologies.orient.core.db.record.ODatabaseRecordAbstract#executeReadRecord}. */ break; case ORecordOperation.UPDATED: database.callbackHooks(TYPE.AFTER_UPDATE, iRecord); break; case ORecordOperation.DELETED: database.callbackHooks(TYPE.AFTER_DELETE, iRecord); break; } } catch (Throwable t) { switch (iStatus) { case ORecordOperation.CREATED: database.callbackHooks(TYPE.CREATE_FAILED, iRecord); break; case ORecordOperation.UPDATED: database.callbackHooks(TYPE.UPDATE_FAILED, iRecord); break; case ORecordOperation.DELETED: database.callbackHooks(TYPE.DELETE_FAILED, iRecord); break; } if (t instanceof RuntimeException) throw (RuntimeException) t; else throw new ODatabaseException("Error on saving record " + iRecord.getIdentity(), t); } }