public void put(ByteArray key, Versioned<byte[]> value) throws PersistenceFailureException { StoreUtils.assertValidKey(key); boolean doCommit = false; Connection conn = null; PreparedStatement insert = null; PreparedStatement select = null; ResultSet results = null; String insertSql = "insert into " + name + " (key_, version_, value_) values (?, ?, ?)"; String selectSql = "select key_, version_ from " + name + " where key_ = ?"; try { conn = datasource.getConnection(); conn.setAutoCommit(false); // check for superior versions select = conn.prepareStatement(selectSql); select.setBytes(1, key.get()); results = select.executeQuery(); while (results.next()) { byte[] thisKey = results.getBytes("key_"); VectorClock version = new VectorClock(results.getBytes("version_")); Occured occured = value.getVersion().compare(version); if (occured == Occured.BEFORE) throw new ObsoleteVersionException( "Attempt to put version " + value.getVersion() + " which is superceeded by " + version + "."); else if (occured == Occured.AFTER) delete(conn, thisKey, version.toBytes()); } // Okay, cool, now put the value insert = conn.prepareStatement(insertSql); insert.setBytes(1, key.get()); VectorClock clock = (VectorClock) value.getVersion(); insert.setBytes(2, clock.toBytes()); insert.setBytes(3, value.getValue()); insert.executeUpdate(); doCommit = true; } catch (SQLException e) { if (e.getErrorCode() == MYSQL_ERR_DUP_KEY || e.getErrorCode() == MYSQL_ERR_DUP_ENTRY) { throw new ObsoleteVersionException("Key or value already used."); } else { throw new PersistenceFailureException("Fix me!", e); } } finally { if (conn != null) { try { if (doCommit) conn.commit(); else conn.rollback(); } catch (SQLException e) { } } tryClose(results); tryClose(insert); tryClose(select); tryClose(conn); } }
@Override @Test public void testPutVersioned() { VectorClock vc = new VectorClock(); vc.incrementVersion(this.nodeId, System.currentTimeMillis()); VectorClock initialVC = vc.clone(); client.put("k", new Versioned<String>("v", vc)); Versioned<String> v = client.get("k"); assertEquals("GET should return the version set by PUT.", "v", v.getValue()); VectorClock expected = initialVC.clone(); expected.incrementVersion(this.nodeId, System.currentTimeMillis()); assertEquals( "The version should be incremented after a put.", expected.getEntries(), ((VectorClock) v.getVersion()).getEntries()); try { client.put("k", new Versioned<String>("v", initialVC)); fail("Put of obsolete version should throw exception."); } catch (ObsoleteVersionException e) { // this is good } // PUT of a concurrent version should succeed client.put( "k", new Versioned<String>( "v2", new VectorClock().incremented(nodeId + 1, time.getMilliseconds()))); assertEquals("GET should return the new value set by PUT.", "v2", client.getValue("k")); assertEquals( "GET should return the new version set by PUT.", expected.incremented(nodeId + 1, time.getMilliseconds()), client.get("k").getVersion()); }
@Override @Test public void testFetchedEqualsPut() throws Exception { System.out.println(" Testing Fetchhed equals put "); ByteArray key = getKey(); Store<ByteArray, byte[], byte[]> store = getStore(); VectorClock clock = getClock(1, 1, 2, 3, 3, 4); byte[] value = getValue(); System.out.println("Value chosen : " + value); List<Versioned<byte[]>> resultList = store.get(key, null); assertNotNull("Null result list obtained from a get request", resultList); assertEquals("Store not empty at start!", 0, resultList.size()); Versioned<byte[]> versioned = new Versioned<byte[]>(value, clock); store.put(key, versioned, null); List<Versioned<byte[]>> found = store.get(key, null); assertEquals("Should only be one version stored.", 1, found.size()); System.out.println("individual bytes"); System.out.println("input"); printBytes(versioned.getValue()); System.out.println("found"); printBytes(found.get(0).getValue()); assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue())); }
@Override protected Versioned<Slop> computeNext() { try { Versioned<Slop> head = null; if (!shutDown) { head = slopQueue.take(); if (head.equals(END)) { shutDown = true; isComplete = true; } else { slopsDone++; if (slopsDone % voldemortConfig.getSlopBatchSize() == 0) { shutDown = true; } writeThrottler.maybeThrottle(writtenLast); writtenLast = slopSize(head); deleteBatch.add(Pair.create(head.getValue().makeKey(), head.getVersion())); return head; } } return endOfData(); } catch (Exception e) { logger.error("Got an exception " + e); return endOfData(); } }
/** * @param keyBytes: keyName strings serialized as bytes eg. 'cluster.xml' * @return List of values (only 1 for Metadata) versioned byte[] eg. UTF bytes for cluster xml * definitions * @throws VoldemortException */ public List<Versioned<byte[]>> get(ByteArray keyBytes) throws VoldemortException { try { String key = ByteUtils.getString(keyBytes.get(), "UTF-8"); if (METADATA_KEYS.contains(key)) { List<Versioned<byte[]>> values = Lists.newArrayList(); // Get the cached value and convert to string Versioned<String> value = convertObjectToString(key, metadataCache.get(key)); values.add( new Versioned<byte[]>( ByteUtils.getBytes(value.getValue(), "UTF-8"), value.getVersion())); return values; } else { throw new VoldemortException("Unhandled Key:" + key + " for MetadataStore get()"); } } catch (Exception e) { throw new VoldemortException( "Failed to read metadata key:" + ByteUtils.getString(keyBytes.get(), "UTF-8") + " delete config/.temp config/.version directories and restart.", e); } }
public VoldemortOperation makePutOperation(String key, Versioned<byte[]> versioned) { return new VoldemortOperation( VoldemortOpCode.PUT_OP_CODE, key, versioned.getValue(), (VectorClock) versioned.getVersion()); }
public List<Version> getVersions(ByteArray key) { List<Versioned<byte[]>> values = get(key); List<Version> versions = new ArrayList<Version>(values.size()); for (Versioned<?> value : values) { versions.add(value.getVersion()); } return versions; }
@Override public V getValue(K key, V defaultValue) { Versioned<V> retVal = get(key); if (retVal == null) { return defaultValue; } else { return retVal.getValue(); } }
private static void printVersioned(Versioned<Object> v) { if (v == null) { System.out.println("null"); } else { System.out.print(v.getVersion()); System.out.print(": "); printObject(v.getValue()); System.out.println(); } }
private void checkValues(Versioned<byte[]> value, List<Versioned<byte[]>> list, ByteArray key) { assertEquals("should return exactly one value ", 1, list.size()); assertEquals( "should return the last saved version", value.getVersion(), list.get(0).getVersion()); assertEquals( "should return the last saved value (key:" + ByteUtils.getString(key.get(), "UTF-8") + ")", new String(value.getValue()), new String(list.get(0).getValue())); }
/** * A write through put to inner-store. * * @param keyBytes: keyName strings serialized as bytes eg. 'cluster.xml' * @param valueBytes: versioned byte[] eg. UTF bytes for cluster xml definitions * @throws VoldemortException */ public synchronized void put(ByteArray keyBytes, Versioned<byte[]> valueBytes) throws VoldemortException { String key = ByteUtils.getString(keyBytes.get(), "UTF-8"); Versioned<String> value = new Versioned<String>( ByteUtils.getString(valueBytes.getValue(), "UTF-8"), valueBytes.getVersion()); Versioned<Object> valueObject = convertStringToObject(key, value); this.put(key, valueObject); }
public void put(ByteArray key, Versioned<byte[]> value) throws PersistenceFailureException { StoreUtils.assertValidKey(key); DatabaseEntry keyEntry = new DatabaseEntry(key.get()); boolean succeeded = false; Transaction transaction = null; Cursor cursor = null; try { transaction = this.environment.beginTransaction(null, null); // Check existing values // if there is a version obsoleted by this value delete it // if there is a version later than this one, throw an exception DatabaseEntry valueEntry = new DatabaseEntry(); cursor = bdbDatabase.openCursor(transaction, null); for (OperationStatus status = cursor.getSearchKey(keyEntry, valueEntry, LockMode.RMW); status == OperationStatus.SUCCESS; status = cursor.getNextDup(keyEntry, valueEntry, LockMode.RMW)) { VectorClock clock = new VectorClock(valueEntry.getData()); Occured occured = value.getVersion().compare(clock); if (occured == Occured.BEFORE) throw new ObsoleteVersionException( "Key '" + new String(hexCodec.encode(key.get())) + "' " + value.getVersion().toString() + " is obsolete," + " current version is " + clock + "."); else if (occured == Occured.AFTER) // best effort delete of obsolete previous value! cursor.delete(); } // Okay so we cleaned up all the prior stuff, so now we are good to // insert the new thing valueEntry = new DatabaseEntry(versionedSerializer.toBytes(value)); OperationStatus status = cursor.put(keyEntry, valueEntry); if (status != OperationStatus.SUCCESS) throw new PersistenceFailureException("Put operation failed with status: " + status); succeeded = true; } catch (DatabaseException e) { logger.error(e); throw new PersistenceFailureException(e); } finally { attemptClose(cursor); if (succeeded) attemptCommit(transaction); else attemptAbort(transaction); } }
public static void main(String[] args) { String bootstrapUrl = "tcp://localhost:6666"; StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl)); // create a client that executes operations on a single store StoreClient store = factory.getStoreClient("kevoree"); store.put("pompier1", "capteurs"); Versioned<String> data = store.get("pompier1"); System.out.println(data.getValue() + " " + data.getVersion()); store.delete("pompier1", data.getVersion()); }
/** * Returns the approximate size of slop to help in throttling * * @param slopVersioned The versioned slop whose size we want * @return Size in bytes */ private int slopSize(Versioned<Slop> slopVersioned) { int nBytes = 0; Slop slop = slopVersioned.getValue(); nBytes += slop.getKey().length(); nBytes += ((VectorClock) slopVersioned.getVersion()).sizeInBytes(); switch (slop.getOperation()) { case PUT: { nBytes += slop.getValue().length; break; } case DELETE: { break; } default: logger.error("Unknown slop operation: " + slop.getOperation()); } return nBytes; }
public VAdminProto.DeletePartitionEntriesResponse handleDeletePartitionEntries( VAdminProto.DeletePartitionEntriesRequest request) { VAdminProto.DeletePartitionEntriesResponse.Builder response = VAdminProto.DeletePartitionEntriesResponse.newBuilder(); ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iterator = null; try { String storeName = request.getStore(); List<Integer> partitions = request.getPartitionsList(); StorageEngine<ByteArray, byte[]> storageEngine = getStorageEngine(storeRepository, storeName); VoldemortFilter filter = (request.hasFilter()) ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter(); RoutingStrategy routingStrategy = metadataStore.getRoutingStrategy(storageEngine.getName()); EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec()); iterator = storageEngine.entries(); int deleteSuccess = 0; while (iterator.hasNext()) { Pair<ByteArray, Versioned<byte[]>> entry = iterator.next(); ByteArray key = entry.getFirst(); Versioned<byte[]> value = entry.getSecond(); throttler.maybeThrottle(key.length() + valueSize(value)); if (checkKeyBelongsToDeletePartition(key.get(), partitions, routingStrategy) && filter.accept(key, value)) { if (storageEngine.delete(key, value.getVersion())) deleteSuccess++; } } response.setCount(deleteSuccess); } catch (VoldemortException e) { response.setError(ProtoUtils.encodeError(errorCodeMapper, e)); logger.error( "handleDeletePartitionEntries failed for request(" + request.toString() + ")", e); } finally { if (null != iterator) iterator.close(); } return response.build(); }
public void simple() { StoreClient<String, Doc> client = _factory.getStoreClient("test"); Versioned<Doc> v = client.get("key"); if (v == null) { Doc d = new Doc("name", "geir"); d.add("x", 1); v = new Versioned<Doc>(d); } // update the value client.put("key", v); v = client.get("key"); System.out.println("value : " + v.getValue()); System.out.println("clock : " + v.getVersion()); }
@Test public void testGetWithBinaryData() throws Exception { Store<ByteArray, byte[], byte[]> store = getStore(); byte[] allPossibleBytes = getAllPossibleBytes(); ByteArray key = new ByteArray(allPossibleBytes); VectorClock vc = getClock(0, 0); Versioned<byte[]> versioned = new Versioned<byte[]>(allPossibleBytes, vc); store.put(key, versioned, null); List<Versioned<byte[]>> found = store.get(key, null); assertEquals("Should only be one version stored.", 1, found.size()); System.out.println("individual bytes"); System.out.println("input"); printBytes(versioned.getValue()); System.out.println("found"); printBytes(found.get(0).getValue()); assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue())); }
/** * helper function to convert strings to bytes as needed. * * @param key * @param value */ @SuppressWarnings("unchecked") public void put(String key, Versioned<Object> value) { if (METADATA_KEYS.contains(key)) { // try inserting into inner store first putInner(key, convertObjectToString(key, value)); // cache all keys if innerStore put succeeded metadataCache.put(key, value); // do special stuff if needed if (CLUSTER_KEY.equals(key)) { updateRoutingStrategies((Cluster) value.getValue(), getStoreDefList()); } else if (STORES_KEY.equals(key)) { updateRoutingStrategies(getCluster(), (List<StoreDefinition>) value.getValue()); } } else { throw new VoldemortException("Unhandled Key:" + key + " for MetadataStore put()"); } }
public void put(ByteArray key, Versioned<byte[]> versioned) throws VoldemortException { StoreUtils.assertValidKey(key); SocketAndStreams sands = pool.checkout(destination); try { requestFormat.writePutRequest( sands.getOutputStream(), name, key, versioned.getValue(), (VectorClock) versioned.getVersion(), reroute); sands.getOutputStream().flush(); requestFormat.readPutResponse(sands.getInputStream()); } catch (IOException e) { close(sands.getSocket()); throw new UnreachableStoreException( "Failure in put on " + destination + ": " + e.getMessage(), e); } finally { pool.checkin(destination, sands); } }
/** * convert Object to String depending on key. * * <p>StoreRepository takes only StorageEngine<ByteArray,byte[]> and for persistence on disk we * need to convert them to String.<br> * * @param key * @param value * @return */ private Versioned<Object> convertStringToObject(String key, Versioned<String> value) { Object valueObject = null; if (CLUSTER_KEY.equals(key)) { valueObject = clusterMapper.readCluster(new StringReader(value.getValue())); } else if (STORES_KEY.equals(key)) { valueObject = storeMapper.readStoreList(new StringReader(value.getValue())); } else if (SERVER_STATE_KEY.equals(key) || CLUSTER_STATE_KEY.equals(key)) { valueObject = VoldemortState.valueOf(value.getValue()); } else if (NODE_ID_KEY.equals(key)) { valueObject = Integer.parseInt(value.getValue()); } else if (REBALANCING_STEAL_INFO.equals(key)) { String valueString = value.getValue(); if (valueString.startsWith("[")) { valueObject = RebalancerState.create(valueString); } else { valueObject = new RebalancerState(Arrays.asList(RebalancePartitionsInfo.create(valueString))); } } else { throw new VoldemortException( "Unhandled key:'" + key + "' for String to Object serialization."); } return new Versioned<Object>(valueObject, value.getVersion()); }
@Override @Test public void testGetVersions() throws Exception { List<ByteArray> keys = getKeys(2); ByteArray key = keys.get(0); byte[] value = getValue(); VectorClock vc = getClock(0, 0); Store<ByteArray, byte[], byte[]> store = getStore(); store.put(key, Versioned.value(value, vc), null); List<Versioned<byte[]>> versioneds = store.get(key, null); List<Version> versions = store.getVersions(key); assertEquals(1, versioneds.size()); assertTrue(versions.size() > 0); for (int i = 0; i < versions.size(); i++) assertEquals(versioneds.get(0).getVersion(), versions.get(i)); assertEquals(0, store.getVersions(keys.get(1)).size()); }
public synchronized void put(String key, Versioned<String> value) throws VoldemortException { StoreUtils.assertValidKey(key); if (null == value.getValue()) { throw new VoldemortException("metadata cannot be null !!"); } // Check for obsolete version File[] files = getDirectory(key).listFiles(); for (File file : files) { if (file.getName().equals(key)) { VectorClock clock = readVersion(key); if (value.getVersion().compare(clock) == Occured.AFTER) { // continue } else if (value.getVersion().compare(clock) == Occured.BEFORE) { throw new ObsoleteVersionException( "A successor version " + clock + " to this " + value.getVersion() + " exists for key " + key); } else if (value.getVersion().compare(clock) == Occured.CONCURRENTLY) { throw new ObsoleteVersionException("Concurrent Operation not allowed on Metadata."); } } } File keyFile = new File(getDirectory(key), key); VectorClock newClock = (VectorClock) value.getVersion(); if (!keyFile.exists() || keyFile.delete()) { try { FileUtils.writeStringToFile(keyFile, value.getValue(), "UTF-8"); writeVersion(key, newClock); } catch (IOException e) { throw new VoldemortException(e); } } }
/** * Converts Object to byte[] depending on the key * * <p>StoreRepository takes only StorageEngine<ByteArray,byte[]> and for persistence on disk we * need to convert them to String.<br> * * @param key * @param value * @return */ @SuppressWarnings("unchecked") private Versioned<String> convertObjectToString(String key, Versioned<Object> value) { String valueStr = value.getValue().toString(); if (CLUSTER_KEY.equals(key)) { valueStr = clusterMapper.writeCluster((Cluster) value.getValue()); } else if (STORES_KEY.equals(key)) { valueStr = storeMapper.writeStoreList((List<StoreDefinition>) value.getValue()); } else if (REBALANCING_STEAL_INFO.equals(key)) { RebalancerState rebalancerState = (RebalancerState) value.getValue(); valueStr = rebalancerState.toJsonString(); } else if (SERVER_STATE_KEY.equals(key) || CLUSTER_STATE_KEY.equals(key) || NODE_ID_KEY.equals(key)) { valueStr = value.getValue().toString(); } else { throw new VoldemortException( "Unhandled key:'" + key + "' for Object to String serialization."); } return new Versioned<String>(valueStr, value.getVersion()); }
@Override public Version put(K key, Versioned<V> versioned) throws ObsoleteVersionException { clientStore.put(key, versioned, null); return versioned.getVersion(); }
@Override public int hashCode() { return Objects.hashCode(nodeId, key, value.getVersion()); }
public Version getVersion() { return value.getVersion(); }
/** * Sends nested multipart response. Outer multipart wraps all the keys requested. Each key has a * separate multipart for the versioned values. */ @Override public void sendResponse(StoreStats performanceStats, boolean isFromLocalZone, long startTimeInMs) throws Exception { // multiPartKeys is the outer multipart MimeMultipart multiPartKeys = new MimeMultipart(); ByteArrayOutputStream keysOutputStream = new ByteArrayOutputStream(); for (Entry<ByteArray, List<Versioned<byte[]>>> entry : versionedResponses.entrySet()) { ByteArray key = entry.getKey(); String contentLocationKey = "/" + this.storeName + "/" + new String(Base64.encodeBase64(key.get())); // Create the individual body part - for each key requested MimeBodyPart keyBody = new MimeBodyPart(); try { // Add the right headers keyBody.addHeader(CONTENT_TYPE, "application/octet-stream"); keyBody.addHeader(CONTENT_TRANSFER_ENCODING, "binary"); keyBody.addHeader(CONTENT_LOCATION, contentLocationKey); } catch (MessagingException me) { logger.error("Exception while constructing key body headers", me); keysOutputStream.close(); throw me; } // multiPartValues is the inner multipart MimeMultipart multiPartValues = new MimeMultipart(); for (Versioned<byte[]> versionedValue : entry.getValue()) { byte[] responseValue = versionedValue.getValue(); VectorClock vectorClock = (VectorClock) versionedValue.getVersion(); String eTag = RestUtils.getSerializedVectorClock(vectorClock); // Create the individual body part - for each versioned value of // a key MimeBodyPart valueBody = new MimeBodyPart(); try { // Add the right headers valueBody.addHeader(CONTENT_TYPE, "application/octet-stream"); valueBody.addHeader(CONTENT_TRANSFER_ENCODING, "binary"); valueBody.addHeader(RestMessageHeaders.X_VOLD_VECTOR_CLOCK, eTag); valueBody.setContent(responseValue, "application/octet-stream"); multiPartValues.addBodyPart(valueBody); } catch (MessagingException me) { logger.error("Exception while constructing value body part", me); keysOutputStream.close(); throw me; } } try { // Add the inner multipart as the content of the outer body part keyBody.setContent(multiPartValues); multiPartKeys.addBodyPart(keyBody); } catch (MessagingException me) { logger.error("Exception while constructing key body part", me); keysOutputStream.close(); throw me; } } try { multiPartKeys.writeTo(keysOutputStream); } catch (Exception e) { logger.error("Exception while writing mutipart to output stream", e); throw e; } ChannelBuffer responseContent = ChannelBuffers.dynamicBuffer(); responseContent.writeBytes(keysOutputStream.toByteArray()); // Create the Response object HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); // Set the right headers response.setHeader(CONTENT_TYPE, "multipart/binary"); response.setHeader(CONTENT_TRANSFER_ENCODING, "binary"); // Copy the data into the payload response.setContent(responseContent); response.setHeader(CONTENT_LENGTH, response.getContent().readableBytes()); // Write the response to the Netty Channel this.messageEvent.getChannel().write(response); if (performanceStats != null && isFromLocalZone) { recordStats(performanceStats, startTimeInMs, Tracked.GET_ALL); } keysOutputStream.close(); }
public VAdminProto.AddStoreResponse handleAddStore(VAdminProto.AddStoreRequest request) { VAdminProto.AddStoreResponse.Builder response = VAdminProto.AddStoreResponse.newBuilder(); // don't try to add a store in the middle of rebalancing if (metadataStore .getServerState() .equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER) || metadataStore .getServerState() .equals(MetadataStore.VoldemortState.REBALANCING_CLUSTER)) { response.setError( ProtoUtils.encodeError( errorCodeMapper, new VoldemortException("Rebalancing in progress"))); return response.build(); } try { // adding a store requires decoding the passed in store string StoreDefinitionsMapper mapper = new StoreDefinitionsMapper(); StoreDefinition def = mapper.readStore(new StringReader(request.getStoreDefinition())); synchronized (lock) { // only allow a single store to be created at a time. We'll see concurrent errors when // writing the // stores.xml file out otherwise. (see ConfigurationStorageEngine.put for details) if (!storeRepository.hasLocalStore(def.getName())) { // open the store storageService.openStore(def); // update stores list in metadata store (this also has the // effect of updating the stores.xml file) List<StoreDefinition> currentStoreDefs; List<Versioned<byte[]>> v = metadataStore.get(MetadataStore.STORES_KEY); if (((v.size() > 0) ? 1 : 0) > 0) { Versioned<byte[]> currentValue = v.get(0); currentStoreDefs = mapper.readStoreList( new StringReader(ByteUtils.getString(currentValue.getValue(), "UTF-8"))); } else { currentStoreDefs = Lists.newArrayList(); } currentStoreDefs.add(def); try { metadataStore.put(MetadataStore.STORES_KEY, currentStoreDefs); } catch (Exception e) { throw new VoldemortException(e); } } else { throw new StoreOperationFailureException( String.format("Store '%s' already exists on this server", def.getName())); } } } catch (VoldemortException e) { response.setError(ProtoUtils.encodeError(errorCodeMapper, e)); logger.error("handleAddStore failed for request(" + request.toString() + ")", e); } return response.build(); }
static int valueSize(Versioned<byte[]> value) { return value.getValue().length + ((VectorClock) value.getVersion()).sizeInBytes() + 1; }
@Override public boolean delete(K key) { Versioned<V> versioned = get(key); if (versioned == null) return false; return this.clientStore.delete(key, versioned.getVersion()); }