public List<Person> all() { try { // Java Scan ScanPolicy policy = new ScanPolicy(); policy.concurrentNodes = true; policy.priority = Priority.LOW; policy.includeBinData = true; final List<Person> persons = new ArrayList<Person>(); client.scanAll( policy, "test", "demo", new ScanCallback() { @Override public void scanCallback(Key key, Record record) throws AerospikeException { persons.add(getPerson(key, record)); } }); return persons; } catch (AerospikeException e) { System.out.println("EXCEPTION - Message: " + e.getMessage()); return new ArrayList<Person>(); } } //
@Override public int delete(String table, String key) { try { as.delete(writePolicy, new Key(NAMESPACE, SET, key)); return OK; } catch (AerospikeException e) { return RESULT_CODE_MAPPER.get(e.getResultCode()); } }
@Override public int read( String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) { try { if (fields != null) { as.get(policy, new Key(NAMESPACE, SET, key), fields.toArray(new String[fields.size()])); return OK; } else { as.get(policy, new Key(NAMESPACE, SET, key)); return OK; } } catch (AerospikeException e) { return RESULT_CODE_MAPPER.get(e.getResultCode()); } }
@Override public int update(String table, String key, HashMap<String, ByteIterator> values) { Bin[] bins = new Bin[values.size()]; int index = 0; for (Map.Entry<String, ByteIterator> entry : values.entrySet()) { bins[index] = new Bin(entry.getKey(), entry.getValue().toArray()); index++; } try { as.put(writePolicy, new Key(NAMESPACE, SET, key), bins); return OK; } catch (AerospikeException e) { return e.getResultCode(); // return RESULT_CODE_MAPPER.get(e.getResultCode()); } }
public void scanAllTweetsForAllUsers() { try { // Java Scan ScanPolicy policy = new ScanPolicy(); policy.concurrentNodes = true; policy.priority = Priority.LOW; policy.includeBinData = true; client.scanAll( policy, "test", "tweets", new ScanCallback() { public void scanCallback(Key key, Record record) throws AerospikeException { console.printf(record.getValue("tweet") + "\n"); } }, "tweet"); } catch (AerospikeException e) { System.out.println("EXCEPTION - Message: " + e.getMessage()); System.out.println("EXCEPTION - StackTrace: " + UtilityService.printStackTrace(e)); } } // scanAllTweetsForAllUsers
private boolean processLine() { log.debug("processing File:line " + Utils.getFileName(fileName) + this.lineNumber); bins = new ArrayList<Bin>(); boolean lineProcessed = false; long errorTotal = 0; try { if (columns.size() != counters.write.colTotal) { if (columns.size() < counters.write.colTotal) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Number of column mismatch:Columns in data file is less than number of column in config file."); } else { throw new ParseException(lineNumber); } } // retrieve set name first for (ColumnDefinition metadataColumn : this.metadataMapping) { if (metadataColumn.staticValue && metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.SET)) { this.set = metadataColumn.binValueHeader; } else { String metadataRawText = this.columns.get(metadataColumn.getBinValuePos()); if (metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.SET)) { if (this.set == null) { this.set = metadataRawText; } } } } // use set name to create key for (ColumnDefinition metadataColumn : this.metadataMapping) { if (metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.KEY)) { String metadataRawText = this.columns.get(metadataColumn.getBinValuePos()); if (metadataColumn.getSrcType() == SrcColumnType.INTEGER) { Long integer = Long.parseLong(metadataRawText); this.key = new Key(this.nameSpace, this.set, integer); } else { this.key = new Key(this.nameSpace, this.set, metadataRawText); } } } for (ColumnDefinition binColumn : this.binMapping) { Bin bin = null; if (!binColumn.staticName) { binColumn.binNameHeader = this.columns.get(binColumn.binNamePos); } if (!binColumn.staticValue) { String binRawText = null; if (binColumn.binValueHeader != null && binColumn.binValueHeader.toLowerCase().equals(Constants.SYSTEM_TIME)) { SimpleDateFormat sdf = new SimpleDateFormat(binColumn.getEncoding()); // dd/MM/yyyy Date now = new Date(); binRawText = sdf.format(now); } else { binRawText = this.columns.get(binColumn.getBinValuePos()); } if (binRawText.equals("")) continue; switch (binColumn.getSrcType()) { case INTEGER: // Server stores all integer type data in 64bit so use long Long integer; try { integer = Long.parseLong(binRawText); bin = new Bin(binColumn.getBinNameHeader(), integer); } catch (Exception pi) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Integer/Long Parse Error:" + pi); } break; case FLOAT: /** Floating type data can be stored as 8 byte byte array. */ try { float binfloat = Float.parseFloat(binRawText); byte[] byteFloat = ByteBuffer.allocate(8).putFloat(binfloat).array(); bin = new Bin(binColumn.getBinNameHeader(), byteFloat); } catch (Exception e) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Floating number Parse Error:" + e); } break; case STRING: bin = new Bin(binColumn.getBinNameHeader(), binRawText); break; case BLOB: if (binColumn.getDstType().equals(DstColumnType.BLOB)) { if (binColumn.encoding.equalsIgnoreCase(Constants.HEX_ENCODING)) bin = new Bin(binColumn.getBinNameHeader(), this.toByteArray(binRawText)); // TODO } break; case LIST: /* * Assumptions * 1. Items are separated by a colon ',' * 2. Item value will be a string * 3. List will be in double quotes * * No support for nested maps or nested lists * */ List<String> list = new ArrayList<String>(); String[] listValues = binRawText.split(Constants.LIST_DELEMITER, -1); if (listValues.length > 0) { for (String value : listValues) { list.add(value.trim()); } bin = Bin.asList(binColumn.getBinNameHeader(), list); } else { bin = null; log.error("Error: Cannot parse to a list: " + binRawText); } break; case MAP: /* * Asumptions: * 1. Items are separated by a colon ',' * 2. Name value pairs are separated by equals ':' * 3. Map key is a string * 4. Map value will be a string * 5. Map will be in double quotes * * No support for nested maps or nested lists * */ Map<String, Object> map = new HashMap<String, Object>(); String[] mapValues = binRawText.split(Constants.MAP_DELEMITER, -1); if (mapValues.length > 0) { for (String value : mapValues) { String[] kv = value.split(Constants.MAPKEY_DELEMITER); if (kv.length != 2) log.error("Error: Cannot parse map <k,v> using: " + kv); else map.put(kv[0].trim(), kv[1].trim()); } log.debug(map.toString()); bin = Bin.asMap(binColumn.getBinNameHeader(), map); } else { bin = null; log.error("Error: Cannot parse to a map: " + binRawText); } break; case JSON: try { log.debug(binRawText); if (jsonParser == null) jsonParser = new JSONParser(); Object obj = jsonParser.parse(binRawText); if (obj instanceof JSONArray) { JSONArray jsonArray = (JSONArray) obj; bin = Bin.asList(binColumn.getBinNameHeader(), jsonArray); } else { JSONObject jsonObj = (JSONObject) obj; bin = Bin.asMap(binColumn.getBinNameHeader(), jsonObj); } } catch (ParseException e) { log.error("Failed to parse JSON", e); } break; case TIMESTAMP: if (binColumn.getDstType().equals(DstColumnType.INTEGER)) { DateFormat format = new SimpleDateFormat(binColumn.getEncoding()); try { Date formatDate = format.parse(binRawText); long miliSecondForDate = formatDate.getTime() - timeZoneOffset; if (binColumn.getEncoding().contains(".SSS") && binColumn.binValueHeader.toLowerCase().equals(Constants.SYSTEM_TIME)) { // We need time in miliseconds so no need to change it to seconds } else { miliSecondForDate = miliSecondForDate / 1000; } bin = new Bin(binColumn.getBinNameHeader(), miliSecondForDate); log.trace("Date format:" + binRawText + " in seconds:" + miliSecondForDate); } catch (java.text.ParseException e) { e.printStackTrace(); } } else if (binColumn.getDstType().equals(DstColumnType.STRING)) { bin = new Bin(binColumn.getBinNameHeader(), binRawText); } break; default: } } else { bin = new Bin(binColumn.getBinNameHeader(), binColumn.getBinValueHeader()); } if (bin != null) { bins.add(bin); } } lineProcessed = true; log.trace( "Formed key and bins for line " + lineNumber + " Key: " + this.key.userKey + " Bins:" + this.bins.toString()); } catch (AerospikeException ae) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Aerospike Bin processing Error:" + ae.getResultCode()); if (log.isDebugEnabled()) { ae.printStackTrace(); } counters.write.processingErrors.getAndIncrement(); counters.write.recordProcessed.addAndGet(this.lineSize); errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + counters.write.processingErrors.get()); if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) { System.exit(-1); } } catch (ParseException pe) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Parsing Error:" + pe); if (log.isDebugEnabled()) { pe.printStackTrace(); } counters.write.processingErrors.getAndIncrement(); counters.write.recordProcessed.addAndGet(this.lineSize); errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + counters.write.processingErrors.get()); if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) { System.exit(-1); } } catch (Exception e) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Unknown Error:" + e); if (log.isDebugEnabled()) { e.printStackTrace(); } counters.write.processingErrors.getAndIncrement(); counters.write.recordProcessed.addAndGet(this.lineSize); errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + counters.write.processingErrors.get()); if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) { System.exit(-1); } } return lineProcessed; }
/** * writes a record to the Aerospike Cluster * * @throws AerospikeException */ private int writeToAs() { int value = 0; long errorTotal = 0; if (this.client != null) { try { if (!bins.isEmpty()) { this.client.put(this.writePolicy, this.key, this.bins.toArray(new Bin[bins.size()])); counters.write.recordProcessed.addAndGet(this.lineSize); counters.write.writeCount.getAndIncrement(); log.trace("Wrote line " + lineNumber + " Key: " + this.key.userKey + " to Aerospike"); value = 1; } else { log.trace("No bins to insert"); } } catch (AerospikeException ae) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + "Aerospike Write Error:" + ae.getResultCode()); if (log.isDebugEnabled()) { ae.printStackTrace(); } counters.write.recordProcessed.addAndGet(this.lineSize); counters.write.writeErrors.getAndIncrement(); errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + counters.write.processingErrors.get()); switch (ae.getResultCode()) { case ResultCode.TIMEOUT: counters.write.writeTimeouts.getAndIncrement(); break; case ResultCode.KEY_EXISTS_ERROR: counters.write.writeKeyExists.getAndIncrement(); break; default: } if ((this.abortErrorCount != 0 && this.abortErrorCount < errorTotal)) { System.exit(-1); } } catch (Exception e) { log.error( "File:" + Utils.getFileName(this.fileName) + " Line:" + lineNumber + " Write Error:" + e); if (log.isDebugEnabled()) { e.printStackTrace(); } counters.write.recordProcessed.addAndGet(this.lineSize); counters.write.writeErrors.getAndIncrement(); errorTotal = (counters.write.readErrors.get() + counters.write.writeErrors.get() + counters.write.processingErrors.get()); if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) { System.exit(-1); } } } return value; }
/** * Do the main operation in emulate Mode: (1) Write a new URL Site Visit Record to the base DB (2) * Check to see if the corresponding User Record is in the Segmented Cache -- If not, write a new * User Record with the CACHE TTL -- UPDATE THE USER RECORD WITH A NEW (FULL) LDT Load (Scan the * existing LDT in the base DB and write to the cache LDT) (3) Probe the Base DB to see if the LDT * data is consistent. * * @param opNum */ private void doOperation(int opNum, ILdtOperations ldtOps) { final String meth = "doOperation()"; boolean recordPresent = false; String baseSet; String cacheSet; try { int customerSeed = random.nextInt(this.customerMax); CustomerRecord custRec = new CustomerRecord(console, customerSeed); long userSeed = getUserRecordSeed(this.userMax); UserRecord userRec = new UserRecord(console, dbOps, custRec.getCustomerID(), (int) userSeed); SiteVisitEntry sve = new SiteVisitEntry( console, custRec.getCustomerID(), userRec.getUserID(), opNum, LDT_BIN, this.timeToLive); baseSet = userRec.getCustomerBaseSet(); cacheSet = userRec.getCustomerCacheSet(); // Write the Site Visit to Storage -- which is hidden behind // this interface because there can be multiple implementations // of the LDT. sve.toStorage(client, baseNamespace, baseSet, ldtOps); // Check to see if the UserRecord is in the Segment Cache. If it is, // then add to the Cache LDT. If it is not, then create a new // User Record in the Segment, and populate the LDT Info (the Site // Visit Data) with the LDT data from the DB User Record. recordPresent = userRec.updateCache(client, cacheNamespace); if (recordPresent) { sve.toStorage(client, cacheNamespace, cacheSet, ldtOps); } else { sve.reloadCache(client, baseNamespace, cacheNamespace, ldtOps); } String keyStr = userRec.getUserID(); // At predetermined milestones, perform various actions // Show Simple Status at regular internals. For the regular large // scale tests with 100 threads, we won't hit this very often. if ((opNum + threadNumber) % 1000 == 0) { console.debug( "<%s:%s> Thread(%d) Cust#(%d) BaseSet(%s) User#(%d) UserID(%s) Iteration(%d)", CLASSNAME, meth, threadNumber, customerSeed, baseSet, userSeed, keyStr, opNum); } // Do a heavy duty scan less frequently. if ((opNum + threadNumber) % 2000 == 0) { Key baseKey = new Key(baseNamespace, baseSet, keyStr); Key cacheKey = new Key(cacheNamespace, cacheSet, keyStr); console.debug( "<%s:%s> <<SCAN TEST>> Thread(%d) Cust#(%d) BaseSet(%s) CacheSet(%s) User#(%d) UserID(%s) Iteration(%d)", CLASSNAME, meth, threadNumber, customerSeed, baseSet, cacheSet, userSeed, keyStr, opNum); List<Map<String, Object>> baseResult = null; List<Map<String, Object>> cacheResult = null; int baseResultSize = 0; int baseCheckSize = 0; int cacheResultSize = 0; int cacheCheckSize = 0; try { baseResult = ldtOps.scanLDT(baseKey); cacheResult = ldtOps.scanLDT(cacheKey); cacheCheckSize = ldtOps.ldtSize(cacheKey, LDT_BIN); if (baseResult != null) { baseResultSize = baseResult.size(); baseCheckSize = ldtOps.ldtSize(baseKey, LDT_BIN); if (baseResultSize != baseCheckSize) { console.error( "<%s:%s> <<BASE SCAN Size Error>> Thread(%d) Cust#(%d) BaseSet(%s) UserID(%s) ScanSide(%d) LDT Size(%d)", CLASSNAME, meth, threadNumber, customerSeed, baseSet, keyStr, baseResultSize, baseCheckSize); } } if (cacheResult != null) { cacheResultSize = cacheResult.size(); cacheCheckSize = ldtOps.ldtSize(cacheKey, LDT_BIN); if (cacheResultSize != cacheCheckSize) { console.error( "<%s:%s> <<CACHE SCAN Size Error>> Thread(%d) Cust#(%d) CacheSet(%s) UserID(%s) ScanSide(%d) LDT Size(%d)", CLASSNAME, meth, threadNumber, customerSeed, cacheSet, keyStr, cacheResultSize, cacheCheckSize); } } } catch (AerospikeException ae) { console.error( "Aerospike Error Code(%d) Error Message(%s)", ae.getResultCode(), ae.getMessage()); console.info("Keep on Truckin"); } console.debug( "<%s:%s> <<SCAN RESULT>> Thread(%d) Cust#(%d) BaseSet(%s) CacheSet(%s) UserID(%s) BaseLDT(%d) CacheLDT(%d)", CLASSNAME, meth, threadNumber, customerSeed, baseSet, cacheSet, keyStr, baseResultSize, cacheResultSize); } } catch (AerospikeException ae) { console.error( "Aerospike Error Code(%d) Error Message(%s)", ae.getResultCode(), ae.getMessage()); } catch (Exception e) { e.printStackTrace(); console.error( "[%s] Problem with Thread(%d) Customer Record: Seed(%d)", "Emulate: doOperation(): ", threadNumber, opNum); } }