private Company toCompany(final Row row) {
final Company company = new Company();
final Person person = new Person(row.getString("first_name"), row.getString("last_name"));
company.setPerson(person);
company.setAddress(row.getString("address"));
company.setBrandOpinions(row.getMap("brand_opinions", String.class, String.class));
return company;
}
private static IndexMetadata build(ColumnMetadata column, Row row) {
if (row == null) return null;
String type = row.getString(INDEX_TYPE);
if (type == null) return null;
IndexMetadata index = new IndexMetadata(column, type, row.getString(INDEX_NAME));
return index;
}
private static void checkExecuteResultSet(ResultSet rs, String key) {
assertTrue(!rs.isExhausted());
Row row = rs.one();
assertTrue(rs.isExhausted());
assertEquals(key, row.getString("k"));
assertEquals("foo", row.getString("t"));
assertEquals(42, row.getInt("i"));
assertEquals(24.03f, row.getFloat("f"), 0.1f);
}
static ColumnMetadata build(TableMetadata tm, Row row) {
try {
String name = row.getString(COLUMN_NAME);
AbstractType<?> t = TypeParser.parse(row.getString(VALIDATOR));
ColumnMetadata cm = new ColumnMetadata(tm, name, Codec.rawTypeToDataType(t), row);
tm.add(cm);
return cm;
} catch (RequestValidationException e) {
// The server will have validated the type
throw new RuntimeException(e);
}
}
@Override
public int run(SparkConf conf, CommandLine cli) throws Exception {
long startMs = System.currentTimeMillis();
conf.set("spark.ui.enabled", "false");
JavaSparkContext jsc = new JavaSparkContext(conf);
SQLContext sqlContext = new SQLContext(jsc);
long diffMs = (System.currentTimeMillis() - startMs);
System.out.println(">> took " + diffMs + " ms to create SQLContext");
Map<String, String> options = new HashMap<>();
options.put("zkhost", "localhost:9983");
options.put("collection", "ml20news");
options.put("query", "content_txt:[* TO *]");
options.put("fields", "content_txt");
DataFrame solrData = sqlContext.read().format("solr").options(options).load();
DataFrame sample = solrData.sample(false, 0.1d, 5150).select("content_txt");
List<Row> rows = sample.collectAsList();
System.out.println(">> loaded " + rows.size() + " docs to classify");
StructType schema = sample.schema();
CrossValidatorModel cvModel = CrossValidatorModel.load("ml-pipeline-model");
PipelineModel bestModel = (PipelineModel) cvModel.bestModel();
int r = 0;
startMs = System.currentTimeMillis();
for (Row next : rows) {
Row oneRow = RowFactory.create(next.getString(0));
DataFrame oneRowDF =
sqlContext.createDataFrame(Collections.<Row>singletonList(oneRow), schema);
DataFrame scored = bestModel.transform(oneRowDF);
Row scoredRow = scored.collect()[0];
String predictedLabel = scoredRow.getString(scoredRow.fieldIndex("predictedLabel"));
// an acutal app would save the predictedLabel
// System.out.println(">> for row["+r+"], model returned "+scoredRows.length+" rows,
// "+scoredRows[0]);
r++;
}
diffMs = (System.currentTimeMillis() - startMs);
System.out.println(">> took " + diffMs + " ms to score " + rows.size() + " docs");
return 0;
}
@Test
public void testGetString() throws Exception {
List<Query.Type> types = Arrays.asList(Query.Type.ENUM, Query.Type.SET);
for (Query.Type type : types) {
try (Cursor cursor =
new SimpleCursor(
QueryResult.newBuilder()
.addFields(Field.newBuilder().setName("col1").setType(type).build())
.addFields(Field.newBuilder().setName("null").setType(type).build())
.addRows(
Query.Row.newBuilder()
.addLengths("val123".length())
.addLengths(-1) // SQL NULL
.setValues(ByteString.copyFromUtf8("val123")))
.build())) {
Row row = cursor.next();
Assert.assertNotNull(row);
Assert.assertEquals("val123", row.getString("col1"));
Assert.assertFalse(row.wasNull());
Assert.assertEquals(null, row.getString("null"));
Assert.assertTrue(row.wasNull());
}
}
}
@Override
public int compare(Row row1, Row row2) {
String item1 = row1.getString(0);
String item2 = row2.getString(0);
return item1.compareTo(item2);
}
public static void main(String[] args) {
for (int i = 0; i < args.length; ++i) {
if ("-endpoint".equals(args[i])) {
endpoint = args[++i];
} else if ("-keyspace".equals(args[i])) {
keyspace = args[++i];
} else if ("-table".equals(args[i])) {
table = args[++i];
} else {
System.err.println("ConsistencyTest -endpoint hostname -keyspace keyspace -table table");
}
}
if (endpoint == null || keyspace == null || table == null) {
System.err.println("ConsistencyTest -endpoint hostname -keyspace keyspace -table table");
System.exit(1);
}
String[] endpoints = endpoint.split(":");
try {
cluster =
Cluster.builder()
.addContactPoints(endpoints)
.withRetryPolicy(DefaultRetryPolicy.INSTANCE)
.withLoadBalancingPolicy(new TokenAwarePolicy(new DCAwareRoundRobinPolicy()))
.build();
session = cluster.connect(keyspace);
// work with ONE for both write and read
long s1 = System.currentTimeMillis();
Statement insert =
QueryBuilder.insertInto(table)
.value("user_id", 100)
.value("fname", "john")
.value("lname", "yamada")
.value("number", 100);
insert.setConsistencyLevel(ConsistencyLevel.ONE);
session.execute(insert);
long e1 = System.currentTimeMillis();
System.out.println("write ONE time taken (ms) : " + (e1 - s1));
long s2 = System.currentTimeMillis();
Statement select =
QueryBuilder.select().all().from(table).where(QueryBuilder.eq("user_id", 100));
select.setConsistencyLevel(ConsistencyLevel.ONE);
results = session.execute(select);
for (Row row : results) {
System.out.format(
"%d %s %s %d\n",
row.getInt("user_id"),
row.getString("fname"),
row.getString("lname"),
row.getInt("number"));
}
long e2 = System.currentTimeMillis();
System.out.println("read ONE time taken (ms) : " + (e2 - s2));
Statement delete = QueryBuilder.delete().from(table).where(QueryBuilder.eq("user_id", 100));
results = session.execute(delete);
// work with QUORUM for both write and read
long s3 = System.currentTimeMillis();
Statement insert2 =
QueryBuilder.insertInto(table)
.value("user_id", 200)
.value("fname", "john")
.value("lname", "yamada")
.value("number", 100);
insert2.setConsistencyLevel(ConsistencyLevel.QUORUM);
session.execute(insert2);
long e3 = System.currentTimeMillis();
System.out.println("write QUORUM time taken (ms) : " + (e3 - s3));
long s4 = System.currentTimeMillis();
Statement select2 =
QueryBuilder.select().all().from(table).where(QueryBuilder.eq("user_id", 200));
select2.setConsistencyLevel(ConsistencyLevel.QUORUM);
results = session.execute(select2);
for (Row row : results) {
System.out.format(
"%d %s %s %d\n",
row.getInt("user_id"),
row.getString("fname"),
row.getString("lname"),
row.getInt("number"));
}
long e4 = System.currentTimeMillis();
System.out.println("read QUORUM time taken (ms) : " + (e4 - s4));
Statement delete2 = QueryBuilder.delete().from(table).where(QueryBuilder.eq("user_id", 200));
results = session.execute(delete2);
// Clean up the connection by closing it
cluster.close();
} catch (Exception e) {
e.printStackTrace();
}
}
