Esempio n. 1
0
  @Test
  public void testIndexerApi() throws Exception {
    LilyClient client = lilyProxy.getLilyServerProxy().getClient();

    final Repository repository = client.getRepository();

    // Create a field type and record type
    String NS = "org.lilyproject.client.test";
    TypeManager typeManager = repository.getTypeManager();
    FieldType fieldType =
        typeManager.newFieldType("STRING", new QName(NS, "indexfield"), Scope.VERSIONED);
    fieldType = typeManager.createFieldType(fieldType);

    RecordType recordType = typeManager.newRecordType(new QName(NS, "indexrt"));
    recordType.addFieldTypeEntry(fieldType.getId(), true);
    recordType = typeManager.createRecordType(recordType);

    // Create a record
    Record record = repository.newRecord();
    record.setId(repository.getIdGenerator().newRecordId("indexrecord"));
    record.setRecordType(recordType.getName());
    record.setField(fieldType.getName(), "value");
    repository.create(record);

    // explicitly index the record (if this succeeds, the test succeeded to verify that we can
    // access the indexer through lily-client)
    client.getIndexer().index(record.getId());
  }
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    repoSetup.setupCore();
    repoSetup.setupRepository();

    repositoryManager = repoSetup.getRepositoryManager();
    repository = repoSetup.getRepositoryManager().getRepository(Table.RECORD.name);

    TypeManager typeManager = repository.getTypeManager();
    typeManager.createFieldType("STRING", new QName("ns", "stringField"), Scope.NON_VERSIONED);
    typeManager.createFieldType(
        "LIST<STRING>", new QName("ns", "stringListField"), Scope.NON_VERSIONED);
    typeManager.createFieldType("LONG", new QName("ns", "longField"), Scope.NON_VERSIONED);
    typeManager
        .recordTypeBuilder()
        .defaultNamespace("ns")
        .name("rt")
        .fieldEntry()
        .name("stringField")
        .add()
        .fieldEntry()
        .name("stringListField")
        .add()
        .fieldEntry()
        .name("longField")
        .add()
        .create();
  }
Esempio n. 3
0
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    lilyProxy = new LilyProxy();

    InputStream is = BatchBuildTest.class.getResourceAsStream("solrschema.xml");
    byte[] solrSchema = IOUtils.toByteArray(is);
    IOUtils.closeQuietly(is);

    lilyProxy.start(solrSchema);

    solrProxy = lilyProxy.getSolrProxy();
    solrServer = solrProxy.getSolrServer();
    lilyServerProxy = lilyProxy.getLilyServerProxy();
    lilyClient = lilyServerProxy.getClient();
    repository = lilyClient.getRepository();

    typeManager = repository.getTypeManager();
    FieldType ft1 =
        typeManager.createFieldType(
            "STRING", new QName("batchindex-test", "field1"), Scope.NON_VERSIONED);
    FieldType ft2 =
        typeManager.createFieldType(
            "LINK", new QName("batchindex-test", "linkField"), Scope.NON_VERSIONED);
    typeManager
        .recordTypeBuilder()
        .defaultNamespace("batchindex-test")
        .name("rt1")
        .fieldEntry()
        .use(ft1)
        .add()
        .fieldEntry()
        .use(ft2)
        .add()
        .create();

    model = lilyServerProxy.getIndexerModel();

    is = BatchBuildTest.class.getResourceAsStream("indexerconf.xml");
    byte[] indexerConfiguration = IOUtils.toByteArray(is);
    IOUtils.closeQuietly(is);

    IndexDefinition index = model.newIndex(INDEX_NAME);
    Map<String, String> solrShards = new HashMap<String, String>();
    solrShards.put("shard1", "http://localhost:8983/solr");
    index.setSolrShards(solrShards);
    index.setConfiguration(indexerConfiguration);
    index.setUpdateState(IndexUpdateState.DO_NOT_SUBSCRIBE);
    model.addIndex(index);
  }
Esempio n. 4
0
  public Record read(RecordId recordId, Long version, QName... fieldNames)
      throws RepositoryException, InterruptedException {
    FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
    List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);

    return read(recordId, version, fields, fieldTypes);
  }
Esempio n. 5
0
  public List<Record> readVersions(RecordId recordId, List<Long> versions, QName... fieldNames)
      throws RepositoryException, InterruptedException {
    ArgumentValidator.notNull(recordId, "recordId");
    ArgumentValidator.notNull(versions, "versions");

    if (versions.isEmpty()) return new ArrayList<Record>();

    Collections.sort(versions);

    FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
    List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);

    Long lowestRequestedVersion = versions.get(0);
    Long highestRequestedVersion = versions.get(versions.size() - 1);
    int numberOfVersionsToRetrieve = (int) (highestRequestedVersion - lowestRequestedVersion + 1);
    Result result = getRow(recordId, highestRequestedVersion, numberOfVersionsToRetrieve, fields);
    Long latestVersion = recdec.getLatestVersion(result);

    // Drop the versions that are higher than the latestVersion
    List<Long> validVersions = new ArrayList<Long>();
    for (Long version : versions) {
      if (version > latestVersion) break;
      validVersions.add(version);
    }
    return recdec.decodeRecords(recordId, validVersions, result, fieldTypes);
  }
Esempio n. 6
0
 @Override
 public BlobAccess getBlob(RecordId recordId, Long version, QName fieldName, int... indexes)
     throws RepositoryException, InterruptedException {
   Record record = read(recordId, version, fieldName);
   FieldType fieldType = typeManager.getFieldTypeByName(fieldName);
   return blobManager.getBlobAccess(record, fieldName, fieldType, indexes);
 }
Esempio n. 7
0
  public List<Record> readVersions(
      RecordId recordId, Long fromVersion, Long toVersion, QName... fieldNames)
      throws RepositoryException, InterruptedException {
    ArgumentValidator.notNull(recordId, "recordId");
    ArgumentValidator.notNull(fromVersion, "fromVersion");
    ArgumentValidator.notNull(toVersion, "toVersion");
    if (fromVersion > toVersion) {
      throw new IllegalArgumentException(
          "fromVersion '"
              + fromVersion
              + "' must be smaller or equal to toVersion '"
              + toVersion
              + "'");
    }

    FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
    List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);

    int numberOfVersionsToRetrieve = (int) (toVersion - fromVersion + 1);
    Result result = getRow(recordId, toVersion, numberOfVersionsToRetrieve, fields);
    if (fromVersion < 1L) fromVersion = 1L; // Put the fromVersion to a sensible value
    Long latestVersion = recdec.getLatestVersion(result);
    if (latestVersion < toVersion)
      toVersion = latestVersion; // Limit the toVersion to the highest possible version
    List<Long> versionsToRead = new ArrayList<Long>();
    for (long version = fromVersion; version <= toVersion; version++) {
      versionsToRead.add(version);
    }
    return recdec.decodeRecords(recordId, versionsToRead, result, fieldTypes);
  }
Esempio n. 8
0
  public IdRecord readWithIds(RecordId recordId, Long version, List<SchemaId> fieldIds)
      throws RepositoryException, InterruptedException {
    FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
    List<FieldType> fields = getFieldTypesFromIds(fieldIds, fieldTypes);

    return readWithIds(recordId, version, fields, fieldTypes);
  }
Esempio n. 9
0
  @Test
  public void testClearDerefMap() throws Exception {
    DerefMap derefMap =
        DerefMapHbaseImpl.create(
            INDEX_NAME, lilyProxy.getHBaseProxy().getConf(), null, repository.getIdGenerator());

    Record linkedRecord =
        repository
            .recordBuilder()
            .id("deref-test-linkedrecord")
            .recordType(rt1.getName())
            .field(ft1.getName(), "deref test linkedrecord")
            .create();

    Record record =
        repository
            .recordBuilder()
            .id("deref-test-main")
            .recordType(rt1.getName())
            .field(ft1.getName(), "deref test main")
            .field(ft2.getName(), new Link(linkedRecord.getId()))
            .create();

    SchemaId vtag = typeManager.getFieldTypeByName(VersionTag.LAST).getId();
    DependantRecordIdsIterator it = null;

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(!it.hasNext());
    } finally {
      it.close();
    }

    setBatchIndexConf(
        getResourceAsByteArray("batchIndexConf-testClearDerefmap-false.json"), null, false);

    buildAndCommit();

    QueryResponse response = solrServer.query(new SolrQuery("field1:deref\\ test\\ main"));
    assertEquals(1, response.getResults().size());

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(it.hasNext());
    } finally {
      it.close();
    }

    setBatchIndexConf(
        null, getResourceAsByteArray("batchIndexConf-testClearDerefmap-true.json"), true);
    waitForIndexAndCommit(BUILD_TIMEOUT);

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(!it.hasNext());
    } finally {
      it.close();
    }
  }
 public SolrInputDocument build() throws InterruptedException, RepositoryException {
   solrDoc.setField("lily.id", recordId.toString());
   solrDoc.setField("lily.table", table);
   solrDoc.setField("lily.key", key);
   solrDoc.setField("lily.vtagId", vtag.toString());
   solrDoc.setField("lily.vtag", typeManager.getFieldTypeById(vtag).getName().getName());
   solrDoc.setField("lily.version", version);
   return solrDoc;
 }
Esempio n. 11
0
  private FieldType getFieldType(QName qname)
      throws IndexerConfException, InterruptedException, RepositoryException {

    if (systemFields.isSystemField(qname)) {
      return systemFields.get(qname);
    }

    return typeManager.getFieldTypeByName(qname);
  }
Esempio n. 12
0
  /** Test scanners: scanners work directly on HBase, also remotely. */
  @Test
  public void testScanners() throws Exception {
    LilyClient client = lilyProxy.getLilyServerProxy().getClient();

    // Obtain a repository
    Repository repository = client.getRepository();
    IdGenerator idGenerator = repository.getIdGenerator();

    String NS = "org.lilyproject.client.test";

    // Create a field type and record type
    TypeManager typeManager = repository.getTypeManager();
    FieldType fieldType =
        typeManager.newFieldType("STRING", new QName(NS, "scanfield"), Scope.VERSIONED);
    fieldType = typeManager.createFieldType(fieldType);

    RecordType recordType = typeManager.newRecordType(new QName(NS, "scanrt"));
    recordType.addFieldTypeEntry(fieldType.getId(), true);
    recordType = typeManager.createRecordType(recordType);

    // Create some records
    for (int i = 0; i < 10; i++) {
      Record record = repository.newRecord();
      record.setId(repository.getIdGenerator().newRecordId("A" + i));
      record.setRecordType(new QName(NS, "scanrt"));
      record.setField(new QName(NS, "scanfield"), "value " + i);
      repository.create(record);
    }

    // Do a scan
    RecordScan scan = new RecordScan();
    scan.setStartRecordId(idGenerator.newRecordId("A"));
    scan.setStopRecordId(idGenerator.newRecordId("B"));

    RecordScanner scanner = repository.getScanner(scan);
    int i = 0;
    while (scanner.next() != null) {
      i++;
    }

    assertEquals("Number of scanned records", 10, i);
  }
Esempio n. 13
0
  /**
   * Creates a record with a blob using a Repository obtained via LilyClient. This verifies that the
   * blobs work remotely and it was able to retrieve the blob stores config from ZooKeeper.
   */
  @Test
  public void testBlob() throws Exception {
    LilyClient client = lilyProxy.getLilyServerProxy().getClient();

    // Obtain a repository
    Repository repository = client.getRepository();

    String NS = "org.lilyproject.client.test";

    // Create a blob field type and record type
    TypeManager typeManager = repository.getTypeManager();
    ValueType blobType = typeManager.getValueType("BLOB");
    FieldType blobFieldType =
        typeManager.newFieldType(blobType, new QName(NS, "data"), Scope.VERSIONED);
    blobFieldType = typeManager.createFieldType(blobFieldType);

    RecordType recordType = typeManager.newRecordType(new QName(NS, "file"));
    recordType.addFieldTypeEntry(blobFieldType.getId(), true);
    recordType = typeManager.createRecordType(recordType);

    // Upload a blob that, based upon the current default config, should end up in HBase
    //  (> 5000 bytes and < 200000 bytes)
    byte[] data = makeBlobData(10000);
    Blob blob = new Blob("application/octet-stream", (long) data.length, null);
    OutputStream blobStream = repository.getOutputStream(blob);
    IOUtils.copy(new ByteArrayInputStream(data), blobStream);
    blobStream.close();
    assertTrue(blob.getValue() != null);

    // Create a record with this blob
    Record record = repository.newRecord();
    record.setRecordType(new QName(NS, "file"));
    record.setField(new QName(NS, "data"), blob);
    record = repository.create(record);
  }
Esempio n. 14
0
  private Set<SchemaId> parseVersionTags(String vtagsSpec)
      throws IndexerConfException, InterruptedException {
    Set<SchemaId> vtags = new HashSet<SchemaId>();

    if (vtagsSpec == null) return vtags;

    for (String tag : COMMA_SPLITTER.split(vtagsSpec)) {
      try {
        vtags.add(typeManager.getFieldTypeByName(VersionTag.qname(tag)).getId());
      } catch (FieldTypeNotFoundException e) {
        throw new IndexerConfException("unknown vtag used in indexer configuration: " + tag);
      } catch (RepositoryException e) {
        throw new IndexerConfException("error loading field type for vtag: " + tag, e);
      }
    }

    return vtags;
  }
Esempio n. 15
0
 public ListValueType(TypeManager typeManager, String typeParams)
     throws RepositoryException, InterruptedException {
   ArgumentValidator.notNull(typeParams, "typeParams");
   this.fullName = NAME + "<" + typeParams + ">";
   this.valueType = typeManager.getValueType(typeParams);
 }
Esempio n. 16
0
 @Before
 public void setup() throws Exception {
   this.ft1 = typeManager.getFieldTypeByName(new QName("batchindex-test", "field1"));
   this.ft2 = typeManager.getFieldTypeByName(new QName("batchindex-test", "linkField"));
   this.rt1 = typeManager.getRecordTypeByName(new QName("batchindex-test", "rt1"), null);
 }
Esempio n. 17
0
 @Override
 public InputStream getInputStream(Record record, QName fieldName, int... indexes)
     throws RepositoryException, InterruptedException {
   FieldType fieldType = typeManager.getFieldTypeByName(fieldName);
   return blobManager.getBlobAccess(record, fieldName, fieldType, indexes).getInputStream();
 }
  @Test
  public void testOne() throws Exception {
    LilyClient client = lilyProxy.getLilyServerProxy().getClient();

    //
    // Create some records
    //
    Repository repository = client.getRepository();
    TypeManager typeManager = repository.getTypeManager();
    IdGenerator idGenerator = repository.getIdGenerator();

    FieldType ft1 =
        typeManager.createFieldType("STRING", new QName("test", "field1"), Scope.NON_VERSIONED);
    FieldType ft2 =
        typeManager.createFieldType("LINK", new QName("test", "field2"), Scope.NON_VERSIONED);

    RecordType rt1 =
        typeManager
            .recordTypeBuilder()
            .defaultNamespace("test")
            .name("rt1")
            .fieldEntry()
            .use(ft1)
            .add()
            .fieldEntry()
            .use(ft2)
            .add()
            .create();

    for (int i = 0; i < 300; i++) {
      repository
          .recordBuilder()
          .recordType(rt1.getName())
          .field(ft1.getName(), "foo bar bar")
          .field(ft2.getName(), new Link(idGenerator.newRecordId()))
          .create();
    }

    Assert.assertTrue(
        "Processing messages took too long", lilyProxy.waitSepEventsProcessed(60000L));

    //
    // Count number of records in each region
    //
    for (String tableName : TABLE_NAMES) {
      HTable table = new HTable(lilyProxy.getHBaseProxy().getConf(), tableName);
      for (HRegionInfo regionInfo : table.getRegionsInfo().keySet()) {
        Scan scan = new Scan();
        scan.setStartRow(regionInfo.getStartKey());
        scan.setStopRow(regionInfo.getEndKey());

        ResultScanner scanner = table.getScanner(scan);
        int count = 0;
        for (Result result : scanner) {
          // System.out.println("result = " + Arrays.toString(result.getRow()));
          count++;
        }

        assertTrue(
            String.format(
                "Number of records in region '%s' is %d, expected between 60 and 140, "
                    + "start key is '%s', end key is '%s'",
                regionInfo.getRegionNameAsString(),
                count,
                Bytes.toStringBinary(regionInfo.getStartKey()),
                Bytes.toStringBinary(regionInfo.getEndKey())),
            count >= 60 && count <= 140);
      }
    }
  }
Esempio n. 19
0
  private ResultScanner createHBaseResultScanner(RecordScan scan)
      throws RepositoryException, InterruptedException {
    Scan hbaseScan = new Scan();

    hbaseScan.setMaxVersions(1);

    if (scan.getRawStartRecordId() != null) {
      hbaseScan.setStartRow(scan.getRawStartRecordId());
    } else if (scan.getStartRecordId() != null) {
      hbaseScan.setStartRow(scan.getStartRecordId().toBytes());
    }

    if (scan.getRawStopRecordId() != null) {
      hbaseScan.setStopRow(scan.getRawStopRecordId());
    } else if (scan.getStopRecordId() != null) {
      hbaseScan.setStopRow(scan.getStopRecordId().toBytes());
    }

    // Filters
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);

    // filter out deleted records
    filterList.addFilter(REAL_RECORDS_FILTER);

    // add user's filter
    if (scan.getRecordFilter() != null) {
      Filter filter = filterFactory.createHBaseFilter(scan.getRecordFilter(), this, filterFactory);
      filterList.addFilter(filter);
    }

    hbaseScan.setFilter(filterList);

    hbaseScan.setCaching(scan.getCaching());

    hbaseScan.setCacheBlocks(scan.getCacheBlocks());

    ReturnFields returnFields = scan.getReturnFields();
    if (returnFields != null && returnFields.getType() != ReturnFields.Type.ALL) {
      RecordDecoder.addSystemColumnsToScan(hbaseScan);
      switch (returnFields.getType()) {
        case ENUM:
          for (QName field : returnFields.getFields()) {
            FieldTypeImpl fieldType = (FieldTypeImpl) typeManager.getFieldTypeByName(field);
            hbaseScan.addColumn(RecordCf.DATA.bytes, fieldType.getQualifier());
          }
          break;
        case NONE:
          // nothing to add
          break;
        default:
          throw new RuntimeException("Unrecognized ReturnFields type: " + returnFields.getType());
      }
    } else {
      hbaseScan.addFamily(RecordCf.DATA.bytes);
    }

    ResultScanner hbaseScanner;
    try {
      hbaseScanner = recordTable.getScanner(hbaseScan);
    } catch (IOException e) {
      throw new RecordException("Error creating scanner", e);
    }
    return hbaseScanner;
  }