Exemplo n.º 1
0
  /** @param args */
  public static void main(String[] args) throws IOException {

    //		File filepor = new File("D:\\Company_File\\log4j-1216\\Java2\\batchImport.log");
    //		if (filepor.exists()) {
    //			filepor.delete();// 删除日志文件
    //		}
    //		filepor = null;
    PropertyConfigurator.configure(
        "F:\\work\\WorkSpace_Eclipse\\WorkSpace_Eclipse\\MassPick\\WebContent\\WEB-INF\\log4j.properties");
    long da = System.currentTimeMillis();
    File file = new File("G:\\Data\\十二月\\中国裁判文书网最新文书(每日下载)\\HTML\\HTML-zgcpwsw20151214-20151217");
    // 查PG省市县/区
    Bucket bucket = null;
    bucket = connectionBucket(bucket);
    AdministrationUtils util = new AdministrationUtils();
    util.initData(); // 查询行政区
    try {
      show(file, bucket, util);
    } catch (Exception e) {
      logger.error(e.getMessage());
    } finally {
      //			file = null;
      util = null;
      bucket.close();
      cluster2 = null;
    }
    logger.info(count + ":数量");
    logger.info("所有文件总耗时" + (((System.currentTimeMillis() - da) / 1000) / 60) + "分钟");
  }
  public static boolean flushCleanup(Bucket bucket, long start) {
    for (long i = 0; i <= 999999999; i++) {
      String id = new String("__flush_marker_" + i);
      JsonObject content = JsonObject.empty().put("data", "temp");
      JsonDocument doc = JsonDocument.create(id, content);
      bucket.upsert(doc);

      bucket.remove(id);
    }

    return true;
  }
  @Test
  public final void givenRandomPerson_whenCreate_thenPersonPersisted() {
    // create person
    Person person = randomPerson();
    personService.create(person);

    // check results
    assertNotNull(person.getId());
    assertNotNull(bucket.get(person.getId()));

    // cleanup
    bucket.remove(person.getId());
  }
  @BeforeClass
  public static void setUpClass() {

    // Connect
    bucket = BucketFactory.getBucket();

    // Clean
    try {
      bucket.remove("count::dmaier");
      bucket.remove("count::dostrovsky");
    } catch (DocumentDoesNotExistException e) {
      LOG.warning("Could not remove a nonexistent document");
    }
  }
Exemplo n.º 5
0
  /**
   * To delete a design document, if the design document was not existent also true is returned
   *
   * @param designDoc
   * @return
   */
  public static boolean deleteDesignDoc(String designDoc) {
    if (designDocExists()) {
      return client.bucketManager().removeDesignDocument(designDoc);
    }

    return true;
  }
 public void run(Statement statement) {
   try {
     QueryResult result = bucket.query(statement);
     printout(result, false);
   } catch (Exception e) {
     LOGGER.error("Error while issuing statement " + demoName(), e);
   }
 }
 // issue a Query
 public void run(Query q, boolean shouldPause) {
   try {
     QueryResult result = bucket.query(q);
     printout(result, shouldPause);
   } catch (Exception e) {
     LOGGER.error("Error while issuing " + demoName(), e);
   }
 }
 @After
 public void deleteDoc() {
   try {
     bucket.remove(uuid);
   } catch (DocumentDoesNotExistException e) {
     // ignore
   }
 }
  @Test
  public final void givenRandomPerson_whenDelete_thenPersonNotInBucket() {
    // create and insert person document
    String id = insertRandomPersonDocument().id();

    // delete person and check results
    personService.delete(id);
    assertNull(bucket.get(id));
  }
  @Test
  public void testConstructorWithEmptyCollectionOverwrites() {
    JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test", "test2"));
    bucket.upsert(preExisting);

    Map<String, Object> map =
        new CouchbaseMap<Object>(uuid, bucket, Collections.<String, Object>emptyMap());

    assertEquals(0, map.size());
  }
  @Test
  public final void givenId_whenRead_thenReturnsPerson() {
    // create and insert person document
    String id = insertRandomPersonDocument().id();

    // read person and check results
    assertNotNull(personService.read(id));

    // cleanup
    bucket.remove(id);
  }
  @Test
  public final void givenNewHometown_whenUpdate_thenNewHometownPersisted() {
    // create and insert person document
    JsonDocument doc = insertRandomPersonDocument();

    // update person
    Person expected = converter.fromDocument(doc);
    String updatedHomeTown = RandomStringUtils.randomAlphabetic(12);
    expected.setHomeTown(updatedHomeTown);
    personService.update(expected);

    // check results
    JsonDocument actual = bucket.get(expected.getId());
    assertNotNull(actual);
    assertNotNull(actual.content());
    assertEquals(expected.getHomeTown(), actual.content().getString("homeTown"));

    // cleanup
    bucket.remove(expected.getId());
  }
  @Test
  public void testConstructorWithPreExistingDocument() {
    JsonDocument preExisting =
        JsonDocument.create(uuid, JsonObject.create().put("test", 123).put("foo", "bar"));
    bucket.upsert(preExisting);

    Map<String, Object> map = new CouchbaseMap<Object>(uuid, bucket);

    assertEquals(2, map.size());
    assertTrue(map.containsKey("foo"));
    assertTrue(map.containsValue(123));
  }
  @Test
  public void testConstructorWithCollectionDataOverwrites() {
    JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test", "test2"));
    bucket.upsert(preExisting);

    Map<String, Object> map =
        new CouchbaseMap<Object>(uuid, bucket, Collections.singletonMap("foo", "bar"));

    assertEquals(1, map.size());
    assertTrue(map.containsKey("foo"));
    assertEquals("bar", map.get("foo"));
  }
  @Test
  public final void givenPersons_whenInsertBulk_thenPersonsAreInserted() {

    // create some persons
    List<Person> persons = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
      persons.add(randomPerson());
    }

    // perform bulk insert
    personService.createBulk(persons);

    // check results
    for (Person person : persons) {
      assertNotNull(bucket.get(person.getId()));
    }

    // cleanup
    for (Person person : persons) {
      bucket.remove(person.getId());
    }
  }
  @Test
  public void testConstructorWithPreExistingDocumentOfWrongTypeFails() {
    JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test"));
    bucket.upsert(preExisting);

    Map<String, Object> map = new CouchbaseMap<Object>(uuid, bucket);
    try {
      map.size();
      fail("Expected TranscodingException");
    } catch (TranscodingException e) {
      // expected
    }
  }
  @Test
  public final void givenPersons_whenUpdateBulk_thenPersonsAreUpdated() {

    List<String> ids = new ArrayList<>();

    // add some person documents
    for (int i = 0; i < 5; i++) {
      ids.add(insertRandomPersonDocument().id());
    }

    // load persons from Couchbase
    List<Person> persons = new ArrayList<>();
    for (String id : ids) {
      persons.add(converter.fromDocument(bucket.get(id)));
    }

    // modify persons
    for (Person person : persons) {
      person.setHomeTown(RandomStringUtils.randomAlphabetic(10));
    }

    // perform bulk update
    personService.updateBulk(persons);

    // check results
    for (Person person : persons) {
      JsonDocument doc = bucket.get(person.getId());
      assertEquals(person.getName(), doc.content().getString("name"));
      assertEquals(person.getHomeTown(), doc.content().getString("homeTown"));
    }

    // cleanup
    for (String id : ids) {
      bucket.remove(id);
    }
  }
Exemplo n.º 18
0
  /**
   * To create a view
   *
   * @param designDocName
   * @param viewDefs
   */
  public static void createViews(String designDocName, List<ViewDef> viewDefs) {
    // Check if the Design document is available, otherwise create it
    if (!designDocExists(designDocName)) {

      List<View> views = new ArrayList<>();

      for (View view : viewDefs) {

        views.add(view);
      }

      DesignDocument designDoc = DesignDocument.create(designDocName, views);

      client.bucketManager().insertDesignDocument(designDoc);
    }
  }
  @Test
  public void givenIds_whenDeleteBulk_thenPersonsAreDeleted() {

    List<String> ids = new ArrayList<>();

    // add some person documents
    for (int i = 0; i < 5; i++) {
      ids.add(insertRandomPersonDocument().id());
    }

    // perform bulk delete
    personService.deleteBulk(ids);

    // check results
    for (String id : ids) {
      assertNull(bucket.get(id));
    }
  }
Exemplo n.º 20
0
  /**
   * Queries all documents of a view with an optional range parameter
   *
   * @param designDocName
   * @param viewName
   * @param startKey
   * @param endKey
   * @return
   */
  public static ViewResult query(
      String designDocName, String viewName, String startKey, String endKey) {
    ViewResult result;

    // Perform the query
    ViewQuery query = ViewQuery.from(designDocName, viewName).inclusiveEnd(true).stale(Stale.FALSE);

    if (startKey != null) {
      query = query.startKey(startKey);
    }

    if (endKey != null) {
      query = query.endKey(endKey);
    }

    result = client.query(query);

    return result;
  }
  @Test
  public final void givenIds_whenReadBulk_thenReturnsOnlyPersonsWithMatchingIds() {
    List<String> ids = new ArrayList<>();

    // add some person documents
    for (int i = 0; i < 5; i++) {
      ids.add(insertRandomPersonDocument().id());
    }

    // perform bulk read
    List<Person> persons = personService.readBulk(ids);

    // check results
    for (Person person : persons) {
      assertTrue(ids.contains(person.getId()));
    }

    // cleanup
    for (String id : ids) {
      bucket.remove(id);
    }
  }
Exemplo n.º 22
0
  /** 裁判文书 抓取word,HTML修改court桶 */
  public static boolean updateJsonData(
      List<ArchivesVO> list, Bucket bucket, AdministrationUtils util) throws Exception {
    if (null == list || list.size() <= 0) {
      return false;
    }
    //		util.initData(); // 查询行政区
    String[] array = null;
    JsonDocument doc = null;
    JsonObject obj2 = null;
    com.google.gson.JsonObject json = null;
    Gson gson = new Gson();
    ArchivesVO archs = null;
    try {
      for (ArchivesVO arch : list) {
        SUM++;
        // 查询数据
        doc = JsonDocument.create(arch.getUuid()); // 获取ID
        obj2 = bucket.get(doc) == null ? null : bucket.get(doc).content();
        if (obj2 == null) {
          logger.info("匹配不到UUID:" + arch.getUuid());
          continue;
        }
        archs = new ArchivesVO();
        json = gson.fromJson(obj2.toString(), com.google.gson.JsonObject.class);
        archs = gson.fromJson(json, ArchivesVO.class);

        if (null != arch.getTitle() && !"".equals(arch.getTitle())) {
          archs.setTitle(arch.getTitle());
        }
        if (null != obj2.get("title") && !"".equals(obj2.get("title"))) {
          archs.setTitle(obj2.get("title").toString()); // 标题
        }
        if (null != arch.getCaseNum() && !"".equals(arch.getCaseNum())) {
          archs.setCaseNum(arch.getCaseNum());
        }
        if (null != obj2.get("caseNum") && !"".equals(obj2.get("caseNum"))) {
          archs.setCaseNum(obj2.get("caseNum").toString()); // 案号
        }
        if (null != arch.getCourtName() && !"".equals(arch.getCourtName())) {
          archs.setCourtName(arch.getCourtName());
        }
        if (null != obj2.get("courtName") && !"".equals(obj2.get("courtName"))) {
          archs.setCourtName(obj2.get("courtName").toString()); // 法院名
        }
        if (null != arch.getCatalog() && !"".equals(arch.getCatalog())) {
          archs.setCatalog(arch.getCatalog());
        }
        if (null != obj2.get("catalog") && !"".equals(obj2.get("catalog"))) {
          archs.setCatalog(obj2.get("catalog").toString()); // 分类
        }
        if (null != arch.getApproval() && !"".equals(arch.getApproval())) {
          archs.setApproval(arch.getApproval());
        }
        if (null != obj2.get("approval") && !"".equals(obj2.get("approval"))) {
          archs.setApproval(obj2.get("approval").toString()); // 审批结果
        }
        if (null != arch.getCaseCause() && !"".equals(arch.getCaseCause())) {
          archs.setCaseCause(arch.getCaseCause());
        }
        if (null != obj2.get("caseCause") && !"".equals(obj2.get("caseCause"))) {
          archs.setCaseCause(obj2.get("caseCause").toString()); // 案由
        }
        if (null != arch.getPlaintiff() && !"".equals(arch.getPlaintiff())) {
          archs.setPlaintiff(arch.getPlaintiff());
        }
        if (null != obj2.get("plaintiff") && !"".equals(obj2.get("plaintiff"))) {
          archs.setPlaintiff(obj2.get("plaintiff").toString()); // 原告
        }

        if (null != arch.getDefendant() && !"".equals(arch.getDefendant())) {
          archs.setDefendant(arch.getDefendant());
        }
        if (null != obj2.get("defendant") && !"".equals(obj2.get("defendant"))) {
          archs.setDefendant(obj2.get("defendant").toString()); // 被告
        }

        if (null != arch.getApprovalDate() && !"".equals(arch.getApprovalDate())) {
          archs.setApprovalDate(arch.getApprovalDate());
        }
        if (null != obj2.get("approvalDate") && !"".equals(obj2.get("approvalDate"))) {
          archs.setApprovalDate(obj2.get("approvalDate").toString()); // 审结日期
        }

        if (null != arch.getSummary() && !"".equals(arch.getSummary())) {
          archs.setSummary(arch.getSummary());
        }
        if (null != obj2.get("summary") && !"".equals(obj2.get("summary"))) {
          archs.setSummary(obj2.get("summary").toString()); // 摘要
        }

        if (null != obj2.get("detailLink") && !"".equals(obj2.get("detailLink"))) {
          archs.setDetailLink(obj2.get("detailLink").toString()); // url
        }

        if (null != obj2.get("publishDate") && !"".equals(obj2.get("publishDate"))) {
          archs.setPublishDate(getReplaceAllDate(obj2.get("publishDate").toString())); // 发布日期
        }

        if (null != obj2.get("province") && !"".equals(obj2.get("province"))) {
          archs.setProvince(obj2.get("province").toString()); // 省
        }
        if (null != obj2.get("city") && !"".equals(obj2.get("city"))) {
          archs.setCity(obj2.get("city").toString()); // 市
        }
        if (null != obj2.get("area") && !"".equals(obj2.get("area"))) {
          archs.setArea(obj2.get("area").toString()); // 县
        }
        if (null != archs.getCourtName() && !"".equals(archs.getCourtName())) {
          array = util.utils(arch.getCourtName());
        }
        if (null != obj2.get("courtName") && !"".equals(obj2.get("courtName"))) {
          array = util.utils(obj2.get("courtName").toString());
        }
        if (null != array) {
          if (null != array[0] && !"".equals(array[0])) {
            archs.setProvince(array[0]);
          }
          if (null != array[1] && !"".equals(array[1])) {
            archs.setCity(array[1]);
          }
          if (null != array[2] && !"".equals(array[2])) {
            archs.setArea(array[2]);
          }
        }

        if (null != obj2.get("collectDate") && !"".equals(obj2.get("collectDate"))) {
          archs.setCollectDate(getReplaceAllDate(obj2.get("collectDate").toString())); // 采集时间
        }
        if (null != obj2.get("suitDate") && !"".equals(obj2.get("suitDate"))) {
          archs.setSuitDate(obj2.get("suitDate").toString()); // 起诉日期
        }
        String jsonss = gson.toJson(archs);
        doc = JsonDocument.create(arch.getUuid(), JsonObject.fromJson(jsonss));
        logger.info("更新条数:" + SUM + "---省:" + array[0] + "---市:" + array[1] + "---县/区:" + array[2]);
        bucket.upsert(doc);
      }
    } catch (Exception e) {
      logger.error(e.getMessage());
      return false;
    } finally {
      array = null;
      gson = null;
      json = null;
      archs = null;
      obj2 = null;
      doc = null;
    }
    return true;
  }
 public void closeBucket(Bucket bucket) {
   bucket.close();
 }
Exemplo n.º 24
0
 public static boolean entireBucketFlush(Bucket bucket) {
   bucket.bucketManager().flush();
   return true;
 }
Exemplo n.º 25
0
 public static boolean singleDocCleanup(Bucket bucket, String id) {
   bucket.remove(id);
   return true;
 }
Exemplo n.º 26
0
 public static boolean docCleanup(Bucket bucket, long start) {
   for (long i = start; i <= 999999999; i++) {
     if (bucket.get(Long.toString(i)) != null) bucket.remove(Long.toString(i));
   }
   return true;
 }
Exemplo n.º 27
0
 /**
  * Check if the design document is accessible
  *
  * @param name
  * @return
  */
 public static boolean designDocExists(String name) {
   DesignDocument designDoc = null;
   designDoc = client.bucketManager().getDesignDocument(name);
   return designDoc != null;
 }
 private JsonDocument insertRandomPersonDocument() {
   Person expected = randomPersonWithId();
   JsonDocument doc = converter.toDocument(expected);
   return bucket.insert(doc);
 }