/** @param args */
public static void main(String[] args) throws IOException {
// File filepor = new File("D:\\Company_File\\log4j-1216\\Java2\\batchImport.log");
// if (filepor.exists()) {
// filepor.delete();// 删除日志文件
// }
// filepor = null;
PropertyConfigurator.configure(
"F:\\work\\WorkSpace_Eclipse\\WorkSpace_Eclipse\\MassPick\\WebContent\\WEB-INF\\log4j.properties");
long da = System.currentTimeMillis();
File file = new File("G:\\Data\\十二月\\中国裁判文书网最新文书（每日下载）\\HTML\\HTML-zgcpwsw20151214-20151217");
// 查PG省市县/区
Bucket bucket = null;
bucket = connectionBucket(bucket);
AdministrationUtils util = new AdministrationUtils();
util.initData(); // 查询行政区
try {
show(file, bucket, util);
} catch (Exception e) {
logger.error(e.getMessage());
} finally {
// file = null;
util = null;
bucket.close();
cluster2 = null;
}
logger.info(count + ":数量");
logger.info("所有文件总耗时" + (((System.currentTimeMillis() - da) / 1000) / 60) + "分钟");
}
public static boolean flushCleanup(Bucket bucket, long start) {
for (long i = 0; i <= 999999999; i++) {
String id = new String("__flush_marker_" + i);
JsonObject content = JsonObject.empty().put("data", "temp");
JsonDocument doc = JsonDocument.create(id, content);
bucket.upsert(doc);
bucket.remove(id);
}
return true;
}
@Test
public final void givenRandomPerson_whenCreate_thenPersonPersisted() {
// create person
Person person = randomPerson();
personService.create(person);
// check results
assertNotNull(person.getId());
assertNotNull(bucket.get(person.getId()));
// cleanup
bucket.remove(person.getId());
}
@BeforeClass
public static void setUpClass() {
// Connect
bucket = BucketFactory.getBucket();
// Clean
try {
bucket.remove("count::dmaier");
bucket.remove("count::dostrovsky");
} catch (DocumentDoesNotExistException e) {
LOG.warning("Could not remove a nonexistent document");
}
}
/**
* To delete a design document, if the design document was not existent also true is returned
*
* @param designDoc
* @return
*/
public static boolean deleteDesignDoc(String designDoc) {
if (designDocExists()) {
return client.bucketManager().removeDesignDocument(designDoc);
}
return true;
}
public void run(Statement statement) {
try {
QueryResult result = bucket.query(statement);
printout(result, false);
} catch (Exception e) {
LOGGER.error("Error while issuing statement " + demoName(), e);
}
}
// issue a Query
public void run(Query q, boolean shouldPause) {
try {
QueryResult result = bucket.query(q);
printout(result, shouldPause);
} catch (Exception e) {
LOGGER.error("Error while issuing " + demoName(), e);
}
}
@After
public void deleteDoc() {
try {
bucket.remove(uuid);
} catch (DocumentDoesNotExistException e) {
// ignore
}
}
@Test
public final void givenRandomPerson_whenDelete_thenPersonNotInBucket() {
// create and insert person document
String id = insertRandomPersonDocument().id();
// delete person and check results
personService.delete(id);
assertNull(bucket.get(id));
}
@Test
public void testConstructorWithEmptyCollectionOverwrites() {
JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test", "test2"));
bucket.upsert(preExisting);
Map<String, Object> map =
new CouchbaseMap<Object>(uuid, bucket, Collections.<String, Object>emptyMap());
assertEquals(0, map.size());
}
@Test
public final void givenId_whenRead_thenReturnsPerson() {
// create and insert person document
String id = insertRandomPersonDocument().id();
// read person and check results
assertNotNull(personService.read(id));
// cleanup
bucket.remove(id);
}
@Test
public final void givenNewHometown_whenUpdate_thenNewHometownPersisted() {
// create and insert person document
JsonDocument doc = insertRandomPersonDocument();
// update person
Person expected = converter.fromDocument(doc);
String updatedHomeTown = RandomStringUtils.randomAlphabetic(12);
expected.setHomeTown(updatedHomeTown);
personService.update(expected);
// check results
JsonDocument actual = bucket.get(expected.getId());
assertNotNull(actual);
assertNotNull(actual.content());
assertEquals(expected.getHomeTown(), actual.content().getString("homeTown"));
// cleanup
bucket.remove(expected.getId());
}
@Test
public void testConstructorWithPreExistingDocument() {
JsonDocument preExisting =
JsonDocument.create(uuid, JsonObject.create().put("test", 123).put("foo", "bar"));
bucket.upsert(preExisting);
Map<String, Object> map = new CouchbaseMap<Object>(uuid, bucket);
assertEquals(2, map.size());
assertTrue(map.containsKey("foo"));
assertTrue(map.containsValue(123));
}
@Test
public void testConstructorWithCollectionDataOverwrites() {
JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test", "test2"));
bucket.upsert(preExisting);
Map<String, Object> map =
new CouchbaseMap<Object>(uuid, bucket, Collections.singletonMap("foo", "bar"));
assertEquals(1, map.size());
assertTrue(map.containsKey("foo"));
assertEquals("bar", map.get("foo"));
}
@Test
public final void givenPersons_whenInsertBulk_thenPersonsAreInserted() {
// create some persons
List<Person> persons = new ArrayList<>();
for (int i = 0; i < 5; i++) {
persons.add(randomPerson());
}
// perform bulk insert
personService.createBulk(persons);
// check results
for (Person person : persons) {
assertNotNull(bucket.get(person.getId()));
}
// cleanup
for (Person person : persons) {
bucket.remove(person.getId());
}
}
@Test
public void testConstructorWithPreExistingDocumentOfWrongTypeFails() {
JsonArrayDocument preExisting = JsonArrayDocument.create(uuid, JsonArray.from("test"));
bucket.upsert(preExisting);
Map<String, Object> map = new CouchbaseMap<Object>(uuid, bucket);
try {
map.size();
fail("Expected TranscodingException");
} catch (TranscodingException e) {
// expected
}
}
@Test
public final void givenPersons_whenUpdateBulk_thenPersonsAreUpdated() {
List<String> ids = new ArrayList<>();
// add some person documents
for (int i = 0; i < 5; i++) {
ids.add(insertRandomPersonDocument().id());
}
// load persons from Couchbase
List<Person> persons = new ArrayList<>();
for (String id : ids) {
persons.add(converter.fromDocument(bucket.get(id)));
}
// modify persons
for (Person person : persons) {
person.setHomeTown(RandomStringUtils.randomAlphabetic(10));
}
// perform bulk update
personService.updateBulk(persons);
// check results
for (Person person : persons) {
JsonDocument doc = bucket.get(person.getId());
assertEquals(person.getName(), doc.content().getString("name"));
assertEquals(person.getHomeTown(), doc.content().getString("homeTown"));
}
// cleanup
for (String id : ids) {
bucket.remove(id);
}
}
/**
* To create a view
*
* @param designDocName
* @param viewDefs
*/
public static void createViews(String designDocName, List<ViewDef> viewDefs) {
// Check if the Design document is available, otherwise create it
if (!designDocExists(designDocName)) {
List<View> views = new ArrayList<>();
for (View view : viewDefs) {
views.add(view);
}
DesignDocument designDoc = DesignDocument.create(designDocName, views);
client.bucketManager().insertDesignDocument(designDoc);
}
}
@Test
public void givenIds_whenDeleteBulk_thenPersonsAreDeleted() {
List<String> ids = new ArrayList<>();
// add some person documents
for (int i = 0; i < 5; i++) {
ids.add(insertRandomPersonDocument().id());
}
// perform bulk delete
personService.deleteBulk(ids);
// check results
for (String id : ids) {
assertNull(bucket.get(id));
}
}
/**
* Queries all documents of a view with an optional range parameter
*
* @param designDocName
* @param viewName
* @param startKey
* @param endKey
* @return
*/
public static ViewResult query(
String designDocName, String viewName, String startKey, String endKey) {
ViewResult result;
// Perform the query
ViewQuery query = ViewQuery.from(designDocName, viewName).inclusiveEnd(true).stale(Stale.FALSE);
if (startKey != null) {
query = query.startKey(startKey);
}
if (endKey != null) {
query = query.endKey(endKey);
}
result = client.query(query);
return result;
}
@Test
public final void givenIds_whenReadBulk_thenReturnsOnlyPersonsWithMatchingIds() {
List<String> ids = new ArrayList<>();
// add some person documents
for (int i = 0; i < 5; i++) {
ids.add(insertRandomPersonDocument().id());
}
// perform bulk read
List<Person> persons = personService.readBulk(ids);
// check results
for (Person person : persons) {
assertTrue(ids.contains(person.getId()));
}
// cleanup
for (String id : ids) {
bucket.remove(id);
}
}
/** 裁判文书 抓取word，HTML修改court桶 */
public static boolean updateJsonData(
List<ArchivesVO> list, Bucket bucket, AdministrationUtils util) throws Exception {
if (null == list || list.size() <= 0) {
return false;
}
// util.initData(); // 查询行政区
String[] array = null;
JsonDocument doc = null;
JsonObject obj2 = null;
com.google.gson.JsonObject json = null;
Gson gson = new Gson();
ArchivesVO archs = null;
try {
for (ArchivesVO arch : list) {
SUM++;
// 查询数据
doc = JsonDocument.create(arch.getUuid()); // 获取ID
obj2 = bucket.get(doc) == null ? null : bucket.get(doc).content();
if (obj2 == null) {
logger.info("匹配不到UUID:" + arch.getUuid());
continue;
}
archs = new ArchivesVO();
json = gson.fromJson(obj2.toString(), com.google.gson.JsonObject.class);
archs = gson.fromJson(json, ArchivesVO.class);
if (null != arch.getTitle() && !"".equals(arch.getTitle())) {
archs.setTitle(arch.getTitle());
}
if (null != obj2.get("title") && !"".equals(obj2.get("title"))) {
archs.setTitle(obj2.get("title").toString()); // 标题
}
if (null != arch.getCaseNum() && !"".equals(arch.getCaseNum())) {
archs.setCaseNum(arch.getCaseNum());
}
if (null != obj2.get("caseNum") && !"".equals(obj2.get("caseNum"))) {
archs.setCaseNum(obj2.get("caseNum").toString()); // 案号
}
if (null != arch.getCourtName() && !"".equals(arch.getCourtName())) {
archs.setCourtName(arch.getCourtName());
}
if (null != obj2.get("courtName") && !"".equals(obj2.get("courtName"))) {
archs.setCourtName(obj2.get("courtName").toString()); // 法院名
}
if (null != arch.getCatalog() && !"".equals(arch.getCatalog())) {
archs.setCatalog(arch.getCatalog());
}
if (null != obj2.get("catalog") && !"".equals(obj2.get("catalog"))) {
archs.setCatalog(obj2.get("catalog").toString()); // 分类
}
if (null != arch.getApproval() && !"".equals(arch.getApproval())) {
archs.setApproval(arch.getApproval());
}
if (null != obj2.get("approval") && !"".equals(obj2.get("approval"))) {
archs.setApproval(obj2.get("approval").toString()); // 审批结果
}
if (null != arch.getCaseCause() && !"".equals(arch.getCaseCause())) {
archs.setCaseCause(arch.getCaseCause());
}
if (null != obj2.get("caseCause") && !"".equals(obj2.get("caseCause"))) {
archs.setCaseCause(obj2.get("caseCause").toString()); // 案由
}
if (null != arch.getPlaintiff() && !"".equals(arch.getPlaintiff())) {
archs.setPlaintiff(arch.getPlaintiff());
}
if (null != obj2.get("plaintiff") && !"".equals(obj2.get("plaintiff"))) {
archs.setPlaintiff(obj2.get("plaintiff").toString()); // 原告
}
if (null != arch.getDefendant() && !"".equals(arch.getDefendant())) {
archs.setDefendant(arch.getDefendant());
}
if (null != obj2.get("defendant") && !"".equals(obj2.get("defendant"))) {
archs.setDefendant(obj2.get("defendant").toString()); // 被告
}
if (null != arch.getApprovalDate() && !"".equals(arch.getApprovalDate())) {
archs.setApprovalDate(arch.getApprovalDate());
}
if (null != obj2.get("approvalDate") && !"".equals(obj2.get("approvalDate"))) {
archs.setApprovalDate(obj2.get("approvalDate").toString()); // 审结日期
}
if (null != arch.getSummary() && !"".equals(arch.getSummary())) {
archs.setSummary(arch.getSummary());
}
if (null != obj2.get("summary") && !"".equals(obj2.get("summary"))) {
archs.setSummary(obj2.get("summary").toString()); // 摘要
}
if (null != obj2.get("detailLink") && !"".equals(obj2.get("detailLink"))) {
archs.setDetailLink(obj2.get("detailLink").toString()); // url
}
if (null != obj2.get("publishDate") && !"".equals(obj2.get("publishDate"))) {
archs.setPublishDate(getReplaceAllDate(obj2.get("publishDate").toString())); // 发布日期
}
if (null != obj2.get("province") && !"".equals(obj2.get("province"))) {
archs.setProvince(obj2.get("province").toString()); // 省
}
if (null != obj2.get("city") && !"".equals(obj2.get("city"))) {
archs.setCity(obj2.get("city").toString()); // 市
}
if (null != obj2.get("area") && !"".equals(obj2.get("area"))) {
archs.setArea(obj2.get("area").toString()); // 县
}
if (null != archs.getCourtName() && !"".equals(archs.getCourtName())) {
array = util.utils(arch.getCourtName());
}
if (null != obj2.get("courtName") && !"".equals(obj2.get("courtName"))) {
array = util.utils(obj2.get("courtName").toString());
}
if (null != array) {
if (null != array[0] && !"".equals(array[0])) {
archs.setProvince(array[0]);
}
if (null != array[1] && !"".equals(array[1])) {
archs.setCity(array[1]);
}
if (null != array[2] && !"".equals(array[2])) {
archs.setArea(array[2]);
}
}
if (null != obj2.get("collectDate") && !"".equals(obj2.get("collectDate"))) {
archs.setCollectDate(getReplaceAllDate(obj2.get("collectDate").toString())); // 采集时间
}
if (null != obj2.get("suitDate") && !"".equals(obj2.get("suitDate"))) {
archs.setSuitDate(obj2.get("suitDate").toString()); // 起诉日期
}
String jsonss = gson.toJson(archs);
doc = JsonDocument.create(arch.getUuid(), JsonObject.fromJson(jsonss));
logger.info("更新条数：" + SUM + "---省：" + array[0] + "---市：" + array[1] + "---县/区：" + array[2]);
bucket.upsert(doc);
}
} catch (Exception e) {
logger.error(e.getMessage());
return false;
} finally {
array = null;
gson = null;
json = null;
archs = null;
obj2 = null;
doc = null;
}
return true;
}
public void closeBucket(Bucket bucket) {
bucket.close();
}
public static boolean entireBucketFlush(Bucket bucket) {
bucket.bucketManager().flush();
return true;
}
public static boolean singleDocCleanup(Bucket bucket, String id) {
bucket.remove(id);
return true;
}
public static boolean docCleanup(Bucket bucket, long start) {
for (long i = start; i <= 999999999; i++) {
if (bucket.get(Long.toString(i)) != null) bucket.remove(Long.toString(i));
}
return true;
}
/**
* Check if the design document is accessible
*
* @param name
* @return
*/
public static boolean designDocExists(String name) {
DesignDocument designDoc = null;
designDoc = client.bucketManager().getDesignDocument(name);
return designDoc != null;
}
private JsonDocument insertRandomPersonDocument() {
Person expected = randomPersonWithId();
JsonDocument doc = converter.toDocument(expected);
return bucket.insert(doc);
}