@Test
@Ignore
public void preFetchShopUrlsFromTable() {
int offset = 0;
EntityManagerFactory emf = Persistence.createEntityManagerFactory("ShopsPU");
EntityManager em = emf.createEntityManager();
List<Shops> models =
null; // where sh.placetypes like '%clothing%' and sh.shopurl is not null and sh.shopurl <>
// ''
// EmailScraper emailFinder = new EmailScraper();
while ((models = getIterableModels(em, offset, Shops.class)).size() > 0) {
em.getTransaction().begin();
for (Shops model : models) {
Logger.getLogger(NewEmptyJUnitTest.class.getName())
.log(Level.INFO, "Url: " + model.getShopurl());
crawlForEmailAddresses(model.getShopurl());
}
em.flush();
em.clear();
em.getTransaction().commit();
offset += models.size();
}
}
