public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
long all_time_bgn = gplx.core.envs.System_.Ticks();
wiki.Init_by_wiki();
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
ctx.Init(wiki, src_conn);
// merge data
int hash_len = wkr_hash.Len();
for (int i = 0; i < hash_len; ++i) {
if (prog_wkr.Canceled()) break;
Merge2_wkr wkr = (Merge2_wkr) wkr_hash.Get_at(i);
// if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
long wkr_time_bgn = gplx.core.envs.System_.Ticks();
wkr.Merge_data(ctx, prog_wkr);
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|"
+ wkr.Tbl().Tbl_name()
+ "|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
}
if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|-1|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
}
private void Init_bldr_bmks() {
if (!resume_enabled) // clear cfg entries if resume disabled; note that disabled by default;
// DATE:2014-10-24
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);
Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
String tier_id_str = bmk_hash.Get_by(Cfg_tier_id_bmk).To_str_or(null);
if (tier_id_str == null) { // bmks not found; new db;
bldr_conn.Txn_bgn("bldr__fsdb_make__bldr_conn");
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk, tier_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk, page_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk, lnki_id_bmk);
bldr_conn.Txn_end();
if (tier_id_bmk == -1) tier_id_bmk = 0;
if (page_id_bmk == -1) page_id_bmk = 0;
if (lnki_id_bmk == -1) lnki_id_bmk = 0;
} else {
if (tier_id_bmk == -1) {
tier_id_bmk = Int_.parse(tier_id_str);
usr_dlg.Note_many("", "", "restoring from bmk: tier_id=~{0}", tier_id_bmk);
}
if (page_id_bmk == -1) {
page_id_bmk = bmk_hash.Get_by(Cfg_page_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: page_id=~{0}", page_id_bmk);
}
if (lnki_id_bmk == -1) {
lnki_id_bmk = bmk_hash.Get_by(Cfg_lnki_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: lnki_id=~{0}", lnki_id_bmk);
}
}
tier_id_val = tier_id_bmk;
page_id_val = page_id_bmk;
lnki_id_val = lnki_id_bmk;
}
public void Cmd_bgn(Xob_bldr bldr) {
Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_xfer_temp_tbl.Create_table(conn);
Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
conn.Txn_bgn();
DataRdr rdr = conn.Exec_sql_as_rdr(Sql_select);
Xob_xfer_temp_itm temp_itm = new Xob_xfer_temp_itm();
Xof_img_size img_size = new Xof_img_size();
byte[] cur_ttl = Bry_.Empty;
byte cur_repo = Byte_.Max_value_127;
while (rdr.MoveNextPeer()) {
temp_itm.Clear();
temp_itm.Load(rdr);
if (Bry_.Eq(cur_ttl, temp_itm.Orig_file_ttl())) { // same ttl; DATE:2015-03-22
if (temp_itm.Orig_repo()
!= cur_repo) // if repo is different, ignore 2nd; handles images in both repos; take 1st
// only (which should be local)
continue;
} else { // new ttl; update ttl, repo
cur_ttl = temp_itm.Orig_file_ttl();
cur_repo = temp_itm.Orig_repo();
}
if (temp_itm.Chk(img_size)) temp_itm.Insert(trg_stmt, img_size);
}
conn.Txn_end();
}
public void Merge_core(Xow_wiki wiki, Io_url src_url) {
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
new Merge_wkr__core().Copy_to_temp(null, wiki, src_conn);
Gfo_invk_.Invk_by_val(
wiki.App().Wiki_mgri(), gplx.xowa.wikis.Xoa_wiki_mgr_.Invk__import_by_url, src_url);
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
// fails b/c no Main_Page; Gfo_invk_.Invk_by_msg(wiki.App().Gui__tab_mgr(),
// gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus",
// true).Add("site", wiki.Domain_str()).Add("page", String_.new_u8(wiki.Props().Main_page())));
}
public void Select_tasks_steps(
Xobc_task_step_hash task_step_hash,
Xobc_step_map_tbl step_map_tbl,
byte[] wiki_abrv,
String wiki_date) {
task_step_hash.Clear();
Db_rdr rdr =
conn.Stmt_sql(
String_.Concat_lines_nl_skip_last(
"SELECT DISTINCT sm.task_id, sm.step_id",
"FROM " + tbl_name + " imps",
" JOIN " + step_map_tbl.Tbl_name() + " sm ON sm.step_id = imps.step_id",
"WHERE imps.wiki_abrv = ?",
"AND imps.wiki_date = ?"))
.Crt_bry_as_str(fld_wiki_abrv, wiki_abrv)
.Crt_str(fld_wiki_date, wiki_date)
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
task_step_hash.Add(rdr.Read_int("task_id"), rdr.Read_int("step_id"));
}
} finally {
rdr.Rls();
}
}
public void Insert(
Xoh_page hpg, Xoh_stat_itm hzip, int wtxt_len, int row_orig_len, int row_zip_len) {
Xopg_module_mgr js_mgr = hpg.Head_mgr();
if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds);
stmt_insert.Clear()
.Val_int(fld_page_id, hpg.Page_id())
.Val_int(fld_wtxt_len, wtxt_len)
.Val_int(fld_row_orig_len, row_orig_len)
.Val_int(fld_row_zip_len, row_zip_len)
.Val_int(fld_body_len, Len_or_0(hpg.Db().Html().Html_bry()))
.Val_int(fld_display_ttl_len, Len_or_0(hpg.Display_ttl()))
.Val_int(fld_content_sub_len, Len_or_0(hpg.Content_sub()))
.Val_int(fld_sidebar_div_len, Len_or_0(hpg.Sidebar_div()))
.Val_bool_as_byte(fld_js_math, js_mgr.Math_exists())
.Val_bool_as_byte(fld_js_imap, js_mgr.Imap_exists())
.Val_bool_as_byte(fld_js_packed, js_mgr.Gallery_packed_exists())
.Val_bool_as_byte(fld_js_hiero, js_mgr.Hiero_exists())
.Val_int(fld_a_rhs, hzip.A_rhs())
.Val_int(fld_lnki_text_n, hzip.Lnki_text_n())
.Val_int(fld_lnki_text_y, hzip.Lnki_text_y())
.Val_int(fld_lnke_free, hzip.Lnke__free())
.Val_int(fld_lnke_auto, hzip.Lnke__auto())
.Val_int(fld_lnke_text, hzip.Lnke__text())
.Val_int(fld_hdr_1, hzip.Hdr_1())
.Val_int(fld_hdr_2, hzip.Hdr_2())
.Val_int(fld_hdr_3, hzip.Hdr_3())
.Val_int(fld_hdr_4, hzip.Hdr_4())
.Val_int(fld_hdr_5, hzip.Hdr_5())
.Val_int(fld_hdr_6, hzip.Hdr_6())
.Val_int(fld_img_full, hzip.Img_full())
.Exec_insert();
}
@Override
public void Cmd_end() {
usr_dlg.Note_many(
"",
"",
"fsdb_make.done: count=~{0} rate=~{1}",
exec_count,
Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn))
.To_str("#,###.000"));
if (src_fsdb_wkr != null) {
src_fsdb_wkr.Mnt_mgr()
.Mnts__get_main()
.Txn_end(); // NOTE: src_fsdb_wkr will be null if no src db defined
}
trg_atr_fil.Conn().Txn_end();
trg_atr_fil.Conn().Rls_conn();
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1
trg_cfg_mgr.Tbl().Conn().Txn_end();
trg_cfg_mgr.Tbl().Conn().Rls_conn();
if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
trg_bin_fil.Conn().Txn_end();
trg_bin_fil.Conn().Rls_conn();
}
if (exec_done) {
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make); // delete bmks for future reruns; DATE:2014-08-20
Io_mgr.Instance.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.file.make.cfg.gfs"))
.MissingFails_off()
.Exec();
}
bldr_conn.Rls_conn();
}
public void Insert(
int step_id,
int host_id,
byte[] wiki_abrv,
String wiki_date,
String import_name,
int import_type,
byte zip_type,
byte[] md5,
long size_zip,
long size_raw,
long prog_size_end,
int prog_count_end) {
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
insert_stmt.Clear()
.Val_int(fld_step_id, step_id)
.Val_int(fld_host_id, host_id)
.Val_bry_as_str(fld_wiki_abrv, wiki_abrv)
.Val_str(fld_wiki_date, wiki_date)
.Val_str(fld_import_name, import_name)
.Val_int(fld_import_type, import_type)
.Val_byte(fld_import_zip, zip_type)
.Val_long(fld_import_size_zip, size_zip)
.Val_long(fld_import_size_raw, size_raw)
.Val_bry_as_str(fld_import_md5, md5)
.Val_long(fld_prog_size_end, prog_size_end)
.Val_int(fld_prog_count_end, prog_count_end)
.Exec_insert();
}
public int Get_pending_count() {
Db_rdr rdr =
make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0")
.Exec_select__rls_auto();
try {
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
} finally {
rdr.Rls();
}
}
public Xoud_site_row[] Select_all() {
List_adp rv = List_adp_.new_();
Db_rdr rdr = conn.Stmt_select(tbl_name, flds).Exec_select__rls_auto();
try {
while (rdr.Move_next()) rv.Add(new_row(rdr));
return (Xoud_site_row[]) rv.To_ary_and_clear(Xoud_site_row.class);
} finally {
rdr.Rls();
}
}
public void Update(
int site_id, int priority, String domain, String name, String path, String xtn) {
Db_stmt stmt = conn.Stmt_update_exclude(tbl_name, flds, fld_site_id);
stmt.Val_int(fld_site_priority, priority)
.Val_str(fld_site_domain, domain)
.Val_str(fld_site_name, name)
.Val_str(fld_site_path, path)
.Val_str(fld_site_xtn, xtn)
.Crt_int(fld_site_id, site_id)
.Exec_update();
}
public Xobc_import_step_itm Select_one(int step_id) {
Db_rdr rdr =
conn.Stmt_select(tbl_name, flds, fld_step_id)
.Crt_int(fld_step_id, step_id)
.Exec_select__rls_auto();
try {
return (rdr.Move_next()) ? New_itm(rdr) : Xobc_import_step_itm.Null;
} finally {
rdr.Rls();
}
}
public void Insert(
int site_id, int priority, String domain, String name, String path, String xtn) {
Db_stmt stmt = conn.Stmt_insert(tbl_name, flds);
stmt.Val_int(fld_site_id, site_id)
.Val_int(fld_site_priority, priority)
.Val_str(fld_site_domain, domain)
.Val_str(fld_site_name, name)
.Val_str(fld_site_path, path)
.Val_str(fld_site_xtn, xtn)
.Exec_insert();
}
public Xoud_site_tbl(Db_conn conn) {
this.conn = conn;
fld_site_id = flds.Add_int_pkey("site_id");
fld_site_priority =
flds.Add_int("site_priority"); // EX: 0=default; 1+ is order if 0 is unavailable
fld_site_domain =
flds.Add_str("site_domain", 255); // EX: en.wikipedia.org; NOTE: no protocol (https:)
fld_site_name = flds.Add_str("site_name", 255); // EX: English Wikipedia
fld_site_path = flds.Add_str("site_path", 255); // EX: ~{xowa_root}/wiki/en.wikipedia.org/
fld_site_xtn = flds.Add_text("site_xtn");
conn.Rls_reg(this);
}
public Xobc_import_step_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "import_step";
this.fld_step_id = flds.Add_int_pkey("step_id");
this.fld_host_id = flds.Add_int("host_id");
this.fld_wiki_abrv = flds.Add_str("wiki_abrv", 255);
this.fld_wiki_date = flds.Add_str("wiki_date", 8);
this.fld_import_name = flds.Add_str("import_name", 255);
this.fld_import_type = flds.Add_int("import_type");
this.fld_import_zip = flds.Add_byte("import_zip");
this.fld_import_size_zip = flds.Add_long("import_size_zip");
this.fld_import_size_raw = flds.Add_long("import_size_raw");
this.fld_import_md5 = flds.Add_str("import_md5", 48);
this.fld_prog_size_end = flds.Add_long("prog_size_end");
this.fld_prog_count_end = flds.Add_long("prog_count_end");
conn.Rls_reg(this);
}
private int Load_from_db(List_adp list, int uid_prv) {
// prepare for page_tbl
String sql =
String_.Format(
String_.Concat_lines_nl_skip_last // ANSI.Y
(
"SELECT mp.xomp_uid",
", pp.page_id",
", pp.page_namespace",
", pp.page_title",
", pp.page_text_db_id",
"FROM xomp_page mp",
" JOIN <page_db>page pp ON mp.page_id = pp.page_id",
"WHERE mp.xomp_uid > {0}",
"AND mp.page_status = 0",
"LIMIT {1}"),
uid_prv,
num_pages_per_load);
this.attach_mgr.Conn_links_(
new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
// run page_tbl
int rv = -1;
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
attach_mgr.Attach();
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
rv = rdr.Read_int("xomp_uid");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
ppg.Init_by_page(
rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title"), text_db_id);
list.Add(ppg);
text_db_loader.Add(text_db_id, ppg);
}
} finally {
rdr.Rls();
}
attach_mgr.Detach();
text_db_loader.Load();
return rv;
}
public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
List_adp list = List_adp_.New();
Db_rdr rdr =
conn.Stmt_sql(
Db_sql_.Make_by_fmt(
String_.Ary(
"SELECT s.*",
"FROM import_step s",
" JOIN step_map sm ON s.step_id = sm.step_id",
"WHERE sm.task_id = {0}"),
task_id))
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
list.Add(New_itm(rdr));
}
} finally {
rdr.Rls();
}
return (Xobc_import_step_itm[]) list.To_ary_and_clear(Xobc_import_step_itm.class);
}
public void Delete(int step_id) {
conn.Stmt_delete(tbl_name, fld_step_id).Crt_int(fld_step_id, step_id).Exec_delete();
}
public void Create_tbl() {
conn.Ddl_create_tbl(Db_meta_tbl.new_(tbl_name, flds));
}
public int Exec_qry(Db_conn conn) {
return conn.Exec_qry(this);
}
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
}
@Override
public void Cmd_run() {
wiki.Init_by_wiki();
Xow_db_file core_db = wiki.Data__core_mgr().Db__core();
Db_conn core_db_conn = core_db.Conn();
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Instance;
usr_dlg.Plog_many("", "", "creating page_filter");
if (!core_db_conn.Meta_tbl_exists("page_filter")) {
core_db_conn.Meta_tbl_create(
Dbmeta_tbl_itm.New(
"page_filter",
new Dbmeta_fld_itm[] {
Dbmeta_fld_itm.new_int("page_id").Primary_y_(),
Dbmeta_fld_itm.new_int("page_text_db_id")
},
Dbmeta_idx_itm.new_normal_by_tbl(
"page_filter", "db_id__page", "page_text_db_id", "page_id"),
Dbmeta_idx_itm.new_normal_by_tbl("page_filter", "page_id", "page_id")));
}
core_db_conn.Exec_sql_plog_ntx(
"finding missing redirects",
String_.Concat_lines_nl_skip_last(
"INSERT INTO page_filter (page_id, page_text_db_id)",
"SELECT ptr.page_id, ptr.page_text_db_id",
"FROM page ptr",
" LEFT JOIN page orig ON ptr.page_redirect_id = orig.page_id",
"WHERE ptr.page_is_redirect = 1",
"AND orig.page_id IS NULL",
"UNION",
"SELECT ptr.page_id, ptr.page_text_db_id",
"FROM page ptr",
"WHERE ptr.page_is_redirect = 1",
"AND ptr.page_redirect_id = -1",
";"));
try {
Xow_db_file[] db_file_ary =
core_db.Tbl__db().Select_all(wiki.Data__core_mgr().Props(), wiki.Fsys_mgr().Root_dir());
int len = db_file_ary.length;
for (int i = 0; i < len; ++i) {
boolean db_file_is_text = Bool_.N, db_file_is_cat = Bool_.N, db_file_is_search = Bool_.N;
Xow_db_file db_file = db_file_ary[i];
switch (db_file.Tid()) {
case Xow_db_file_.Tid__core:
case Xow_db_file_.Tid__wiki_solo:
case Xow_db_file_.Tid__text_solo:
if (wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot())
continue; // if mode is lot, then "core" db does not have text, cat, search; skip;
// DATE:2016-01-31
db_file_is_text = db_file_is_cat = db_file_is_search = Bool_.Y;
break;
case Xow_db_file_.Tid__text:
db_file_is_text = Bool_.Y;
break;
case Xow_db_file_.Tid__cat:
db_file_is_cat = Bool_.Y;
break;
case Xow_db_file_.Tid__search_core:
db_file_is_search = Bool_.Y;
break;
}
int db_id = db_file.Id();
if (db_file_is_text)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting text: " + db_id,
"DELETE FROM <data_db>text WHERE page_id IN (SELECT page_id FROM page_filter WHERE page_text_db_id = {0});");
if (db_file_is_cat)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting cat: " + db_id,
"DELETE FROM <data_db>cat_link WHERE cl_from IN (SELECT page_id FROM page_filter);");
if (db_file_is_search)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting search:" + db_id,
"DELETE FROM <data_db>search_link WHERE page_id IN (SELECT page_id FROM page_filter);");
if (db_file_is_text || db_file_is_cat || db_file_is_search) db_file.Conn().Env_vacuum();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many(
"", "", "fatal error during page deletion: err=~{0}", Err_.Message_gplx_log(e));
}
core_db_conn.Exec_sql_plog_ntx(
"deleting from table: page",
"DELETE FROM page WHERE page_id IN (SELECT page_id FROM page_filter);");
// core_db_conn.Meta_tbl_delete("page_filter");
core_db_conn.Env_vacuum();
usr_dlg.Plog_many("", "", "");
}
public void Delete(int site_id) {
Db_stmt stmt = conn.Stmt_delete(tbl_name, fld_site_id);
stmt.Crt_int(fld_site_id, site_id).Exec_delete();
}
public Xoh_stat_tbl(Db_conn conn) {
this.conn = conn;
this.Create_tbl();
conn.Stmt_delete(tbl_name).Exec_delete(); // always zap table
conn.Rls_reg(this);
}
private void Assert_exists() {
assert_exists = false;
if (!conn.Meta_tbl_exists(sys_tbl.Tbl_name())) sys_tbl.Create_tbl();
}
public void Create_tbl() {
conn.Meta_tbl_create(
Dbmeta_tbl_itm.New(
tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "pkey", fld_page_id)));
}