private void Init_bldr_bmks() {
if (!resume_enabled) // clear cfg entries if resume disabled; note that disabled by default;
// DATE:2014-10-24
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);
Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
String tier_id_str = bmk_hash.Get_by(Cfg_tier_id_bmk).To_str_or(null);
if (tier_id_str == null) { // bmks not found; new db;
bldr_conn.Txn_bgn("bldr__fsdb_make__bldr_conn");
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk, tier_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk, page_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk, lnki_id_bmk);
bldr_conn.Txn_end();
if (tier_id_bmk == -1) tier_id_bmk = 0;
if (page_id_bmk == -1) page_id_bmk = 0;
if (lnki_id_bmk == -1) lnki_id_bmk = 0;
} else {
if (tier_id_bmk == -1) {
tier_id_bmk = Int_.parse(tier_id_str);
usr_dlg.Note_many("", "", "restoring from bmk: tier_id=~{0}", tier_id_bmk);
}
if (page_id_bmk == -1) {
page_id_bmk = bmk_hash.Get_by(Cfg_page_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: page_id=~{0}", page_id_bmk);
}
if (lnki_id_bmk == -1) {
lnki_id_bmk = bmk_hash.Get_by(Cfg_lnki_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: lnki_id=~{0}", lnki_id_bmk);
}
}
tier_id_val = tier_id_bmk;
page_id_val = page_id_bmk;
lnki_id_val = lnki_id_bmk;
}
public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
long all_time_bgn = gplx.core.envs.System_.Ticks();
wiki.Init_by_wiki();
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
ctx.Init(wiki, src_conn);
// merge data
int hash_len = wkr_hash.Len();
for (int i = 0; i < hash_len; ++i) {
if (prog_wkr.Canceled()) break;
Merge2_wkr wkr = (Merge2_wkr) wkr_hash.Get_at(i);
// if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
long wkr_time_bgn = gplx.core.envs.System_.Ticks();
wkr.Merge_data(ctx, prog_wkr);
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|"
+ wkr.Tbl().Tbl_name()
+ "|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
}
if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|-1|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
}
public void Merge_core(Xow_wiki wiki, Io_url src_url) {
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
new Merge_wkr__core().Copy_to_temp(null, wiki, src_conn);
Gfo_invk_.Invk_by_val(
wiki.App().Wiki_mgri(), gplx.xowa.wikis.Xoa_wiki_mgr_.Invk__import_by_url, src_url);
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
// fails b/c no Main_Page; Gfo_invk_.Invk_by_msg(wiki.App().Gui__tab_mgr(),
// gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus",
// true).Add("site", wiki.Domain_str()).Add("page", String_.new_u8(wiki.Props().Main_page())));
}
@Override
public void Cmd_end() {
usr_dlg.Note_many(
"",
"",
"fsdb_make.done: count=~{0} rate=~{1}",
exec_count,
Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn))
.To_str("#,###.000"));
if (src_fsdb_wkr != null) {
src_fsdb_wkr.Mnt_mgr()
.Mnts__get_main()
.Txn_end(); // NOTE: src_fsdb_wkr will be null if no src db defined
}
trg_atr_fil.Conn().Txn_end();
trg_atr_fil.Conn().Rls_conn();
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1
trg_cfg_mgr.Tbl().Conn().Txn_end();
trg_cfg_mgr.Tbl().Conn().Rls_conn();
if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
trg_bin_fil.Conn().Txn_end();
trg_bin_fil.Conn().Rls_conn();
}
if (exec_done) {
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make); // delete bmks for future reruns; DATE:2014-08-20
Io_mgr.Instance.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.file.make.cfg.gfs"))
.MissingFails_off()
.Exec();
}
bldr_conn.Rls_conn();
}
public void Select_tasks_steps(
Xobc_task_step_hash task_step_hash,
Xobc_step_map_tbl step_map_tbl,
byte[] wiki_abrv,
String wiki_date) {
task_step_hash.Clear();
Db_rdr rdr =
conn.Stmt_sql(
String_.Concat_lines_nl_skip_last(
"SELECT DISTINCT sm.task_id, sm.step_id",
"FROM " + tbl_name + " imps",
" JOIN " + step_map_tbl.Tbl_name() + " sm ON sm.step_id = imps.step_id",
"WHERE imps.wiki_abrv = ?",
"AND imps.wiki_date = ?"))
.Crt_bry_as_str(fld_wiki_abrv, wiki_abrv)
.Crt_str(fld_wiki_date, wiki_date)
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
task_step_hash.Add(rdr.Read_int("task_id"), rdr.Read_int("step_id"));
}
} finally {
rdr.Rls();
}
}
public void Insert(
int step_id,
int host_id,
byte[] wiki_abrv,
String wiki_date,
String import_name,
int import_type,
byte zip_type,
byte[] md5,
long size_zip,
long size_raw,
long prog_size_end,
int prog_count_end) {
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
insert_stmt.Clear()
.Val_int(fld_step_id, step_id)
.Val_int(fld_host_id, host_id)
.Val_bry_as_str(fld_wiki_abrv, wiki_abrv)
.Val_str(fld_wiki_date, wiki_date)
.Val_str(fld_import_name, import_name)
.Val_int(fld_import_type, import_type)
.Val_byte(fld_import_zip, zip_type)
.Val_long(fld_import_size_zip, size_zip)
.Val_long(fld_import_size_raw, size_raw)
.Val_bry_as_str(fld_import_md5, md5)
.Val_long(fld_prog_size_end, prog_size_end)
.Val_int(fld_prog_count_end, prog_count_end)
.Exec_insert();
}
public int Get_pending_count() {
Db_rdr rdr =
make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0")
.Exec_select__rls_auto();
try {
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
} finally {
rdr.Rls();
}
}
public Xobc_import_step_itm Select_one(int step_id) {
Db_rdr rdr =
conn.Stmt_select(tbl_name, flds, fld_step_id)
.Crt_int(fld_step_id, step_id)
.Exec_select__rls_auto();
try {
return (rdr.Move_next()) ? New_itm(rdr) : Xobc_import_step_itm.Null;
} finally {
rdr.Rls();
}
}
public Xobc_import_step_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "import_step";
this.fld_step_id = flds.Add_int_pkey("step_id");
this.fld_host_id = flds.Add_int("host_id");
this.fld_wiki_abrv = flds.Add_str("wiki_abrv", 255);
this.fld_wiki_date = flds.Add_str("wiki_date", 8);
this.fld_import_name = flds.Add_str("import_name", 255);
this.fld_import_type = flds.Add_int("import_type");
this.fld_import_zip = flds.Add_byte("import_zip");
this.fld_import_size_zip = flds.Add_long("import_size_zip");
this.fld_import_size_raw = flds.Add_long("import_size_raw");
this.fld_import_md5 = flds.Add_str("import_md5", 48);
this.fld_prog_size_end = flds.Add_long("prog_size_end");
this.fld_prog_count_end = flds.Add_long("prog_count_end");
conn.Rls_reg(this);
}
private int Load_from_db(List_adp list, int uid_prv) {
// prepare for page_tbl
String sql =
String_.Format(
String_.Concat_lines_nl_skip_last // ANSI.Y
(
"SELECT mp.xomp_uid",
", pp.page_id",
", pp.page_namespace",
", pp.page_title",
", pp.page_text_db_id",
"FROM xomp_page mp",
" JOIN <page_db>page pp ON mp.page_id = pp.page_id",
"WHERE mp.xomp_uid > {0}",
"AND mp.page_status = 0",
"LIMIT {1}"),
uid_prv,
num_pages_per_load);
this.attach_mgr.Conn_links_(
new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
// run page_tbl
int rv = -1;
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
attach_mgr.Attach();
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
rv = rdr.Read_int("xomp_uid");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
ppg.Init_by_page(
rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title"), text_db_id);
list.Add(ppg);
text_db_loader.Add(text_db_id, ppg);
}
} finally {
rdr.Rls();
}
attach_mgr.Detach();
text_db_loader.Load();
return rv;
}
public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
List_adp list = List_adp_.New();
Db_rdr rdr =
conn.Stmt_sql(
Db_sql_.Make_by_fmt(
String_.Ary(
"SELECT s.*",
"FROM import_step s",
" JOIN step_map sm ON s.step_id = sm.step_id",
"WHERE sm.task_id = {0}"),
task_id))
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
list.Add(New_itm(rdr));
}
} finally {
rdr.Rls();
}
return (Xobc_import_step_itm[]) list.To_ary_and_clear(Xobc_import_step_itm.class);
}
public void Delete(int step_id) {
conn.Stmt_delete(tbl_name, fld_step_id).Crt_int(fld_step_id, step_id).Exec_delete();
}
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
}