private void Init_bldr_bmks() {
   if (!resume_enabled) // clear cfg entries if resume disabled; note that disabled by default;
     // DATE:2014-10-24
     bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);
   Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
   String tier_id_str = bmk_hash.Get_by(Cfg_tier_id_bmk).To_str_or(null);
   if (tier_id_str == null) { // bmks not found; new db;
     bldr_conn.Txn_bgn("bldr__fsdb_make__bldr_conn");
     bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk, tier_id_bmk);
     bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk, page_id_bmk);
     bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk, lnki_id_bmk);
     bldr_conn.Txn_end();
     if (tier_id_bmk == -1) tier_id_bmk = 0;
     if (page_id_bmk == -1) page_id_bmk = 0;
     if (lnki_id_bmk == -1) lnki_id_bmk = 0;
   } else {
     if (tier_id_bmk == -1) {
       tier_id_bmk = Int_.parse(tier_id_str);
       usr_dlg.Note_many("", "", "restoring from bmk: tier_id=~{0}", tier_id_bmk);
     }
     if (page_id_bmk == -1) {
       page_id_bmk = bmk_hash.Get_by(Cfg_page_id_bmk).To_int();
       usr_dlg.Note_many("", "", "restoring from bmk: page_id=~{0}", page_id_bmk);
     }
     if (lnki_id_bmk == -1) {
       lnki_id_bmk = bmk_hash.Get_by(Cfg_lnki_id_bmk).To_int();
       usr_dlg.Note_many("", "", "restoring from bmk: lnki_id=~{0}", lnki_id_bmk);
     }
   }
   tier_id_val = tier_id_bmk;
   page_id_val = page_id_bmk;
   lnki_id_val = lnki_id_bmk;
 }
Example #2
0
  public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
    long all_time_bgn = gplx.core.envs.System_.Ticks();
    wiki.Init_by_wiki();
    Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
    ctx.Init(wiki, src_conn);

    // merge data
    int hash_len = wkr_hash.Len();
    for (int i = 0; i < hash_len; ++i) {
      if (prog_wkr.Canceled()) break;
      Merge2_wkr wkr = (Merge2_wkr) wkr_hash.Get_at(i);
      // if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
      long wkr_time_bgn = gplx.core.envs.System_.Ticks();
      wkr.Merge_data(ctx, prog_wkr);
      Gfo_log_.Instance.Info(
          "merge.wkr.done",
          "data",
          src_url.NameAndExt()
              + "|"
              + wkr.Tbl().Tbl_name()
              + "|"
              + gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
    }
    if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
    Gfo_log_.Instance.Info(
        "merge.wkr.done",
        "data",
        src_url.NameAndExt()
            + "|-1|"
            + gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
    src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
  }
Example #3
0
 public void Merge_core(Xow_wiki wiki, Io_url src_url) {
   Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
   new Merge_wkr__core().Copy_to_temp(null, wiki, src_conn);
   Gfo_invk_.Invk_by_val(
       wiki.App().Wiki_mgri(), gplx.xowa.wikis.Xoa_wiki_mgr_.Invk__import_by_url, src_url);
   src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
   // fails b/c no Main_Page; Gfo_invk_.Invk_by_msg(wiki.App().Gui__tab_mgr(),
   // gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus",
   // true).Add("site", wiki.Domain_str()).Add("page", String_.new_u8(wiki.Props().Main_page())));
 }
 @Override
 public void Cmd_end() {
   usr_dlg.Note_many(
       "",
       "",
       "fsdb_make.done: count=~{0} rate=~{1}",
       exec_count,
       Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn))
           .To_str("#,###.000"));
   if (src_fsdb_wkr != null) {
     src_fsdb_wkr.Mnt_mgr()
         .Mnts__get_main()
         .Txn_end(); // NOTE: src_fsdb_wkr will be null if no src db defined
   }
   trg_atr_fil.Conn().Txn_end();
   trg_atr_fil.Conn().Rls_conn();
   if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1
   trg_cfg_mgr.Tbl().Conn().Txn_end();
   trg_cfg_mgr.Tbl().Conn().Rls_conn();
   if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
     trg_bin_fil.Conn().Txn_end();
     trg_bin_fil.Conn().Rls_conn();
   }
   if (exec_done) {
     bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make); // delete bmks for future reruns; DATE:2014-08-20
     Io_mgr.Instance.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.file.make.cfg.gfs"))
         .MissingFails_off()
         .Exec();
   }
   bldr_conn.Rls_conn();
 }
 public void Select_tasks_steps(
     Xobc_task_step_hash task_step_hash,
     Xobc_step_map_tbl step_map_tbl,
     byte[] wiki_abrv,
     String wiki_date) {
   task_step_hash.Clear();
   Db_rdr rdr =
       conn.Stmt_sql(
               String_.Concat_lines_nl_skip_last(
                   "SELECT  DISTINCT sm.task_id, sm.step_id",
                   "FROM    " + tbl_name + " imps",
                   "        JOIN " + step_map_tbl.Tbl_name() + " sm ON sm.step_id = imps.step_id",
                   "WHERE   imps.wiki_abrv = ?",
                   "AND     imps.wiki_date = ?"))
           .Crt_bry_as_str(fld_wiki_abrv, wiki_abrv)
           .Crt_str(fld_wiki_date, wiki_date)
           .Exec_select__rls_auto();
   try {
     while (rdr.Move_next()) {
       task_step_hash.Add(rdr.Read_int("task_id"), rdr.Read_int("step_id"));
     }
   } finally {
     rdr.Rls();
   }
 }
 public void Insert(
     int step_id,
     int host_id,
     byte[] wiki_abrv,
     String wiki_date,
     String import_name,
     int import_type,
     byte zip_type,
     byte[] md5,
     long size_zip,
     long size_raw,
     long prog_size_end,
     int prog_count_end) {
   if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
   insert_stmt.Clear()
       .Val_int(fld_step_id, step_id)
       .Val_int(fld_host_id, host_id)
       .Val_bry_as_str(fld_wiki_abrv, wiki_abrv)
       .Val_str(fld_wiki_date, wiki_date)
       .Val_str(fld_import_name, import_name)
       .Val_int(fld_import_type, import_type)
       .Val_byte(fld_import_zip, zip_type)
       .Val_long(fld_import_size_zip, size_zip)
       .Val_long(fld_import_size_raw, size_raw)
       .Val_bry_as_str(fld_import_md5, md5)
       .Val_long(fld_prog_size_end, prog_size_end)
       .Val_int(fld_prog_count_end, prog_count_end)
       .Exec_insert();
 }
 public int Get_pending_count() {
   Db_rdr rdr =
       make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0")
           .Exec_select__rls_auto();
   try {
     return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
   } finally {
     rdr.Rls();
   }
 }
 public Xobc_import_step_itm Select_one(int step_id) {
   Db_rdr rdr =
       conn.Stmt_select(tbl_name, flds, fld_step_id)
           .Crt_int(fld_step_id, step_id)
           .Exec_select__rls_auto();
   try {
     return (rdr.Move_next()) ? New_itm(rdr) : Xobc_import_step_itm.Null;
   } finally {
     rdr.Rls();
   }
 }
 public Xobc_import_step_tbl(Db_conn conn) {
   this.conn = conn;
   this.tbl_name = "import_step";
   this.fld_step_id = flds.Add_int_pkey("step_id");
   this.fld_host_id = flds.Add_int("host_id");
   this.fld_wiki_abrv = flds.Add_str("wiki_abrv", 255);
   this.fld_wiki_date = flds.Add_str("wiki_date", 8);
   this.fld_import_name = flds.Add_str("import_name", 255);
   this.fld_import_type = flds.Add_int("import_type");
   this.fld_import_zip = flds.Add_byte("import_zip");
   this.fld_import_size_zip = flds.Add_long("import_size_zip");
   this.fld_import_size_raw = flds.Add_long("import_size_raw");
   this.fld_import_md5 = flds.Add_str("import_md5", 48);
   this.fld_prog_size_end = flds.Add_long("prog_size_end");
   this.fld_prog_count_end = flds.Add_long("prog_count_end");
   conn.Rls_reg(this);
 }
  private int Load_from_db(List_adp list, int uid_prv) {
    // prepare for page_tbl
    String sql =
        String_.Format(
            String_.Concat_lines_nl_skip_last // ANSI.Y
                (
                "SELECT  mp.xomp_uid",
                ",       pp.page_id",
                ",       pp.page_namespace",
                ",       pp.page_title",
                ",       pp.page_text_db_id",
                "FROM    xomp_page mp",
                "        JOIN <page_db>page pp ON mp.page_id = pp.page_id",
                "WHERE   mp.xomp_uid > {0}",
                "AND     mp.page_status = 0",
                "LIMIT   {1}"),
            uid_prv,
            num_pages_per_load);
    this.attach_mgr.Conn_links_(
        new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
    sql = attach_mgr.Resolve_sql(sql);

    // run page_tbl
    int rv = -1;
    Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
    attach_mgr.Attach();
    Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
    try {
      while (rdr.Move_next()) {
        rv = rdr.Read_int("xomp_uid");
        int text_db_id = rdr.Read_int("page_text_db_id");
        Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
        ppg.Init_by_page(
            rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title"), text_db_id);
        list.Add(ppg);
        text_db_loader.Add(text_db_id, ppg);
      }
    } finally {
      rdr.Rls();
    }
    attach_mgr.Detach();

    text_db_loader.Load();
    return rv;
  }
 public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
   List_adp list = List_adp_.New();
   Db_rdr rdr =
       conn.Stmt_sql(
               Db_sql_.Make_by_fmt(
                   String_.Ary(
                       "SELECT  s.*",
                       "FROM    import_step s",
                       "        JOIN step_map sm ON s.step_id = sm.step_id",
                       "WHERE   sm.task_id = {0}"),
                   task_id))
           .Exec_select__rls_auto();
   try {
     while (rdr.Move_next()) {
       list.Add(New_itm(rdr));
     }
   } finally {
     rdr.Rls();
   }
   return (Xobc_import_step_itm[]) list.To_ary_and_clear(Xobc_import_step_itm.class);
 }
 public void Delete(int step_id) {
   conn.Stmt_delete(tbl_name, fld_step_id).Crt_int(fld_step_id, step_id).Exec_delete();
 }
 public void Create_tbl() {
   conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
 }