コード例 #1
0
 private void Download_itm(Xodb_tbl_oimg_xfer_itm fsdb) {
   try {
     tier_id_val = fsdb.Lnki_tier_id();
     page_id_val = fsdb.Lnki_page_id();
     lnki_id_val = fsdb.Lnki_id();
     fsdb.Orig_repo_name_(
         fsdb.Orig_repo_id() == Xof_repo_tid_.Tid__local
             ? wiki.Domain_bry()
             : Xow_domain_itm_.Bry__commons);
     Download_exec(fsdb);
     ++exec_count;
     if (exec_count % progress_interval == 0) Print_progress(fsdb);
     if (exec_count % poll_interval == 0) poll_mgr.Poll();
     if (exec_count % commit_interval == 0) Txn_sav();
     if (exec_count % delete_interval == 0) Delete_files();
   } catch (Exception exc) {
     ++exec_fail;
     usr_dlg.Warn_many(
         "",
         "",
         "download error; ttl=~{0} w=~{1} err=~{2}",
         fsdb.Orig_ttl(),
         fsdb.Lnki_w(),
         Err_.Message_gplx_full(exc));
   }
 }
コード例 #2
0
 public static Fsdb_db_mgr new_src_bin_db_mgr(Xow_wiki wiki, String version) {
   String domain_str = wiki.Domain_str();
   Fsdb_db_mgr rv = null;
   Io_url url = null;
   if (String_.Eq(version, "v1")) {
     url =
         wiki.Fsys_mgr()
             .File_dir()
             .OwnerDir()
             .GenSubDir(domain_str + "-prv"); // v1: EX: /xowa/file/en.wikipedia.org-prv/
     rv = new Fsdb_db_mgr__v1(url);
   } else if (String_.Eq(version, "v2")) {
     url = wiki.Fsys_mgr().Root_dir().GenSubDir("prv"); // v2: EX: /xowa/wiki/en.wikipedia.org/prv/
     rv = Fsdb_db_mgr_.new_detect(wiki, url, url); // note that v2 is prioritized over v1
   } else throw Err_.new_wo_type("fsdb.make:unknown fsdb_type", "version", version);
   if (rv == null)
     throw Err_.new_wo_type(
         "fsdb.make:source fsdb not found", "version", version, "url", url.Raw());
   return rv;
 }
コード例 #3
0
 @Override
 public void Cmd_run() {
   wiki.Init_by_wiki();
   Xow_db_file core_db = wiki.Data__core_mgr().Db__core();
   Db_conn core_db_conn = core_db.Conn();
   Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Instance;
   usr_dlg.Plog_many("", "", "creating page_filter");
   if (!core_db_conn.Meta_tbl_exists("page_filter")) {
     core_db_conn.Meta_tbl_create(
         Dbmeta_tbl_itm.New(
             "page_filter",
             new Dbmeta_fld_itm[] {
               Dbmeta_fld_itm.new_int("page_id").Primary_y_(),
               Dbmeta_fld_itm.new_int("page_text_db_id")
             },
             Dbmeta_idx_itm.new_normal_by_tbl(
                 "page_filter", "db_id__page", "page_text_db_id", "page_id"),
             Dbmeta_idx_itm.new_normal_by_tbl("page_filter", "page_id", "page_id")));
   }
   core_db_conn.Exec_sql_plog_ntx(
       "finding missing redirects",
       String_.Concat_lines_nl_skip_last(
           "INSERT INTO page_filter (page_id, page_text_db_id)",
           "SELECT  ptr.page_id, ptr.page_text_db_id",
           "FROM    page ptr",
           "        LEFT JOIN page orig ON ptr.page_redirect_id = orig.page_id",
           "WHERE   ptr.page_is_redirect = 1",
           "AND     orig.page_id IS NULL",
           "UNION",
           "SELECT  ptr.page_id, ptr.page_text_db_id",
           "FROM    page ptr",
           "WHERE   ptr.page_is_redirect = 1",
           "AND     ptr.page_redirect_id = -1",
           ";"));
   try {
     Xow_db_file[] db_file_ary =
         core_db.Tbl__db().Select_all(wiki.Data__core_mgr().Props(), wiki.Fsys_mgr().Root_dir());
     int len = db_file_ary.length;
     for (int i = 0; i < len; ++i) {
       boolean db_file_is_text = Bool_.N, db_file_is_cat = Bool_.N, db_file_is_search = Bool_.N;
       Xow_db_file db_file = db_file_ary[i];
       switch (db_file.Tid()) {
         case Xow_db_file_.Tid__core:
         case Xow_db_file_.Tid__wiki_solo:
         case Xow_db_file_.Tid__text_solo:
           if (wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot())
             continue; // if mode is lot, then "core" db does not have text, cat, search; skip;
           // DATE:2016-01-31
           db_file_is_text = db_file_is_cat = db_file_is_search = Bool_.Y;
           break;
         case Xow_db_file_.Tid__text:
           db_file_is_text = Bool_.Y;
           break;
         case Xow_db_file_.Tid__cat:
           db_file_is_cat = Bool_.Y;
           break;
         case Xow_db_file_.Tid__search_core:
           db_file_is_search = Bool_.Y;
           break;
       }
       int db_id = db_file.Id();
       if (db_file_is_text)
         Run_sql(
             core_db_conn,
             db_file.Url(),
             db_id,
             "deleting text: " + db_id,
             "DELETE FROM <data_db>text WHERE page_id IN (SELECT page_id FROM page_filter WHERE page_text_db_id = {0});");
       if (db_file_is_cat)
         Run_sql(
             core_db_conn,
             db_file.Url(),
             db_id,
             "deleting cat: " + db_id,
             "DELETE FROM <data_db>cat_link WHERE cl_from IN (SELECT page_id FROM page_filter);");
       if (db_file_is_search)
         Run_sql(
             core_db_conn,
             db_file.Url(),
             db_id,
             "deleting search:" + db_id,
             "DELETE FROM <data_db>search_link WHERE page_id IN (SELECT page_id FROM page_filter);");
       if (db_file_is_text || db_file_is_cat || db_file_is_search) db_file.Conn().Env_vacuum();
     }
   } catch (Exception e) {
     Gfo_usr_dlg_.Instance.Warn_many(
         "", "", "fatal error during page deletion: err=~{0}", Err_.Message_gplx_log(e));
   }
   core_db_conn.Exec_sql_plog_ntx(
       "deleting from table: page",
       "DELETE FROM page WHERE page_id IN (SELECT page_id FROM page_filter);");
   // core_db_conn.Meta_tbl_delete("page_filter");
   core_db_conn.Env_vacuum();
   usr_dlg.Plog_many("", "", "");
 }