public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
long all_time_bgn = gplx.core.envs.System_.Ticks();
wiki.Init_by_wiki();
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
ctx.Init(wiki, src_conn);
// merge data
int hash_len = wkr_hash.Len();
for (int i = 0; i < hash_len; ++i) {
if (prog_wkr.Canceled()) break;
Merge2_wkr wkr = (Merge2_wkr) wkr_hash.Get_at(i);
// if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
long wkr_time_bgn = gplx.core.envs.System_.Ticks();
wkr.Merge_data(ctx, prog_wkr);
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|"
+ wkr.Tbl().Tbl_name()
+ "|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
}
if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
Gfo_log_.Instance.Info(
"merge.wkr.done",
"data",
src_url.NameAndExt()
+ "|-1|"
+ gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
}
public boolean Chk(Xowe_wiki wiki) {
if (!wiki.App().App_type().Uid_is_gui())
return true; // NOTE: ignore during Server / Console modes; DATE:2015-04-01
if (in_progress_hash.Has(wiki.Domain_bry()))
return true; // NOTE: ignore if currently building; different bldr commands call
// wiki.Init_assert() which may lead to fals checks;
Io_url url = url_(wiki);
if (!Io_mgr.I.ExistsFil(url)) return true;
Xoae_app app = wiki.Appe();
app.Usr_dlg().Log_many("", "", "import.marker: marker found: url=~{0}", url.Raw());
byte[] incompete_msg_bry =
app.Usere()
.Msg_mgr()
.Val_by_key_args(Bry_.new_a7("api-xowa.import.core.incomplete"), wiki.Domain_str());
int rslt = app.Gui_mgr().Kit().Ask_yes_no_cancel("", "", String_.new_u8(incompete_msg_bry));
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes:
Xob_cleanup_cmd.Delete_wiki_sql(wiki);
End(wiki);
return false; // delete wiki
case Gfui_dlg_msg_.Btn_no:
End(wiki);
return true; // delete marker
case Gfui_dlg_msg_.Btn_cancel:
return true; // noop
default:
throw Err_.unhandled(rslt);
}
}
private void Download_fils(Io_url css_dir, String[] ary) {
int ary_len = ary.length;
for (int i = 0; i < ary_len; i++) {
String src = ary[i];
Io_url trg =
css_dir.GenSubFil_nest(
Op_sys.Cur().Fsys_http_frag_to_url_str(Replace_invalid_chars_str(src)));
if (Io_mgr.Instance.ExistsFil(trg)) continue;
download_wkr.Download(true, "https://" + src, trg, "download: " + src); // ILN
if (Io_mgr.Instance.QueryFil(trg).Size()
== 0) { // warn if 0 byte files downloaded; DATE:2015-07-06
Xoa_app_.Usr_dlg()
.Warn_many("", "", "css.download; 0 byte file downloaded; file=~{0}", trg.Raw());
}
}
}
public static Xob_db_file New(Io_url dir, String name) {
Io_url url = dir.GenSubFil(name);
Db_conn_bldr_data conn_data = Db_conn_bldr.Instance.Get_or_new(url);
Db_conn conn = conn_data.Conn();
Xob_db_file rv = new Xob_db_file(url, conn);
if (conn_data.Created()) rv.Tbl__cfg().Create_tbl();
return rv;
}
public static String Resolve_file(boolean use_file_protocol, Io_url root_dir, String file) {
String rv = file;
// resolve relative urls; EX: "./a.js" -> "/xowa/wiki/simple.wikipedia.org/bin/script/a.js"
if (String_.Has_at_bgn(rv, "./")) {
// remove "./"
rv = String_.Mid(rv, 2);
if (use_file_protocol) rv = root_dir.To_http_file_str() + rv;
else {
// if fsys_url && wnt, replace "\" with "/"
if (Op_sys.Cur().Tid_is_wnt())
rv = String_.Replace(rv, Op_sys.Lnx.Fsys_dir_spr_str(), Op_sys.Wnt.Fsys_dir_spr_str());
rv = root_dir.Xto_api() + Op_sys.Cur().Fsys_dir_spr_str() + rv;
}
}
return rv;
}
private static void New_subs(Io_url owner_dir, List_adp list, IoItmList subs, byte[] dir_cmd) {
subs.Sort();
int len = subs.Len();
int list_total = list.Len();
byte[] owner_dir_bry = owner_dir.RawBry();
for (int i = 0; i < len; ++i) {
IoItm_base src = (IoItm_base) subs.Get_at(i);
Xow_import_doc trg = null;
if (src.Type_dir()) {
byte[] trg_url = src.Url().RawBry();
trg =
new Xow_import_doc(
Bool_.Y,
Bool_.N,
list_total + i,
owner_dir_bry,
trg_url,
Bry_.new_u8(src.Url().NameAndExt_noDirSpr()),
Bry_.Empty,
Bry_.Empty,
dir_cmd,
Ary_empty);
} else {
IoItmFil src_as_fil = (IoItmFil) src;
String size_str = Io_size_.To_str(src_as_fil.Size(), "#,###");
boolean is_xowa_core =
gplx.xowa.wikis.data.Xow_db_file__core_.Is_core_fil_name(
owner_dir.NameOnly(), src.Url().NameAndExt());
trg =
new Xow_import_doc(
Bool_.N,
is_xowa_core,
list_total + i,
owner_dir_bry,
src.Url().RawBry(),
Bry_.new_u8(src.Name()),
Bry_.new_u8(src_as_fil.ModifiedTime().XtoStr_fmt("yyyy-MM-dd")),
Bry_.new_u8(size_str),
dir_cmd,
Ary_empty);
}
list.Add(trg);
}
}
public void Chk(byte[] wiki_domain, Io_url css_fil) {
this.wiki_domain = wiki_domain;
List_adp img_list = List_adp_.New();
byte[] old_bry = Io_mgr.Instance.LoadFilBry(css_fil);
byte[] rel_url_prefix = Bry_.Add(Bry_fwd_slashes, wiki_domain);
byte[] new_bry = Convert_to_local_urls(rel_url_prefix, old_bry, img_list);
Io_url img_dir = css_fil.OwnerDir();
Download_fils(img_dir, img_list.To_str_ary());
Io_mgr.Instance.SaveFilBry(css_fil, new_bry);
}
public static Xopg_tag_itm New_js_file(Io_url src_url) {
byte[] src = src_url.To_http_file_bry();
return new Xopg_tag_itm(
Tid__js_file,
Gfh_tag_.Bry__script,
src,
null,
Tag__type_js,
Keyval_.new_("src", String_.new_u8(src)));
}
public static Xopg_tag_itm New_css_file(Io_url href_url) {
byte[] href = href_url.To_http_file_bry();
return new Xopg_tag_itm(
Tid__css_file,
Gfh_tag_.Bry__link,
href,
null,
Tag__type_css,
Tag__rel_stylesheet,
Keyval_.new_("href", String_.new_u8(href)));
}
public static Fsdb_db_mgr new_src_bin_db_mgr(Xow_wiki wiki, String version) {
String domain_str = wiki.Domain_str();
Fsdb_db_mgr rv = null;
Io_url url = null;
if (String_.Eq(version, "v1")) {
url =
wiki.Fsys_mgr()
.File_dir()
.OwnerDir()
.GenSubDir(domain_str + "-prv"); // v1: EX: /xowa/file/en.wikipedia.org-prv/
rv = new Fsdb_db_mgr__v1(url);
} else if (String_.Eq(version, "v2")) {
url = wiki.Fsys_mgr().Root_dir().GenSubDir("prv"); // v2: EX: /xowa/wiki/en.wikipedia.org/prv/
rv = Fsdb_db_mgr_.new_detect(wiki, url, url); // note that v2 is prioritized over v1
} else throw Err_.new_wo_type("fsdb.make:unknown fsdb_type", "version", version);
if (rv == null)
throw Err_.new_wo_type(
"fsdb.make:source fsdb not found", "version", version, "url", url.Raw());
return rv;
}
private static IoItmFil Get_itm_by_filters(Io_url dir, String... filter_ary) {
// create array of matches based on filters
int match_ary_len = filter_ary.length;
Criteria_ioMatch[] match_ary = new Criteria_ioMatch[match_ary_len];
for (int i = 0; i < match_ary_len; ++i)
match_ary[i] = Criteria_ioMatch.parse_(true, filter_ary[i], dir.Info().CaseSensitive());
// get files and check each file for match
IoItmFil rv = null;
IoItmHash itm_hash = Io_mgr.I.QueryDir_args(dir).ExecAsItmHash();
int len = itm_hash.Count();
for (int i = 0; i < len; ++i) {
IoItm_base itm = itm_hash.Get_at(i);
for (int j = 0; j < match_ary_len; ++j) {
if (itm.Type_fil() && match_ary[j].Matches(itm.Url()))
rv =
(IoItmFil)
itm; // NOTE: this will return the last match; useful for getting latest dump when
// multiple dumps are in one dir; (assuming latest should alphabetize last)
}
}
return rv;
}
public boolean Matches(Object compObj) {
Io_url comp = (Io_url) compObj;
boolean rv = pattern.Matches(comp.XtoCaseNormalized());
return match ? rv : !rv;
}
public boolean Handles(Io_url url) {
return String_.In(url.Ext(), zip_exts);
}
public int compareTo(Object obj) {
return gplx.CompareAble_.Compare(url.Raw(), ((Io_fil) obj).Url().Raw());
}
public Io_stream_rdr New_rdr(Io_url url) {
return stdout_enabled
? Io_stream_rdr_process.new_(
stdout_process.Exe_url(), url, stdout_process.Xto_process_bldr_args(url.Raw()))
: Io_stream_rdr_.bzip2_(url);
}
public void Exec() {
if (String_.Eq(text, "") && append) return; // no change; don't bother writing to disc
IoEnginePool.Instance.Get_by(url.Info().EngineKey()).SaveFilText_api(this);
}
public Gfui_dlg_file Init_dir_(Io_url v) {
under.setFilterPath(v.Xto_api());
return this;
}