public Xow_search_tkn[] Scan(byte[] src) {
this.src = src;
this.src_len = src.length;
tkns.Clear();
pos = 0;
txt_bgn = -1;
while (pos < src_len) {
byte cur_b = src[pos];
Object cur_obj = trie.Match_bgn_w_byte(cur_b, src, pos, src_len);
if (cur_obj == null) { // text character
if (txt_bgn == -1) txt_bgn = pos; // 1st character not set; set it
++pos;
} else { // AND, OR, (, ), -, \s, "
int pos_end = trie.Match_pos();
byte cur_tid = ((Byte_obj_val) cur_obj).Val();
if (Cur_join_is_word(cur_tid, pos_end))
continue; // ignore words containing "and", "or"; EX: "random"; "for"
if (txt_bgn != -1) { // pending word; create
Tkns_add_word(Xow_search_tkn.Tid_word, txt_bgn, pos);
txt_bgn = -1;
}
switch (cur_tid) {
case Xow_search_tkn.Tid_space: // discard spaces
pos = Bry_finder.Find_fwd_while(src, pos, src_len, Byte_ascii.Space);
break;
case Xow_search_tkn.Tid_quote: // find end quote and add as word
int quote_bgn = pos + 1;
int quote_end = Bry_finder.Find_fwd(src, Byte_ascii.Quote, quote_bgn, src_len);
if (quote_end == Bry_.NotFound)
throw Err_.new_fmt_("could not find end quote: {0}", String_.new_u8(src));
Tkns_add_word(Xow_search_tkn.Tid_word_quoted, quote_bgn, quote_end);
pos = quote_end + 1; // +1 to place after quote
break;
case Xow_search_tkn.Tid_not:
Tkns_add_word(Xow_search_tkn.Tid_not, pos, pos_end);
pos = pos_end;
break;
case Xow_search_tkn.Tid_paren_bgn:
case Xow_search_tkn.Tid_paren_end:
case Xow_search_tkn.Tid_and:
case Xow_search_tkn.Tid_or:
tkns.Add(new_tkn(cur_tid, pos, pos_end));
pos = pos_end;
break;
default:
throw Err_.unhandled(cur_tid);
}
}
}
if (txt_bgn != -1) { // pending word; create
Tkns_add_word(Xow_search_tkn.Tid_word, txt_bgn, pos);
txt_bgn = -1;
}
return (Xow_search_tkn[]) tkns.To_ary_and_clear(Xow_search_tkn.class);
}
public static String Xto_name(byte tid) {
switch (tid) {
case Tid_unknown:
return "unknown";
case Tid_show:
return "show";
case Tid_all:
return "all";
case Tid_err:
return "err";
case Tid_add:
return "add";
case Tid_title:
return "title";
case Tid_raw:
return "raw";
case Tid_descrip:
return "descrip";
case Tid_del:
return "del";
case Tid_macro:
return "macro";
case Tid_name:
return "name";
case Tid_lang:
return "lang";
default:
throw Err_.unhandled(tid);
}
}
public boolean Chk(Xowe_wiki wiki) {
if (!wiki.App().App_type().Uid_is_gui())
return true; // NOTE: ignore during Server / Console modes; DATE:2015-04-01
if (in_progress_hash.Has(wiki.Domain_bry()))
return true; // NOTE: ignore if currently building; different bldr commands call
// wiki.Init_assert() which may lead to fals checks;
Io_url url = url_(wiki);
if (!Io_mgr.I.ExistsFil(url)) return true;
Xoae_app app = wiki.Appe();
app.Usr_dlg().Log_many("", "", "import.marker: marker found: url=~{0}", url.Raw());
byte[] incompete_msg_bry =
app.Usere()
.Msg_mgr()
.Val_by_key_args(Bry_.new_a7("api-xowa.import.core.incomplete"), wiki.Domain_str());
int rslt = app.Gui_mgr().Kit().Ask_yes_no_cancel("", "", String_.new_u8(incompete_msg_bry));
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes:
Xob_cleanup_cmd.Delete_wiki_sql(wiki);
End(wiki);
return false; // delete wiki
case Gfui_dlg_msg_.Btn_no:
End(wiki);
return true; // delete marker
case Gfui_dlg_msg_.Btn_cancel:
return true; // noop
default:
throw Err_.unhandled(rslt);
}
}
public static void Val_by_obj(Db_stmt stmt, String key, Object val) {
int tid = Type_adp_.To_tid_obj(val);
switch (tid) {
case Type_adp_.Tid__bool:
stmt.Val_bool_as_byte(key, Bool_.cast(val));
break;
case Type_adp_.Tid__byte:
stmt.Val_byte(key, Byte_.cast(val));
break;
case Type_adp_.Tid__int:
stmt.Val_int(key, Int_.cast(val));
break;
case Type_adp_.Tid__long:
stmt.Val_long(key, Long_.cast(val));
break;
case Type_adp_.Tid__float:
stmt.Val_float(key, Float_.cast(val));
break;
case Type_adp_.Tid__double:
stmt.Val_double(key, Double_.cast(val));
break;
case Type_adp_.Tid__str:
stmt.Val_str(key, String_.cast(val));
break;
case Type_adp_.Tid__bry:
stmt.Val_bry(key, Bry_.cast(val));
break;
default:
throw Err_.new_unhandled_default(tid);
}
}
public static short cast(Object obj) {
try {
return (Short) obj;
} catch (Exception exc) {
throw Err_.new_type_mismatch_w_exc(exc, short.class, obj);
}
} // #<>(short)obj~(Short)obj
private void Download_itm(Xodb_tbl_oimg_xfer_itm fsdb) {
try {
tier_id_val = fsdb.Lnki_tier_id();
page_id_val = fsdb.Lnki_page_id();
lnki_id_val = fsdb.Lnki_id();
fsdb.Orig_repo_name_(
fsdb.Orig_repo_id() == Xof_repo_tid_.Tid__local
? wiki.Domain_bry()
: Xow_domain_itm_.Bry__commons);
Download_exec(fsdb);
++exec_count;
if (exec_count % progress_interval == 0) Print_progress(fsdb);
if (exec_count % poll_interval == 0) poll_mgr.Poll();
if (exec_count % commit_interval == 0) Txn_sav();
if (exec_count % delete_interval == 0) Delete_files();
} catch (Exception exc) {
++exec_fail;
usr_dlg.Warn_many(
"",
"",
"download error; ttl=~{0} w=~{1} err=~{2}",
fsdb.Orig_ttl(),
fsdb.Lnki_w(),
Err_.Message_gplx_full(exc));
}
}
public boolean ReadYn_toggle(String k, boolean cur) {
Object rv = ReadOr(k, "!");
if (rv == Nil) ThrowNotFound(k);
if (!parse) throw Err_.new_wo_type("only parse supported");
String rv_str = (String) rv;
return (String_.Eq(rv_str, "!")) ? !cur : Yn.parse(rv_str);
}
public void Init_by_wiki(Xow_wiki wiki) {
if (init) return;
try {
init = true;
// if (wiki.File__fsdb_mode().Tid_v0()) return;
this.url_bldr = Xof_url_bldr.new_v2();
this.repo_mgr = wiki.File__repo_mgr();
Fsdb_db_mgr fsdb_core = wiki.File__fsdb_core();
// Fsdb_db_mgr fsdb_core = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(),
// wiki.Fsys_mgr().File_dir());
if (fsdb_core == null) return;
fsdb_enabled = true;
mnt_mgr.Ctor_by_load(fsdb_core);
this.bin_mgr =
new Xof_bin_mgr(
mnt_mgr,
repo_mgr,
wiki.App().File__img_mgr().Wkr_resize_img(),
wiki.App().Wmf_mgr().Download_wkr().Download_xrg().Download_fmt());
bin_mgr.Wkrs__add(Xof_bin_wkr__fsdb_sql.new_(mnt_mgr));
bin_mgr.Wkrs__add(Xof_bin_wkr__http_wmf.new_(wiki));
} catch (Exception e) {
throw Err_.new_exc(e, "xo", "failed to initialize fsdb_mgr}", "wiki", wiki.Domain_str());
}
}
public void Special__gen(Xoa_app app, Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
int slash_pos =
Bry_find_.Find_fwd(ttl.Page_txt_wo_qargs(), Xoa_ttl.Subpage_spr); // check for slash
byte[] special_name =
slash_pos == Bry_find_.Not_found
? ttl
.Base_txt_wo_qarg() // no slash found; use base_txt; ignore qry args and just get
// page_names; EX: Search/Earth?fulltext=y;
// Allpages?from=Earth...
: Bry_.Mid(
ttl.Page_txt_wo_qargs(),
0,
slash_pos); // slash found; use root page; EX: Special:ItemByTitle/enwiki/Earth
special_name =
Xoa_ttl.Replace_spaces(special_name); // handle spaces; EX:Spezial:Zufällige_Seite
Xow_special_page special = (Xow_special_page) hash.Get_by_bry(special_name);
if (special != null) { // special found; generate it;
special = special.Special__clone();
page.Db().Page().Modified_on_(Datetime_now.Get());
try {
special.Special__gen(wiki, page, url, ttl);
} catch (Exception e) {
Gfo_log_.Instance.Warn(
"failed to generate special page",
"url",
url.To_str(),
"err",
Err_.Message_gplx_log(e));
}
}
}
public boolean Procs_exec(int key, Scrib_proc_args args, Scrib_proc_rslt rslt) {
switch (key) {
case Proc_expr:
return Expr(args, rslt);
default:
throw Err_.new_unhandled(key);
}
}
public String Xml_outer() {
Transformer transformer = transformer_();
StringWriter writer = new StringWriter();
try {
transformer.transform(new DOMSource(xnde), new StreamResult(writer));
} catch (TransformerException e) {
throw Err_.err_key_(e, XmlDoc_.Err_XmlException, "failed to get xml string");
}
return writer.toString();
}
private boolean Cur_join_is_word(
byte cur_tid,
int
pos_end) { // extra logic to handle and / or occuring in unquoted strings; EX: "random";
// "for"
switch (cur_tid) {
default:
return false; // only look at AND, OR, -
case Xow_search_tkn.Tid_and:
case Xow_search_tkn.Tid_or:
case Xow_search_tkn.Tid_not:
break;
}
boolean join_is_word = true;
if (txt_bgn == -1) { // no pending word;
if (cur_tid == Xow_search_tkn.Tid_not)
return false; // NOT is only operator if no pending tkn; EX: -abc -> NOT abc; a-b -> a-b
byte nxt_b = pos_end < src_len ? src[pos_end] : Byte_ascii.Nil;
Object nxt_obj = trie.Match_bgn_w_byte(nxt_b, src, pos_end, src_len);
if (nxt_obj == null) // next tkn is text; join must be word
join_is_word = true;
else { // next tkn is tkn
byte nxt_tid = ((Byte_obj_val) nxt_obj).Val();
switch (nxt_tid) {
case Xow_search_tkn.Tid_space:
case Xow_search_tkn.Tid_quote:
case Xow_search_tkn.Tid_paren_bgn:
case Xow_search_tkn.Tid_paren_end:
join_is_word =
false; // next tkn is sym; and/or is not word; EX: a AND ; a AND"b"; a AND(b)
break;
case Xow_search_tkn.Tid_not:
case Xow_search_tkn.Tid_and:
case Xow_search_tkn.Tid_or:
join_is_word = true; // next tkn is and or not; and/or is word; EX: andor; oror; or-abc;
break;
default:
throw Err_.unhandled(cur_tid);
}
}
} else { // pending word; cur join must be word; EX: "grand": "and" invoked and "gr" pending
join_is_word = true;
}
if (join_is_word) {
if (txt_bgn == -1) txt_bgn = pos; // 1st character not set; set it
pos = pos_end;
return true;
}
if (txt_bgn != -1) {
Tkns_add_word(Xow_search_tkn.Tid_word, txt_bgn, pos); // create word
txt_bgn = -1;
}
return false;
}
static Transformer transformer_() {
TransformerFactory transformerfactory = TransformerFactory.newInstance();
Transformer transformer = null;
try {
transformer = transformerfactory.newTransformer();
} catch (TransformerConfigurationException e) {
throw Err_.err_key_(e, XmlDoc_.Err_XmlException, "failed to get create transformer");
}
transformer.setOutputProperty("omit-xml-declaration", "yes");
return transformer;
}
public static Fsdb_db_mgr new_src_bin_db_mgr(Xow_wiki wiki, String version) {
String domain_str = wiki.Domain_str();
Fsdb_db_mgr rv = null;
Io_url url = null;
if (String_.Eq(version, "v1")) {
url =
wiki.Fsys_mgr()
.File_dir()
.OwnerDir()
.GenSubDir(domain_str + "-prv"); // v1: EX: /xowa/file/en.wikipedia.org-prv/
rv = new Fsdb_db_mgr__v1(url);
} else if (String_.Eq(version, "v2")) {
url = wiki.Fsys_mgr().Root_dir().GenSubDir("prv"); // v2: EX: /xowa/wiki/en.wikipedia.org/prv/
rv = Fsdb_db_mgr_.new_detect(wiki, url, url); // note that v2 is prioritized over v1
} else throw Err_.new_wo_type("fsdb.make:unknown fsdb_type", "version", version);
if (rv == null)
throw Err_.new_wo_type(
"fsdb.make:source fsdb not found", "version", version, "url", url.Raw());
return rv;
}
static int XtoPow10(int v) {
if (v > -1 && v < 10) return 10;
else if (v > 9 && v < 100) return 100;
else if (v > 99 && v < 1000) return 1000;
else if (v > 999 && v < 10000) return 10000;
else if (v > 9999 && v < 100000) return 100000;
else if (v > 99999 && v < 1000000) return 1000000;
else if (v > 999999 && v < 10000000) return 10000000;
else if (v > 9999999 && v < 100000000) return 100000000;
else if (v > 99999999 && v < 1000000000) return 1000000000;
else throw Err_.new_("value must be between 0 and 1 billion").Add("v", v);
}
public byte[] Cartouche_img(
Xoh_wtr_ctx hctx,
boolean bgn,
byte[] glyph) { // render open / close cartouche; note that MW has two branches, but they are
// both the same
int height = (int) ((Hiero_html_mgr.Max_height * Hiero_html_mgr.scale) / 100);
Hiero_phoneme_itm phoneme_itm = phoneme_mgr.Get_by_key(glyph);
if (phoneme_itm == null) throw Err_.new_fmt_("missing phoneme: {0}", String_.new_u8(glyph));
byte[] code = phoneme_itm.Gardiner_code();
byte[] title = bgn ? Html_entity_.Lt_bry : Html_entity_.Gt_bry;
return cartouche_img_fmtr.Bld_bry_many(temp_bfr, hiero_img_dir, code, height, title);
}
public static byte To_update(byte cur) {
switch (cur) {
case Tid_create: // ignore update if item is already marked for create
case Tid_delete: // ignore update if item is already marked for delete (might want to throw
// error)
return cur;
case Tid_ignore: // must mark for update
case Tid_update: // return self
return Tid_update;
default:
throw Err_.unhandled(cur);
}
}
public static byte[] XmlTag_itm(byte b) {
switch (b) {
case List_itmTyp_ul:
case List_itmTyp_ol:
return Byt_li;
case List_itmTyp_dt:
return Byt_dt;
case List_itmTyp_dd:
return Byt_dd;
default:
throw Err_.unhandled(b);
}
}
public static byte Char_lst(byte b) {
switch (b) {
case List_itmTyp_ul:
return Byte_ascii.Star;
case List_itmTyp_ol:
return Byte_ascii.Hash;
case List_itmTyp_dt:
return Byte_ascii.Semic;
case List_itmTyp_dd:
return Byte_ascii.Colon;
default:
throw Err_.unhandled(b);
}
}
public Btrie_fast_mgr Add(byte[] key, Object val) {
if (val == null)
throw Err_.new_("null objects cannot be registered").Add("key", String_.new_u8(key));
int key_len = key.length;
int key_end = key_len - 1;
ByteTrieItm_fast cur = root;
for (int i = 0; i < key_len; i++) {
byte b = key[i];
ByteTrieItm_fast nxt = cur.Ary_find(b);
if (nxt == null) nxt = cur.Ary_add(b, null);
if (i == key_end) nxt.Val_set(val);
cur = nxt;
}
return this;
}
public static GfoMsg root_leafArgs_(String[] ary, Keyval... kvAry) {
int len = Array_.Len(ary);
if (len == 0) throw Err_.new_invalid_arg("== 0", "@len", len);
GfoMsg root = new GfoMsg_base().ctor_(ary[0], false);
GfoMsg owner = root;
for (int i = 1; i < len; i++) {
String key = ary[i];
GfoMsg cur = new GfoMsg_base().ctor_(key, false);
owner.Subs_add(cur);
owner = cur;
}
for (int i = 0; i < kvAry.length; i++) {
Keyval kv = kvAry[i];
owner.Add(kv.Key(), kv.Val());
}
return root;
}
public static byte[] Write_wdata_links(List_adp slink_list, Xowe_wiki wiki, Xoa_ttl ttl, Wdata_external_lang_links_data external_links_mgr) {
try {
switch (wiki.Domain_tid()) {
case Xow_domain_type_.Tid_home: // home will never be in wikidata
case Xow_domain_type_.Tid_wikidata: // wikidata will never be in wikidata
return Qid_null;
}
Wdata_wiki_mgr wdata_mgr = wiki.Appe().Wiki_mgr().Wdata_mgr();
Wdata_doc doc = wdata_mgr.Pages_get(wiki, ttl); if (doc == null) return Qid_null; // no links
boolean external_links_mgr_enabled = external_links_mgr.Enabled();
Ordered_hash links = doc.Slink_list();
Bry_bfr tmp_bfr = wiki.Appe().Utl__bfr_mkr().Get_k004();
Xow_wiki_abrv wiki_abrv = new Xow_wiki_abrv();
int len = links.Count();
for (int i = 0; i < len; i++) {
Wdata_sitelink_itm slink = (Wdata_sitelink_itm)links.Get_at(i);
byte[] xwiki_key = slink.Site();
Xow_wiki_abrv_.parse_(wiki_abrv, xwiki_key, 0, xwiki_key.length);
if (wiki_abrv.Domain_tid() == Xow_wiki_abrv_.Tid_null) {
wiki.Appe().Usr_dlg().Warn_many("", "", "unknown wiki in wikidata: ttl=~{0} wiki=~{1}", ttl.Page_db_as_str(), String_.new_u8(xwiki_key));
continue;
}
if (wiki_abrv.Domain_tid() != wiki.Domain_tid()) continue; // ignore wikis in a different domain; EX: looking at enwiki:Earth, and wikidata has dewikiquote; ignore dewikiquote; DATE:2014-06-21
byte[] lang_key = wiki_abrv.Lang_itm().Key();
if (external_links_mgr_enabled && external_links_mgr.Langs_hide(lang_key, 0, lang_key.length)) continue;
tmp_bfr.Add(lang_key);
tmp_bfr.Add_byte(Byte_ascii.Colon);
tmp_bfr.Add(slink.Name());
Xoa_ttl slink_ttl = Xoa_ttl.parse_(wiki, tmp_bfr.Xto_bry_and_clear());
if (slink_ttl == null) continue; // invalid ttl
Xow_xwiki_itm xwiki_itm = slink_ttl.Wik_itm();
if ( xwiki_itm == null // not a known xwiki; EX: [[zzz:abc]]
|| Bry_.Eq(xwiki_itm.Domain_bry(), wiki.Domain_bry()) // skip if same as self; i.e.: do not include links to enwiki if already in enwiki
) continue;
slink.Page_ttl_(slink_ttl);
slink_list.Add(slink);
}
tmp_bfr.Mkr_rls();
if (external_links_mgr_enabled && external_links_mgr.Sort())
slink_list.Sort_by(Xoa_ttl_sorter._);
return doc.Qid();
} catch (Exception e) {Err_.Noop(e); return Qid_null;}
}
public static Xoa_app_mode parse(String s) {
Object o = type_hash.Get_by(Bry_.new_u8(s));
if (o == null)
return Itm_cmd; // default to cmd as per early behaviour; handles mistaken "--app_mode wrong"
int tid = ((Int_obj_val) o).Val();
switch (tid) {
case Tid_cmd:
return Itm_cmd;
case Tid_gui:
return Itm_gui;
case Tid_http:
return Itm_http;
case Tid_tcp:
return Itm_tcp;
case Tid_file:
return Itm_file;
default:
throw Err_.new_unhandled(tid);
}
}
@Override
public void Cmd_run() {
wiki.Init_by_wiki();
Xow_db_file core_db = wiki.Data__core_mgr().Db__core();
Db_conn core_db_conn = core_db.Conn();
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Instance;
usr_dlg.Plog_many("", "", "creating page_filter");
if (!core_db_conn.Meta_tbl_exists("page_filter")) {
core_db_conn.Meta_tbl_create(
Dbmeta_tbl_itm.New(
"page_filter",
new Dbmeta_fld_itm[] {
Dbmeta_fld_itm.new_int("page_id").Primary_y_(),
Dbmeta_fld_itm.new_int("page_text_db_id")
},
Dbmeta_idx_itm.new_normal_by_tbl(
"page_filter", "db_id__page", "page_text_db_id", "page_id"),
Dbmeta_idx_itm.new_normal_by_tbl("page_filter", "page_id", "page_id")));
}
core_db_conn.Exec_sql_plog_ntx(
"finding missing redirects",
String_.Concat_lines_nl_skip_last(
"INSERT INTO page_filter (page_id, page_text_db_id)",
"SELECT ptr.page_id, ptr.page_text_db_id",
"FROM page ptr",
" LEFT JOIN page orig ON ptr.page_redirect_id = orig.page_id",
"WHERE ptr.page_is_redirect = 1",
"AND orig.page_id IS NULL",
"UNION",
"SELECT ptr.page_id, ptr.page_text_db_id",
"FROM page ptr",
"WHERE ptr.page_is_redirect = 1",
"AND ptr.page_redirect_id = -1",
";"));
try {
Xow_db_file[] db_file_ary =
core_db.Tbl__db().Select_all(wiki.Data__core_mgr().Props(), wiki.Fsys_mgr().Root_dir());
int len = db_file_ary.length;
for (int i = 0; i < len; ++i) {
boolean db_file_is_text = Bool_.N, db_file_is_cat = Bool_.N, db_file_is_search = Bool_.N;
Xow_db_file db_file = db_file_ary[i];
switch (db_file.Tid()) {
case Xow_db_file_.Tid__core:
case Xow_db_file_.Tid__wiki_solo:
case Xow_db_file_.Tid__text_solo:
if (wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot())
continue; // if mode is lot, then "core" db does not have text, cat, search; skip;
// DATE:2016-01-31
db_file_is_text = db_file_is_cat = db_file_is_search = Bool_.Y;
break;
case Xow_db_file_.Tid__text:
db_file_is_text = Bool_.Y;
break;
case Xow_db_file_.Tid__cat:
db_file_is_cat = Bool_.Y;
break;
case Xow_db_file_.Tid__search_core:
db_file_is_search = Bool_.Y;
break;
}
int db_id = db_file.Id();
if (db_file_is_text)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting text: " + db_id,
"DELETE FROM <data_db>text WHERE page_id IN (SELECT page_id FROM page_filter WHERE page_text_db_id = {0});");
if (db_file_is_cat)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting cat: " + db_id,
"DELETE FROM <data_db>cat_link WHERE cl_from IN (SELECT page_id FROM page_filter);");
if (db_file_is_search)
Run_sql(
core_db_conn,
db_file.Url(),
db_id,
"deleting search:" + db_id,
"DELETE FROM <data_db>search_link WHERE page_id IN (SELECT page_id FROM page_filter);");
if (db_file_is_text || db_file_is_cat || db_file_is_search) db_file.Conn().Env_vacuum();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many(
"", "", "fatal error during page deletion: err=~{0}", Err_.Message_gplx_log(e));
}
core_db_conn.Exec_sql_plog_ntx(
"deleting from table: page",
"DELETE FROM page WHERE page_id IN (SELECT page_id FROM page_filter);");
// core_db_conn.Meta_tbl_delete("page_filter");
core_db_conn.Env_vacuum();
usr_dlg.Plog_many("", "", "");
}
public void Val_from_args(Hash_adp args) {
throw Err_.new_unimplemented();
}
public byte[] Convert_to_local_urls(byte[] rel_url_prefix, byte[] src, List_adp list) {
try {
int src_len = src.length;
int prv_pos = 0;
Bry_bfr bfr = Bry_bfr_.New_w_size(src_len);
Hash_adp img_hash = Hash_adp_bry.cs();
while (true) {
int url_pos = Bry_find_.Find_fwd(src, Bry_url, prv_pos);
if (url_pos == Bry_find_.Not_found) {
bfr.Add_mid(src, prv_pos, src_len);
break;
} // no more "url("; exit;
int bgn_pos = url_pos + Bry_url_len; // set bgn_pos after "url("
byte bgn_byte = src[bgn_pos];
byte end_byte = Byte_ascii.Null;
boolean quoted = true;
switch (bgn_byte) { // find end_byte
case Byte_ascii.Quote:
case Byte_ascii.Apos: // quoted; end_byte is ' or "
end_byte = bgn_byte;
++bgn_pos;
break;
default: // not quoted; end byte is ")"
end_byte = Byte_ascii.Paren_end;
quoted = false;
break;
}
int end_pos = Bry_find_.Find_fwd(src, end_byte, bgn_pos, src_len);
if (end_pos
== Bry_find_.Not_found) { // unclosed "url("; exit since nothing else will be found
usr_dlg.Warn_many(
GRP_KEY,
"parse.invalid_url.end_missing",
"could not find end_sequence for 'url(': bgn='~{0}' end='~{1}'",
prv_pos,
String_.new_u8__by_len(src, prv_pos, prv_pos + 25));
bfr.Add_mid(src, prv_pos, src_len);
break;
}
if (end_pos - bgn_pos == 0) { // empty; "url()"; ignore
usr_dlg.Warn_many(
GRP_KEY,
"parse.invalid_url.empty",
"'url(' is empty: bgn='~{0}' end='~{1}'",
prv_pos,
String_.new_u8__by_len(src, prv_pos, prv_pos + 25));
bfr.Add_mid(src, prv_pos, bgn_pos);
prv_pos = bgn_pos;
continue;
}
byte[] img_raw = Bry_.Mid(src, bgn_pos, end_pos);
int img_raw_len = img_raw.length;
if (Bry_.Has_at_bgn(img_raw, Bry_data_image, 0, img_raw_len)) { // base64
bfr.Add_mid(src, prv_pos, end_pos); // nothing to download; just add entire String
prv_pos = end_pos;
continue;
}
int import_url_end =
Import_url_chk(
rel_url_prefix,
src,
src_len,
prv_pos,
url_pos,
img_raw,
bfr); // check for embedded stylesheets via @import tag
if (import_url_end != Bry_find_.Not_found) {
prv_pos = import_url_end;
continue;
}
byte[] img_cleaned = Xob_url_fixer.Fix(wiki_domain, img_raw, img_raw_len);
if (img_cleaned == null) { // could not clean img
usr_dlg.Warn_many(
GRP_KEY,
"parse.invalid_url.clean_failed",
"could not extract valid http src: bgn='~{0}' end='~{1}'",
prv_pos,
String_.new_u8(img_raw));
bfr.Add_mid(src, prv_pos, bgn_pos);
prv_pos = bgn_pos;
continue;
}
if (!img_hash.Has(img_cleaned)) { // only add unique items for download;
img_hash.Add_as_key_and_val(img_cleaned);
list.Add(String_.new_u8(img_cleaned));
}
img_cleaned =
Replace_invalid_chars(
Bry_.Copy(
img_cleaned)); // NOTE: must call ByteAry.Copy else img_cleaned will change
// *inside* hash
bfr.Add_mid(src, prv_pos, bgn_pos);
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
bfr.Add(img_cleaned);
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
prv_pos = end_pos;
}
return bfr.To_bry_and_clear();
} catch (Exception e) {
usr_dlg.Warn_many(
"",
"",
"failed to convert local_urls: ~{0} ~{1}",
String_.new_u8(rel_url_prefix),
Err_.Message_gplx_full(e));
return src;
}
}
void ThrowNotFound(String k) {
throw Err_.new_wo_type("arg not found in msg", "k", k, "counter", counter, "args", args);
}
@gplx.Internal
@Override
protected GxwCheckListBox checkListBox_(KeyValHash ctorArgs) {
throw Err_.not_implemented_();
}
private void Ident_props(int apos_len) {
typ = apos_len;
switch (apos_len) {
case Xop_apos_tkn_.Len_ital:
{
switch (state) {
case Xop_apos_tkn_.State_i:
cmd = Xop_apos_tkn_.Cmd_i_end;
state = Xop_apos_tkn_.State_nil;
break;
case Xop_apos_tkn_.State_bi:
cmd = Xop_apos_tkn_.Cmd_i_end;
state = Xop_apos_tkn_.State_b;
break;
case Xop_apos_tkn_.State_ib:
cmd = Xop_apos_tkn_.Cmd_bi_end__b_bgn;
state = Xop_apos_tkn_.State_b;
break;
case Xop_apos_tkn_.State_dual:
cmd = Xop_apos_tkn_.Cmd_i_end;
state = Xop_apos_tkn_.State_b;
dual_cmd = Xop_apos_tkn_.Cmd_bi_bgn;
break;
case Xop_apos_tkn_.State_b:
cmd = Xop_apos_tkn_.Cmd_i_bgn;
state = Xop_apos_tkn_.State_bi;
break;
case Xop_apos_tkn_.State_nil:
cmd = Xop_apos_tkn_.Cmd_i_bgn;
state = Xop_apos_tkn_.State_i;
break;
default:
throw Err_.unhandled(state);
}
break;
}
case Xop_apos_tkn_.Len_bold:
{
switch (state) {
case Xop_apos_tkn_.State_b:
cmd = Xop_apos_tkn_.Cmd_b_end;
state = Xop_apos_tkn_.State_nil;
break;
case Xop_apos_tkn_.State_bi:
cmd = Xop_apos_tkn_.Cmd_ib_end__i_bgn;
state = Xop_apos_tkn_.State_i;
break;
case Xop_apos_tkn_.State_ib:
cmd = Xop_apos_tkn_.Cmd_b_end;
state = Xop_apos_tkn_.State_i;
break;
case Xop_apos_tkn_.State_dual:
cmd = Xop_apos_tkn_.Cmd_b_end;
state = Xop_apos_tkn_.State_i;
break; // NOTE: dual_cmd = Cmd_ib_bgn is implied
case Xop_apos_tkn_.State_i:
cmd = Xop_apos_tkn_.Cmd_b_bgn;
state = Xop_apos_tkn_.State_ib;
break;
case Xop_apos_tkn_.State_nil:
cmd = Xop_apos_tkn_.Cmd_b_bgn;
state = Xop_apos_tkn_.State_b;
break;
default:
throw Err_.unhandled(state);
}
break;
}
case Xop_apos_tkn_.Len_dual:
{
switch (state) {
case Xop_apos_tkn_.State_b:
cmd = Xop_apos_tkn_.Cmd_b_end__i_bgn;
state = Xop_apos_tkn_.State_i;
break;
case Xop_apos_tkn_.State_i:
cmd = Xop_apos_tkn_.Cmd_i_end__b_bgn;
state = Xop_apos_tkn_.State_b;
break;
case Xop_apos_tkn_.State_bi:
cmd = Xop_apos_tkn_.Cmd_ib_end;
state = Xop_apos_tkn_.State_nil;
break;
case Xop_apos_tkn_.State_ib:
cmd = Xop_apos_tkn_.Cmd_bi_end;
state = Xop_apos_tkn_.State_nil;
break;
case Xop_apos_tkn_.State_dual:
cmd = Xop_apos_tkn_.Cmd_bi_end;
state = Xop_apos_tkn_.State_nil;
break; // NOTE: dual_cmd = Cmd_ib_bgn is implied
case Xop_apos_tkn_.State_nil:
cmd = Xop_apos_tkn_.Cmd_ib_bgn;
state = Xop_apos_tkn_.State_dual;
break;
default:
throw Err_.unhandled(state);
}
break;
}
default:
throw Err_.unhandled(apos_len);
}
}
public static Xot_defn Get_prototype(int id) {
switch (id) {
case Xol_kwd_grp_.Id_utc_year:
case Xol_kwd_grp_.Id_utc_month_int_len2:
case Xol_kwd_grp_.Id_utc_month_int:
case Xol_kwd_grp_.Id_utc_day_int_len2:
case Xol_kwd_grp_.Id_utc_day_int:
case Xol_kwd_grp_.Id_utc_hour:
case Xol_kwd_grp_.Id_utc_time:
case Xol_kwd_grp_.Id_utc_timestamp:
case Xol_kwd_grp_.Id_utc_week:
case Xol_kwd_grp_.Id_utc_dow:
return Pft_func_date_int.Utc;
case Xol_kwd_grp_.Id_utc_month_abrv:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_utc,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_abrv_jan - Int_.Base1);
case Xol_kwd_grp_.Id_utc_month_name:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_utc,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_name_january - Int_.Base1);
case Xol_kwd_grp_.Id_utc_month_gen:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_utc,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_gen_january - Int_.Base1);
case Xol_kwd_grp_.Id_utc_day_name:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_utc,
DateAdp_.SegIdx_dayOfWeek,
Xol_msg_itm_.Id_dte_dow_name_sunday);
case Xol_kwd_grp_.Id_lcl_year:
case Xol_kwd_grp_.Id_lcl_month_int_len2:
case Xol_kwd_grp_.Id_lcl_month_int:
case Xol_kwd_grp_.Id_lcl_day_int_len2:
case Xol_kwd_grp_.Id_lcl_day_int:
case Xol_kwd_grp_.Id_lcl_hour:
case Xol_kwd_grp_.Id_lcl_time:
case Xol_kwd_grp_.Id_lcl_timestamp:
case Xol_kwd_grp_.Id_lcl_week:
case Xol_kwd_grp_.Id_lcl_dow:
return Pft_func_date_int.Lcl;
case Xol_kwd_grp_.Id_lcl_month_abrv:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_lcl,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_abrv_jan - Int_.Base1);
case Xol_kwd_grp_.Id_lcl_month_name:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_lcl,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_name_january - Int_.Base1);
case Xol_kwd_grp_.Id_lcl_month_gen:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_lcl,
DateAdp_.SegIdx_month,
Xol_msg_itm_.Id_dte_month_gen_january - Int_.Base1);
case Xol_kwd_grp_.Id_lcl_day_name:
return new Pft_func_date_name(
-1,
Pft_func_date_int.Date_tid_lcl,
DateAdp_.SegIdx_dayOfWeek,
Xol_msg_itm_.Id_dte_dow_name_sunday);
case Xol_kwd_grp_.Id_rev_year:
case Xol_kwd_grp_.Id_rev_month_int_len2:
case Xol_kwd_grp_.Id_rev_month_int:
case Xol_kwd_grp_.Id_rev_day_int_len2:
case Xol_kwd_grp_.Id_rev_day_int:
case Xol_kwd_grp_.Id_rev_timestamp:
return Pft_func_date_int.Rev;
case Xol_kwd_grp_.Id_ns_num:
case Xol_kwd_grp_.Id_ns_txt:
case Xol_kwd_grp_.Id_ns_url:
case Xol_kwd_grp_.Id_ns_subj_txt:
case Xol_kwd_grp_.Id_ns_subj_url:
case Xol_kwd_grp_.Id_ns_talk_txt:
case Xol_kwd_grp_.Id_ns_talk_url:
case Xol_kwd_grp_.Id_ttl_full_txt:
case Xol_kwd_grp_.Id_ttl_full_url:
case Xol_kwd_grp_.Id_ttl_page_txt:
case Xol_kwd_grp_.Id_ttl_base_txt:
case Xol_kwd_grp_.Id_ttl_page_url:
case Xol_kwd_grp_.Id_ttl_base_url:
case Xol_kwd_grp_.Id_ttl_leaf_txt:
case Xol_kwd_grp_.Id_ttl_leaf_url:
case Xol_kwd_grp_.Id_ttl_subj_txt:
case Xol_kwd_grp_.Id_ttl_subj_url:
case Xol_kwd_grp_.Id_ttl_talk_txt:
case Xol_kwd_grp_.Id_ttl_talk_url:
return Pfunc_ttl.Instance;
case Xol_kwd_grp_.Id_site_sitename:
case Xol_kwd_grp_.Id_site_servername:
case Xol_kwd_grp_.Id_site_server:
case Xol_kwd_grp_.Id_site_articlepath:
case Xol_kwd_grp_.Id_site_scriptpath:
case Xol_kwd_grp_.Id_site_stylepath:
case Xol_kwd_grp_.Id_site_contentlanguage:
case Xol_kwd_grp_.Id_site_directionmark:
case Xol_kwd_grp_.Id_site_currentversion:
return Pfunc_wiki_props.Instance;
case Xol_kwd_grp_.Id_num_pages:
case Xol_kwd_grp_.Id_num_articles:
case Xol_kwd_grp_.Id_num_files:
case Xol_kwd_grp_.Id_num_edits:
case Xol_kwd_grp_.Id_num_views:
case Xol_kwd_grp_.Id_num_users:
case Xol_kwd_grp_.Id_num_users_active:
case Xol_kwd_grp_.Id_num_admins:
return Pfunc_wiki_stats.Instance;
case Xol_kwd_grp_.Id_page_id:
case Xol_kwd_grp_.Id_rev_id:
case Xol_kwd_grp_.Id_rev_pagesize:
case Xol_kwd_grp_.Id_rev_revisionsize:
case Xol_kwd_grp_.Id_rev_user:
case Xol_kwd_grp_.Id_rev_protectionexpiry:
case Xol_kwd_grp_.Id_rev_protectionlevel:
return Pfunc_rev_props.Instance;
case Xol_kwd_grp_.Id_page_displaytitle:
return Pfunc_displaytitle.Instance;
case Xol_kwd_grp_.Id_page_defaultsort:
return Pfunc_defaultsort.Instance;
case Xol_kwd_grp_.Id_noeditsection:
return Pfunc_noeditsection.Instance;
case Xol_kwd_grp_.Id_site_pagesincategory:
return Pfunc_pagesincategory.Instance;
case Xol_kwd_grp_.Id_url_ns:
return new Pfunc_ns(false);
case Xol_kwd_grp_.Id_url_nse:
return new Pfunc_ns(true);
case Xol_kwd_grp_.Id_url_urlencode:
return new Pfunc_urlencode();
case Xol_kwd_grp_.Id_str_lc:
return new Pfunc_case(Xol_lang_itm.Tid_lower, false);
case Xol_kwd_grp_.Id_str_lcfirst:
return new Pfunc_case(Xol_lang_itm.Tid_lower, true);
case Xol_kwd_grp_.Id_str_uc:
return new Pfunc_case(Xol_lang_itm.Tid_upper, false);
case Xol_kwd_grp_.Id_str_ucfirst:
return new Pfunc_case(Xol_lang_itm.Tid_upper, true);
case Xol_kwd_grp_.Id_str_padleft:
return new Pfunc_pad(Xol_kwd_grp_.Id_str_padleft, false);
case Xol_kwd_grp_.Id_str_padright:
return new Pfunc_pad(Xol_kwd_grp_.Id_str_padright, true);
case Xol_kwd_grp_.Id_str_formatnum:
return new Pf_formatnum();
case Xol_kwd_grp_.Id_str_formatdate:
return new Pft_func_formatdate();
case Xol_kwd_grp_.Id_url_localurl:
return new Pfunc_urlfunc(Xol_kwd_grp_.Id_url_localurl, Pfunc_urlfunc.Tid_local, false);
case Xol_kwd_grp_.Id_url_localurle:
return new Pfunc_urlfunc(Xol_kwd_grp_.Id_url_localurle, Pfunc_urlfunc.Tid_local, true);
case Xol_kwd_grp_.Id_url_fullurl:
return new Pfunc_urlfunc(Xol_kwd_grp_.Id_url_fullurl, Pfunc_urlfunc.Tid_full, false);
case Xol_kwd_grp_.Id_url_fullurle:
return new Pfunc_urlfunc(Xol_kwd_grp_.Id_url_fullurle, Pfunc_urlfunc.Tid_full, true);
case Xol_kwd_grp_.Id_url_canonicalurl:
return new Pfunc_urlfunc(
Xol_kwd_grp_.Id_url_canonicalurl, Pfunc_urlfunc.Tid_canonical, false);
case Xol_kwd_grp_.Id_url_canonicalurle:
return new Pfunc_urlfunc(
Xol_kwd_grp_.Id_url_canonicalurle, Pfunc_urlfunc.Tid_canonical, false);
case Xol_kwd_grp_.Id_url_filepath:
return new Pfunc_filepath();
case Xol_kwd_grp_.Id_url_anchorencode:
return new Pfunc_anchorencode();
case Xol_kwd_grp_.Id_strx_len:
return new Pfunc_len();
case Xol_kwd_grp_.Id_strx_pos:
return new Pfunc_pos();
case Xol_kwd_grp_.Id_strx_rpos:
return new Pfunc_rpos();
case Xol_kwd_grp_.Id_strx_sub:
return new Pfunc_sub();
case Xol_kwd_grp_.Id_strx_count:
return new Pfunc_count();
case Xol_kwd_grp_.Id_strx_replace:
return new Pfunc_replace();
case Xol_kwd_grp_.Id_strx_explode:
return new Pfunc_explode();
case Xol_kwd_grp_.Id_strx_urldecode:
return new Pfunc_urldecode();
case Xol_kwd_grp_.Id_i18n_plural:
return new Pfunc_plural();
case Xol_kwd_grp_.Id_i18n_language:
return new Pfunc_language();
case Xol_kwd_grp_.Id_i18n_int:
return new Pfunc_int();
case Xol_kwd_grp_.Id_i18n_grammar:
return new Pfunc_grammar();
case Xol_kwd_grp_.Id_i18n_gender:
return new Pfunc_gender();
case Xol_kwd_grp_.Id_misc_tag:
return new Pfunc_tag();
case Xol_kwd_grp_.Id_xtn_expr:
return new gplx.xowa.xtns.pfuncs.exprs.Pfunc_expr();
case Xol_kwd_grp_.Id_xtn_if:
return new Pfunc_if();
case Xol_kwd_grp_.Id_xtn_ifeq:
return new Pfunc_ifeq();
case Xol_kwd_grp_.Id_xtn_iferror:
return new Pfunc_iferror();
case Xol_kwd_grp_.Id_xtn_ifexpr:
return new Pfunc_ifexpr();
case Xol_kwd_grp_.Id_xtn_ifexist:
return new Pfunc_ifexist();
case Xol_kwd_grp_.Id_xtn_rel2abs:
return new Pfunc_rel2abs();
case Xol_kwd_grp_.Id_xtn_switch:
return new Pfunc_switch();
case Xol_kwd_grp_.Id_xtn_time:
return Pft_func_time._Utc;
case Xol_kwd_grp_.Id_xtn_timel:
return Pft_func_time._Lcl;
case Xol_kwd_grp_.Id_xtn_titleparts:
return new Pfunc_titleparts();
case Xol_kwd_grp_.Id_subst:
case Xol_kwd_grp_.Id_safesubst:
case Xol_kwd_grp_.Id_msg:
case Xol_kwd_grp_.Id_msgnw:
case Xol_kwd_grp_.Id_raw:
return new Xot_defn_subst((byte) id, Bry_.Empty);
case Xol_kwd_grp_.Id_xowa_dbg:
return new Xop_xowa_dbg();
case Xol_kwd_grp_.Id_xowa:
return new gplx.xowa.xtns.xowa_cmds.Xop_xowa_func();
case Xol_kwd_grp_.Id_xtn_geodata_coordinates:
return gplx.xowa.xtns.geodata.Geo_coordinates_func.Instance;
case Xol_kwd_grp_.Id_lst:
return gplx.xowa.xtns.lst.Lst_pfunc_lst.Prime;
case Xol_kwd_grp_.Id_lstx:
return gplx.xowa.xtns.lst.Lst_pfunc_lstx.Prime;
case Xol_kwd_grp_.Id_lsth:
return gplx.xowa.xtns.lst.Lst_pfunc_lsth.Prime;
case Xol_kwd_grp_.Id_invoke:
return new gplx.xowa.xtns.scribunto.Scrib_invoke_func();
case Xol_kwd_grp_.Id_pagebanner:
return new gplx.xowa.xtns.pagebanners.Pgbnr_func();
case Xol_kwd_grp_.Id_new_window_link:
return new gplx.xowa.xtns.new_window_links.New_window_link_func();
case Xol_kwd_grp_.Id_property:
return new gplx.xowa.xtns.wbases.pfuncs.Wdata_pf_property();
case Xol_kwd_grp_.Id_noexternallanglinks:
return new gplx.xowa.xtns.wbases.pfuncs.Wdata_pf_noExternalLangLinks();
case Xol_kwd_grp_.Id_wbreponame:
return new gplx.xowa.xtns.wbases.pfuncs.Wdata_pf_wbreponame();
case Xol_kwd_grp_.Id_mapSources_deg2dd:
return gplx.xowa.xtns.mapSources.Map_deg2dd_func.Instance;
case Xol_kwd_grp_.Id_mapSources_dd2dms:
return gplx.xowa.xtns.mapSources.Map_dd2dms_func.Instance;
case Xol_kwd_grp_.Id_mapSources_geoLink:
return gplx.xowa.xtns.mapSources.Map_geolink_func.Instance;
case Xol_kwd_grp_.Id_geoCrumbs_isin:
return gplx.xowa.xtns.geoCrumbs.Geoc_isin_func.Instance;
case Xol_kwd_grp_.Id_relatedArticles:
return gplx.xowa.xtns.relatedArticles.Articles_func.Instance;
case Xol_kwd_grp_.Id_insider:
return gplx.xowa.xtns.insiders.Insider_func.Instance;
case Xol_kwd_grp_.Id_massMessage_target:
return gplx.xowa.xtns.massMessage.Message_target_func.Instance;
case Xol_kwd_grp_.Id_categorytree:
return gplx.xowa.xtns.categorytrees.Categorytree_func.Instance;
case Xol_kwd_grp_.Id_pendingChangeLevel:
return gplx.xowa.xtns.flaggedRevs.Pending_change_level_func.Instance;
case Xol_kwd_grp_.Id_pagesUsingPendingChanges:
return gplx.xowa.xtns.flaggedRevs.Pages_using_pending_changes_func.Instance;
case Xol_kwd_grp_.Id_cascadingSources:
return new Pf_func_noop(id);
case Xol_kwd_grp_.Id_bang:
return Pf_func_bang.Instance;
default:
throw Err_.new_unhandled(id);
}
}