public int Make_tkn(
Xop_ctx ctx,
Xop_tkn_mkr tkn_mkr,
Xop_root_tkn root,
byte[] src,
int src_len,
int bgn_pos,
int cur_pos) {
Xop_tkn_itm tkn = tkn_mkr.Brack_bgn(bgn_pos, cur_pos);
ctx.Subs_add_and_stack(root, tkn);
return cur_pos;
}
public void MakeTkn_end(
Xop_ctx ctx,
Xop_tkn_mkr tkn_mkr,
Xop_root_tkn root,
byte[] src,
int src_len,
int bgn_pos,
int cur_pos,
Xop_list_tkn bgn,
byte sub_last) {
// boolean empty_ignored = ctx.Empty_ignored(); // commented; see below; DATE:2014-06-24
Xop_tkn_itm end_tkn =
tkn_mkr.List_end(bgn_pos, bgn.List_itmTyp())
.List_path_(bgn.List_path())
.List_uid_(listId)
.List_sub_last_(sub_last);
ctx.Subs_add(root, end_tkn);
// if (empty_ignored) ctx.Empty_ignore(root, bgn.Tkn_sub_idx()); // commented; code was
// incorrectly deactivating "*a" when "<li>" encountered; PAGE:en.w:Bristol_Bullfinch
// DATE:2014-06-24
ctx.Para().Process_block__bgn_n__end_y(Xop_xnde_tag_.Tag__ul);
}
public int MakeTkn_bgn(
Xop_ctx ctx,
Xop_tkn_mkr tkn_mkr,
Xop_root_tkn root,
byte[] src,
int src_len,
int bgn_pos,
int cur_pos) { // REF.MW: Parser|doBlockLevels
if (bgn_pos == Xop_parser_.Doc_bgn_bos) bgn_pos = 0; // do not allow -1 pos
// pop hdr if exists; EX: \n== a ==\n*b; \n* needs to close hdr
int acsPos = ctx.Stack_idx_typ(Xop_tkn_itm_.Tid_hdr);
if (acsPos != -1)
ctx.Stack_pop_til(root, src, acsPos, true, bgn_pos, cur_pos, Xop_tkn_itm_.Tid_list);
// close apos
ctx.Apos().End_frame(ctx, root, src, bgn_pos, false);
byte symByt =
src[
cur_pos
- 1]; // -1 b/c symByt is byte before curByt; EX: \n*a; cur_pos is at a; want to get
// *
int prvSymLen = curSymLen;
cur_pos = SymAry_fill(src, cur_pos, src_len, symByt);
symByt =
src[
cur_pos
- 1]; // NOTE: get symByt again b/c cur_pos may have changed; EX: "#*"; # may have
// triggered list, but last symByt should be *
if (SymAry_fill_overflow) return ctx.Lxr_make_txt_(cur_pos);
PrvItm_compare();
ctx.Para()
.Process_block__bgn__nl_w_symbol(
ctx,
root,
src,
bgn_pos,
cur_pos - 1,
Xop_xnde_tag_
.Tag__li); // -1 b/c cur_pos includes sym_byte; EX: \n*; pass li; should pass
// correct tag, but for purposes of para_wkr, <li> doesn't matter
if (prvSymMatch) {
PopTil(ctx, tkn_mkr, root, src, src_len, bgn_pos, cur_pos, Bool_.N_byte);
posBldr.MoveNext();
prvSymAry = Xop_list_wkr_.MakeSymAry(curSymAry, curSymLen);
Xop_list_tkn prvItm =
tkn_mkr.List_bgn(bgn_pos, cur_pos, curSymAry[curSymLen - 1], curSymLen)
.List_path_(posBldr.XtoIntAry())
.List_uid_(listId);
ctx.Subs_add_and_stack(root, prvItm);
ctx.Empty_ignored_y_();
} else {
for (int i = prvSymLen; i > commonSymLen; i--) { // close all discontinued itms: EX: ##\n#\n
PopTil(ctx, tkn_mkr, root, src, src_len, bgn_pos, cur_pos, Bool_.Y_byte);
posBldr.MoveUp();
}
if (commonSymLen == 0 && prvSymLen != 0) { // nothing in common; reset list
listId++;
posBldr.Init();
}
if (curSymLen == commonSymLen) { // add another itm if continuing; EX: #\n#\n
PopTil(ctx, tkn_mkr, root, src, src_len, bgn_pos, cur_pos, Bool_.N_byte);
if ((prvSymLen - curSymLen)
> 0 // moving up many levels; do not open new list; just MoveNext; EX: #1\n###3\n##2
&& curSymLen
!= 1) { // do not moveNext if at level 1; this has to do with strange incrementing
// logic in posBldr at rootLvl
posBldr.MoveNext();
} else {
posBldr.MoveUp();
posBldr.MoveDown();
}
prvSymAry = Xop_list_wkr_.MakeSymAry(curSymAry, curSymLen);
symByt = src[cur_pos - 1];
Xop_list_tkn prvItm =
tkn_mkr.List_bgn(bgn_pos, cur_pos, symByt, curSymLen)
.List_path_(posBldr.XtoIntAry())
.List_uid_(listId);
ctx.Subs_add_and_stack(root, prvItm);
ctx.Empty_ignored_y_();
}
for (int i = commonSymLen; i < curSymLen; i++) { // open new itms; EX: #\n##\n
posBldr.MoveDown();
symByt = curSymAry[i];
prvSymAry = Xop_list_wkr_.MakeSymAry(curSymAry, curSymLen);
Xop_list_tkn prvItm =
tkn_mkr.List_bgn(bgn_pos, cur_pos, symByt, i + List_adp_.Base1)
.List_path_(posBldr.XtoIntAry())
.List_uid_(listId);
ctx.Subs_add_and_stack(root, prvItm);
ctx.Empty_ignored_y_();
}
}
if (allDd
&& cur_pos < src_len - 2
&& src[cur_pos] == '{'
&& src[cur_pos + 1] == '|') // NOTE: if indent && next == {| then invoke table; EX: ":::{|"
return ctx.Tblw()
.Make_tkn_bgn(
ctx,
tkn_mkr,
root,
src,
src_len,
cur_pos,
cur_pos + 2,
false,
Xop_tblw_wkr.Tblw_type_tb,
Xop_tblw_wkr.Called_from_list,
-1,
-1); // NOTE: ws_enabled must be set to true; see test for Adinkras; Cato the Elder
else {
dd_chk = symByt == Xop_list_tkn_.List_itmTyp_dt;
return cur_pos;
}
}
public boolean Bld(
Xop_ctx ctx,
Xop_tkn_mkr tkn_mkr,
Xop_arg_wkr wkr,
int wkr_typ,
Xop_root_tkn root,
Xop_tkn_itm tkn,
int bgn_pos,
int cur_pos,
int loop_bgn,
int loop_end,
byte[] src) {
boolean ws_bgn_chk = true, colon_chk = false, itm_is_static = true, key_exists = false;
int ws_bgn_idx = -1, ws_end_idx = -1, cur_itm_subs_len = 0, cur_nde_idx = -1;
Arg_nde_tkn cur_nde = null;
Arg_itm_tkn cur_itm = null;
int brack_count = 0;
Xop_tkn_itm eq_pending = null;
for (int i = loop_bgn;
i < loop_end;
i++) { // loop over subs between bookends; if lnki, all tkns between [[ and ]]; if tmpl, {{
// and }}
Xop_tkn_itm sub = root.Subs_get(i);
int sub_pos_bgn = sub.Src_bgn_grp(root, i);
if (cur_nde == null) {
cur_nde = tkn_mkr.ArgNde(++cur_nde_idx, sub_pos_bgn);
brack_count = 0;
key_exists = false;
}
if (cur_itm == null) {
cur_itm = tkn_mkr.ArgItm(sub_pos_bgn, -1);
itm_is_static = ws_bgn_chk = true;
cur_itm_subs_len = 0;
ws_bgn_idx = ws_end_idx = -1;
if (eq_pending
!= null) { // something like "A==B" encountered; zh.w:Wikipedia:条目评选; DATE:2014-08-27
eq_pending.Src_end_(eq_pending.Src_end() - 1); // remove an "=" EX:"A==B" -> "A","=","=B"
cur_itm.Subs_add_grp(eq_pending, root, i);
cur_itm_subs_len++; // add the tkn to cur_itm
eq_pending = null;
}
}
switch (sub.Tkn_tid()) {
case Xop_tkn_itm_
.Tid_ignore: // comment or *include* tkn; mark itm as non_static for tmpl (forces
// re-eval)
switch (wkr_typ) {
case Xop_arg_wkr_.Typ_tmpl:
case Xop_arg_wkr_.Typ_prm:
itm_is_static = false;
break;
}
break;
case Xop_tkn_itm_
.Tid_para: // NOTE: para can appear in following: [[File:A.png| \n 40px]]; EX:
// w:Supreme_Court_of_the_United_States; DATE:2014-04-05
case Xop_tkn_itm_.Tid_newLine:
case Xop_tkn_itm_.Tid_space:
case Xop_tkn_itm_.Tid_tab: // whitespace
if (ws_bgn_chk)
ws_bgn_idx =
cur_itm_subs_len; // definite ws at bgn; set ws_bgn_idx, and keep setting until text
// tkn reached; handles mixed sequence of \s\n\t where last tkn
// should be ws_bgn_idx
else {
if (ws_end_idx == -1) ws_end_idx = cur_itm_subs_len;
}
; // possible ws at end; may be overriden later; see AdjustWsForTxtTkn
break;
case Xop_tkn_itm_.Tid_colon:
if (wkr_typ
== Xop_arg_wkr_
.Typ_tmpl) { // treat colons as text; tmpl will do its own : parsing for 1st arg;
// NOTE: must do ws check else 2nd colon will break; EX: "{{#ifeq:
// :|a|b|c}}"; DATE:2013-12-10
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
} else {
if (cur_nde_idx == 0
&& !colon_chk) { // if 1st arg, mark colon pos; needed for lnki; EX: [[Category:A]];
// {{#ifeq:1}}
colon_chk = true;
cur_nde.Arg_colon_pos_(sub_pos_bgn);
}
}
break;
case Xop_tkn_itm_.Tid_brack_bgn:
++brack_count;
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
case Xop_tkn_itm_.Tid_brack_end:
--brack_count;
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
case Xop_tkn_itm_.Tid_eq:
if (wkr_typ == Xop_arg_wkr_.Typ_tmpl && brack_count > 0) {
} else if (wkr_typ == Xop_arg_wkr_.Typ_prm) {
} // always ignore for prm
else {
if (cur_nde_idx
!= 0 // if 1st arg, treat equal_tkn as txt_tkn; i.e.: eq should not be used to
// separate key/val
&& cur_nde.Eq_tkn()
== Xop_tkn_null
.Null_tkn // only mark key if key is not set; handle multiple-keys; EX:
// {{name|key1=b=c}}; DATE:2014-02-09
) {
Xop_eq_tkn sub_as_eq = (Xop_eq_tkn) sub;
int sub_as_eq_len = sub_as_eq.Eq_len();
boolean eq_is_spr =
sub_as_eq_len
== 1 // eq with len of 1 are considered separators;
// MW.REF:Preprocessor_DOM.php|preprocessToXml; "if ( $count == 1 &&
// $findEquals )" PAGE:en.w:Wikipedia:Picture_of_the_day/June_2014;
// DATE:2014-07-21
|| (cur_itm.Subs_len()
> 0 // or eq.len > 1 that occur later in itm; EX: a==b;
// zh.w:Wikipedia:条目评选; DATE:2014-08-27
&& cur_itm.Subs_get(0).Tkn_tid()
!= Xop_tkn_itm_
.Tid_eq // and 1st tkn is not ==; EX:==a==; 2nd == should not be
// eq b/c 1st == "deactivates" nde; DATE:2014-08-27
);
if (eq_is_spr) {
if (sub_as_eq_len == 1) // =.len == 1
cur_nde.Eq_tkn_(sub); // set as eq tkn
else // =.len > 1
eq_pending =
sub; // do not set as eq tkn; note that Eq_tkn exists for bookkeeping and is
// not printed out,
key_exists = true;
Arg_itm_end(
ctx,
cur_nde,
cur_itm,
ws_bgn_idx,
ws_end_idx,
cur_itm_subs_len,
sub_pos_bgn,
wkr_typ,
key_exists,
true,
itm_is_static,
src,
cur_nde_idx);
cur_nde.Key_tkn_(cur_itm);
cur_itm = null;
continue; // do not add tkn to cur_itm
}
}
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
}
break;
case Xop_tkn_itm_.Tid_pipe:
if (cur_nde_idx == 0 && ws_bgn_chk && !colon_chk && wkr_typ == Xop_arg_wkr_.Typ_tmpl)
return false; // 1st arg, but no name; EX: "{{|a}}", "{{ }}"; disregard if lnki, since
// "[[|a]]" is valid
if (wkr_typ == Xop_arg_wkr_.Typ_tmpl && brack_count > 0) {
break;
} else {
Arg_itm_end(
ctx,
cur_nde,
cur_itm,
ws_bgn_idx,
ws_end_idx,
cur_itm_subs_len,
sub_pos_bgn,
wkr_typ,
key_exists,
false,
itm_is_static,
src,
cur_nde_idx);
cur_nde.Val_tkn_(cur_itm);
if (!wkr.Args_add(ctx, src, tkn, cur_nde, cur_nde_idx))
return false; // NOTE: if invalid, exit now; lnki_wkr expects false if any argument is
// invalid; DATE:2014-06-06
cur_nde = null;
cur_itm = null;
key_exists = false; // reset
continue; // do not add tkn to cur_itm
}
case Xop_tkn_itm_
.Tid_tmpl_prm: // nested prm (3 {) or invk (2 {); mark itm_is_static = false and treat
// tkn as txt
case Xop_tkn_itm_.Tid_tmpl_invk:
itm_is_static = false;
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
case Xop_tkn_itm_.Tid_xnde:
Xop_xnde_tkn sub_as_xnde = (Xop_xnde_tkn) sub;
switch (sub_as_xnde.Tag().Id()) {
case Xop_xnde_tag_.Tid__noinclude:
case Xop_xnde_tag_.Tid__includeonly:
case Xop_xnde_tag_.Tid__onlyinclude:
itm_is_static = false;
break;
}
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
default:
if (ws_bgn_chk) ws_bgn_chk = false;
else ws_end_idx = -1; // INLINE: AdjustWsForTxtTkn
break;
}
cur_itm.Subs_add_grp(sub, root, i);
cur_itm_subs_len++;
}
if (brack_count > 0) return false;
if (cur_nde == null) // occurs when | is last tkn; EX: {{name|a|}};
cur_nde = tkn_mkr.ArgNde(++cur_nde_idx, bgn_pos);
if (cur_itm == null) { // occurs when = is last tkn; EX: {{name|a=}};
cur_itm = tkn_mkr.ArgItm(bgn_pos, -1);
itm_is_static = ws_bgn_chk = true;
cur_itm_subs_len = 0;
ws_bgn_idx = ws_end_idx = -1;
key_exists = false;
}
Arg_itm_end(
ctx,
cur_nde,
cur_itm,
ws_bgn_idx,
ws_end_idx,
cur_itm_subs_len,
bgn_pos,
wkr_typ,
key_exists,
false,
itm_is_static,
src,
cur_nde_idx);
cur_nde.Val_tkn_(cur_itm);
return wkr.Args_add(ctx, src, tkn, cur_nde, cur_nde_idx);
}
public int Process_pre(
Xop_ctx ctx,
Xop_tkn_mkr tkn_mkr,
Xop_root_tkn root,
byte[] src,
int src_len,
int bgn_pos,
int cur_pos,
int txt_pos) {
Dd_clear(ctx);
Btrie_slim_mgr tblw_ws_trie = ctx.App().Utl_trie_tblw_ws();
Object o = tblw_ws_trie.Match_bgn(src, txt_pos, src_len);
if (o != null) { // tblw_ws found
Xop_tblw_ws_itm ws_itm = (Xop_tblw_ws_itm) o;
byte tblw_type = ws_itm.Tblw_type();
switch (tblw_type) {
case Xop_tblw_ws_itm.Type_nl: // \n\s
if (cur_mode == Mode_pre) { // already in pre; just process "\n\s"
ctx.Subs_add(root, tkn_mkr.NewLine(bgn_pos, bgn_pos, Xop_nl_tkn.Tid_char, 1));
prv_nl_pos =
bgn_pos; // NOTE: must update prv_nl_pos; PAGE:en.w:Preferred_number DATE:2014-06-24
return txt_pos;
}
break;
case Xop_tblw_ws_itm.Type_xnde:
int nxt_pos = tblw_ws_trie.Match_pos();
if (nxt_pos < src_len) { // bounds check
switch (src[
nxt_pos]) { // check that next char is "end" of xnde name; guard against false
// matches like "<trk" PAGE:de.v:Via_Jutlandica/Gpx DATE:2014-11-29
case Byte_ascii.Space:
case Byte_ascii.Nl:
case Byte_ascii.Tab: // whitespace
case Byte_ascii.Slash:
case Byte_ascii.Gt: // end node
case Byte_ascii.Quote:
case Byte_ascii.Apos: // quotes
if (bgn_pos != Xop_parser_.Doc_bgn_bos)
ctx.Para().Process_nl(ctx, root, src, bgn_pos, cur_pos);
return ctx.Xnde().Make_tkn(ctx, tkn_mkr, root, src, src_len, txt_pos, txt_pos + 1);
}
}
break;
default:
{
int tblw_rv =
ctx.Tblw()
.Make_tkn_bgn(
ctx,
tkn_mkr,
root,
src,
src_len,
bgn_pos,
txt_pos + ws_itm.Hook_len(),
false,
tblw_type,
Xop_tblw_wkr.Called_from_pre,
-1,
-1);
if (tblw_rv
!= -1) // \n\s| is valid tblw tkn and processed; otherwise process pre-code below;
// EX:w:Wikipedia:WikiProject_History/CategoryExample; DATE:2014-04-14
return tblw_rv;
break;
}
}
}
// NOTE: pre lxr emulates MW for "\n\s" by (1) calling Process nl for "\n"; (2) anticipating
// next line by setting prv_ws_bgn
// EX: "\na\n b\n"; note that "\n " is cur
if (bgn_pos != Xop_parser_.Doc_bgn_bos) // if bos, then don't close 1st para
Process_nl(
ctx,
root,
src,
bgn_pos,
bgn_pos + 1); // note that tkn is \n\s; so, bgn_pos -> bgn_pos + 1 is \n ...
if (cur_mode == Mode_pre) // in pre_mode
ctx.Subs_add(
root,
tkn_mkr.Space(
root, cur_pos,
txt_pos)); // cur_pos to start after \s; do not capture "\s" in "\n\s"; (not sure why
// not before \s)
prv_ws_bgn = txt_pos - cur_pos + 1;
return txt_pos;
}