Exemplo n.º 1
0
  public static Keyval[] new_() {
    Ordered_hash translated = Ordered_hash_.New_bry();
    List_adp untranslated = List_adp_.New();
    Add_itm_many(
        translated,
        Xol_lang_stub_.Id_en,
        Xol_lang_stub_.Id_de,
        Xol_lang_stub_.Id_pl,
        Xol_lang_stub_.Id_zh_hans,
        Xol_lang_stub_
            .Id_zh_hant); // add langs with translations first, so they alphabetize to top of list
    int len = Xol_lang_stub_.Id__max;
    for (int i = 0; i < len; i++) { // add rest of langs, but sort by code
      Xol_lang_stub itm = Xol_lang_stub_.Get_by_id(i);
      if (translated.Has(itm.Key())) continue;
      untranslated.Add(itm);
    }
    untranslated.Sort_by(Xol_lang_stub_.Comparer_key);

    Keyval[] rv = new Keyval[len];
    int translated_max = translated.Count();
    for (int i = 0; i < translated_max; i++) rv[i] = new_itm((Xol_lang_stub) translated.Get_at(i));

    for (int i = translated_max; i < len; i++)
      rv[i] = new_itm((Xol_lang_stub) untranslated.Get_at(i - translated_max));
    return rv;
  }
Exemplo n.º 2
0
  public static void Find_pages(List_adp rv, Xowe_wiki wiki, Dpl_itm itm) {
    rv.Clear();
    List_adp includes = itm.Ctg_includes();
    if (includes == null) return;
    int includes_len = includes.Count();
    Ordered_hash old_regy = Ordered_hash_.New(),
        new_regy = Ordered_hash_.New(),
        cur_regy = Ordered_hash_.New();
    Xodb_load_mgr load_mgr = wiki.Db_mgr().Load_mgr();
    Xowd_page_itm tmp_page = new Xowd_page_itm();
    Int_obj_ref tmp_id = Int_obj_ref.New_zero();
    List_adp del_list = List_adp_.New();
    int ns_filter = itm.Ns_filter();
    Ordered_hash exclude_pages = Ordered_hash_.New();
    Find_excludes(exclude_pages, wiki, load_mgr, tmp_page, tmp_id, itm.Ctg_excludes());

    for (int i = 0; i < includes_len; i++) { // loop over includes
      byte[] include = (byte[]) includes.Get_at(i);
      cur_regy.Clear();
      del_list.Clear();
      Find_pages_in_ctg(cur_regy, wiki, load_mgr, tmp_page, tmp_id, include);
      Del_old_pages_not_in_cur(i, tmp_id, old_regy, cur_regy, del_list);
      Add_cur_pages_also_in_old(i, tmp_id, old_regy, cur_regy, new_regy, exclude_pages, ns_filter);
      old_regy = new_regy;
      new_regy = Ordered_hash_.New();
    }
    int pages_len = old_regy.Count();
    for (int i = 0; i < pages_len; i++) { // loop over old and create pages
      Int_obj_ref old_id = (Int_obj_ref) old_regy.Get_at(i);
      rv.Add(new Xowd_page_itm().Id_(old_id.Val()));
    }
    wiki.Db_mgr().Load_mgr().Load_by_ids(Cancelable_.Never, rv, 0, pages_len);
    rv.Sort_by(Xowd_page_itm_sorter.IdAsc);
  }
Exemplo n.º 3
0
  private static void Find_pages_in_ctg(
      Ordered_hash rv,
      Xowe_wiki wiki,
      Xodb_load_mgr load_mgr,
      Xowd_page_itm tmp_page,
      Int_obj_ref tmp_id,
      byte[] ctg_ttl) {
    Xoctg_catpage_ctg ctg =
        wiki.Ctg__catpage_mgr()
            .Get_or_load_or_null(
                Xoctg_catpage_url.New__blank(),
                wiki.Ttl_parse(gplx.xowa.wikis.nss.Xow_ns_.Tid__category, ctg_ttl),
                -1);
    if (ctg == null) return;

    // loop grps to get grp
    for (byte ctg_tid = 0; ctg_tid < Xoa_ctg_mgr.Tid___max; ++ctg_tid) {
      Xoctg_catpage_grp ctg_grp = ctg.Grp_by_tid(ctg_tid);
      int itms_len = ctg_grp.Itms__len();

      // loop itms in grp and add to hash
      for (int i = 0; i < itms_len; ++i) {
        Xoctg_catpage_itm ctg_itm = ctg_grp.Itms__get_at(i);
        int itm_page_id = ctg_itm.Page_id();
        if (rv.Has(tmp_id.Val_(itm_page_id))) continue;
        rv.Add(Int_obj_ref.New(itm_page_id), ctg_itm);

        // DELETE: recurse subcategories; PAGE:en.b:XML DATE:2016-09-18
        // if (ctg_tid == Xoa_ctg_mgr.Tid__subc) {
        //	load_mgr.Load_by_id(tmp_page, itm_page_id);
        //	Find_pages_in_ctg(rv, wiki, load_mgr, tmp_page, tmp_id, tmp_page.Ttl_page_db());
        // }
      }
    }
  }
Exemplo n.º 4
0
  public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
    long all_time_bgn = gplx.core.envs.System_.Ticks();
    wiki.Init_by_wiki();
    Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
    ctx.Init(wiki, src_conn);

    // merge data
    int hash_len = wkr_hash.Len();
    for (int i = 0; i < hash_len; ++i) {
      if (prog_wkr.Canceled()) break;
      Merge2_wkr wkr = (Merge2_wkr) wkr_hash.Get_at(i);
      // if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
      long wkr_time_bgn = gplx.core.envs.System_.Ticks();
      wkr.Merge_data(ctx, prog_wkr);
      Gfo_log_.Instance.Info(
          "merge.wkr.done",
          "data",
          src_url.NameAndExt()
              + "|"
              + wkr.Tbl().Tbl_name()
              + "|"
              + gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
    }
    if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
    Gfo_log_.Instance.Info(
        "merge.wkr.done",
        "data",
        src_url.NameAndExt()
            + "|-1|"
            + gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
    src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
  }
Exemplo n.º 5
0
 private static void Add_itm_many(Ordered_hash translated, int... langs) {
   int langs_len = langs.length;
   for (int i = 0; i < langs_len; i++) {
     Xol_lang_stub itm = Xol_lang_stub_.Get_by_id(langs[i]);
     translated.Add_if_dupe_use_nth(itm.Key(), itm);
   }
 }
Exemplo n.º 6
0
 private static void Del_old_pages_not_in_cur(
     int i, Int_obj_ref tmp_id, Ordered_hash old_regy, Ordered_hash cur_regy, List_adp del_list) {
   if (i == 0) return; // skip logic for first ctg (which doesn't have a predecessor)
   int old_len = old_regy.Count();
   for (int j = 0; j < old_len; j++) { // if cur is not in new, del it
     Int_obj_ref old_id = (Int_obj_ref) old_regy.Get_at(j);
     if (!cur_regy.Has(tmp_id.Val_(old_id.Val()))) // old_itm does not exist in cur_regy
     del_list.Add(
           old_id); // remove; EX: (A,B) in old; B only in cur; old should now be (A) only
   }
   int del_len = del_list.Count();
   for (int j = 0; j < del_len; j++) {
     Int_obj_ref old_itm = (Int_obj_ref) del_list.Get_at(j);
     old_regy.Del(tmp_id.Val_(old_itm.Val()));
   }
 }
 @Override
 protected void Fill_stmt(Db_stmt stmt, int bgn, int end) {
   for (int i = bgn; i < end; i++) {
     Xowd_page_itm page = (Xowd_page_itm) hash.Get_at(i);
     stmt.Crt_int(page_tbl.Fld_page_ns(), page.Ns_id());
     stmt.Crt_bry_as_str(page_tbl.Fld_page_title(), page.Ttl_page_db());
   }
 }
 @Override
 protected Xowd_page_itm Get_page_or_null(Xowd_page_itm rdr_page) {
   Xow_ns ns = ns_mgr.Ids_get_or_null(rdr_page.Ns_id());
   if (ns == null)
     return null; // NOTE: ns seems to "randomly" be null when threading during redlinks; guard
                  // against null; DATE:2014-01-03
   byte[] ttl_wo_ns = rdr_page.Ttl_page_db();
   rdr_page.Ttl_(ns, ttl_wo_ns);
   return (Xowd_page_itm) hash.Get_by(rdr_page.Ttl_full_db());
 }
Exemplo n.º 9
0
	public static byte[] Write_wdata_links(List_adp slink_list, Xowe_wiki wiki, Xoa_ttl ttl, Wdata_external_lang_links_data external_links_mgr) {
		try {
			switch (wiki.Domain_tid()) {
				case Xow_domain_type_.Tid_home:		// home will never be in wikidata
				case Xow_domain_type_.Tid_wikidata:	// wikidata will never be in wikidata
					return Qid_null;
			}
			Wdata_wiki_mgr wdata_mgr = wiki.Appe().Wiki_mgr().Wdata_mgr();
			Wdata_doc doc = wdata_mgr.Pages_get(wiki, ttl); if (doc == null) return Qid_null;	// no links
			boolean external_links_mgr_enabled = external_links_mgr.Enabled();
			Ordered_hash links = doc.Slink_list();
			Bry_bfr tmp_bfr = wiki.Appe().Utl__bfr_mkr().Get_k004();
			Xow_wiki_abrv wiki_abrv = new Xow_wiki_abrv();
			int len = links.Count();
			for (int i = 0; i < len; i++) {
				Wdata_sitelink_itm slink = (Wdata_sitelink_itm)links.Get_at(i);
				byte[] xwiki_key = slink.Site();
				Xow_wiki_abrv_.parse_(wiki_abrv, xwiki_key, 0, xwiki_key.length);
				if (wiki_abrv.Domain_tid() == Xow_wiki_abrv_.Tid_null) {
					wiki.Appe().Usr_dlg().Warn_many("", "", "unknown wiki in wikidata: ttl=~{0} wiki=~{1}", ttl.Page_db_as_str(), String_.new_u8(xwiki_key));
					continue;
				}
				if (wiki_abrv.Domain_tid() != wiki.Domain_tid()) continue;	// ignore wikis in a different domain; EX: looking at enwiki:Earth, and wikidata has dewikiquote; ignore dewikiquote; DATE:2014-06-21
				byte[] lang_key = wiki_abrv.Lang_itm().Key();
				if (external_links_mgr_enabled && external_links_mgr.Langs_hide(lang_key, 0, lang_key.length)) continue;
				tmp_bfr.Add(lang_key);
				tmp_bfr.Add_byte(Byte_ascii.Colon);
				tmp_bfr.Add(slink.Name());
				Xoa_ttl slink_ttl = Xoa_ttl.parse_(wiki, tmp_bfr.Xto_bry_and_clear());
				if (slink_ttl == null) continue;								// invalid ttl
				Xow_xwiki_itm xwiki_itm = slink_ttl.Wik_itm();
				if (	xwiki_itm == null									// not a known xwiki; EX: [[zzz:abc]]
					||	Bry_.Eq(xwiki_itm.Domain_bry(), wiki.Domain_bry())	// skip if same as self; i.e.: do not include links to enwiki if already in enwiki
					) continue;
				slink.Page_ttl_(slink_ttl);
				slink_list.Add(slink);
			}
			tmp_bfr.Mkr_rls();
			if (external_links_mgr_enabled && external_links_mgr.Sort())
				slink_list.Sort_by(Xoa_ttl_sorter._);
			return doc.Qid();
		} catch (Exception e) {Err_.Noop(e); return Qid_null;}
	}
Exemplo n.º 10
0
 private static void Add_cur_pages_also_in_old(
     int i,
     Int_obj_ref tmp_id,
     Ordered_hash old_regy,
     Ordered_hash cur_regy,
     Ordered_hash new_regy,
     Ordered_hash exclude_pages,
     int ns_filter) {
   int found_len = cur_regy.Count();
   for (int j = 0; j < found_len; j++) { // if new_page is in cur, add it
     Xoctg_catpage_itm cur_itm = (Xoctg_catpage_itm) cur_regy.Get_at(j);
     Xoa_ttl cur_ttl = cur_itm.Page_ttl();
     if (cur_ttl == null) continue;
     if (ns_filter != Dpl_itm.Ns_filter_null && ns_filter != cur_ttl.Ns().Id()) continue;
     tmp_id.Val_(cur_itm.Page_id()); // set tmp_id, since it will be used at least once
     if (exclude_pages.Has(tmp_id)) continue; // ignore excluded pages
     if (i != 0) { // skip logic for first ctg (which doesn't have a predecessor)
       if (!old_regy.Has(tmp_id)) continue; // cur_itm not in old_regy; ignore
     }
     new_regy.Add_as_key_and_val(Int_obj_ref.New(cur_itm.Page_id()));
   }
 }
Exemplo n.º 11
0
  @Override
  public Xobc_xodir_dir[] Get_dirs(Xoa_app app) {
    Ordered_hash hash = Ordered_hash_.New();

    // add default app dir
    String default_data_dir = context.getApplicationInfo().dataDir + "/";
    String selected =
        app.User()
            .User_db_mgr()
            .Cfg()
            .Get_app_str_or(Xobc_xodir_cfg.Key__selected_dir, default_data_dir);
    Xobc_xodir_dir itm =
        new Xobc_xodir_dir(
            String_.Eq(selected, default_data_dir), false, Bry_.new_u8(default_data_dir));
    hash.Add(default_data_dir, itm);

    // add "guessed" external dirs
    Sdcard_detection_strategy[] strategy_ary =
        new Sdcard_detection_strategy[] {
          new Strategy__storage_dir(-1),
          new Strategy__envvar__external_storage(-1),
          new Strategy__envvar__secondary_storage(
              -1) // NOTE: needed for BLU-STUDIO C 5+5 LTE; DATE: 2016-06-28
        };
    int len = strategy_ary.length;
    String root_rhs = Sdcard_detection_mgr.Get_root_rhs() + "/";
    for (int i = 0; i < len; ++i) {
      Sdcard_detection_strategy strategy = strategy_ary[i];
      String[] dir_strs = strategy.Get_dirs(context);
      for (String dir_str : dir_strs) {
        if (dir_str == null) continue;
        dir_str += root_rhs;
        if (hash.Has(dir_str)) continue;
        File dir = new File(dir_str);
        if (!dir.exists())
          continue; // dir doesn't exist; don't bother trying to create b/c permission won't be
                    // available at 4.4+
        if (!Sdcard_detection_mgr.Dir_is_writeable(dir)) continue;
        itm = new Xobc_xodir_dir(String_.Eq(selected, dir_str), false, Bry_.new_u8(dir_str));
        hash.Add(dir_str, itm);
      }
    }

    // add custom dir
    String custom =
        app.User()
            .User_db_mgr()
            .Cfg()
            .Get_app_str_or(Xobc_xodir_cfg.Key__custom_dir, "(choose your own folder)");
    itm = new Xobc_xodir_dir(String_.Eq(selected, custom), true, Bry_.new_u8(custom));
    hash.Add(custom, itm);
    return (Xobc_xodir_dir[]) hash.To_ary(Xobc_xodir_dir.class);
  }
Exemplo n.º 12
0
 public Wbase_claim_grp Get_at(int i) {
   return (Wbase_claim_grp) hash.Get_at(i);
 }
Exemplo n.º 13
0
 public int Len() {
   return hash.Count();
 }
Exemplo n.º 14
0
 public void Add(Wbase_claim_grp itm) {
   hash.Add(itm.Id_ref(), itm);
 }
Exemplo n.º 15
0
 private static Ordered_hash Make_wkrs(Merge2_wkr... wkrs) {
   Ordered_hash rv = Ordered_hash_.New();
   for (Merge2_wkr wkr : wkrs) rv.Add(wkr.Tbl().Tbl_name(), wkr);
   return rv;
 }