Bump version from 0.1.2-alpha.0 to 0.1.2-alpha.1

app.py

@@ -107,7 +107,7 @@ loggu.remove()
 _ = (
     "<green>{time:YY-MM-DD HH:mm:ss}</green> | "
     "<level>{level: <5}</level> | <level>{message}</level> "
-    "<cyan>{name}</cyan>:<cyan>{line}</cyan>"
+    "<cyan>{module}.{name}</cyan>:<cyan>{line}</cyan>"
 )
 loggu.add(
     sys.stderr,
@@ -131,11 +131,15 @@ st.set_page_config(  # type: ignore
 pd.set_option("display.precision", 2)
 pd.options.display.float_format = "{:,.2f}".format
 
+sourcetype = "upload"
+if set_loglevel() <= 10:
+    sourcetype = "urls"
+
 _ = dict(
     beetype="ezbee",
-    sourcetype="upload",
+    sourcetype=sourcetype,
     sourcecount=2,
-    sent_ali=None,
+    sentali=None,
     src_filename="",
     tgt_filename="",
     src_fileio=b"",
@@ -148,6 +152,7 @@ _ = dict(
     df_a=None,
     df_s_a=None,
     count=1,
+    updated=False,
 )
 if "ns" not in state:
     state.ns = SimpleNamespace(**_)
@@ -186,7 +191,8 @@ def main():
     if set_loglevel() <= 10:
         st.markdown(state.ns.count)
     loggu.debug(f" run: {state.ns.count}")
+    logger.debug(f" run: {state.ns.count}")
     state.ns.count += 1
-
+    state.ns.updated = False
 
 main()

data/test_zh_t.txt

@@ -1,74 +1,36 @@
 呼嘯山莊
-
--------------------------------------------------- ------------------------------
-
+--------------------------------------------------------------------------------
 第二章
-
- 中文
-
-
-我想就在書房邊呼磨山霧了，想不想踩著雜草到。
-
-注意 注意 注意 吃 吃 飯 午飯 吃 並且 當作 所 房子 午 飯 的 附屬物 管家 , 一 位 的 這 位 的 這 位 的 這 位 的 這 位 並 開 意 的 太太 能 或 願 意 意 我 在 五 點 鐘 時 用 意 意 , 在 懷 懷 的 聯 系上一樓，邁一個進了屋子地上，有掃帚正在她她正在堆堆的煤渣搞一團散佈的立刻回頭了。場場初降鵝毛鵝毛大雪。
-
-在荒涼的涼涼裡，而而在山頂的樹叢中，踩著陸地的。到我的手指骨都痛了，狗狂吠起來。
-
-“倒霉的人至少就沒有禮了，我就直接為我家就這樣了。”我還在搖家裡孤零零地打電話。它。苦臉的約瑟夫從穀倉的一個圓窗裡探出頭來。
-
-“你幹嗎？”他大叫。 “主人在牛欄裡，你找他說話，就從這條路口繞過去。”
-
-“屋裡開門嗎？”我也叫起來。
-
-“除了老婆沒有別人。你就是騰到夜裡，她也不會開。”
-
-“為什麼？你就不能告訴她我是誰，呃，約瑟夫？”
-
+英文
+昨天下午又冷又有霧。 我想就在書房爐邊消磨一下午，不想踩著雜草污泥到呼嘯山莊了。
+但是，吃過午飯（注意——我在十二點與一點鐘之間吃午飯，而可以當作這所房子的附屬物的管家婆，一比特慈祥的家後卻不能，或者並不願理解我請求在五點鐘開飯的用意），在我懷着這個懶惰的想法上了樓，邁進屋子的時候，看見一個女僕跪在地上，身邊是掃帚和煤鬥。 她正在用一堆堆煤渣封火，搞起一片彌漫的灰塵。 這景象立刻把我趕回頭了。 我拿了帽子，走了四裡路，到達了希刺克厲夫的花園口口，剛好躲過了一場今年初降的鹅毛大雪。
+在那荒凉的山頂上，土地由於結了一層黑冰而凍得堅硬，冷空氣使我四肢發抖。 我弄不開門鏈，就跳進去，順著兩邊種著蔓延的醋栗樹叢的石路跑去。 我白白地敲了半天門，一直敲到我的手指骨都痛了，狗也狂吠起來。
+“倒楣的人家！” 我心裡直叫，“只為你這樣無禮待客，就該一輩子跟人群隔離。我至少還不會在白天把門閂住。我才不管呢——我要進去！” 如此决定了。 我就抓住門閂，使勁搖它。 苦臉的約瑟夫從穀倉的一個圓窗裏探出頭來。
+“你幹嗎？” 他大叫。 “主人在牛欄裏，你要是找他說話，就從這條路口繞過去。”
+“屋裡沒人開門嗎？” 我也叫起來。
+“除了家後沒有別人。你就是鬧騰到夜裡，她也不會開。”
+“為什麼？你就不能告訴她我是誰嗎，呃，約瑟夫？”
 “別找我！我才不管這些閒事呢，”這個腦袋咕嚕著，又不見了。
-
-雪開始大 握住 門柄 門柄 又 沒有 一個 外衣 外衣 烤 ， ， 一草 耙 ， 在 院子 裡 他 招呼 我 著 走 走 一 穿過 一個 一個 和 一 鋪 一地 的 、 有 抽水機抽水機 抽水機 鴿籠 鴿籠 到 到 我 使 接待 過 的 那 個 的 、 屋子老婆以前從來沒有她的料。這麼一她我的等候，以為我會叫坐下。望望我，往的椅子一靠，不動，也不出。
-
-“天氣真！壞我我是老婆的老婆，說不定你的僕人大吃大喝，大勁才聽到，聽見了！我的”
-
-眼睛一瞪——也不開開。 總是一冷冷瞪她的神眸冷眼瞪住，使，而且不愉快
-
-“坐下吧，”那粗聲粗氣說，“他來了。”
-
-我咳了一聲，叫那惡狗朱諾。臨到第二次會面，它總算賞臉，搖起尾巴人埆礆。服從表柆示。衘礆。
-
-“好漂亮的狗我又開始說話了。”你是不是反對那些呢！
-
-“這一些我的女刺客克厲克民的回答，比希希親可愛的回答的主人公更冷冷的說那些人。”
-
-“，，我所愛的就是在一堆人身上！”
-
-“誰會愛這些東西那才怪呢！”她輕蔑地說。
-
-倒霉，原來又是一堆死兔子。我輕咳一聲，向火爐湊近，又把今晚天氣不好的話評論一通。
-
-“你本來就不會。。她說，站起來去拿”該台上的兩個茶葉罐。
-
-我沒有她的小官現在還沒有全身出現纖細的透明狀態，她的臉還沒有明顯的淡麗，而且她的身體還沒有明顯的淡麗。捲髮還不如說說，垂垂垂垂的她脖子上說說，輕而易舉地在與眼神能拒一些和說，要使人抗拒。一，而那張眼神 眼神 眼神 不是自然的。
-
-她夠不到。 動了 一動 ，她 猛 猛 猛 猛 地 向 像 像 守財奴 守財奴 他 他 的 金子 金子
-
-“我不要你幫忙，”她怒氣沖沖地說，“我自己得到了。”
-
-“對不起！”我連忙回答。
-
-“是請你來喫茶的嗎？”她問，把一條圍裙系在那乾淨的���衣服上，就這樣站著，拿一盒茶葉〣邌候茶葉〣邌候茶葉〣邌
-
-“我很想喝杯茶。”我回答。
-
-“是請你來的嗎？”她又問。
-
-“沒有，”我說，勉強笑一笑。 “您請我喝茶。”
-
-
-
-
+雪開始下大了。 我握住門柄又試一回。 這時一個沒穿外衣的年輕人，扛著一根草耙，在後面院子裏出現了。 他招呼我跟著他走，穿過了一個洗衣房和一片鋪平的地，那兒有煤棚、抽水機和鴿籠，我們終於到了我上次被接待過的那間溫暖的、熱鬧的大屋子。 煤、炭和木材混合在一起燃起的熊熊爐火，使這屋子放著光彩。 在準備擺上豐盛晚餐的桌旁，我很高興地看到了那位“家後”，以前我從未料想到會有這麼一個人存在的。 我鞠躬等候，以為她會叫我坐下。 她望望我，往她的椅背一靠，不動，也不出聲。
+“天氣真壞！” 我說，“希刺克厲夫家後，恐怕大門因為您的僕人偷懶而大吃苦頭，我費了好大勁才使他們聽見我敲門！”
+她死不開口。 我瞪眼——她也瞪眼。 反正她總是以一種冷冷的、漠不關心的神氣盯住我，使人十分窘，而且不愉快。
+“坐下吧，”那年輕人粗聲粗氣地說，“他就要來了。”
+我服從了； 輕輕咳了一下，叫喚那惡狗朱諾。 臨到第二次會面，它總算賞臉，搖起尾巴尖，表示認我是熟人了。
+“好漂亮的狗！” 我又開始說話。 “您是不是打算不要這些小的呢，夫人？”
+“那些不是我的，”這可愛可親的女主人說，比希刺克厲夫本人所能回答的腔調還要更冷淡些。
+“啊，您所心愛的是在這一堆裏啦！” 我轉身指著一個看不清楚的靠墊上那一堆像猫似的東西，接著說下去。
+“誰會愛這些東西那才怪呢！” 她輕蔑地說。
+倒楣，原來那是堆死兔子。 我又輕咳一聲，向火爐凑近些，又把今晚天氣不好的話評論一通。
+“你本來就不該出來。” 她說，站起來去拿壁爐臺上的兩個彩色茶葉罐。
+她原先坐在光線被遮住的地方，現在我把她的全身和面貌都看得清清楚楚。 她苗條，顯然還沒有過青春期。 挺好看的體態，還有一張我生平從未有幸見過的絕妙的小臉蛋。 五官纖麗，非常漂亮。 淡黃色的卷髮，或者不如說是金黃色的，松松地垂在她那細嫩的頸上。 至於眼睛，要是眼神能顯得和悅些，就要使人無法抗拒了。 對我這容易動情的心說來倒是常事，因為它們所表現的只是在輕蔑與近似絕望之間的一種情緒，而在那張臉上看見那樣的眼神是特別不自然的。
+她簡直够不到茶葉罐。 我動了一動，想幫她一下。 她猛地扭轉身向我，像守財奴看見別人打算幫他數他的金子一樣。
+“我不要你幫忙，”她怒氣衝衝地說，“我自己拿得到。”
+“對不起！” 我連忙回答。
+“是請你來吃茶的嗎？” 她問，把一條圍裙系在她那乾淨的黑衣服上，就這樣站著，拿一匙茶葉正要往茶壺裏倒。
+“我很想喝杯茶。” 我回答。
+“是請你來的嗎？” 她又問。
+“沒有，”我說，勉强笑一笑。 “您正好請我喝茶。”
 目錄
- 上一章
- 下一章
-
-
+上一章
+下一章
 返回首頁

litbee/__init__.py

@@ -1,5 +1,5 @@
 """Init."""
-__version__ = "0.1.2a0"
+__version__ = "0.1.2a1"
 from .litbee import litbee
 
 __all__ = ("litbee",)

litbee/app.py

@@ -107,7 +107,7 @@ loggu.remove()
 _ = (
     "<green>{time:YY-MM-DD HH:mm:ss}</green> | "
     "<level>{level: <5}</level> | <level>{message}</level> "
-    "<cyan>{name}</cyan>:<cyan>{line}</cyan>"
+    "<cyan>{module}.{name}</cyan>:<cyan>{line}</cyan>"
 )
 loggu.add(
     sys.stderr,
@@ -131,11 +131,15 @@ st.set_page_config(  # type: ignore
 pd.set_option("display.precision", 2)
 pd.options.display.float_format = "{:,.2f}".format
 
+sourcetype = "upload"
+if set_loglevel() <= 10:
+    sourcetype = "urls"
+
 _ = dict(
     beetype="ezbee",
-    sourcetype="upload",
+    sourcetype=sourcetype,
     sourcecount=2,
-    sent_ali=None,
+    sentali=None,
     src_filename="",
     tgt_filename="",
     src_fileio=b"",
@@ -148,6 +152,7 @@ _ = dict(
     df_a=None,
     df_s_a=None,
     count=1,
+    updated=False,
 )
 if "ns" not in state:
     state.ns = SimpleNamespace(**_)
@@ -186,7 +191,8 @@ def main():
     if set_loglevel() <= 10:
         st.markdown(state.ns.count)
     loggu.debug(f" run: {state.ns.count}")
+    logger.debug(f" run: {state.ns.count}")
     state.ns.count += 1
-
+    state.ns.updated = False
 
 main()

litbee/fetch_paste.py

@@ -1,9 +1,44 @@
-"""Display from paste."""
-import pandas as pd
+"""Fetch pasted text and convert to state.ns.list1/list2."""
+# pylint: disable=invalid-name
 import streamlit as st
 from logzero import logger
+from streamlit import session_state as state
 
 
 def fetch_paste():
-    """Display from paste."""
-    st.write("Coming soon")
+    """Fetch from clipboard."""
+    # st.write("Coming soon")
+    with st.form(key="paste_in_form"):
+        _ = st.expander(f"{state.ns.beetype}: Paste text", expanded=True)
+        with _:
+            col1, col2 = st.columns(2)
+            with col1:
+                text1 = st.text_area(
+                    label="Paste your stuff here",
+                    key="paste_text1",
+                    # help=""
+                    height=500,
+                )
+
+            with col2:
+                text2 = st.text_area(
+                    label="Paste your stuff here",
+                    # help=""
+                    key="paste_text2",
+                    height=500,
+                )
+
+        submitted = st.form_submit_button("Submit")
+
+    logger.debug("text1[:10]: %s, text2[:10]: %s", text1[:10], text2[:10])
+
+    list1 = [_.strip() for _ in text1.splitlines() if _.strip()]
+    list2 = [_.strip() for _ in text2.splitlines() if _.strip()]
+
+    state.ns.list1 = list1[:]
+    state.ns.list2 = list2[:]
+
+    logger.debug("len(list1): %s, len(list2): %s", len(list1), len(list2))
+
+    state.ns.updated = True
+    logger.debug("state.ns.updated: %s", state.ns.updated)

litbee/fetch_upload.py

@@ -0,0 +1,128 @@
+"""Fetch upload and convert to list1/list2."""
+from logzero import logger
+import streamlit as st
+from streamlit import session_state as state
+
+
+def fetch_upload():
+    """Fetch upload and convert to list1/list2."""
+    # src_fileio tgt_fileio
+    with st.form(key="upload_in_form"):
+        _ = st.expander(f"{state.ns.beetype}: Pick two files", expanded=True)
+        with _:
+            col1, col2 = st.columns(2)
+            with col1:
+                src_fileio = st.file_uploader(
+                    "Choose source file (utf8 txt)",
+                    type=[
+                        "txt",
+                    ],
+                    key="src_text",
+                    # accept_multiple_files=True,
+                    # accept_multiple_files=False,
+                )
+
+            with col2:
+                tgt_fileio = st.file_uploader(
+                    "Choose target file (utf8 txt)",
+                    type=[
+                        "txt",
+                    ],
+                    key="tgt_text",
+                    # accept_multiple_files=True,
+                )
+        submitted = st.form_submit_button("Submit")
+
+    # logger.debug(" len(src_fileio): %s", len(src_fileio))
+    # logger.debug(" len(tgt_fileio): %s", len(tgt_fileio))
+
+    filename1 = ""
+    if src_fileio:
+        logger.debug(" type(src_fileio): %s", type(src_fileio))
+
+        # for st.file_uploade accept_multiple_files=True
+        if isinstance(src_fileio, list):
+            logger.debug(" len(src_fileio): %s", len(src_fileio))
+            filenames = []
+            try:
+                filenames = [elm.name for elm in src_fileio]  # type: ignore
+            except Exception as exc:
+                logger.error(exc)
+            logger.debug("src_fileio  names: *%s*", filenames)
+
+            # state.ns.src_fileio = src_fileio
+            state.ns.src_file = src_fileio[-1].getvalue().decode()
+            state.ns.src_filename = src_fileio[-1].name
+        else:
+            logger.debug("src_fileio.name: [%s]", src_fileio.name)
+            filenames = [src_fileio.name]
+            logger.debug("src_fileio  names: %s", filenames)
+
+            # state.ns.src_fileio = src_fileio
+            state.ns.src_file = src_fileio.getvalue().decode()
+            state.ns.src_filename = src_fileio.name
+        filename1 = state.ns.src_filename
+
+    filename2 = ""
+    if tgt_fileio:
+        if isinstance(tgt_fileio, list):
+            logger.warning("not set to handle multiple files")
+            logger.warning("set accept_multiple_files=False in the meantime")
+        else:
+            state.ns.tgt_file = tgt_fileio.getvalue().decode()
+            state.ns.tgt_filename = tgt_fileio.name
+            filename2 = tgt_fileio.name
+
+    # proceed when Submit is clicked
+    msg1 = ""
+    if filename1:
+        msg1 += f" file1 {filename1}"
+    msg2 = ""
+    if filename2:
+        msg2 += f" file2 {filename2}"
+    glue = ""
+    if filename1 and filename2:
+        glue = ", "
+
+    upload_placeholder = st.empty()
+    prefix = f" Upload submitted: {msg1}{glue}{msg2}"
+    upload_placeholder.write(prefix)
+
+    # st.write(f"  Submitted upload: {msg1}{glue}{msg2}")
+    if not submitted:
+        return None
+
+    if not (filename1 or filename2):
+        # st.write("|  no file uploaded")
+        upload_placeholder.write(f"{prefix} no file uploaded")
+        return None
+
+    if not filename1:
+        # st.write("|  file1 not ready")
+        upload_placeholder.write(f"{prefix}, file1 not ready")
+        return None
+
+    if not filename2:
+        # st.write("|  file2 not ready")
+        upload_placeholder.write(f"{prefix}, file2 not ready")
+        return None
+
+    try:
+        _ = state.ns.src_file.splitlines()
+        list1 = [elm.strip() for elm in _ if elm.strip()]
+        _ = state.ns.tgt_file.splitlines()
+        list2 = [elm.strip() for elm in _ if elm.strip()]
+    except Exception as exc:
+        logger.error(exc)
+        list1 = [""]
+        list2 = [""]
+
+    logger.debug("len(list1): %s, len(list2): %s", len(list1), len(list2))
+
+    state.ns.list1 = list1[:]
+    state.ns.list2 = list2[:]
+
+    state.ns.updated = True
+    logger.debug("state.ns.updated: %s", state.ns.updated)
+
+    return None

litbee/fetch_urls.py

@@ -1,9 +1,170 @@
-"""Fetch from urls."""
-import pandas as pd
+"""Fetch text from urls and convert to state.ns.list1/list2."""
+# pylint: disable=invalid-name
 import streamlit as st
+from icecream import ic
 from logzero import logger
+from streamlit import session_state as state
+from litbee.url2txt import url2txt
+
+ic.configureOutput(
+    includeContext=True,
+    outputFunction=logger.debug,  # outputFunction=logger.info,
+)
 
 
 def fetch_urls():
-    """Display from urls."""
-    st.write("Coming soon")
+    """Fetch text from urls and convert to state.ns.list1/list2."""
+    beetype = state.ns.beetype
+    sourcecount = state.ns.sourcecount
+    value = ""
+    if beetype == "ezbee":
+        url1 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/test_en.txt"
+        url2 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/test_zh.txt"
+        value = f"{url1} {url2}"
+    if beetype == "dzbee":
+        url1 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-de.txt"
+        url2 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-zh.txt"
+        value = f"{url1} {url2}"
+    if beetype == "debee":
+        url1 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-de.txt"
+        url2 = "https://raw.githubusercontent.com/ffreemt/en-de-zh-txt/master/sternstunden04-en.txt"
+        value = f"{url1} {url2}"
+
+    dict_ = dict(text1="", text2="")
+
+    def fetch_cb():
+        """Fetch text (dict_["text1"|"text2"]) from urls."""
+        ic("fetch_cb")
+        urls = [elm.strip() for elm in text_inp.split(" ") if elm.strip()]
+
+        # supply http:// if not startswith http
+        urls = [elm if elm.startswith("http") else "http://" + elm for elm in urls]
+
+        _ = "\n\t"
+        # st.markdown(f" urls submitted: \n{_.join(urls)}")
+        ic(f" urls submitted: \n{_.join(urls)}")
+
+        # st.write(" TODO: fetch text from urls.")
+
+        if state.ns.sourcecount == 2:  # 2-sep
+            for idx, url in enumerate(urls[:2]):
+                try:
+                    _ = url2txt(url)
+                except Exception as e:
+                    logger.error(e)
+                    _ = str(e)
+                dict_[f"text{idx + 1}"] = _
+                ic(f"{idx + 1}: [{url}] {dict_['text' + str(idx + 1)][:100]}")
+
+            ic(dict_["text1"][:10])
+            ic(dict_["text2"][:10])
+        else:  # 1-mix
+            text1 = ""
+            for url in urls:
+                try:
+                    _ = url2txt(url)
+                except Exception as e:
+                    logger.error(e)
+                    _ = str(e)
+                text1 += _
+            ic(text1[:10])
+            dict_["text1"] = text1[:]
+
+        _ = [elm.strip() for elm in dict_["text1"].splitlines() if elm.strip()]
+        state.ns.list1 = _
+        _ = [elm.strip() for elm in dict_["text2"].splitlines() if elm.strip()]
+        state.ns.list2 = _
+
+        list1 = state.ns.list1
+        list2 = state.ns.list2
+        ic(len(list1), len(list2))
+
+        state.fetched_text1 = dict_["text1"]
+        state.fetched_text2 = dict_["text2"]
+
+        # streamlit complains if an initial value of
+        # a widget with this key is set
+        # state.text_area_urls = text_inp
+
+    # with st.form(key="urls_in_form"):
+    # _ = st.expander(f"{beetype}: Paste urls below and press Ctl+Enter or Space Ctl+Enter to testdrive", expanded=True)
+    # with _:
+    label = f"{beetype}: Paste urls below and press Ctl+Enter or Space Ctl+Enter to testdrive"
+    text_inp = st.text_area(
+        label=label,
+        value=value,
+        key="text_area_urls",
+        height=25,
+        help=" URLs separated by at least a space or a newline（贴网址，空格分开或另起一行, Ctrl+回车提交）",
+        on_change=fetch_cb,
+        # args=(text_inp,),
+    )
+
+    # st.button("Fetch", on_click=fetch_cb, args=(text_inp,))
+
+    def text2lists():
+        """Convert text(s) to list(s)."""
+        if text1:
+            try:
+                list1 = [elm.strip() for elm in text1.splitlines() if elm.strip()]
+                state.ns.list1 = list1[:]
+            except Exception as e:
+                logger.warning("text1 to list1 errors: %s", e)
+
+        if text2:
+            try:
+                list2 = [elm.strip() for elm in text2.splitlines() if elm.strip()]
+                state.ns.list2 = list2[:]
+            except Exception as e:
+                logger.warning("text2 to list2 errors: %s", e)
+
+    # show fetch text(s)
+    text1 = dict_["text1"]
+    text2 = dict_["text2"]
+    if state.ns.sourcecount == 2:  # 2-sep
+        with st.form(key="fetched_2texts_in_form"):
+            _ = st.expander(f"{state.ns.beetype}: fetched text", expanded=True)
+            with _:
+                col1, col2 = st.columns(2)
+                with col1:
+                    text1 = st.text_area(
+                        label="Edit when necessary, click Submit when ready",
+                        key="fetched_text1",
+                        # help=""
+                        height=500,
+                        value=text1,
+                    )
+
+                with col2:
+                    text2 = st.text_area(
+                        label="Edit when necessary, click Submit when ready",
+                        # help=""
+                        key="fetched_text2",
+                        height=500,
+                        value=text2,
+                    )
+
+            submitted = st.form_submit_button(
+                "Submit",
+                on_click=text2lists
+            )
+
+    else:  # 1-mix
+        with st.form(key="fetched_1_text_in_form"):
+            _ = st.expander(f"{state.ns.beetype}: fetched mixed text", expanded=True)
+            with _:
+                text1 = st.text_area(
+                    label="Edit when necessary, click Submit when ready",
+                    key="fetched_mixed_text1",
+                    height=500,
+                    value=text1,
+                )
+            submitted = st.form_submit_button("Submit", on_click=text2lists)
+
+    # _ = """
+    if not submitted:
+        ic("Submit not yet clicked")
+        return
+    # """
+
+    state.ns.updated = True

litbee/home.py

@@ -32,14 +32,26 @@ from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
 # from st_aggrid.grid_options_builder import GridOptionsBuilder
 from streamlit import session_state as state
 
+from litbee.fetch_upload import fetch_upload
+from litbee.fetch_paste import fetch_paste
+from litbee.fetch_urls import fetch_urls
 from litbee.t2s import t2s
 
 
 def home():  # noqa
-    """Fetch content from upload."""
-    # st.write(state.ns.beetype)
+    """Run tasks.
 
-    if state.ns.sourcetype not in ["upload"]:
+    beetype
+
+    sourcetype
+        fetch_upload/fetch_paste, fetch_url
+            sourcecount
+
+    align: para-align/sent-align
+
+    save xlsx/tsv
+    """
+    if state.ns.sourcetype not in ["upload", "paste", "urls"]:
         st.write("Coming soooooooon...")
         return None
 
@@ -47,122 +59,37 @@ def home():  # noqa
         st.write("Coming soon...")
         return None
 
-    # src_fileio tgt_fileio
-    with st.form(key="upload_in_form"):
-        _ = st.expander(f"{state.ns.beetype}: Pick two files", expanded=True)
-        with _:
-            col1, col2 = st.columns(2)
-            with col1:
-                src_fileio = st.file_uploader(
-                    "Choose source file (utf8 txt)",
-                    type=[
-                        "txt",
-                    ],
-                    key="src_text",
-                    # accept_multiple_files=True,
-                    # accept_multiple_files=False,
-                )
-
-            with col2:
-                tgt_fileio = st.file_uploader(
-                    "Choose target file (utf8 txt)",
-                    type=[
-                        "txt",
-                    ],
-                    key="tgt_text",
-                    # accept_multiple_files=True,
-                )
-        submitted = st.form_submit_button("Submit")
-
-    # logger.debug(" len(src_fileio): %s", len(src_fileio))
-    # logger.debug(" len(tgt_fileio): %s", len(tgt_fileio))
-
-    filename1 = ""
-    if src_fileio:
-        logger.debug(" type(src_fileio): %s", type(src_fileio))
-
-        # for st.file_uploade accept_multiple_files=True
-        if isinstance(src_fileio, list):
-            logger.debug(" len(src_fileio): %s", len(src_fileio))
-            filenames = []
-            try:
-                filenames = [elm.name for elm in src_fileio]  # type: ignore
-            except Exception as exc:
-                logger.error(exc)
-            logger.debug("src_fileio  names: *%s*", filenames)
-
-            # state.ns.src_fileio = src_fileio
-            state.ns.src_file = src_fileio[-1].getvalue().decode()
-            state.ns.src_filename = src_fileio[-1].name
-        else:
-            logger.debug("src_fileio.name: [%s]", src_fileio.name)
-            filenames = [src_fileio.name]
-            logger.debug("src_fileio  names: %s", filenames)
-
-            # state.ns.src_fileio = src_fileio
-            state.ns.src_file = src_fileio.getvalue().decode()
-            state.ns.src_filename = src_fileio.name
-        filename1 = state.ns.src_filename
-
-    filename2 = ""
-    if tgt_fileio:
-        if isinstance(tgt_fileio, list):
-            logger.warning("not set to handle multiple files")
-            logger.warning("set accept_multiple_files=False in the meantime")
-        else:
-            state.ns.tgt_file = tgt_fileio.getvalue().decode()
-            state.ns.tgt_filename = tgt_fileio.name
-            filename2 = tgt_fileio.name
-
-    # proceed when Submit is clicked
-    msg1 = ""
-    if filename1:
-        msg1 += f" file1 {filename1}"
-    msg2 = ""
-    if filename2:
-        msg2 += f" file2 {filename2}"
-    glue = ""
-    if filename1 and filename2:
-        glue = ", "
-
-    upload_placeholder = st.empty()
-    prefix = f"  Submitted upload: {msg1}{glue}{msg2}"
-    upload_placeholder.write(prefix)
-
-    # st.write(f"  Submitted upload: {msg1}{glue}{msg2}")
-    if not submitted:
+    # process sourcetype and fetch list1/list2
+    list1 = list2 = []
+    # fetch_upload/fetch_paste
+    if state.ns.sourcetype in ["upload"]:
+        fetch_upload()
+    elif state.ns.sourcetype in ["paste"]:
+        fetch_paste()
+    elif state.ns.sourcetype in ["urls"]:
+        fetch_urls()
+    else:
+        st.warning(f"{state.ns.sourcetype}: Not implemented")
         return None
 
-    if not (filename1 or filename2):
-        # st.write("|  no file uploaded")
-        upload_placeholder.write(f"{prefix} no file uploaded")
-        return None
+    logger.debug("state.ns.updated: %s", state.ns.updated)
 
-    if not filename1:
-        # st.write("|  file1 not ready")
-        upload_placeholder.write(f"{prefix}, file1 not ready")
+    # if not updated, quit: this does not quite work
+    # only prevents the first run/missing upload
+    if not state.ns.updated:
+        logger.debug(" not updated, early exit.")
         return None
 
-    if not filename2:
-        # st.write("|  file2 not ready")
-        upload_placeholder.write(f"{prefix}, file2 not ready")
-        return None
+    list1 = state.ns.list1[:]
+    list2 = state.ns.list2[:]
 
-    try:
-        _ = state.ns.src_file.splitlines()
-        list1 = [elm.strip() for elm in _ if elm.strip()]
-        _ = state.ns.tgt_file.splitlines()
-        list2 = [elm.strip() for elm in _ if elm.strip()]
-    except Exception as exc:
-        logger.error(exc)
-        list1 = [""]
-        list2 = [""]
-    state.ns.list1 = list1[:]
-    state.ns.list2 = list2[:]
+    logger.debug("list1[:3]: %s", list1[:3])
+    logger.debug("list2[:3]: %s", list2[:3])
 
     df = pd.DataFrame(zip_longest(list1, list2, fillvalue=""))
     try:
-        df.columns = ["text1", "text2"]
+        # df.columns = ["text1", "text2"]
+        df.columns = [f"text{i + 1}" for i in range(len(df.columns))]
     except Exception as exc:
         logger.debug("df: \n%s", df)
         logger.error("%s", exc)
@@ -171,6 +98,7 @@ def home():  # noqa
     logger.debug("df: %s", df)
 
     # st.table(df)  # looks alright
+    # equiv to st.markdown(df.to_markdown())?
 
     # stlyed pd dataframe?
     # bigger, no pagination
@@ -186,28 +114,11 @@ def home():  # noqa
         logger.debug(" df.empty, return")
         return None
 
-    # df = pd.DataFrame([["", "", ""]], columns=["text1", "text2", "llh"])
-
-    df_exp = st.expander("to be aligned", expanded=False)
-    with df_exp:
-        st.write(df)  # too small
-
-    _ = """
-    ag_exp = st.expander("done aligned")  # , expanded=False
-    with ag_exp:
-        agdf = AgGrid(
-            df,
-            # fit_columns_on_grid_load=True,
-            editable=True,
-            gridOptions=gridOptions,
-            key="ag_exp",
-        )
-    # """
-
-    list1 = [elm.strip() for elm in df.text1 if elm.strip()]
-    list2 = [elm.strip() for elm in df.text2 if elm.strip()]
-    logger.debug("list1[:3]: %s", list1[:3])
-    logger.debug("list2[:3]: %s", list2[:3])
+    # only show this for upload
+    if state.ns.sourcetype in ["upload"]:
+        _ = st.expander("to be aligned", expanded=False)
+        with _:
+            st.write(df)
 
     logger.info("Processing data... %s", state.ns.beetype)
     # if state.ns.beetype in ["ezbee", "dzbee"]:
@@ -245,12 +156,12 @@ def home():  # noqa
                         # min_samples=min_samples,
                     )
                 except Exception as e:
-                    # logger.error("aset = ezbee(...) exc: %s", e)
                     logger.exception("aset = globals()[state.ns.beetype](...) exc: %s", e)
                     aset = ""
-                    # st.write(e)
                     st.write("Collecting inputs...")
+                    logger.debug("Collecting inputs...")
                     return None
+
         st.success(f"Done, took {t.duration_human}")
 
     else:
@@ -279,8 +190,8 @@ def home():  # noqa
     # aligned_pairs = gen_pairs(list1, list2, aset)
     aligned_pairs = aset2pairs(list1, list2, aset)
     if aligned_pairs:
-        logger.debug("%s...%s", aligned_pairs[:1], aligned_pairs[-1:])
-        # logger.debug("aligned_pairs[:20]: \n%s", aligned_pairs[:20])
+        # logger.debug("%s...%s", aligned_pairs[:1], aligned_pairs[-1:])
+        logger.debug("%s...s", aligned_pairs[:1])
 
     df_a = pd.DataFrame(
         aligned_pairs, columns=["text1", "text2", "llh"], dtype="object"
@@ -323,4 +234,7 @@ def home():  # noqa
             update_mode=GridUpdateMode.MODEL_CHANGED,
         )
 
+    # reset
+    state.ns.updated = False
+
     return None

litbee/pad.txt

@@ -0,0 +1,21 @@
+
+    if text1:
+        try:
+            list1 = [elm.strip() for elm in text1.splitlines() if elm.strip()]
+            state.ns.list1 = list1[:]
+        except Exception as e:
+            logger.warning("text1 to list1 errors: %s", e)
+        st.text_area(
+            label="Your pasted",
+            value=text1,
+            key="pasted_text1",
+            # help=""
+            height=500,
+        )
+
+    if text2:
+        try:
+            list1 = [elm.strip() for elm in text1.splitlines() if elm.strip()]
+            state.ns.list1 = list1[:]
+        except Exception as e:
+            logger.warning("text1 to list1 errors: %s", e)

litbee/settings.py

@@ -9,7 +9,7 @@ from streamlit import session_state as state
 
 
 def settings():
-    """Prep  Settings/Options page.
+    """Prep Settings/Options page.
 
     Refer to options.py
     """
@@ -53,7 +53,7 @@ def settings():
         index=index,
         format_func=lambda x: f"{x:<8} |",
         help="upload: one or two files; paste: from clipboard; urls: from the net",
-        disabled=True,
+        # disabled=True,
     )
     state.ns.sourcetype = sourcetype
 

litbee/url2txt.py

@@ -0,0 +1,97 @@
+"""Fetch text from url."""
+from typing import Optional
+from urllib.parse import urlparse
+
+import html2text
+import httpx
+import streamlit as st
+from logzero import logger
+from readability import Document
+
+
+@st.cache
+def url2txt(
+    url: str,
+    bodywidth: Optional[int] = 5000,
+    remove: bool = False,
+    show_url: bool = True,
+    ignore_links: bool = True,
+) -> str:
+    """Fetch text from url.
+
+    Args:
+        url: netloc from which to fetch text
+        bodywidth: if set to None, fall back to default bodywidth of
+            html2text.HTML2Text
+        remove: remove blank lines if set to True
+        show_url: prepend url if set to True
+        ignore_links: remove [ur](url)
+
+    Return:
+        main body in text
+
+    bodywidth: Optional[int] = 5000
+    remove: bool = False
+    show_url: bool = True
+    ignore_links: bool = True
+    """
+    url = url.strip()
+    if not url.startswith("http"):
+        url = "http://" + url
+
+    logger.info("url: %s", url)
+
+    parsed = urlparse(url)
+    if not parsed.scheme or not parsed.netloc:  # no scheme or netloc present
+        raise Exception(f"Invalid url: {url}")
+
+    try:
+        resp = httpx.get(url, timeout=30)
+        resp.raise_for_status()
+    except Exception as exc:
+        logger.error(exc)
+        raise
+
+    try:
+        content_type = resp.headers["content-type"]
+    except Exception as e:
+        logger.error(e)
+        content_type = ""
+    # output text if text/plain
+    if "text/plain" in content_type:
+        return resp.text
+
+    # handle html and the rest
+    try:
+        doc = Document(resp.text)
+    except Exception as exc:
+        logger.error(exc)
+        raise
+
+    if not doc.summary().strip():
+        raise Exception("No content for some reason...")
+
+    if bodywidth is not None:
+        handle = html2text.HTML2Text(bodywidth=bodywidth)
+    else:
+        handle = html2text.HTML2Text()
+
+    handle.ignore_links = ignore_links
+
+    try:
+        res = handle.handle(doc.summary())
+    except Exception as exc:
+        logger.error(exc)
+        raise
+
+    # remove double blank lines
+    if remove:
+        res = "\n".join(elm for elm in res.splitlines() if elm.strip())
+
+    if not res.strip():  # warn if empty output
+        logger.warning("Output seems to be empty...")
+
+    if show_url:
+        return f"{url}\n# {doc.title()}\n{res}"
+
+    return f"# {doc.title()}\n{res}"

poetry.lock

@@ -37,6 +37,23 @@ toolz = "*"
 [package.extras]
 dev = ["black", "docutils", "ipython", "flake8", "pytest", "sphinx", "mistune (<2.0.0)", "m2r", "vega-datasets", "recommonmark"]
 
+[[package]]
+name = "anyio"
+version = "3.6.1"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+category = "main"
+optional = false
+python-versions = ">=3.6.2"
+
+[package.dependencies]
+idna = ">=2.8"
+sniffio = ">=1.1"
+
+[package.extras]
+doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"]
+test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"]
+trio = ["trio (>=0.16)"]
+
 [[package]]
 name = "appnope"
 version = "0.1.3"
@@ -216,6 +233,14 @@ python-versions = "*"
 [package.dependencies]
 pycparser = "*"
 
+[[package]]
+name = "chardet"
+version = "4.0.0"
+description = "Universal encoding detector for Python 2 and 3"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
 [[package]]
 name = "charset-normalizer"
 version = "2.0.12"
@@ -273,6 +298,14 @@ python-versions = "*"
 [package.extras]
 test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"]
 
+[[package]]
+name = "cssselect"
+version = "1.1.0"
+description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
 [[package]]
 name = "de2en"
 version = "0.1.1"
@@ -519,6 +552,14 @@ python-versions = "*"
 [package.extras]
 test = ["pytest", "sphinx", "sphinx-autobuild", "wheel", "twine"]
 
+[[package]]
+name = "h11"
+version = "0.12.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "hanzidentifier"
 version = "1.0.2"
@@ -530,6 +571,52 @@ python-versions = "*"
 [package.dependencies]
 zhon = ">=1.1.3"
 
+[[package]]
+name = "html2text"
+version = "2020.1.16"
+description = "Turn HTML into equivalent Markdown-structured text."
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "httpcore"
+version = "0.15.0"
+description = "A minimal low-level HTTP client."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+anyio = ">=3.0.0,<4.0.0"
+certifi = "*"
+h11 = ">=0.11,<0.13"
+sniffio = ">=1.0.0,<2.0.0"
+
+[package.extras]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.23.0"
+description = "The next generation HTTP client."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+certifi = "*"
+httpcore = ">=0.15.0,<0.16.0"
+rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
+sniffio = "*"
+
+[package.extras]
+brotli = ["brotlicffi", "brotli"]
+cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10,<13)", "pygments (>=2.0.0,<3.0.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
+
 [[package]]
 name = "icecream"
 version = "2.1.2"
@@ -842,6 +929,20 @@ python-versions = "*"
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
 
+[[package]]
+name = "lxml"
+version = "4.9.0"
+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*"
+
+[package.extras]
+cssselect = ["cssselect (>=0.7)"]
+html5 = ["html5lib"]
+htmlsoup = ["beautifulsoup4"]
+source = ["Cython (>=0.29.7)"]
+
 [[package]]
 name = "markupsafe"
 version = "2.1.1"
@@ -1463,6 +1564,22 @@ python-versions = ">=3.6"
 cffi = {version = "*", markers = "implementation_name == \"pypy\""}
 py = {version = "*", markers = "implementation_name == \"pypy\""}
 
+[[package]]
+name = "readability-lxml"
+version = "0.8.1"
+description = "fast html to text parser (article readability tool) with python 3 support"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+chardet = "*"
+cssselect = "*"
+lxml = "*"
+
+[package.extras]
+test = ["timeout-decorator"]
+
 [[package]]
 name = "regex"
 version = "2022.4.24"
@@ -1489,6 +1606,20 @@ urllib3 = ">=1.21.1,<1.27"
 socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
 use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
 
+[[package]]
+name = "rfc3986"
+version = "1.5.0"
+description = "Validating URI References per RFC 3986"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
+
+[package.extras]
+idna2008 = ["idna"]
+
 [[package]]
 name = "rich"
 version = "12.4.4"
@@ -1612,6 +1743,14 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "sniffio"
+version = "1.2.0"
+description = "Sniff out which async library your code is running under"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
 [[package]]
 name = "snowballstemmer"
 version = "2.2.0"
@@ -2027,7 +2166,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8.3"
-content-hash = "671c809c9707c753d7aab881b8319beb76c2311edd270ca629f40cfd10227faa"
+content-hash = "f594ea8ffe579f3c01e737f1d41548f25e8d16bf5a3dab9344aad0376fb456d9"
 
 [metadata.files]
 about-time = [
@@ -2042,6 +2181,10 @@ altair = [
     {file = "altair-4.2.0-py3-none-any.whl", hash = "sha256:0c724848ae53410c13fa28be2b3b9a9dcb7b5caa1a70f7f217bd663bb419935a"},
     {file = "altair-4.2.0.tar.gz", hash = "sha256:d87d9372e63b48cd96b2a6415f0cf9457f50162ab79dc7a31cd7e024dd840026"},
 ]
+anyio = [
+    {file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"},
+    {file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"},
+]
 appnope = [
     {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"},
     {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"},
@@ -2213,6 +2356,10 @@ cffi = [
     {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"},
     {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"},
 ]
+chardet = [
+    {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"},
+    {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
+]
 charset-normalizer = [
     {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"},
     {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
@@ -2233,6 +2380,10 @@ commonmark = [
     {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"},
     {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"},
 ]
+cssselect = [
+    {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
+    {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"},
+]
 de2en = [
     {file = "de2en-0.1.1-py3-none-any.whl", hash = "sha256:6a04c77a04d3a8d426012b29a4561701416aaede64b650b4a1161c3b803fa928"},
     {file = "de2en-0.1.1.tar.gz", hash = "sha256:830dfc7a168cc1dcaea582655fc738db9add98ded30f56ddaca86b7b23dd6eff"},
@@ -2319,9 +2470,25 @@ gitpython = [
 grapheme = [
     {file = "grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca"},
 ]
+h11 = [
+    {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
+    {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
+]
 hanzidentifier = [
     {file = "hanzidentifier-1.0.2.tar.gz", hash = "sha256:793a298430aa9a9d6ab344dc0ca0ab4bd1161d88c7da941d6554571093003cba"},
 ]
+html2text = [
+    {file = "html2text-2020.1.16-py3-none-any.whl", hash = "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b"},
+    {file = "html2text-2020.1.16.tar.gz", hash = "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"},
+]
+httpcore = [
+    {file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"},
+    {file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"},
+]
+httpx = [
+    {file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"},
+    {file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"},
+]
 icecream = [
     {file = "icecream-2.1.2-py2.py3-none-any.whl", hash = "sha256:04b9cea4d9931cf6960db0430ebf11fa34464ce7152e384ddf50f96d25b201b4"},
     {file = "icecream-2.1.2.tar.gz", hash = "sha256:09300b2d1c678712410cbd47c95198eb1b580f66f311a554ccd6b9e758ece0ee"},
@@ -2406,6 +2573,71 @@ logzero = [
     {file = "logzero-1.7.0-py2.py3-none-any.whl", hash = "sha256:23eb1f717a2736f9ab91ca0d43160fd2c996ad49ae6bad34652d47aba908769d"},
     {file = "logzero-1.7.0.tar.gz", hash = "sha256:7f73ddd3ae393457236f081ffebd044a3aa2e423a47ae6ddb5179ab90d0ad082"},
 ]
+lxml = [
+    {file = "lxml-4.9.0-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b5031d151d6147eac53366d6ec87da84cd4d8c5e80b1d9948a667a7164116e39"},
+    {file = "lxml-4.9.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5d52e1173f52020392f593f87a6af2d4055dd800574a5cb0af4ea3878801d307"},
+    {file = "lxml-4.9.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3af00ee88376022589ceeb8170eb67dacf5f7cd625ea59fa0977d719777d4ae8"},
+    {file = "lxml-4.9.0-cp27-cp27m-win32.whl", hash = "sha256:1057356b808d149bc14eb8f37bb89129f237df488661c1e0fc0376ca90e1d2c3"},
+    {file = "lxml-4.9.0-cp27-cp27m-win_amd64.whl", hash = "sha256:f6d23a01921b741774f35e924d418a43cf03eca1444f3fdfd7978d35a5aaab8b"},
+    {file = "lxml-4.9.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56e19fb6e4b8bd07fb20028d03d3bc67bcc0621347fbde64f248e44839771756"},
+    {file = "lxml-4.9.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4cd69bca464e892ea4ed544ba6a7850aaff6f8d792f8055a10638db60acbac18"},
+    {file = "lxml-4.9.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:94b181dd2777890139e49a5336bf3a9a3378ce66132c665fe8db4e8b7683cde2"},
+    {file = "lxml-4.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:607224ffae9a0cf0a2f6e14f5f6bce43e83a6fbdaa647891729c103bdd6a5593"},
+    {file = "lxml-4.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:11d62c97ceff9bab94b6b29c010ea5fb6831743459bb759c917f49ba75601cd0"},
+    {file = "lxml-4.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:70a198030d26f5e569367f0f04509b63256faa76a22886280eea69a4f535dd40"},
+    {file = "lxml-4.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3cf816aed8125cfc9e6e5c6c31ff94278320d591bd7970c4a0233bee0d1c8790"},
+    {file = "lxml-4.9.0-cp310-cp310-win32.whl", hash = "sha256:65b3b5f12c6fb5611e79157214f3cd533083f9b058bf2fc8a1c5cc5ee40fdc5a"},
+    {file = "lxml-4.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:0aa4cce579512c33373ca4c5e23c21e40c1aa1a33533a75e51b654834fd0e4f2"},
+    {file = "lxml-4.9.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63419db39df8dc5564f6f103102c4665f7e4d9cb64030e98cf7a74eae5d5760d"},
+    {file = "lxml-4.9.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d8e5021e770b0a3084c30dda5901d5fce6d4474feaf0ced8f8e5a82702502fbb"},
+    {file = "lxml-4.9.0-cp35-cp35m-win32.whl", hash = "sha256:f17b9df97c5ecdfb56c5e85b3c9df9831246df698f8581c6e111ac664c7c656e"},
+    {file = "lxml-4.9.0-cp35-cp35m-win_amd64.whl", hash = "sha256:75da29a0752c8f2395df0115ac1681cefbdd4418676015be8178b733704cbff2"},
+    {file = "lxml-4.9.0-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:e4d020ecf3740b7312bacab2cb966bb720fd4d3490562d373b4ad91dd1857c0d"},
+    {file = "lxml-4.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b71c52d69b91af7d18c13aef1b0cc3baee36b78607c711eb14a52bf3aa7c815e"},
+    {file = "lxml-4.9.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cf04a1a38e961d4a764d2940af9b941b66263ed5584392ef875ee9c1e360a3"},
+    {file = "lxml-4.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:915ecf7d486df17cc65aeefdb680d5ad4390cc8c857cf8db3fe241ed234f856a"},
+    {file = "lxml-4.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e564d5a771b4015f34166a05ea2165b7e283635c41b1347696117f780084b46d"},
+    {file = "lxml-4.9.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c2a57755e366e0ac7ebdb3e9207f159c3bf1afed02392ab18453ce81f5ee92ee"},
+    {file = "lxml-4.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:00f3a6f88fd5f4357844dd91a1abac5f466c6799f1b7f1da2df6665253845b11"},
+    {file = "lxml-4.9.0-cp36-cp36m-win32.whl", hash = "sha256:9093a359a86650a3dbd6532c3e4d21a6f58ba2cb60d0e72db0848115d24c10ba"},
+    {file = "lxml-4.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d1690c4d37674a5f0cdafbc5ed7e360800afcf06928c2a024c779c046891bf09"},
+    {file = "lxml-4.9.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:6af7f51a6010748fc1bb71917318d953c9673e4ae3f6d285aaf93ef5b2eb11c1"},
+    {file = "lxml-4.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:eabdbe04ee0a7e760fa6cd9e799d2b020d098c580ba99107d52e1e5e538b1ecb"},
+    {file = "lxml-4.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b1e22f3ee4d75ca261b6bffbf64f6f178cb194b1be3191065a09f8d98828daa9"},
+    {file = "lxml-4.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:53b0410b220766321759f7f9066da67b1d0d4a7f6636a477984cbb1d98483955"},
+    {file = "lxml-4.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d76da27f5e3e9bc40eba6ed7a9e985f57547e98cf20521d91215707f2fb57e0f"},
+    {file = "lxml-4.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:686565ac77ff94a8965c11829af253d9e2ce3bf0d9225b1d2eb5c4d4666d0dca"},
+    {file = "lxml-4.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b62d1431b4c40cda43cc986f19b8c86b1d2ae8918cfc00f4776fdf070b65c0c4"},
+    {file = "lxml-4.9.0-cp37-cp37m-win32.whl", hash = "sha256:4becd16750ca5c2a1b1588269322b2cebd10c07738f336c922b658dbab96a61c"},
+    {file = "lxml-4.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e35a298691b9e10e5a5631f8f0ba605b30ebe19208dc8f58b670462f53753641"},
+    {file = "lxml-4.9.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:aa7447bf7c1a15ef24e2b86a277b585dd3f055e8890ac7f97374d170187daa97"},
+    {file = "lxml-4.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:612ef8f2795a89ba3a1d4c8c1af84d8453fd53ee611aa5ad460fdd2cab426fc2"},
+    {file = "lxml-4.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:1bfb791a8fcdbf55d1d41b8be940393687bec0e9b12733f0796668086d1a23ff"},
+    {file = "lxml-4.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:024684e0c5cfa121c22140d3a0898a3a9b2ea0f0fd2c229b6658af4bdf1155e5"},
+    {file = "lxml-4.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:81c29c8741fa07ecec8ec7417c3d8d1e2f18cf5a10a280f4e1c3f8c3590228b2"},
+    {file = "lxml-4.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6467626fa74f96f4d80fc6ec2555799e97fff8f36e0bfc7f67769f83e59cff40"},
+    {file = "lxml-4.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9cae837b988f44925d14d048fa6a8c54f197c8b1223fd9ee9c27084f84606143"},
+    {file = "lxml-4.9.0-cp38-cp38-win32.whl", hash = "sha256:5a49ad78543925e1a4196e20c9c54492afa4f1502c2a563f73097e2044c75190"},
+    {file = "lxml-4.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:bb7c1b029e54e26e01b1d1d912fc21abb65650d16ea9a191d026def4ed0859ed"},
+    {file = "lxml-4.9.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d0d03b9636f1326772e6854459728676354d4c7731dae9902b180e2065ba3da6"},
+    {file = "lxml-4.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:9af19eb789d674b59a9bee5005779757aab857c40bf9cc313cb01eafac55ce55"},
+    {file = "lxml-4.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:dd00d28d1ab5fa7627f5abc957f29a6338a7395b724571a8cbff8fbed83aaa82"},
+    {file = "lxml-4.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:754a1dd04bff8a509a31146bd8f3a5dc8191a8694d582dd5fb71ff09f0722c22"},
+    {file = "lxml-4.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7679344f2270840dc5babc9ccbedbc04f7473c1f66d4676bb01680c0db85bcc"},
+    {file = "lxml-4.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d882c2f3345261e898b9f604be76b61c901fbfa4ac32e3f51d5dc1edc89da3cb"},
+    {file = "lxml-4.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e97c8fc761ad63909198acc892f34c20f37f3baa2c50a62d5ec5d7f1efc68a1"},
+    {file = "lxml-4.9.0-cp39-cp39-win32.whl", hash = "sha256:cf9ec915857d260511399ab87e1e70fa13d6b2972258f8e620a3959468edfc32"},
+    {file = "lxml-4.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:1254a79f8a67a3908de725caf59eae62d86738f6387b0a34b32e02abd6ae73db"},
+    {file = "lxml-4.9.0-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:03370ec37fe562238d385e2c53089076dee53aabf8325cab964fdb04a9130fa0"},
+    {file = "lxml-4.9.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f386def57742aacc3d864169dfce644a8c396f95aa35b41b69df53f558d56dd0"},
+    {file = "lxml-4.9.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ea3f2e9eb41f973f73619e88bf7bd950b16b4c2ce73d15f24a11800ce1eaf276"},
+    {file = "lxml-4.9.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2d10659e6e5c53298e6d718fd126e793285bff904bb71d7239a17218f6a197b7"},
+    {file = "lxml-4.9.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:fcdf70191f0d1761d190a436db06a46f05af60e1410e1507935f0332280c9268"},
+    {file = "lxml-4.9.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:2b9c2341d96926b0d0e132e5c49ef85eb53fa92ae1c3a70f9072f3db0d32bc07"},
+    {file = "lxml-4.9.0-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:615886ee84b6f42f1bdf1852a9669b5fe3b96b6ff27f1a7a330b67ad9911200a"},
+    {file = "lxml-4.9.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:94f2e45b054dd759bed137b6e14ae8625495f7d90ddd23cf62c7a68f72b62656"},
+    {file = "lxml-4.9.0.tar.gz", hash = "sha256:520461c36727268a989790aef08884347cd41f2d8ae855489ccf40b50321d8d7"},
+]
 markupsafe = [
     {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"},
     {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"},
@@ -2931,6 +3163,10 @@ pyzmq = [
     {file = "pyzmq-23.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4d861ae20040afc17adef33053c328667da78d4d3676b2936788fd031665e3a8"},
     {file = "pyzmq-23.0.0.tar.gz", hash = "sha256:a45f5c0477d12df05ef2e2922b49b7c0ae9d0f4ff9b6bb0d666558df0ef37122"},
 ]
+readability-lxml = [
+    {file = "readability-lxml-0.8.1.tar.gz", hash = "sha256:e51fea56b5909aaf886d307d48e79e096293255afa567b7d08bca94d25b1a4e1"},
+    {file = "readability_lxml-0.8.1-py3-none-any.whl", hash = "sha256:e0d366a21b1bd6cca17de71a4e6ea16fcfaa8b0a5b4004e39e2c7eff884e6305"},
+]
 regex = [
     {file = "regex-2022.4.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f86aef546add4ff1202e1f31e9bb54f9268f17d996b2428877283146bf9bc013"},
     {file = "regex-2022.4.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e944268445b5694f5d41292c9228f0ca46d5a32a67f195d5f8547c1f1d91f4bc"},
@@ -3011,6 +3247,10 @@ requests = [
     {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
     {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
 ]
+rfc3986 = [
+    {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
+    {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
+]
 rich = [
     {file = "rich-12.4.4-py3-none-any.whl", hash = "sha256:d2bbd99c320a2532ac71ff6a3164867884357da3e3301f0240090c5d2fdac7ec"},
     {file = "rich-12.4.4.tar.gz", hash = "sha256:4c586de507202505346f3e32d1363eb9ed6932f0c2f63184dea88983ff4971e2"},
@@ -3146,6 +3386,10 @@ smmap = [
     {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"},
     {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"},
 ]
+sniffio = [
+    {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"},
+    {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"},
+]
 snowballstemmer = [
     {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"},
     {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"},

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litbee"
-version = "0.1.2-alpha.0"
+version = "0.1.2-alpha.1"
 description = "align (en, de, zh) texts via streamlit "
 authors = ["ffreemt"]
 license = "MIT"
@@ -26,6 +26,9 @@ plotly = "^5.8.0"
 hanzidentifier = "^1.0.2"
 opencc-python-reimplemented = "^0.1.6"
 tabulate = "^0.8.9"
+httpx = "^0.23.0"
+html2text = "^2020.1.16"
+readability-lxml = "^0.8.1"
 
 [tool.poe.executor]
 type = "poetry"