Spaces:
Sleeping
Sleeping
| import http.client as http_client | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import string | |
| import traceback | |
| import gradio as gr | |
| import requests | |
| from huggingface_hub import HfApi | |
| hf_api = HfApi() | |
| roots_datasets = {dset.id.split("/")[-1]:dset for dset in hf_api.list_datasets(author="bigscience-data", use_auth_token=os.environ.get("bigscience_data_token"))} | |
| def get_docid_html(docid): | |
| data_org, dataset, docid = docid.split("/") | |
| metadata = roots_datasets[dataset] | |
| if metadata.private: | |
| docid_html = ( | |
| f"<a " | |
| f'class="underline-on-hover"' | |
| f'title="This dataset is private. See the introductory text for more information"' | |
| f'style="color:#AA4A44;"' | |
| f'href="https://huggingface.co/datasets/bigscience-data/{dataset}"' | |
| f'target="_blank"><b>🔒{dataset}</b></a><span style="color: #7978FF;">/{docid}</span>' | |
| ) | |
| else: | |
| docid_html = ( | |
| f"<a " | |
| f'class="underline-on-hover"' | |
| f'title="This dataset is licensed {metadata.tags[0].split(":")[-1]}"' | |
| f'style="color:#2D31FA;"' | |
| f'href="https://huggingface.co/datasets/bigscience-data/{dataset}"' | |
| f'target="_blank"><b>{dataset}</b></a><span style="color: #7978FF;">/{docid}</span>' | |
| ) | |
| return docid_html | |
| PII_TAGS = {"KEY", "EMAIL", "USER", "IP_ADDRESS", "ID", "IPv4", "IPv6"} | |
| PII_PREFIX = "PI:" | |
| def process_pii(text): | |
| for tag in PII_TAGS: | |
| text = text.replace( | |
| PII_PREFIX + tag, | |
| """<b><mark style="background: Fuchsia; color: Lime;">REDACTED {}</mark></b>""".format(tag), | |
| ) | |
| return text | |
| def process_results(results, highlight_terms): | |
| if len(results) == 0: | |
| return """<br><p style='font-family: Arial; color:Silver; text-align: center;'> | |
| No results retrieved.</p><br><hr>""" | |
| results_html = "" | |
| for result in results: | |
| tokens = result["text"].split() | |
| tokens_html = [] | |
| for token in tokens: | |
| if token in highlight_terms: | |
| tokens_html.append("<b>{}</b>".format(token)) | |
| else: | |
| tokens_html.append(token) | |
| tokens_html = " ".join(tokens_html) | |
| tokens_html = process_pii(tokens_html) | |
| meta_html = ( | |
| """ | |
| <p class='underline-on-hover' style='font-size:12px; font-family: Arial; color:#585858; text-align: left;'> | |
| <a href='{}' target='_blank'>{}</a></p>""".format( | |
| result["meta"]["url"], result["meta"]["url"] | |
| ) | |
| if "meta" in result and result["meta"] is not None and "url" in result["meta"] | |
| else "" | |
| ) | |
| docid_html = get_docid_html(result["docid"]) | |
| results_html += """{} | |
| <p style='font-size:14px; font-family: Arial; color:#7978FF; text-align: left;'>Document ID: {}</p> | |
| <p style='font-size:12px; font-family: Arial; color:MediumAquaMarine'>Language: {}</p> | |
| <p style='font-family: Arial;'>{}</p> | |
| <br> | |
| """.format( | |
| meta_html, docid_html, result["lang"], tokens_html | |
| ) | |
| return results_html + "<hr>" | |
| def scisearch(query, language, num_results=10): | |
| try: | |
| query = " ".join(query.split()) | |
| if query == "" or query is None: | |
| return "" | |
| post_data = {"query": query, "k": num_results} | |
| if language != "detect_language": | |
| post_data["lang"] = language | |
| output = requests.post( | |
| os.environ.get("address"), | |
| headers={"Content-type": "application/json"}, | |
| data=json.dumps(post_data), | |
| timeout=60, | |
| ) | |
| payload = json.loads(output.text) | |
| if "err" in payload: | |
| if payload["err"]["type"] == "unsupported_lang": | |
| detected_lang = payload["err"]["meta"]["detected_lang"] | |
| return f""" | |
| <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'> | |
| Detected language <b>{detected_lang}</b> is not supported.<br> | |
| Please choose a language from the dropdown or type another query. | |
| </p><br><hr><br>""" | |
| results = payload["results"] | |
| highlight_terms = payload["highlight_terms"] | |
| if language == "detect_language": | |
| results = list(results.values())[0] | |
| return ( | |
| ( | |
| f"""<p style='font-family: Arial; color:MediumAquaMarine; text-align: center; line-height: 3em'> | |
| Detected language: <b>{results[0]["lang"]}</b></p><br><hr><br>""" | |
| if len(results) > 0 and language == "detect_language" | |
| else "" | |
| ) | |
| + process_results(results, highlight_terms) | |
| ) | |
| if language == "all": | |
| results_html = "" | |
| for lang, results_for_lang in results.items(): | |
| if len(results_for_lang) == 0: | |
| results_html += f"""<p style='font-family: Arial; color:Silver; text-align: left; line-height: 3em'> | |
| No results for language: <b>{lang}</b><hr></p>""" | |
| continue | |
| collapsible_results = f""" | |
| <details> | |
| <summary style='font-family: Arial; color:MediumAquaMarine; text-align: left; line-height: 3em'> | |
| Results for language: <b>{lang}</b><hr> | |
| </summary> | |
| {process_results(results_for_lang, highlight_terms)} | |
| </details>""" | |
| results_html += collapsible_results | |
| return results_html | |
| results = list(results.values())[0] | |
| return process_results(results, highlight_terms) | |
| except Exception as e: | |
| results_html = f""" | |
| <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'> | |
| Raised {type(e).__name__}</p> | |
| <p style='font-size:14px; font-family: Arial; '> | |
| Check if a relevant discussion already exists in the Community tab. If not, please open a discussion. | |
| </p> | |
| """ | |
| print(e) | |
| print(traceback.format_exc()) | |
| return results_html | |
| def flag(query, language, num_results, issue_description): | |
| try: | |
| post_data = {"query": query, "k": num_results, "flag": True, "description": issue_description} | |
| if language != "detect_language": | |
| post_data["lang"] = language | |
| output = requests.post( | |
| os.environ.get("address"), | |
| headers={"Content-type": "application/json"}, | |
| data=json.dumps(post_data), | |
| timeout=120, | |
| ) | |
| results = json.loads(output.text) | |
| except: | |
| print("Error flagging") | |
| return "" | |
| description = """# <p style="text-align: center;"> 🌸 🔎 Bloom Searcher 🔍 🌸 </p> | |
| Tool design for Roots: [URL](https://huggingface.co/spaces/bigscience-data/scisearch/blob/main/roots_search_tool_specs.pdf). | |
| Bloom on Wikipedia: [URL](https://en.wikipedia.org/wiki/BLOOM_(language_model)). | |
| Bloom Video Playlist: [URL](https://www.youtube.com/playlist?list=PLHgX2IExbFouqnsIqziThlPCX_miiDq14). | |
| Access full corpus check [URL](https://forms.gle/qyYswbEL5kA23Wu99). | |
| Big Science - How to get started | |
| Big Science is a 176B parameter new ML model that was trained on a set of datasets for Natural Language processing, and many other tasks that are not yet explored.. Below is the set of the papers, models, links, and datasets around big science which promises to be the best, most recent large model of its kind benefitting all science pursuits. | |
| Model: https://huggingface.co/bigscience/bloom | |
| Papers: | |
| BLOOM: A 176B-Parameter Open-Access Multilingual Language Model https://arxiv.org/abs/2211.05100 | |
| Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism https://arxiv.org/abs/1909.08053 | |
| 8-bit Optimizers via Block-wise Quantization https://arxiv.org/abs/2110.02861 | |
| Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation https://arxiv.org/abs/2108.12409 | |
| https://huggingface.co/models?other=doi:10.57967/hf/0003 | |
| 217 Other Models optimizing use of bloom via specialization: https://huggingface.co/models?other=bloom | |
| Datasets | |
| Universal Dependencies: https://paperswithcode.com/dataset/universal-dependencies | |
| WMT 2014: https://paperswithcode.com/dataset/wmt-2014 | |
| The Pile: https://paperswithcode.com/dataset/the-pile | |
| HumanEval: https://paperswithcode.com/dataset/humaneval | |
| FLORES-101: https://paperswithcode.com/dataset/flores-101 | |
| CrowS-Pairs: https://paperswithcode.com/dataset/crows-pairs | |
| WikiLingua: https://paperswithcode.com/dataset/wikilingua | |
| MTEB: https://paperswithcode.com/dataset/mteb | |
| xP3: https://paperswithcode.com/dataset/xp3 | |
| DiaBLa: https://paperswithcode.com/dataset/diabla | |
| """ | |
| if __name__ == "__main__": | |
| demo = gr.Blocks( | |
| css=".underline-on-hover:hover { text-decoration: underline; } .flagging { font-size:12px; color:Silver; }" | |
| ) | |
| with demo: | |
| with gr.Row(): | |
| gr.Markdown(value=description) | |
| with gr.Row(): | |
| query = gr.Textbox(lines=1, max_lines=1, placeholder="Type your query here...", label="Query") | |
| with gr.Row(): | |
| lang = gr.Dropdown( | |
| choices=[ | |
| "ar", | |
| "ca", | |
| "code", | |
| "en", | |
| "es", | |
| "eu", | |
| "fr", | |
| "id", | |
| "indic", | |
| "nigercongo", | |
| "pt", | |
| "vi", | |
| "zh", | |
| "detect_language", | |
| "all", | |
| ], | |
| value="en", | |
| label="Language", | |
| ) | |
| with gr.Row(): | |
| k = gr.Slider(1, 100, value=10, step=1, label="Max Results") | |
| with gr.Row(): | |
| submit_btn = gr.Button("Submit") | |
| with gr.Row(): | |
| results = gr.HTML(label="Results") | |
| flag_description = """ | |
| <p class='flagging'> | |
| If you choose to flag your search, we will save the query, language and the number of results you requested. | |
| Please consider adding any additional context in the box on the right.</p>""" | |
| with gr.Column(visible=False) as flagging_form: | |
| flag_txt = gr.Textbox( | |
| lines=1, | |
| placeholder="Type here...", | |
| label="""If you choose to flag your search, we will save the query, language and the number of results | |
| you requested. Please consider adding relevant additional context below:""", | |
| ) | |
| flag_btn = gr.Button("Flag Results") | |
| flag_btn.click(flag, inputs=[query, lang, k, flag_txt], outputs=[flag_txt]) | |
| def submit(query, lang, k): | |
| query = query.strip() | |
| if query is None or query == "": | |
| return "", "" | |
| return { | |
| results: scisearch(query, lang, k), | |
| flagging_form: gr.update(visible=True), | |
| } | |
| query.submit(fn=submit, inputs=[query, lang, k], outputs=[results, flagging_form]) | |
| submit_btn.click(submit, inputs=[query, lang, k], outputs=[results, flagging_form]) | |
| demo.launch(enable_queue=True, debug=True) |