feat: add Wikipedia operator #918 (#1516)

### What problem does this PR solve?

Add Wikipedia operator #918

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/locales/en.ts

@@ -579,6 +579,7 @@ The above is the content you need to summarize.`,
       messageDescription:
         'This component is used to send user static information. You can prepare several messages which will be chosen randomly.',
       keywordDescription: `This component is used to extract keywords from user's question. Top N specifies the number of keywords you need to extract.`,
+      wikipediaDescription: `This component is used to get search result from https://www.wikipedia.org/. Typically, it performs as a supplement to knowledgebases. Top N specifies the number of search results you need to adopt.`,
       promptText: `Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
         {input}
   The above is the content you need to summarize.`,
@@ -612,6 +613,7 @@ The above is the content you need to summarize.`,
       messageHistoryWindowSize: 'Message window size',
       messageHistoryWindowSizeTip:
         'The  window size of conversation history that needed to be seen by LLM. The larger the better. But be careful with the maximum content length of LLM.',
+      wikipedia: 'Wikipedia',
     },
     footer: {
       profile: 'All rights reserved @ React',

web/src/locales/zh-traditional.ts

@@ -541,6 +541,7 @@ export default {
       messageDescription:
         '此元件用於向使用者發送靜態訊息。您可以準備幾條訊息，這些訊息將隨機選擇。',
       keywordDescription: `該組件用於從用戶的問題中提取關鍵字。 Top N指定需要提取的關鍵字數量。`,
+      wikipediaDescription: `此元件用於從 https://www.wikipedia.org/ 取得搜尋結果。通常，它充當知識庫的補充。 Top N 指定您需要採用的搜尋結果的數量。`,
       promptText: `請總結以下段落。注意數字，不要胡編亂造。段落如下：
 {input}
 以上就是你需要總結的內容。`,
@@ -573,6 +574,7 @@ export default {
       messageHistoryWindowSize: '歷史訊息視窗大小',
       messageHistoryWindowSizeTip:
         'LLM需要查看的對話記錄的視窗大小。越大越好。但要注意LLM的最大內容長度。',
+      wikipedia: '維基百科',
     },
     footer: {
       profile: '“保留所有權利 @ react”',

web/src/locales/zh.ts

@@ -559,6 +559,7 @@ export default {
       messageDescription:
         '此组件用于向用户发送静态信息。您可以准备几条消息，这些消息将被随机选择。',
       keywordDescription: `该组件用于从用户的问题中提取关键词。Top N指定需要提取的关键词数量。`,
+      wikipediaDescription: `此组件用于从 https://www.wikipedia.org/ 获取搜索结果。通常，它作为知识库的补充。Top N 指定您需要采用的搜索结果数量。`,
       promptText: `请总结以下段落。注意数字，不要胡编乱造。段落如下：
 {input}
 以上就是你需要总结的内容。`,
@@ -591,6 +592,7 @@ export default {
       messageHistoryWindowSize: '历史消息窗口大小',
       messageHistoryWindowSizeTip:
         'LLM 需要查看的对话历史窗口大小。越大越好。但要注意 LLM 的最大内容长度。',
+      wikipedia: '维基百科',
     },
     footer: {
       profile: 'All rights reserved @ React',

web/src/pages/flow/constant.tsx

@@ -1,6 +1,7 @@
 import { ReactComponent as BaiduIcon } from '@/assets/svg/baidu.svg';
 import { ReactComponent as DuckIcon } from '@/assets/svg/duck.svg';
 import { ReactComponent as KeywordIcon } from '@/assets/svg/keyword.svg';
+import { ReactComponent as WikipediaIcon } from '@/assets/svg/wikipedia.svg';
 import { variableEnabledFieldMap } from '@/constants/chat';
 import i18n from '@/locales/config';
 
@@ -33,6 +34,7 @@ export enum Operator {
   KeywordExtract = 'KeywordExtract',
   Baidu = 'Baidu',
   DuckDuckGo = 'DuckDuckGo',
+  Wikipedia = 'Wikipedia',
 }
 
 export const operatorIconMap = {
@@ -47,6 +49,7 @@ export const operatorIconMap = {
   [Operator.KeywordExtract]: KeywordIcon,
   [Operator.DuckDuckGo]: DuckIcon,
   [Operator.Baidu]: BaiduIcon,
+  [Operator.Wikipedia]: WikipediaIcon,
 };
 
 export const operatorMap = {
@@ -107,6 +110,9 @@ export const operatorMap = {
     color: '#aea00c',
   },
   [Operator.Baidu]: {},
+  [Operator.Wikipedia]: {
+    backgroundColor: '#dee0e2',
+  },
 };
 
 export const componentMenuList = [
@@ -140,6 +146,9 @@ export const componentMenuList = [
   {
     name: Operator.Baidu,
   },
+  {
+    name: Operator.Wikipedia,
+  },
 ];
 
 export const initialRetrievalValues = {
@@ -207,6 +216,11 @@ export const initialBaiduValues = {
   top_n: 10,
 };
 
+export const initialWikipediaValues = {
+  top_n: 10,
+  language: 'en',
+};
+
 export const CategorizeAnchorPointPositions = [
   { top: 1, right: 34 },
   { top: 8, right: 18 },
@@ -265,6 +279,7 @@ export const RestrictedUpstreamMap = {
   ],
   [Operator.Baidu]: [Operator.Begin, Operator.Retrieval],
   [Operator.DuckDuckGo]: [Operator.Begin, Operator.Retrieval],
+  [Operator.Wikipedia]: [Operator.Begin, Operator.Retrieval],
 };
 
 export const NodeMap = {
@@ -279,4 +294,288 @@ export const NodeMap = {
   [Operator.KeywordExtract]: 'ragNode',
   [Operator.DuckDuckGo]: 'ragNode',
   [Operator.Baidu]: 'ragNode',
+  [Operator.Wikipedia]: 'ragNode',
 };
+
+export const LanguageOptions = [
+  {
+    value: 'af',
+    label: 'Afrikaans',
+  },
+  {
+    value: 'pl',
+    label: 'Polski',
+  },
+  {
+    value: 'ar',
+    label: 'العربية',
+  },
+  {
+    value: 'ast',
+    label: 'Asturianu',
+  },
+  {
+    value: 'az',
+    label: 'Azərbaycanca',
+  },
+  {
+    value: 'bg',
+    label: 'Български',
+  },
+  {
+    value: 'nan',
+    label: '閩南語 / Bân-lâm-gú',
+  },
+  {
+    value: 'bn',
+    label: 'বাংলা',
+  },
+  {
+    value: 'be',
+    label: 'Беларуская',
+  },
+  {
+    value: 'ca',
+    label: 'Català',
+  },
+  {
+    value: 'cs',
+    label: 'Čeština',
+  },
+  {
+    value: 'cy',
+    label: 'Cymraeg',
+  },
+  {
+    value: 'da',
+    label: 'Dansk',
+  },
+  {
+    value: 'de',
+    label: 'Deutsch',
+  },
+  {
+    value: 'et',
+    label: 'Eesti',
+  },
+  {
+    value: 'el',
+    label: 'Ελληνικά',
+  },
+  {
+    value: 'en',
+    label: 'English',
+  },
+  {
+    value: 'es',
+    label: 'Español',
+  },
+  {
+    value: 'eo',
+    label: 'Esperanto',
+  },
+  {
+    value: 'eu',
+    label: 'Euskara',
+  },
+  {
+    value: 'fa',
+    label: 'فارسی',
+  },
+  {
+    value: 'fr',
+    label: 'Français',
+  },
+  {
+    value: 'gl',
+    label: 'Galego',
+  },
+  {
+    value: 'ko',
+    label: '한국어',
+  },
+  {
+    value: 'hy',
+    label: 'Հայերեն',
+  },
+  {
+    value: 'hi',
+    label: 'हिन्दी',
+  },
+  {
+    value: 'hr',
+    label: 'Hrvatski',
+  },
+  {
+    value: 'id',
+    label: 'Bahasa Indonesia',
+  },
+  {
+    value: 'it',
+    label: 'Italiano',
+  },
+  {
+    value: 'he',
+    label: 'עברית',
+  },
+  {
+    value: 'ka',
+    label: 'ქართული',
+  },
+  {
+    value: 'lld',
+    label: 'Ladin',
+  },
+  {
+    value: 'la',
+    label: 'Latina',
+  },
+  {
+    value: 'lv',
+    label: 'Latviešu',
+  },
+  {
+    value: 'lt',
+    label: 'Lietuvių',
+  },
+  {
+    value: 'hu',
+    label: 'Magyar',
+  },
+  {
+    value: 'mk',
+    label: 'Македонски',
+  },
+  {
+    value: 'arz',
+    label: 'مصرى',
+  },
+  {
+    value: 'ms',
+    label: 'Bahasa Melayu',
+  },
+  {
+    value: 'min',
+    label: 'Bahaso Minangkabau',
+  },
+  {
+    value: 'my',
+    label: 'မြန်မာဘာသာ',
+  },
+  {
+    value: 'nl',
+    label: 'Nederlands',
+  },
+  {
+    value: 'ja',
+    label: '日本語',
+  },
+  {
+    value: 'no',
+    label: 'Norsk (bokmål)',
+  },
+  {
+    value: 'nn',
+    label: 'Norsk (nynorsk)',
+  },
+  {
+    value: 'ce',
+    label: 'Нохчийн',
+  },
+  {
+    value: 'uz',
+    label: 'Oʻzbekcha / Ўзбекча',
+  },
+  {
+    value: 'pt',
+    label: 'Português',
+  },
+  {
+    value: 'kk',
+    label: 'Қазақша / Qazaqşa / قازاقشا',
+  },
+  {
+    value: 'ro',
+    label: 'Română',
+  },
+  {
+    value: 'ru',
+    label: 'Русский',
+  },
+  {
+    value: 'ceb',
+    label: 'Sinugboanong Binisaya',
+  },
+  {
+    value: 'sk',
+    label: 'Slovenčina',
+  },
+  {
+    value: 'sl',
+    label: 'Slovenščina',
+  },
+  {
+    value: 'sr',
+    label: 'Српски / Srpski',
+  },
+  {
+    value: 'sh',
+    label: 'Srpskohrvatski / Српскохрватски',
+  },
+  {
+    value: 'fi',
+    label: 'Suomi',
+  },
+  {
+    value: 'sv',
+    label: 'Svenska',
+  },
+  {
+    value: 'ta',
+    label: 'தமிழ்',
+  },
+  {
+    value: 'tt',
+    label: 'Татарча / Tatarça',
+  },
+  {
+    value: 'th',
+    label: 'ภาษาไทย',
+  },
+  {
+    value: 'tg',
+    label: 'Тоҷикӣ',
+  },
+  {
+    value: 'azb',
+    label: 'تۆرکجه',
+  },
+  {
+    value: 'tr',
+    label: 'Türkçe',
+  },
+  {
+    value: 'uk',
+    label: 'Українська',
+  },
+  {
+    value: 'ur',
+    label: 'اردو',
+  },
+  {
+    value: 'vi',
+    label: 'Tiếng Việt',
+  },
+  {
+    value: 'war',
+    label: 'Winaray',
+  },
+  {
+    value: 'zh',
+    label: '中文',
+  },
+  {
+    value: 'yue',
+    label: '粵語',
+  },
+];

web/src/pages/flow/flow-drawer/index.tsx

@@ -17,6 +17,7 @@ import OperatorIcon from '../operator-icon';
 import RelevantForm from '../relevant-form';
 import RetrievalForm from '../retrieval-form';
 import RewriteQuestionForm from '../rewrite-question-form';
+import WikipediaForm from '../wikipedia-form';
 
 import styles from './index.less';
 
@@ -36,6 +37,7 @@ const FormMap = {
   [Operator.Baidu]: BaiduForm,
   [Operator.DuckDuckGo]: DuckDuckGoForm,
   [Operator.KeywordExtract]: KeywordExtractForm,
+  [Operator.Wikipedia]: WikipediaForm,
 };
 
 const EmptyContent = () => <div>empty</div>;

web/src/pages/flow/hooks.ts

@@ -41,6 +41,7 @@ import {
   initialRelevantValues,
   initialRetrievalValues,
   initialRewriteQuestionValues,
+  initialWikipediaValues,
 } from './constant';
 import { ICategorizeForm, IRelevantForm } from './interface';
 import useGraphStore, { RFState } from './store';
@@ -88,6 +89,7 @@ export const useInitializeOperatorParams = () => {
       },
       [Operator.DuckDuckGo]: initialDuckValues,
       [Operator.Baidu]: initialBaiduValues,
+      [Operator.Wikipedia]: initialWikipediaValues,
     };
   }, [llmId]);
 

web/src/pages/flow/wikipedia-form/index.tsx

@@ -0,0 +1,27 @@
+import TopNItem from '@/components/top-n-item';
+import { useTranslate } from '@/hooks/commonHooks';
+import { Form, Select } from 'antd';
+import { LanguageOptions } from '../constant';
+import { IOperatorForm } from '../interface';
+
+const WikipediaForm = ({ onValuesChange, form }: IOperatorForm) => {
+  const { t } = useTranslate('common');
+
+  return (
+    <Form
+      name="basic"
+      labelCol={{ span: 6 }}
+      wrapperCol={{ span: 18 }}
+      autoComplete="off"
+      form={form}
+      onValuesChange={onValuesChange}
+    >
+      <TopNItem initialValue={10}></TopNItem>
+      <Form.Item label={t('language')} name={'language'}>
+        <Select options={LanguageOptions}></Select>
+      </Form.Item>
+    </Form>
+  );
+};
+
+export default WikipediaForm;