Google検索APIを統合し、住所正規化処理を改善する新しいエンドポイントを追加しました。住所のスペル修正機能を実装し、Gradioタブに新しい機能を追加しました。また、正規化処理のバージョン2を実装し、類似度計算を強化しました。

app.py

@@ -37,6 +37,9 @@ VECTOR_SEARCH_TOKEN = os.environ.get('VECTOR_SEARCH_TOKEN')
 VECTOR_SEARCH_COLLECTION_NAME = os.environ.get('VECTOR_SEARCH_COLLECTION_NAME')
 VECTOR_SEARCH_COLLECTION_NAME_V2 = os.environ.get('VECTOR_SEARCH_COLLECTION_NAME_V2')
 
+GOOGLE_SEARCH_API_KEY = os.environ.get('GOOGLE_SEARCH_API_KEY')
+GOOGLE_SEARCH_ENGINE_ID = os.environ.get('GOOGLE_SEARCH_ENGINE_ID')
+
 MILVUS_CLIENT = MilvusClient(uri=VECTOR_SEARCH_ENDPOINT, token=VECTOR_SEARCH_TOKEN)
 print(f"Connected to DB: {VECTOR_SEARCH_ENDPOINT} successfully")
 
@@ -134,6 +137,16 @@ def measure(label="処理"):
     end = time.time()
     print(f"{label} 実行時間: {end - start:.6f} 秒")
 
+def get_spelling(query_address):
+    # APIリクエストを作成
+    url = f'https://www.googleapis.com/customsearch/v1?key={GOOGLE_SEARCH_API_KEY}&cx={GOOGLE_SEARCH_ENGINE_ID}&q={query_address}'
+
+    # リクエストを送信
+    response = requests.get(url)
+    results = response.json()
+
+    return results.get('spelling', {}).get('correctedQuery', '')
+
 def convert_zenkaku_to_hankaku(text):
     zenkaku_numbers = '０１２３４５６７８９'
     hankaku_numbers = '0123456789'
@@ -497,6 +510,57 @@ def normalize_address(query_address):
         best_address = best_addresses[0]
     return best_address + splitted['building']
 
+def convert_no_to_hyphen(query_address):
+    return re.sub(r'(?<=\d)の(?=\d)', '-', query_address)
+
+def normalize_address_v2(query_address, top_k=1):
+    with measure('convert_zenkaku_to_hankaku'):
+        query_address = convert_zenkaku_to_hankaku(query_address)
+    with measure('split_address_building_with_gpt'):
+        splitted = split_address_building_with_gpt(query_address)
+    with measure('get_spelling'):
+        spelling = get_spelling(splitted['address'])
+        if spelling:
+            splitted['address'] = spelling
+    with measure(''):
+        splitted['address'] = convert_no_to_hyphen(splitted['address'])
+    with measure('preprocess'):
+        preprocessed = preprocess(splitted['address'])
+    with measure('vector_search'):
+        hits = vector_search(preprocessed, 1)
+    with measure('split_address'):
+        splits = {
+            'pref': hits[0][3],
+            'county': hits[0][4],
+            'city': hits[0][5],
+            'ward': hits[0][6],
+            'oaza_cho': hits[0][7],
+            'chome': hits[0][8],
+            'koaza': hits[0][9],
+        }
+    with measure('get_addresses_with_parcel'):
+        addresses = get_addresses_with_parcel(
+                splits['pref'], splits['county'], splits['city'], splits['ward'],
+                splits['oaza_cho'], splits['chome'], splits['koaza'])
+    with measure('get_addresses_with_rsdtdsp'):
+        addresses += get_addresses_with_rsdtdsp(
+                splits['pref'], splits['county'], splits['city'], splits['ward'],
+                splits['oaza_cho'], splits['chome'], splits['koaza'])
+        addresses = list(set(addresses))  # 重複を除去
+    with measure('embed_via_multilingual_e5_large'):
+        embeds = embed_via_multilingual_e5_large([splitted['address']] + addresses)
+        query_embed = [embeds[0]]
+        address_embeds = embeds[1:]
+    with measure('cosine'):
+        # コサイン類似度を計算
+        similarities = cosine_similarity(query_embed, address_embeds)
+
+        best_match_indices = np.argsort(similarities[0])[-top_k:][::-1]  # 上位Kのインデックスを取得
+        best_addresses = [addresses[i] for i in best_match_indices]
+        best_similarities = similarities[0][best_match_indices]
+
+    return splitted, hits, splits, best_addresses, best_similarities
+
 
 # =========================
 #  FastAPI definition
@@ -514,7 +578,6 @@ app = FastAPI(
 # ---------------------------
 # リクエスト・レスポンス定義
 # ---------------------------
-
 class CompareAddressesRequest(BaseModel):
     address1: str = Field(..., description="比較する最初の住所", example="東京 墨田区 押上 １丁目１-1")
     address2: str = Field(..., description="比較する2番目の住所", example="東京 墨田区 押上 1-１-1")
@@ -532,7 +595,6 @@ class NormalizeAddressResponse(BaseModel):
 # ---------------------------
 # エンドポイント定義
 # ---------------------------
-
 @app.post(
     "/compare-two-addresses",
     response_model=CompareAddressesResponse,
@@ -559,7 +621,6 @@ async def compare_two_addresses_api(request: CompareAddressesRequest):
     result = compare_two_addresses(request.address1, request.address2)
     return {"result": result}
 
-
 @app.post(
     "/normalize-address",
     response_model=NormalizeAddressResponse,
@@ -585,11 +646,37 @@ async def normalize_address_api(request: NormalizeAddressRequest):
     normalized = normalize_address(request.query_address)
     return {"normalized": normalized}
 
+@app.post(
+    "/normalize-address-v2",
+    response_model=NormalizeAddressResponse,
+    summary="住所を正規化する",
+    description="指定された住所を正規化し、正規化後の住所を返します。",
+    responses={
+        200: {
+            "description": "正規化結果の返却",
+            "content": {
+                "application/json": {
+                    "example": {
+                        "normalized": "東京都千代田区一丁目1番"
+                    }
+                }
+            }
+        }
+    }
+)
+async def normalize_address_v2_api(request: NormalizeAddressRequest):
+    """
+    - **query_address**: 正規化する住所
+    """
+    _, __, ___, bests, _____ = normalize_address_v2(request.query_address)
+    return {"normalized": bests[0]}
+
 
 # =========================
 #  Gradio tabs definition
 # =========================
 examples = [
+        '東京都中央区みなと3の12の10、プレサンスロゼ東京港301。',
         '東京都荒川区１−5−6荒川マンション102',
         '福岡市中央区天神1の11の2',
         '私の住所は京都府京都市右京区太秦青木元町４－１０です。',
@@ -685,6 +772,22 @@ def create_function_test_tab():
                 inputs=[in_tb],
                 outputs=[out_tb],
             )
+    def create_normalize_address__v2_tab():
+        with gr.Tab("normalize_address_v2"):
+            in_tb = gr.Textbox(label='住所', placeholder='住所を入力してください')
+            gr.Examples(examples=examples, inputs=[in_tb])
+            out_tb = gr.Textbox(label='アウトプット')
+            exe_button = gr.Button(value='実行', variant='primary')
+
+            def f(query_address):
+                splitted, __, ___, bests, _____ = normalize_address_v2(query_address)
+                return bests[0] + splitted['building']
+
+            exe_button.click(
+                fn=f,
+                inputs=[in_tb],
+                outputs=[out_tb],
+            )
     def create_split_address_tab():
         with gr.Tab("split_address"):
             in_tb = gr.Textbox(label='住所', placeholder='住所を入力してください')
@@ -767,10 +870,23 @@ def create_function_test_tab():
                 inputs=[in_tb],
                 outputs=[out_df],
             )
+    def create_get_spelling_tab():
+        with gr.Tab("create_get_spelling_tab"):
+            in_tb = gr.Textbox(label='住所', placeholder='住所を入力してください')
+            gr.Examples(examples=examples, inputs=[in_tb])
+            out_tb = gr.Textbox(label='アウトプット')
+            exe_button = gr.Button(value='実行', variant='primary')
+            exe_button.click(
+                fn=get_spelling,
+                inputs=[in_tb],
+                outputs=[out_tb],
+            )
 
     with gr.Tab("関数テスト"):
         create_normalize_address_tab()
+        create_normalize_address__v2_tab()
         create_compare_two_addresses_tab()
+        create_get_spelling_tab()
         create_get_addresses_with_parcel_tab()
         create_vector_search()
         create_remove_left_of_pref_tab()
@@ -899,9 +1015,42 @@ def create_vector_search_tab():
             outputs=[search_result_df, chiban_result_df, result_tb, result_df],
         )
 
+def create_vector_search_v2_tab():
+    with gr.Tab("ベクトル検索V2"):
+        with gr.Row():
+            with gr.Column():
+                address_input = gr.Textbox(label='住所', placeholder='検索したい住所を入力してください')
+                gr.Examples(examples=examples, inputs=[address_input])
+                top_k_input = gr.Slider(minimum=1, maximum=100, step=1, value=5, label='検索数top-k')
+                search_button = gr.Button(value='検索', variant='primary')
+        result_tb = gr.Textbox(label='正規化後')
+        result_df = gr.Dataframe(label="正規化後（分割）", wrap=True)
+        search_result_df = gr.Dataframe(label="町丁目まで検索結果")
+        chiban_result_df = gr.Dataframe(label="地番・住居表示検索結果")
+
+        def search_address(query_address, top_k):
+            splitted, hits, splits, best_addresses, best_similarities = normalize_address_v2(query_address, top_k)
+            search_result_df = pd.DataFrame(hits, columns=['Top-k', '類似度', '住所', '都道府県', '郡', '市区町村', '政令市区', '大字・町', '丁目', '小字'])
+            result_df = pd.DataFrame([splits.values()], columns=splits.keys())
+            chiban_result_df = pd.DataFrame({
+                'Top-k': range(1, len(best_similarities) + 1),
+                '類似度': best_similarities,
+                '住所': [best_address + splitted['building'] for best_address in best_addresses]
+            })
+            best_address = best_addresses[0] + splitted['building']
+
+            return search_result_df, chiban_result_df, best_address, result_df
+
+        search_button.click(
+            fn=search_address,
+            inputs=[address_input, top_k_input],
+            outputs=[search_result_df, chiban_result_df, result_tb, result_df],
+        )
+
 with gr.Blocks() as demo:
     create_function_test_tab()
     create_vector_search_tab()
+    create_vector_search_v2_tab()
     create_digital_agency_tab()
 
 app = gr.mount_gradio_app(app, demo, path='/')