住所検索機能を拡張し、検索結果に都道府県、郡、市区町村、政令市区、大字・町、丁目、小字を追加。データフレームの構造を更新し、住所生成のロジックを改善。処理時間の測定を強化し、重複アドレスを除去する機能を追加。

app.py

@@ -15,7 +15,6 @@ import time
 from contextlib import contextmanager
 import numpy as np
 
-
 # .envファイルを読み込む
 load_dotenv()
 
@@ -174,16 +173,23 @@ def search_via_milvus(query_vector, top_k, collection_name, thresh=0.0):
         search_params=search_params,
         limit=top_k,
         anns_field='embedding',
-        output_fields=['address'],
+        output_fields=['address', 'pref', 'county', 'city', 'ward', 'oaza_cho', 'chome', 'koaza'],
     )[0]
 
     hits = []
     for i, result in enumerate(results, start=1):
         distance = result['distance']
         address = result['entity'].get('address')
+        pref = result['entity'].get('pref')
+        county = result['entity'].get('county')
+        city = result['entity'].get('city')
+        ward = result['entity'].get('ward')
+        oaza_cho = result['entity'].get('oaza_cho')
+        chome = result['entity'].get('chome')
+        koaza = result['entity'].get('koaza')
 
         if distance >= thresh:
-            hits.append([i, distance, address])
+            hits.append([i, distance, address, pref, county, city, ward, oaza_cho, chome, koaza])
 
     return hits
 
@@ -373,42 +379,61 @@ def create_vector_search_tab():
                 preprocessed = preprocess(query_address)
             with measure('vector_search'):
                 hits = vector_search(preprocessed, top_k)
-                normalized = hits[0][-1]
-                search_result_df = pd.DataFrame(hits, columns=['Top-k', '類似度', '住所'])
+                search_result_df = pd.DataFrame(hits, columns=['Top-k', '類似度', '住所', '都道府県', '郡', '市区町村', '政令市区', '大字・町', '丁目', '小字'])
             with measure('split_address'):
-                splits = split_address(normalized)
-                data = {
-                    'pref': splits['pref'],
-                    'city': splits['city'],
-                    'town': splits['town'],
-                    'addr': splits['addr'],
+                splits = {
+                    'pref': hits[0][3],
+                    'county': hits[0][4],
+                    'city': hits[0][5],
+                    'ward': hits[0][6],
+                    'oaza_cho': hits[0][7],
+                    'chome': hits[0][8],
+                    'koaza': hits[0][9],
                 }
-                result_df = pd.DataFrame([data])
+                result_df = pd.DataFrame([splits.values()], columns=splits.keys())
             with measure('load city_all_file'):
                 target_dir = Path(r'C:\Users\taish\Development\whispercustom\projects\abr-geocoder\temp\download')
                 city_all_file = target_dir / 'mt_city_all.csv'
                 city_all_df = pd.read_csv(city_all_file)
                 city_all_df_temp = city_all_df[city_all_df['pref'] == splits['pref']]
-                lg_codes = city_all_df_temp[city_all_df_temp['county'].fillna('') + city_all_df_temp['city'].fillna('') + city_all_df_temp['ward'].fillna('') == splits['city']]['lg_code'].values
+                city_name1 = city_all_df_temp['county'].fillna('') + city_all_df_temp['city'].fillna('') + city_all_df_temp['ward'].fillna('')
+                city_name2 = splits['county'] + splits['city'] + splits['ward']
+                lg_codes = city_all_df_temp[city_name1 == city_name2]['lg_code'].values
                 if len(lg_codes) > 1:
                     raise Exception('Too many lg_code')
                 lg_code = lg_codes[0]
             with measure('load parcel_city_file'):
                 parcel_city_file = target_dir / f'mt_parcel_city{lg_code:06d}.csv'
                 if not os.path.exists(parcel_city_file):
-                    raise gr.Error('Too many lg_code')
+                    # raise gr.Error('Too many lg_code')
+                    raise Exception('Too many lg_code')
                 parcel_city_df = pd.read_csv(parcel_city_file)
-                parcel_city_df = parcel_city_df[parcel_city_df['city'].fillna('') + parcel_city_df['ward'].fillna('') == splits['city']]
-                parcel_city_df = parcel_city_df[parcel_city_df['oaza_cho'].fillna('') + parcel_city_df['chome'].fillna('') == splits['town']]
 
                 cities = parcel_city_df['city'].fillna('')
                 wards = parcel_city_df['ward'].fillna('')
                 oaza_chos = parcel_city_df['oaza_cho'].fillna('')
                 chomes = parcel_city_df['chome'].fillna('')
                 koazas = parcel_city_df['koaza'].fillna('')
-                prc_num1s = parcel_city_df['prc_num1'].fillna(9999).astype(int).astype(str).replace('9999', '')
-                prc_num2s = parcel_city_df['prc_num2'].fillna(9999).astype(int).astype(str).replace('9999', '')
-                prc_num3s = parcel_city_df['prc_num3'].fillna(9999).astype(int).astype(str).replace('9999', '')
+
+                city_name1 = cities + wards
+                city_name2 = splits['county'] + splits['city'] + splits['ward']
+                city_mask = city_name1 == city_name2
+
+                town_name1 = oaza_chos + chomes
+                town_name2 = splits['oaza_cho'] + splits['chome']
+                town_mask = town_name1 == town_name2
+
+                koaza_mask = koazas == splits['koaza']
+                parcel_city_df_filtered = parcel_city_df[city_mask & town_mask & koaza_mask]
+
+                cities = parcel_city_df_filtered['city'].fillna('')
+                wards = parcel_city_df_filtered['ward'].fillna('')
+                oaza_chos = parcel_city_df_filtered['oaza_cho'].fillna('')
+                chomes = parcel_city_df_filtered['chome'].fillna('')
+                koazas = parcel_city_df_filtered['koaza'].fillna('')
+                prc_num1s = parcel_city_df_filtered['prc_num1'].fillna(9999).astype(int).astype(str).replace('9999', '')
+                prc_num2s = parcel_city_df_filtered['prc_num2'].fillna(9999).astype(int).astype(str).replace('9999', '')
+                prc_num3s = parcel_city_df_filtered['prc_num3'].fillna(9999).astype(int).astype(str).replace('9999', '')
 
                 # アドレスを生成
                 addresses = [
@@ -419,6 +444,47 @@ def create_vector_search_tab():
                         cities, wards, oaza_chos, chomes, koazas, prc_num1s, prc_num2s, prc_num3s
                     )
                 ]
+            with measure('load rsdtdsp_file'):
+                pref_code = ('%06d' % lg_code)[0:2]
+                rsdtdsp_dir = Path(rf'G:\マイドライブ\Development\Dataset\Misc\japanese_address\rsdt\original')
+                rsdtdsp_file = rsdtdsp_dir / f'mt_rsdtdsp_rsdt_pref{pref_code}.csv\mt_rsdtdsp_rsdt_pref{pref_code}.csv'
+                if not os.path.exists(rsdtdsp_file):
+                    # raise gr.Error(f'Not found: {rsdtdsp_file}')
+                    raise Exception(f'Not found: {rsdtdsp_file}')
+                rsdtdsp_df = pd.read_csv(rsdtdsp_file)
+
+                city_name1 = rsdtdsp_df['city'].fillna('') + rsdtdsp_df['ward'].fillna('')
+                city_name2 = splits['county'] + splits['city'] + splits['ward']
+                city_mask = city_name1 == city_name2
+
+                town_name1 = rsdtdsp_df['oaza_cho'].fillna('') + rsdtdsp_df['chome'].fillna('')
+                town_name2 = splits['oaza_cho'] + splits['chome']
+                town_mask = town_name1 == town_name2
+
+                koaza_mask = rsdtdsp_df['koaza'].fillna('') == splits['koaza']
+
+                rsdtdsp_df_filtered = rsdtdsp_df[city_mask & town_mask & koaza_mask]
+
+                cities = rsdtdsp_df_filtered['city'].fillna('')
+                wards = rsdtdsp_df_filtered['ward'].fillna('')
+                oaza_chos = rsdtdsp_df_filtered['oaza_cho'].fillna('')
+                chomes = rsdtdsp_df_filtered['chome'].fillna('')
+                koazas = rsdtdsp_df_filtered['koaza'].fillna('')
+                blk_nums = rsdtdsp_df_filtered['blk_num'].fillna(9999).astype(int).astype(str).replace('9999', '')
+                rsdt_nums = rsdtdsp_df_filtered['rsdt_num'].fillna(9999).astype(int).astype(str).replace('9999', '')
+                rsdt_num2s = rsdtdsp_df_filtered['rsdt_num2'].fillna(9999).astype(int).astype(str).replace('9999', '')
+
+                # アドレスを生成
+                addresses += [
+                    f"{splits['pref']}{city}{ward}{oaza_cho}{chome}{koaza}{blk_num}" +
+                    (f"-{rsdt_num}" if rsdt_num else '') +
+                    (f"-{rsdt_num2}" if rsdt_num2 else '')
+                    for city, ward, oaza_cho, chome, koaza, blk_num, rsdt_num, rsdt_num2 in zip(
+                        cities, wards, oaza_chos, chomes, koazas, blk_nums, rsdt_nums, rsdt_num2s
+                    )
+                ]
+
+                addresses = list(set(addresses))  # 重複を除去
             with measure('query_embed'):
                 query_embed = embed_via_multilingual_e5_large([query_address])
             with measure('address_embeds'):