Modified name parsing to use bionomia directly

Dockerfile

@@ -18,15 +18,15 @@ RUN unzip /data/gbif-occs.zip -d /data
 RUN ls -l /data
 COPY ./tab2csv.py /code/tab2csv.py
 
-# Setup to parse collector names using Bionomia utils (reqs Ruby)
-# Install ruby
-RUN \
-  apt-get update && \
-  apt-get install -y ruby
-RUN gem install dwc_agent
+## Setup to parse collector names using Bionomia utils (reqs Ruby)
+## Install ruby
+#RUN \
+#  apt-get update && \
+#  apt-get install -y ruby
+#RUN gem install dwc_agent
 
 #COPY ./extractcollectorname.py /code/extractcollectorname.py
-RUN python tab2csv.py --limit 1000 --createcols /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc.csv 
+RUN python tab2csv.py --createcols /data/${GBIF_DOWNLOAD_ID}.csv /data/gbifocc.csv 
 #RUN python extractcollectorname.py /data/gbifocc-temp.csv /data/gbifocc.csv 
 RUN csvs-to-sqlite /data/gbifocc.csv /code/gbifocc.db
 RUN ls -l /code

tab2csv.py

@@ -1,18 +1,51 @@
 import argparse
 import pandas as pd
-import bananompy
+import requests
 from tqdm import tqdm
 tqdm.pandas()
 
-def getFirstFamilyName(s):
+def getFirstFamilyName(recordedBy):
     firstFamilyName = None
-    parsed = bananompy.parse(s)
+    parsed = bananompy.parse(recordedBy)
     try:
         firstFamilyName = parsed[0]['parsed'][0]['family']
     except:
         pass
     return firstFamilyName
 
+def getFirstFamilyNames(recordedBy_l):
+    # post to bionomia
+    bionomia_parse_endpoint_url = "https://api.bionomia.net/parse.json"
+    data = dict()
+    data['names'] = '\r\n'.join(recordedBy_l)
+    r = requests.post(bionomia_parse_endpoint_url, data=data)
+    parsed_results = r.json()
+    results = dict()
+    for parsed_result in parsed_results:
+        try:
+            results[parsed_result['original']] = parsed_result['parsed'][0]['family']
+        except:
+            results[parsed_result['original']] = None
+    return results
+
+def getFirstFamilyNameBulk(df, 
+                            recordedByColName="recordedBy", 
+                            firstFamilyNameColName="recordedBy_first_familyname",
+                            batchsize=500):
+    results = dict()
+    recordedBy_l = []
+    for s in tqdm(df[recordedByColName].values):
+        if len(recordedBy_l) == batchsize:
+            # send it
+            results.update(getFirstFamilyNames(recordedBy_l))
+            # clear for next iteration
+            recordedBy_l = []
+        recordedBy_l.append(s)
+    if len(recordedBy_l) > 0:
+        results.update(getFirstFamilyNames(recordedBy_l))
+    df[firstFamilyNameColName] = df[recordedByColName].map(results)
+    return df
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument("inputfile")
@@ -30,7 +63,8 @@ if __name__ == '__main__':
     if args.createcols:
         # Extract unique recordedBy values
         df_rb = df[['recordedBy']].drop_duplicates()
-        df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
+        df_rb = getFirstFamilyNameBulk(df_rb)
+        #df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
         # Apply back to main dataframe
         df = pd.merge(left = df, right=df_rb, left_on='recordedBy', right_on='recordedBy', how='left')
         # Add column holding collector name and number