removed uncleaned summaries

all_summaries.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57a94e016723f692a3f4b4ad2b61f509cb77d0b3f14a2d2da9287d9d24e2dd42
-size 26408635
+oid sha256:ee0536b4e7e2297521a1f11b6f18e788d77f70129c8651fbbb3e7044782e3675
+size 28540700

related_summaries.py

@@ -2,6 +2,7 @@ import pandas as pd
 import torch
 from transformers import AutoTokenizer, AutoModel, set_seed
 from tqdm import tqdm
+from utils import clean_text
 
 from model import MimicTransformer
 set_seed(42)
@@ -17,7 +18,7 @@ mimic.eval()
 mimic.cuda()
 tokenizer = mimic.tokenizer
 
-summaries = pd.read_csv('all_summaries.csv')['SUMMARIES']
+summaries = pd.read_csv('all_summaries_backup.csv')['SUMMARIES']
 
 def mean_pooling(model_output, attention_mask):
     token_embeddings = model_output[0] #First element of model_output contains all token embeddings
@@ -34,10 +35,20 @@ def get_model_outputs(text):
 
 return_tensors = torch.zeros(size=(10000, 738))
 
-for i, summary in tqdm(enumerate(summaries[:10000])):
+non_defunct_summaries = []
+
+for i, summary in tqdm(enumerate(summaries[:50000])):
+    cleaned = clean_text(summary)
+    if len(non_defunct_summaries) == 10000:
+        break
+    if len(cleaned) > 100:
+        non_defunct_summaries.append(cleaned)
+
+for i, summary in tqdm(enumerate(non_defunct_summaries)):
     res = get_model_outputs(text=summary)
     return_tensors[i, :] = res.detach().cpu()
 # sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
 # sentence_embeddings = sentence_embeddings/sentence_embeddings.norm(dim=1)[:,None]
 
+pd.DataFrame(data={'SUMMARIES':non_defunct_summaries}).to_csv('all_summaries.csv', index=False)
 torch.save(return_tensors, f='discharge_embeddings.pt')

utils.py

@@ -3,6 +3,7 @@ import json
 import pandas as pd
 import ssl
 import torch
+import re
 from pprint import pprint
 from captum.attr import visualization
 
@@ -20,6 +21,50 @@ class PyTMinMaxScalerVectorized(object):
         tensor.mul_(scale).sub_(tensor.min(dim=0, keepdim=True)[0])
         return tensor
 
+def find_end(text):
+    """Find the end of the report."""
+    ends = [len(text)]
+    patterns = [
+        re.compile(r'BY ELECTRONICALLY SIGNING THIS REPORT', re.I),
+        re.compile(r'\n {3,}DR.', re.I),
+        re.compile(r'[ ]{1,}RADLINE ', re.I),
+        re.compile(r'.*electronically signed on', re.I),
+        re.compile(r'M\[0KM\[0KM')
+    ]
+    for pattern in patterns:
+        matchobj = pattern.search(text)
+        if matchobj:
+            ends.append(matchobj.start())
+    return min(ends)
+
+def pattern_repl(matchobj):
+    """
+    Return a replacement string to be used for match object
+    """
+    return ' '.rjust(len(matchobj.group(0)))
+
+def clean_text(text):
+    """
+    Clean text
+    """
+
+    # Replace [**Patterns**] with spaces.
+    text = re.sub(r'\[\*\*.*?\*\*\]', pattern_repl, text)
+    # Replace `_` with spaces.
+    text = re.sub(r'_', ' ', text)
+
+    start = 0
+    end = find_end(text)
+    new_text = ''
+    if start > 0:
+        new_text += ' ' * start
+    new_text = text[start:end]
+
+    # make sure the new text has the same length of old text.
+    if len(text) - end > 0:
+        new_text += ' ' * (len(text) - end)
+    return new_text
+
 def get_drg_link(drg_code):
     return f'https://www.aapc.com/codes/icd9-codes/{drg_code}'