Update test script

.gitignore

@@ -98,8 +98,8 @@ docs/**/*.html
 .bash_env
 **/*secret*
 **/*private*
-/call_history.csv
-/call_history.txt
-/output.csv
-/call_history_bash.csv
-/call_history_sentiment_bash.csv
+/bad_response.txt
+/data/call_history_sentiment_1_bash.csv
+/data/call_history_text2int_1_bash.csv
+/data/processed_call_history_sentiment_1_bash.csv
+/data/processed_call_history_text2int_1_bash.csv

plot_calls.py

@@ -1,29 +1,77 @@
 from datetime import datetime
+
 import matplotlib.pyplot as plt
 import pandas as pd
 
+pd.set_option('display.max_columns', None)
+pd.set_option('display.max_rows', None)
+
+log_files = [
+    'call_history_sentiment_1_bash.csv',
+    'call_history_text2int_1_bash.csv',
+]
+
+for log_file in log_files:
+    path_ = f"./data/{log_file}"
+    df = pd.read_csv(filepath_or_buffer=path_, sep=";")
+    df["elapsed"] = df["finished"].apply(lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f")) - df["started"].apply(
+        lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f"))
+    df["elapsed"] = df["elapsed"].apply(lambda x: x.total_seconds())
+    df["success"] = df["outputs"].apply(lambda x: 0 if "Time-out" in x else 1)
+    df.to_csv(f"./data/processed_{log_file}", index=False, sep=";")
+
+    student_numbers = sorted(df['active_students'].unique())
+
+    result = df.groupby(['active_students', 'success']) \
+        .agg({
+        'elapsed': ['mean', 'median', 'min', 'max'],
+        'success': ['count'],
+    })
+
+    print(f"Results for {log_file}")
+    print(result, "\n")
+
+    title = None
+    if "sentiment" in log_file.lower():
+        title = "API result for 'sentiment-analysis' endpoint"
+    elif "text2int" in log_file.lower():
+        title = "API result for 'text2int' endpoint"
+
+    for student_number in student_numbers:
+        try:
+            failed_calls = result.loc[(student_number, 0), 'success'][0]
+        except:
+            failed_calls = 0
+        successful_calls = result.loc[(student_number, 1), 'success'][0]
+        percentage = (successful_calls / (failed_calls + successful_calls)) * 100
+        print(f"Percentage of successful API calls for {student_number} students: {percentage.__round__(2)}")
 
-# pd.set_option('display.max_columns', None)
-# pd.set_option('display.max_rows', None)
+    rows = len(student_numbers)
 
-df = pd.read_csv(filepath_or_buffer='call_history_bash.csv', sep=";")
-df["elapsed"] = df["finished"].apply(lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f")) - df["started"].apply(lambda x: datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f"))
-df["elapsed"] = df["elapsed"].apply(lambda x: x.total_seconds())
-df.to_csv("output.csv", index=False, sep=";")
+    # plt.figure(figsize=(16, 10))
+    # for index, student_number in enumerate(student_numbers, 1):
+    #     data = df[df["active_students"] == student_number]
+    #     fig = plt.subplot(rows, 2, 2 * index - 1)
+    #     plt.title("y=seconds, x=active students", x=0.75, y=0.75)
+    #     plt.boxplot(x=data["elapsed"], labels=[student_number])
+    #     plt.subplot(rows, 2, 2 * index)
+    #     plt.title("y=count of seconds, x=seconds", x=0.75, y=0.75)
+    #     plt.hist(x=data["elapsed"], bins=25, edgecolor='white')
 
-student_numbers = df['active_students'].unique()
+    fig, axs = plt.subplots(rows, 2)  # (rows, columns)
 
-plt.figure(figsize=(16, 10))
-rows = len(student_numbers)
+    for index, student_number in enumerate(student_numbers):
+        data = df[df["active_students"] == student_number]
+        axs[index][0].boxplot(x=data["elapsed"])  # axs[row][column]
+        # axs[index][0].set_title(f'Boxplot for {student_number} students')
+        axs[index][0].set_xlabel(f'student number {student_number}')
+        axs[index][0].set_ylabel('Elapsed time (s)')
 
-for index, student_number in enumerate(student_numbers, 1):
-    data = df[df["active_students"] == student_number]
-    plt.subplot(rows, 2, 2 * index - 1)
-    plt.title("y=seconds, x=active students", x=0.75, y=0.75)
-    plt.boxplot(x=data["elapsed"], labels=[student_number])
-    plt.subplot(rows, 2, 2 * index)
-    plt.title("y=count of seconds, x=seconds", x=0.75, y=0.75)
-    plt.hist(x=data["elapsed"], bins=25, edgecolor='white')
+        axs[index][1].hist(x=data["elapsed"], bins=25)  # axs[row][column]
+        # axs[index][1].set_title(f'Histogram for {student_number} students')
+        axs[index][1].set_xlabel('seconds')
+        axs[index][1].set_ylabel('Count of API calls')
 
+    fig.suptitle(title, fontsize=16)
 
 plt.show()

test_api.sh

@@ -3,11 +3,10 @@
 LOG_FILE_NAME="call_history_bash.csv"
 
 if [[ ! -f "$LOG_FILE_NAME" ]]; then
-    # Creation of column names if the file does not exits
-    echo "student_id;active_students;endpoint;inputs;outputs;started;finished" > $LOG_FILE_NAME
+  # Creation of column names if the file does not exits
+  echo "student_id;active_students;endpoint;inputs;outputs;started;finished" >$LOG_FILE_NAME
 fi
 
-
 data_list_1() {
   responses=(
     "one hundred forty five"
@@ -30,48 +29,54 @@ data_list_2() {
   echo "${responses[$1]}"
 }
 
-# endpoints: "text2int" "text2int-preprocessed" "sentiment-analysis"
+# endpoints: "text2int" "sentiment-analysis"
 # selected endpoint to test
-endpoint="text2int"
+endpoint="sentiment-analysis"
 
-create_random_delay () {
+create_random_delay() {
   # creates a random delay for given arguments
   echo "scale=8; $RANDOM/32768*$1" | bc
 }
 
-
 simulate_student() {
   # Student simulator waits randomly between 0-10s after an interaction.
   # Based on 100 interactions per student
   for i in {1..100}; do
+
+    random_value=$((RANDOM % 5))
+    text=$(data_list_2 $random_value)
+    data='{"data": ["'$text'"]}'
+
     start_=$(date +"%F %T.%6N")
-    url="https://tangibleai-mathtext.hf.space/run/$3"
 
-    response=$(curl --silent -X POST "$url" -H 'Content-Type: application/json' -d "$4")
+    url="https://tangibleai-mathtext.hf.space/run/$3"
+    response=$(curl --silent --connect-timeout 30 --max-time 30 -X POST "$url" -H 'Content-Type: application/json' -d "$data")
 
-    if [[ "$response" == *"504"* ]]; then
+    if [[ "$response" == *"Time-out"* ]]; then
+      echo "$response" >>bad_response.txt
+      response="504 Gateway Time-out"
+    elif [[ -z "$response" ]]; then
+      echo "No response" >>bad_response.txt
       response="504 Gateway Time-out"
     fi
 
     end_=$(date +"%F %T.%6N")
-    printf "%s;%s;%s;%s;%s;%s;%s\n" "$1" "$2" "$3" "$4" "$response" "$start_" "$end_" >>$LOG_FILE_NAME
+
+    printf "%s;%s;%s;%s;%s;%s;%s\n" "$1" "$2" "$3" "$data" "$response" "$start_" "$end_" >>$LOG_FILE_NAME
     sleep "$(create_random_delay 10)"
+
   done
 }
 
 echo "start: $(date)"
 
-active_students=250  # the number of students using the system at the same time
+active_students=250 # the number of students using the system at the same time
 
 i=1
-while [[ "$i" -le "$active_students" ]]
-do
-  random_value=$((RANDOM % 5))
-  text=$(data_list_2 $random_value)
-  data='{"data": ["'$text'"]}'
-  simulate_student "student$i" "$active_students" "$endpoint" "$data" &
-  sleep "$(create_random_delay 1)"  # adding a random delay between students
-  i=$(( "$i" + 1 ))
+while [[ "$i" -le "$active_students" ]]; do
+  simulate_student "student$i" "$active_students" "$endpoint" &
+  sleep "$(create_random_delay 1)" # adding a random delay between students
+  i=$(("$i" + 1))
 done
 
 wait