Upload 2 files

TMIDIX.py

@@ -8,7 +8,7 @@ r'''############################################################################
 #	Tegridy MIDI X Module (TMIDI X / tee-midi eks)
 #	Version 1.0
 #
-#   NOTE: TMIDI X Module starts after the partial MIDI.py module @ line 1342
+#   NOTE: TMIDI X Module starts after the partial MIDI.py module @ line 1438
 #
 #	Based upon MIDI.py module v.6.7. by Peter Billam / pjb.com.au
 #
@@ -21,19 +21,19 @@ r'''############################################################################
 #
 ###################################################################################
 ###################################################################################
-#       Copyright 2025 Project Los Angeles / Tegridy Code
+#   Copyright 2025 Project Los Angeles / Tegridy Code
 #
-#       Licensed under the Apache License, Version 2.0 (the "License");
-#       you may not use this file except in compliance with the License.
-#       You may obtain a copy of the License at
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
 #
-#           http://www.apache.org/licenses/LICENSE-2.0
+#       http://www.apache.org/licenses/LICENSE-2.0
 #
-#       Unless required by applicable law or agreed to in writing, software
-#       distributed under the License is distributed on an "AS IS" BASIS,
-#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#       See the License for the specific language governing permissions and
-#       limitations under the License.
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
 ###################################################################################
 ###################################################################################
 #
@@ -1446,8 +1446,9 @@ def _encode(events_lol, unknown_callback=None, never_add_eot=False,
 #	pjb.com.au
 #
 #	Project Los Angeles
-#	Tegridy Code 2021
-# https://github.com/Tegridy-Code/Project-Los-Angeles
+#	Tegridy Code 2025
+#
+#   https://github.com/Tegridy-Code/Project-Los-Angeles
 #
 ###################################################################################
 ###################################################################################
@@ -1490,6 +1491,10 @@ import math
 
 import matplotlib.pyplot as plt
 
+import psutil
+
+from collections import defaultdict
+
 ###################################################################################
 #
 # Original TMIDI Tegridy helper functions
@@ -11182,7 +11187,182 @@ def rle_decode_ones(encoding, size=(128, 128)):
     return matrix
 
 ###################################################################################
-#  
+
+def vertical_list_search(list_of_lists, trg_list):
+
+    src_list = list_of_lists
+    
+    if not src_list or not trg_list:
+        return []
+    
+    num_rows = len(src_list)
+    k = len(trg_list)
+    
+    row_sets = [set(row) for row in src_list]
+    
+    results = []
+    
+    for start in range(num_rows - k + 1):
+        valid = True
+        
+        for offset, target in enumerate(trg_list):
+
+            if target not in row_sets[start + offset]:
+                valid = False
+                break
+                
+        if valid:
+            results.append(list(range(start, start + k)))
+            
+    return results
+    
+###################################################################################
+
+def smooth_values(values, window_size=3):
+
+    smoothed = []
+    
+    for i in range(len(values)):
+
+        start = max(0, i - window_size // 2)
+        end = min(len(values), i + window_size // 2 + 1)
+        
+        window = values[start:end]
+        
+        smoothed.append(int(sum(window) / len(window)))
+        
+    return smoothed
+
+###################################################################################
+
+def is_mostly_wide_peaks_and_valleys(values, 
+                                     min_range=32, 
+                                     threshold=0.7, 
+                                     smoothing_window=5
+                                    ):
+
+    if not values:
+        return False
+
+    smoothed_values = smooth_values(values, smoothing_window)
+
+    value_range = max(smoothed_values) - min(smoothed_values)
+    
+    if value_range < min_range:
+        return False
+
+    if all(v == smoothed_values[0] for v in smoothed_values):
+        return False
+
+    trend_types = []
+    
+    for i in range(1, len(smoothed_values)):
+        if smoothed_values[i] > smoothed_values[i - 1]:
+            trend_types.append(1)
+            
+        elif smoothed_values[i] < smoothed_values[i - 1]:
+            trend_types.append(-1)
+            
+        else:
+            trend_types.append(0)
+
+    trend_count = trend_types.count(1) + trend_types.count(-1)
+
+    proportion = trend_count / len(trend_types)
+
+    return proportion >= threshold
+
+###################################################################################
+
+def system_memory_utilization(return_dict=False):
+
+    if return_dict:
+        return dict(psutil.virtual_memory()._asdict())
+
+    else:
+        print('RAM memory % used:', psutil.virtual_memory()[2])
+        print('RAM Used (GB):', psutil.virtual_memory()[3]/(1024**3))
+
+###################################################################################
+
+def create_files_list(datasets_paths=['./'],
+                      files_exts=['.mid', '.midi', '.kar', '.MID', '.MIDI', '.KAR'],
+                      randomize_files_list=True,
+                      verbose=True
+                     ):
+    if verbose:
+        print('=' * 70)
+        print('Searching for files...')
+        print('This may take a while on a large dataset in particular...')
+        print('=' * 70)
+
+    filez_set = defaultdict(None)
+
+    files_exts = tuple(files_exts)
+    
+    for dataset_addr in tqdm.tqdm(datasets_paths):
+        for dirpath, dirnames, filenames in os.walk(dataset_addr):
+            for file in filenames:
+                if file not in filez_set and file.endswith(files_exts):
+                    filez_set[os.path.join(dirpath, file)] = None
+    
+    filez = list(filez_set.keys())
+
+    if verbose:
+        print('Done!')
+        print('=' * 70)
+    
+    if filez:
+        if randomize_files_list:
+            
+            if verbose:
+                print('Randomizing file list...')
+                
+            random.shuffle(filez)
+            
+            if verbose:
+                print('Done!')
+                print('=' * 70)
+                
+        if verbose:
+            print('Found', len(filez), 'files.')
+            print('=' * 70)
+ 
+    else:
+        if verbose:
+            print('Could not find any files...')
+            print('Please check dataset dirs and files extensions...')
+            print('=' * 70)
+        
+    return filez
+
+###################################################################################
+
+def has_consecutive_trend(nums, count):
+    
+    if len(nums) < count:
+        return False
+    
+    increasing_streak = 1
+    decreasing_streak = 1
+
+    for i in range(1, len(nums)):
+        if nums[i] > nums[i - 1]:
+            increasing_streak += 1
+            decreasing_streak = 1
+            
+        elif nums[i] < nums[i - 1]:
+            decreasing_streak += 1
+            increasing_streak = 1
+            
+        else:
+            increasing_streak = decreasing_streak = 1
+        
+        if increasing_streak == count or decreasing_streak == count:
+            return True
+    
+    return False
+
+###################################################################################
 # This is the end of the TMIDI X Python module
-#
 ###################################################################################

TPLOTS.py

@@ -4,12 +4,12 @@ r'''############################################################################
 ################################################################################
 #
 #
-#	      Tegridy Plots Python Module (TPLOTS)
-#	      Version 1.0
+#	    Tegridy Plots Python Module (TPLOTS)
+#	    Version 1.0
 #
-#	      Project Los Angeles
+#	    Project Los Angeles
 #
-#	      Tegridy Code 2024
+#	    Tegridy Code 2025
 #
 #       https://github.com/asigalov61/tegridy-tools
 #
@@ -33,20 +33,20 @@ r'''############################################################################
 ################################################################################
 ################################################################################
 #
-# Critical dependencies
+#       Critical dependencies
 #
-# !pip install numpy
-# !pip install scipy
-# !pip install matplotlib
-# !pip install networkx
-# !pip3 install scikit-learn
+#       !pip install numpy==1.24.4
+#       !pip install scipy
+#       !pip install matplotlib
+#       !pip install networkx
+#       !pip3 install scikit-learn
 #
 ################################################################################
 #
-# Future critical dependencies
+#       Future critical dependencies
 #
-# !pip install umap-learn
-# !pip install alphashape
+#       !pip install umap-learn
+#       !pip install alphashape
 #
 ################################################################################
 '''
@@ -1254,6 +1254,113 @@ def plot_parsons_code(parsons_code,
 
   plt.close()
   
+################################################################################
+
+def plot_tokens_embeddings_constellation(tokens_embeddings,
+                                         start_token,
+                                         end_token,
+                                         plot_size=(10, 10),
+                                         labels_size=12,
+                                         show_grid=False,
+                                         save_plot=''):
+    
+    """
+    Plots token embeddings constellation using MST and graph layout
+    without dimensionality reduction.
+    """
+
+    distance_matrix = metrics.pairwise_distances(tokens_embeddings[start_token:end_token], metric='cosine')
+    
+    token_labels = [str(i) for i in range(start_token, end_token)]
+
+    mst = minimum_spanning_tree(distance_matrix).toarray()
+    
+    n = distance_matrix.shape[0]
+    G = nx.Graph()
+    
+    for i in range(n):
+        for j in range(n):
+            if mst[i, j] > 0:
+                weight = 1 / (distance_matrix[i, j] + 1e-8)
+                G.add_edge(i, j, weight=weight)
+    
+    pos = nx.kamada_kawai_layout(G, weight='weight')
+    
+    points = np.array([pos[i] for i in range(n)])
+    
+    plt.figure(figsize=plot_size)
+    plt.scatter(points[:, 0], points[:, 1], color='blue')
+    
+    for i, label in enumerate(token_labels):
+        plt.annotate(label, (points[i, 0], points[i, 1]),
+                     textcoords="offset points",
+                     xytext=(0, 10),
+                     ha='center',
+                     fontsize=labels_size)
+    
+    for i in range(n):
+        for j in range(n):
+            if mst[i, j] > 0:
+                plt.plot([points[i, 0], points[j, 0]],
+                         [points[i, 1], points[j, 1]],
+                         'k--', alpha=0.5)
+    
+    plt.title('Token Embeddings Constellation with MST', fontsize=labels_size)
+    plt.grid(show_grid)
+    
+    if save_plot:
+        plt.savefig(save_plot, bbox_inches="tight")
+        plt.close()
+        
+    else:
+        plt.show()
+        
+    plt.close()
+    
+################################################################################
+
+def find_token_path(tokens_embeddings, 
+                    start_token, 
+                    end_token, 
+                    verbose=False
+                   ):
+    
+    """
+    Finds the path of tokens between start_token and end_token using
+    the Minimum Spanning Tree (MST) derived from the distance matrix.
+    """
+
+    distance_matrix = metrics.pairwise_distances(tokens_embeddings, metric='cosine')
+    
+    token_labels = [str(i) for i in range(len(distance_matrix))]
+    
+    if verbose:
+        print('Total number of tokens:', len(distance_matrix))
+    
+    mst = minimum_spanning_tree(distance_matrix).toarray()
+    
+    n = distance_matrix.shape[0]
+    G = nx.Graph()
+    
+    for i in range(n):
+        for j in range(n):
+            if mst[i, j] > 0:
+                weight = 1 / (distance_matrix[i, j] + 1e-8)
+                G.add_edge(i, j, weight=weight)
+    
+    try:
+        start_idx = token_labels.index(str(start_token))
+        end_idx = token_labels.index(str(end_token))
+        
+    except ValueError:
+        raise ValueError("Start or end token not found in the provided token labels.")
+    
+    path_indices = nx.shortest_path(G, source=start_idx, target=end_idx)
+    
+    token_path = [int(token_labels[idx]) for idx in path_indices]
+    
+    return token_path
+
 ################################################################################
 # [WIP] Future dev functions
 ################################################################################
@@ -1363,7 +1470,53 @@ plt.show()
 '''
 
 ################################################################################
-#
+
+def plot_tree_horizontal(data):
+    
+    """
+    Given data as a list of levels (each level is a tuple or list of 
+    displacements for each branch), this function computes the cumulative 
+    value per branch (starting from 0) and plots each branch
+    with the tree level mapped to the x-axis and the cumulative value mapped 
+    to the y-axis. This gives a left-to-right tree with branches spanning up 
+    (positive) and down (negative).
+    
+    Parameters:
+        data (list of tuple/list): Each element represents a tree level.
+                                   It is assumed every level has the same length.
+    """
+    
+    # Convert data to a NumPy array with shape (n_levels, n_branches)
+    data = np.array(data)
+    n_levels, n_branches = data.shape
+
+    # Compute cumulative sums along each branch.
+    # Each branch starts at 0 at level 0.
+    cum = np.zeros((n_levels + 1, n_branches))
+    for i in range(n_levels):
+        cum[i + 1, :] = cum[i, :] + data[i, :]
+    
+    plt.figure(figsize=(12, 8))
+    
+    # Plot each branch as a line. For branch j:
+    #   - x coordinates are the tree levels (0 to n_levels)
+    #   - y coordinates are the corresponding cumulative values.
+    for j in range(n_branches):
+        x = np.arange(n_levels + 1)
+        y = cum[:, j]
+        plt.plot(x, y, marker='o', label=f'Branch {j}')
+    
+    plt.title("Horizontal Tree Visualization: Branches Spanning Up and Down", fontsize=14)
+    plt.xlabel("Tree Level (Left = Root)")
+    plt.ylabel("Cumulative Value (Up = Positive, Down = Negative)")
+    
+    # Add a horizontal line at y=0 to emphasize the center.
+    plt.axhline(0, color="gray", linestyle="--")
+    
+    #plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
+    plt.tight_layout()
+    plt.show()
+    
+################################################################################
 # This is the end of TPLOTS Python modules
-#
 ################################################################################