Spaces:
Sleeping
Sleeping
“Transcendental-Programmer”
commited on
Commit
·
3de89cd
1
Parent(s):
4932e02
feat: Implement client model training and data generation
Browse files- src/client/data_handler.py +55 -0
- src/client/model.py +121 -0
src/client/data_handler.py
ADDED
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""data_handler.py module."""
|
2 |
+
|
3 |
+
import numpy as np
|
4 |
+
import pandas as pd
|
5 |
+
from typing import Tuple, Dict
|
6 |
+
import tensorflow as tf
|
7 |
+
from sklearn.preprocessing import StandardScaler
|
8 |
+
|
9 |
+
class FinancialDataHandler:
|
10 |
+
def __init__(self, config: Dict):
|
11 |
+
"""Initialize the data handler with configuration."""
|
12 |
+
self.batch_size = config['data']['batch_size']
|
13 |
+
self.shuffle_buffer = config['data']['shuffle_buffer']
|
14 |
+
self.prefetch_buffer = config['data']['prefetch_buffer']
|
15 |
+
self.scaler = StandardScaler()
|
16 |
+
|
17 |
+
def simulate_financial_data(self, num_samples: int = 1000) -> pd.DataFrame:
|
18 |
+
"""Generate synthetic financial data for testing."""
|
19 |
+
np.random.seed(42)
|
20 |
+
|
21 |
+
data = {
|
22 |
+
'transaction_amount': np.random.lognormal(mean=4.0, sigma=1.0, size=num_samples),
|
23 |
+
'account_balance': np.random.normal(loc=10000, scale=5000, size=num_samples),
|
24 |
+
'transaction_frequency': np.random.poisson(lam=5, size=num_samples),
|
25 |
+
'credit_score': np.random.normal(loc=700, scale=50, size=num_samples).clip(300, 850),
|
26 |
+
'days_since_last_transaction': np.random.exponential(scale=7, size=num_samples)
|
27 |
+
}
|
28 |
+
|
29 |
+
return pd.DataFrame(data)
|
30 |
+
|
31 |
+
def preprocess_data(self, data: pd.DataFrame) -> tf.data.Dataset:
|
32 |
+
"""Preprocess the data and convert to TensorFlow dataset."""
|
33 |
+
# Standardize the features
|
34 |
+
scaled_data = self.scaler.fit_transform(data)
|
35 |
+
|
36 |
+
# Convert to TensorFlow dataset
|
37 |
+
dataset = tf.data.Dataset.from_tensor_slices(scaled_data)
|
38 |
+
|
39 |
+
# Apply dataset transformations
|
40 |
+
dataset = dataset.shuffle(self.shuffle_buffer)
|
41 |
+
dataset = dataset.batch(self.batch_size)
|
42 |
+
dataset = dataset.prefetch(self.prefetch_buffer)
|
43 |
+
|
44 |
+
return dataset
|
45 |
+
|
46 |
+
def get_client_data(self) -> Tuple[tf.data.Dataset, StandardScaler]:
|
47 |
+
"""Get preprocessed client data and scaler."""
|
48 |
+
# Simulate client data
|
49 |
+
raw_data = self.simulate_financial_data()
|
50 |
+
|
51 |
+
# Preprocess data
|
52 |
+
dataset = self.preprocess_data(raw_data)
|
53 |
+
|
54 |
+
return dataset, self.scaler
|
55 |
+
|
src/client/model.py
ADDED
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""model.py module."""
|
2 |
+
|
3 |
+
from typing import Dict, List
|
4 |
+
import tensorflow as tf
|
5 |
+
import numpy as np
|
6 |
+
import logging
|
7 |
+
|
8 |
+
class FederatedClient:
|
9 |
+
def __init__(self, client_id: int, config: Dict):
|
10 |
+
"""Initialize the federated client."""
|
11 |
+
self.client_id = client_id
|
12 |
+
self.config = config.get('client', {})
|
13 |
+
self.model = self._build_model()
|
14 |
+
|
15 |
+
def start(self):
|
16 |
+
"""Start the federated client process."""
|
17 |
+
logger = logging.getLogger(__name__)
|
18 |
+
logger.info(f"Client {self.client_id} started")
|
19 |
+
logger.info(f"Client config: {self.config}")
|
20 |
+
|
21 |
+
try:
|
22 |
+
# Simulate some data
|
23 |
+
logger.info("Generating training data...")
|
24 |
+
X, y = self._generate_dummy_data()
|
25 |
+
logger.info(f"Generated data shapes - X: {X.shape}, y: {y.shape}")
|
26 |
+
|
27 |
+
# Train locally
|
28 |
+
logger.info("Starting local training...")
|
29 |
+
history = self.train_local((X, y))
|
30 |
+
|
31 |
+
# Log training metrics
|
32 |
+
losses = history.get('loss', [])
|
33 |
+
logger.info("\nTraining Progress Summary:")
|
34 |
+
logger.info("-" * 30)
|
35 |
+
for epoch, loss in enumerate(losses, 1):
|
36 |
+
logger.info(f"Epoch {epoch:2d}/{len(losses)}: loss = {loss:.4f}")
|
37 |
+
|
38 |
+
final_loss = losses[-1]
|
39 |
+
logger.info(f"\nTraining completed - Final loss: {final_loss:.4f}")
|
40 |
+
|
41 |
+
# Log model summary in a simpler format
|
42 |
+
logger.info("\nModel Architecture:")
|
43 |
+
logger.info("-" * 30)
|
44 |
+
logger.info("Layer (Output Shape) -> Params")
|
45 |
+
total_params = 0
|
46 |
+
for layer in self.model.layers:
|
47 |
+
params = layer.count_params()
|
48 |
+
total_params += params
|
49 |
+
logger.info(f"{layer.name} {layer.output.shape} -> {params:,} params")
|
50 |
+
logger.info(f"Total Parameters: {total_params:,}")
|
51 |
+
|
52 |
+
except Exception as e:
|
53 |
+
logger.error(f"Error during client execution: {str(e)}")
|
54 |
+
raise
|
55 |
+
|
56 |
+
def _generate_dummy_data(self):
|
57 |
+
"""Generate dummy data for testing."""
|
58 |
+
num_samples = 100
|
59 |
+
input_dim = 32 # Match with model's input dimension
|
60 |
+
|
61 |
+
# Generate input data
|
62 |
+
X = tf.random.normal((num_samples, input_dim))
|
63 |
+
# Generate target data (for this example, we'll predict the sum of inputs)
|
64 |
+
y = tf.reduce_sum(X, axis=1, keepdims=True)
|
65 |
+
|
66 |
+
return X, y
|
67 |
+
|
68 |
+
def _build_model(self):
|
69 |
+
"""Build the initial model architecture."""
|
70 |
+
input_dim = 32 # Match with data generation
|
71 |
+
model = tf.keras.Sequential([
|
72 |
+
tf.keras.layers.Input(shape=(input_dim,)),
|
73 |
+
tf.keras.layers.Dense(128, activation='relu'),
|
74 |
+
tf.keras.layers.Dense(64, activation='relu'),
|
75 |
+
tf.keras.layers.Dense(1) # Output layer for regression
|
76 |
+
])
|
77 |
+
model.compile(
|
78 |
+
optimizer=tf.keras.optimizers.Adam(
|
79 |
+
learning_rate=self.config.get('training', {}).get('learning_rate', 0.001)
|
80 |
+
),
|
81 |
+
loss='mse'
|
82 |
+
)
|
83 |
+
return model
|
84 |
+
|
85 |
+
def train_local(self, data):
|
86 |
+
"""Train the model on local data."""
|
87 |
+
logger = logging.getLogger(__name__)
|
88 |
+
X, y = data
|
89 |
+
|
90 |
+
# Log training parameters
|
91 |
+
logger.info(f"\nTraining Parameters:")
|
92 |
+
logger.info("-" * 50)
|
93 |
+
logger.info(f"Input shape: {X.shape}")
|
94 |
+
logger.info(f"Output shape: {y.shape}")
|
95 |
+
logger.info(f"Batch size: {self.config.get('data', {}).get('batch_size', 32)}")
|
96 |
+
logger.info(f"Epochs: {self.config.get('training', {}).get('local_epochs', 5)}")
|
97 |
+
logger.info(f"Learning rate: {self.config.get('training', {}).get('learning_rate', 0.001)}")
|
98 |
+
logger.info("-" * 50)
|
99 |
+
|
100 |
+
class LogCallback(tf.keras.callbacks.Callback):
|
101 |
+
def on_epoch_end(self, epoch, logs=None):
|
102 |
+
logger.info(f"Epoch {epoch + 1} - loss: {logs['loss']:.4f}")
|
103 |
+
|
104 |
+
# Enable verbose mode for training
|
105 |
+
history = self.model.fit(
|
106 |
+
X, y,
|
107 |
+
batch_size=self.config.get('data', {}).get('batch_size', 32),
|
108 |
+
epochs=self.config.get('training', {}).get('local_epochs', 5),
|
109 |
+
verbose=0, # Disable default verbose output
|
110 |
+
callbacks=[LogCallback()] # Use our custom callback
|
111 |
+
)
|
112 |
+
return history.history
|
113 |
+
|
114 |
+
def get_weights(self) -> List:
|
115 |
+
"""Get the model weights."""
|
116 |
+
return self.model.get_weights()
|
117 |
+
|
118 |
+
def set_weights(self, weights: List):
|
119 |
+
"""Update local model with global weights."""
|
120 |
+
self.model.set_weights(weights)
|
121 |
+
|