Top 3 Custom Metrics Data Scientists Should Build for Finance: A Tutorial

Introduction

What You Miss Out On With Traditional Metrics

Credit Card Fraud Detection

💡

For a more in-depth look at these post-deployment challenges and strategies for addressing them, refer to this blog that details maintaining and improving fraud detection systems.

Monitoring fraud detection models

Learn common reasons why fraud detection models degrade after deployment in production, and follow a hands-on tutorial to resolve these issues.

https://www.nannyml.com/blog/monitor-fraud-detection-models

Accuracy

Balanced Accuracy

import pandas as pd
from sklearn.metrics import balanced_accuracy_score
import numpy as np

def calculate(
    chunk_data,
    y_pred
) -> float:
    y_true = chunk_data['target'].to_numpy()
    y_pred = np.asarray(y_pred)

    data = pd.DataFrame({'y_true': y_true, 'y_pred': y_pred})
    data.dropna(axis=0, inplace=True, subset=['y_true'])

    y_true, y_pred = data['y_true'].to_numpy(), data['y_pred'].to_numpy()
    
    if len(y_true) == 0:
        return np.nan

    return balanced_accuracy_score(y_true, y_pred)

import pandas as pd
import numpy as np

def estimate(
        estimated_target_probabilities,
        y_pred
) -> float:
    y_pred = np.asarray(y_pred)
    estimated_target_probabilities = estimated_target_probabilities.to_numpy().ravel()

    data = pd.DataFrame({
        'estimated_target_probabilities': estimated_target_probabilities,
        'y_pred': y_pred
    })
    data.dropna(axis=0, inplace=True)

    estimated_target_probabilities = data['estimated_target_probabilities'].to_numpy()
    y_pred = data['y_pred'].to_numpy()
    
    tp = np.sum(np.where(y_pred == 1, estimated_target_probabilities, 0))
    fp = np.sum(np.where(y_pred == 1, 1 - estimated_target_probabilities, 0))
    fn = np.sum(np.where(y_pred == 0, estimated_target_probabilities, 0))
    tn = np.sum(np.where(y_pred == 0, 1 - estimated_target_probabilities, 0))
    
    sensitivity = tp / (tp + fn) if (tp + fn) > 0 else 0
    specificity = tn / (tn + fp) if (tn + fp) > 0 else 0
    balanced_accuracy = (sensitivity + specificity) / 2
    
    return balanced_accuracy

Prevalence

import pandas as pd
import numpy as np

def calculate(
    chunk_data,
    y_pred
) -> float:
    y_true = chunk_data['target'].to_numpy()
    y_pred = np.asarray(y_pred)

    data = pd.DataFrame({'y_true': y_true, 'y_pred': y_pred})
    data.dropna(axis=0, inplace=True, subset=['y_true'])

    y_true, y_pred = data['y_true'].to_numpy(), data['y_pred'].to_numpy()

    if len(y_true) == 0:
        return np.nan

    # Calculate Prevalence
    true_positives = np.sum((y_true == 1) & (y_pred == 1))
    false_negatives = np.sum((y_true == 1) & (y_pred == 0))
    total_positive_cases = true_positives + false_negatives
    total_instances = len(y_true)

    prevalence = total_positive_cases / total_instances if total_instances > 0 else np.nan
    return prevalence

import numpy as np
import pandas as pd

def estimate(
    estimated_target_probabilities: pd.DataFrame,
    y_pred: pd.Series,
    y_pred_proba: pd.DataFrame,
) -> float:
    
    estimated_target_probabilities = estimated_target_probabilities.to_numpy().ravel()
    y_pred = np.asarray(y_pred)
    data = pd.DataFrame({
        'estimated_target_probabilities': estimated_target_probabilities,
        'y_pred': y_pred
    })
    data.dropna(axis=0, inplace=True)
    if len(data) == 0:
        return np.nan
    prevalence = np.mean(data['estimated_target_probabilities'])
    return prevalence

Adding Business Context in the Mix

Stock Market Analysis and Prediction

RMSE

Sharpe Index

📌

The book every data scientist needs on their desk.

Metrics are arguably the most important part of data science work, yet they are rarely taught in courses or university degrees.

Order here: https://www.nannyml.com/metrics

import numpy as np 
import pandas as pd

def loss(
        y_true: pd.Series,
        y_pred: pd.Series,
        chunk_data: pd.DataFrame
    ) -> np.ndarray:
    y_true = chunk_data['y_true'].to_numpy()
    y_pred = chunk_data['y_pred'].to_numpy()
    
    risk_free_rate = 0.01
    
    # Calculate returns without using pct_change
    actual_returns = np.diff(y_true) / y_true[:-1]
    predicted_returns = np.diff(y_pred) / y_pred[:-1]
    
    # Add a 0 at the beginning to match the original length
    actual_returns = np.insert(actual_returns, 0, 0)
    predicted_returns = np.insert(predicted_returns, 0, 0)
    
    excess_returns = predicted_returns - (risk_free_rate / 252)
    std_excess_return = np.std(excess_returns)
    
    if std_excess_return == 0:
        return np.zeros(len(excess_returns))
    
    excess_returns_by_std = excess_returns / std_excess_return
    
    return excess_returns_by_std

import numpy as np
import pandas as pd 

def aggregate(
    loss: np.ndarray,
) -> float:
    return loss.mean()

Conclusion

Post-deployment Data Science is a data science vertical that deals with monitoring and maintaining production-grade machine learning models. Covariate Shift, Concept Drift, and Data Quality are a trinity of issues that your model can face over time.

To know how you can get started, Speak with a NannyML founder today and get a demo.

 

Read More…

Estimating Model Performance Without Labels

Blogs about all things data science and machine learning, post deployment. Most popular topics: data drift, model monitoring.

https://www.nannyml.com/blog/machine-learning-performance-estimation

Covariate Shift: What it is and How to Detect it

Covariate shift occurs when the distribution of the model’s input features in production changes compared to what the model saw during training and validation.

https://www.nannyml.com/blog/detecting-covariate-shift-multivariate-approach

No Ground Truth: How to Monitor ML Models

Learn how monitoring machine learning models is possible without access to target data. Understand nannyML Performance Estimation algorithm.

https://www.nannyml.com/blog/model-monitoring-without-ground-truth

References