Evaluation

Evaluate the performance of predictive models.

BaseBettor(betting_markets=None, init_cash=None, stake=None)

Bases: MultiOutputMixin, ClassifierMixin, BaseEstimator

The base class for bettors.

A bettor turns probabilities into bets. It places a bet when the model gives an outcome a higher probability than its price implies. To build one, implement _fit and _predict_proba.

Parameters:

Name	Type	Description	Default
betting_markets	list[str] | None	Select the betting markets from the ones included in the data.	None
init_cash	float | None	The initial cash to use when betting.	None
stake	float | None	The stake of each bet.	None

Examples:

>>> import numpy as np
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> from sportsbet.evaluation import BaseBettor, backtest
>>>
>>> class BaseRateBettor(BaseBettor):
...     'A bettor of your own, knowing only how often each outcome has happened.'
...
...     def _fit(self, X, Y, O):
...         # `Y` carries the markets it was told to bet, in the order it was told them.
...         self.rates_ = Y.mean().to_numpy()
...         return self
...
...     def _predict_proba(self, X):
...         rates = np.tile(self.rates_, (len(X), 1))
...         return rates / rates.sum(axis=1, keepdims=True)
>>>
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> bettor = BaseRateBettor(betting_markets=['home_win', 'draw', 'away_win'])
>>> results = backtest(bettor, X, Y, O)
>>> 'Yield percentage per bet' in results.columns
True
>>> # `bet` gives the value bets, one row per match and one column per market.
>>> bettor.fit(X, Y, O).bet(X, O).shape
(380, 3)

Source code in src/sportsbet/evaluation/_base.py

def __init__(
    self: Self,
    betting_markets: list[str] | None = None,
    init_cash: float | None = None,
    stake: float | None = None,
) -> None:
    self.betting_markets = betting_markets
    self.init_cash = init_cash
    self.stake = stake

bet(X, O)

Predict the value bets for the provided input data and odds.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required
O	DataFrame	The odds data.	required

Returns:

Name	Type	Description
B	BoolData	The value bets.

Raises:

Type	Description
TypeError	If X or O are not pandas dataframes, or X has no date index.
ValueError	If the O column names are malformed, or do not include the selected betting markets.

Source code in src/sportsbet/evaluation/_base.py

def bet(self: Self, X: pd.DataFrame, O: pd.DataFrame) -> BoolData:
    """Predict the value bets for the provided input data and odds.

    Args:
        X:
            The input data.

        O:
            The odds data.

    Returns:
        B:
            The value bets.

    Raises:
        TypeError:
            If `X` or `O` are not pandas dataframes, or `X` has no date index.

        ValueError:
            If the `O` column names are malformed, or do not include the selected
            betting markets.
    """
    Y_proba_pred = self.predict_proba(self._append_odds_data(X, O))
    X, O, O_betting_markets = self._validate_X_O(X, O)
    if not set(O_betting_markets).issuperset(self.betting_markets_):
        error_msg = 'Odds data do not include selected betting markets.'
        raise ValueError(error_msg)
    O = O[self._get_feature_names_odds(O)]
    B_pred = Y_proba_pred * O > 1
    B_pred_selected = []
    for events in self.complementary_events_:
        events_indices = np.where(np.isin(self.betting_markets_, events))[0]
        if events_indices.size > 0:
            estimated_returns = np.nan_to_num(
                (O.iloc[:, events_indices] * Y_proba_pred[:, events_indices] - 1).to_numpy(),
            )
            estimated_returns += [eps * self.TOL for eps in range(estimated_returns.shape[1])]
            mask = estimated_returns != np.max(estimated_returns, axis=1).reshape(-1, 1)
            B_pred_events = B_pred.iloc[:, events_indices].copy()
            B_pred_events[mask] = False
            B_pred_selected.append(B_pred_events)
    return pd.concat(B_pred_selected, axis=1).to_numpy()

fit(X, Y, O=None)

Fit the bettor to the input data and multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required
Y	DataFrame	The multi-output targets.	required
O	DataFrame | None	The odds data.	None

Returns:

Name	Type	Description
self	Self	The fitted bettor object.

Raises:

Type	Description
TypeError	If X, Y or O are not pandas dataframes, or X has no date index.
ValueError	If the Y or O column names are malformed, or the output and odds markets are not compatible.

Source code in src/sportsbet/evaluation/_base.py

def fit(self: Self, X: pd.DataFrame, Y: pd.DataFrame, O: pd.DataFrame | None = None) -> Self:
    """Fit the bettor to the input data and multi-output targets.

    Args:
        X:
            The input data.

        Y:
            The multi-output targets.

        O:
            The odds data.

    Returns:
        self:
            The fitted bettor object.

    Raises:
        TypeError:
            If `X`, `Y` or `O` are not pandas dataframes, or `X` has no date index.

        ValueError:
            If the `Y` or `O` column names are malformed, or the output and odds
            markets are not compatible.
    """
    X, Y, Y_betting_markets = self._validate_X_Y(X, Y)
    if O is not None:
        X, O, O_betting_markets = self._validate_X_O(X, O)
        _check_markets_compatible(Y_betting_markets, O_betting_markets)
    X_fit = self._append_odds_data(X, O)
    self._check(X_fit, Y, O, Y_betting_markets)
    return self._fit(X_fit, Y[self.feature_names_out_], O[self.feature_names_odds_] if O is not None else None)

predict(X)

Predict class labels for multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required

Returns:

Name	Type	Description
Y	BoolData	The positive class labels.

Source code in src/sportsbet/evaluation/_base.py

def predict(self: Self, X: pd.DataFrame) -> BoolData:
    """Predict class labels for multi-output targets.

    Args:
        X:
            The input data.

    Returns:
        Y:
            The positive class labels.
    """
    decision_threshold = 0.5
    Y_pred = self.predict_proba(X) > decision_threshold
    return Y_pred

predict_proba(X)

Predict class probabilities for multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required

Returns:

Name	Type	Description
Y	Data	The positive class probabilities.

Raises:

Type	Description
TypeError	If the predicted probabilities and selected betting markets have incompatible shapes.

Source code in src/sportsbet/evaluation/_base.py

def predict_proba(self: Self, X: pd.DataFrame) -> Data:
    """Predict class probabilities for multi-output targets.

    Args:
        X:
            The input data.

    Returns:
        Y:
            The positive class probabilities.

    Raises:
        TypeError:
            If the predicted probabilities and selected betting markets have
            incompatible shapes.
    """
    check_is_fitted(self)
    _check_feature_names(self, X, reset=False)
    if X.empty:
        return np.empty((0, self.betting_markets_.size), dtype=float)
    Y_proba_pred = self._predict_proba(X)
    Y_proba_pred = Y_proba_pred.reshape(Y_proba_pred.shape[0], -1)
    if Y_proba_pred.shape[1] != self.betting_markets_.size:
        error_msg = 'Predicted probabilities and selected betting markets have incompatible shapes.'
        raise TypeError(error_msg)
    Y_proba_pred = self._normalize_proba(Y_proba_pred)
    return Y_proba_pred

score(X, Y, O)

Return the annual sharpe ratio on the given data.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required
Y	DataFrame	The output data.	required
O	DataFrame	The odds data.	required

Returns:

Name	Type	Description
score	float	Annual sharpe ratio of predicted value bets.

Raises:

Type	Description
TypeError	If X, Y or O are not pandas dataframes, or X has no date index.
ValueError	If the Y or O column names are malformed, or the output and odds markets are not compatible.

Source code in src/sportsbet/evaluation/_base.py

def score(self: Self, X: pd.DataFrame, Y: pd.DataFrame, O: pd.DataFrame) -> float:
    """Return the annual sharpe ratio on the given data.

    Args:
        X:
            The input data.

        Y:
            The output data.

        O:
            The odds data.

    Returns:
        score:
            Annual sharpe ratio of predicted value bets.

    Raises:
        TypeError:
            If `X`, `Y` or `O` are not pandas dataframes, or `X` has no date index.

        ValueError:
            If the `Y` or `O` column names are malformed, or the output and odds
            markets are not compatible.
    """
    check_is_fitted(self)
    X, Y, Y_betting_markets = self._validate_X_Y(X, Y)
    X, O, O_betting_markets = self._validate_X_O(X, O)
    _check_markets_compatible(Y_betting_markets, O_betting_markets)
    value_bets = self.bet(X, O)
    Y = Y[self.feature_names_out_]
    O = O[self._get_feature_names_odds(O)]
    returns = np.sum(
        np.nan_to_num(
            (Y.to_numpy().astype(int) * O.to_numpy().astype(float) - 1) * value_bets.astype(int),
        ),
        axis=1,
    )
    returns = pd.DataFrame(returns).set_index(X.index).groupby('date').sum()
    dates = pd.DataFrame(pd.date_range(returns.index.min(), returns.index.max()), columns=['date'])
    returns = dates.merge(returns.reset_index(), how='left')
    returns_mean, returns_std = returns[0].fillna(0).mean(), returns[0].fillna(0).std()
    if returns_std == 0 or np.isnan(returns_std):
        max_sharpe_ratio = 100.0
        return max_sharpe_ratio if returns_mean > 0 else -max_sharpe_ratio
    return np.sqrt(365) * returns_mean / returns_std

BettorGridSearchCV(estimator, param_grid, *, scoring=None, n_jobs=None, refit=True, cv=TSCV, verbose=0, pre_dispatch='2*n_jobs', error_score=np.nan, return_train_score=False)

Bases: GridSearchCV, BaseBettor

Search a bettor's parameter grid with cross-validation.

It optimizes the bettor's parameters by cross-validated grid search over the parameter grid.

Read more in the user guide.

Parameters:

Name	Type	Description	Default
estimator	BaseBettor	This is assumed to implement the bettor interface.	required
param_grid	dict | list	Dictionary with parameters names (str) as keys and lists of parameter settings to try as values, or a list of such dictionaries, in which case the grids spanned by each dictionary in the list are explored. This enables searching over any sequence of parameter settings.	required
scoring	str | Callable | list | tuple | dict[str, Callable] | None	Strategy to evaluate the performance of the cross-validated model on the test set. If scoring represents a single score, one can use: a single string a callable (see :ref:scoring) that returns a single value If scoring represents multiple scores, one can use: a list or tuple of unique strings a callable returning a dictionary where the keys are the metric names and the values are the metric scores a dictionary with metric names as keys and callables a values	None
n_jobs	int | None	Number of jobs to run in parallel. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors.	None
refit	bool | str | Callable	Refit an estimator using the best found parameters on the whole dataset. For multiple metric evaluation, this needs to be a str denoting the scorer that would be used to find the best parameters for refitting the estimator at the end. Where there are considerations other than maximum score in choosing a best estimator, refit can be set to a function which returns the selected best_index_ given cv_results_. In that case, the best_estimator_ and best_params_ will be set according to the returned best_index_ while the best_score_ attribute will not be available. The refitted estimator is made available at the best_estimator_ attribute and permits using predict directly on this BettorGridSearchCV instance. Also for multiple metric evaluation, the attributes best_index_, best_score_ and best_params_ will only be available if refit is set and all of them will be determined w.r.t this specific scorer. See scoring parameter to know more about multiple metric evaluation.	True
cv	TimeSeriesSplit	Provides train/test indices to split time series data samples that are observed at fixed time intervals, in train/test sets.	TSCV
verbose	int	Controls the verbosity: the higher, the more messages.	0
pre_dispatch	int | str	Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - `None`, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'	'2*n_jobs'
error_score	str | float | int	Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error.	nan
return_train_score	bool	If False, the cv_results_ attribute will not include training scores. Computing training scores is used to get insights on how different parameter settings impact the overfitting/underfitting trade-off. However computing the scores on the training set can be computationally expensive and is not strictly required to select the parameters that yield the best generalization performance.	False

Attributes:

Name	Type	Description
cv_results_		A dict with keys as column headers and values as columns, that can be imported into a pandas DataFrame. The key 'params' is used to store a list of parameter settings dicts for all the parameter candidates. The mean_fit_time, std_fit_time, mean_score_time and std_score_time are all in seconds. For multi-metric evaluation, the scores for all the scorers are available in the cv_results_ dict at the keys ending with that scorer's name.
best_estimator_		Estimator that was chosen by the search, i.e. estimator which gave highest score (or smallest loss if specified) on the left out data. Not available if refit=False.
best_score_		Mean cross-validated score of the best_estimator For multi-metric evaluation, this is present only if refit is specified. This attribute is not available if refit is a function.
best_params_		Parameter setting that gave the best results on the hold out data. For multi-metric evaluation, this is present only if refit is specified.
best_index_		The index (of the cv_results_ arrays) which corresponds to the best candidate parameter setting. For multi-metric evaluation, this is present only if refit is specified.
scorer_		Scorer function used on the held out data to choose the best parameters for the model. For multi-metric evaluation, this attribute holds the validated scoring dict which maps the scorer key to the scorer callable.
n_splits_		The number of cross-validation splits (folds/iterations).
refit_time_		Seconds used for refitting the best model on the whole dataset. This is present only if refit is not False.
multimetric_		Whether or not the scorers compute several metrics.
classes_	list	The classes labels. This is present only if refit is specified and the underlying estimator is a classifier.
n_features_in_	list	Number of features seen during fit. Only defined if best_estimator_ is defined and that best_estimator_ exposes n_features_in_ when fit.
feature_names_in_	list	Names of features seen during fit. Only defined if best_estimator_ is defined and that best_estimator_ exposes feature_names_in_ when fit.

Examples:

>>> from sportsbet.evaluation import BettorGridSearchCV, OddsComparisonBettor, backtest
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> from sklearn.model_selection import TimeSeriesSplit
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> bettor = BettorGridSearchCV(
...     estimator=OddsComparisonBettor(),
...     param_grid={'alpha': [0.02, 0.05, 0.1]},
...     cv=TimeSeriesSplit(2),
... )
>>> results = backtest(bettor, X, Y, O, cv=TimeSeriesSplit(2))
>>> 'Number of bets' in results.columns
True

Source code in src/sportsbet/evaluation/_model_selection.py

def __init__(
    self: Self,
    estimator: BaseBettor,
    param_grid: dict | list,
    *,
    scoring: str | Callable | list | tuple | dict[str, Callable] | None = None,
    n_jobs: int | None = None,
    refit: bool | str | Callable = True,
    cv: TimeSeriesSplit = TSCV,
    verbose: int = 0,
    pre_dispatch: int | str = '2*n_jobs',
    error_score: str | float | int = np.nan,
    return_train_score: bool = False,
) -> None:
    GridSearchCV.__init__(
        self,
        estimator=estimator,
        param_grid=param_grid,
        scoring=scoring,
        n_jobs=n_jobs,
        refit=refit,
        cv=cv,
        verbose=verbose,
        pre_dispatch=pre_dispatch,
        error_score=error_score,
        return_train_score=return_train_score,
    )

bet(X, O)

Predict the value bets for the provided input data and odds.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required
O	DataFrame	The odds data.	required

Returns:

Name	Type	Description
B	BoolData	The value bets.

Source code in src/sportsbet/evaluation/_model_selection.py

def bet(self: Self, X: pd.DataFrame, O: pd.DataFrame) -> BoolData:
    """Predict the value bets for the provided input data and odds.

    Args:
        X:
            The input data.

        O:
            The odds data.

    Returns:
        B:
            The value bets.
    """
    self._check_attr('bet', False, True)
    return self.best_estimator_.bet(X, O)

fit(X, Y, O=None)

Fit the bettor to the input data and multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required
Y	DataFrame	The multi-output targets.	required
O	DataFrame | None	The odds data.	None

Returns:

Name	Type	Description
self	Self	The fitted bettor object.

Source code in src/sportsbet/evaluation/_model_selection.py

def fit(self: Self, X: pd.DataFrame, Y: pd.DataFrame, O: pd.DataFrame | None = None) -> Self:
    """Fit the bettor to the input data and multi-output targets.

    Args:
        X:
            The input data.

        Y:
            The multi-output targets.

        O:
            The odds data.

    Returns:
        self:
            The fitted bettor object.
    """
    self._fit(X, Y, O)
    if hasattr(self, 'best_estimator_'):
        self.init_cash_ = self.best_estimator_.init_cash_
        self.stake_ = self.best_estimator_.stake_
        self.betting_markets_ = self.best_estimator_.betting_markets_
        self.feature_names_out_ = self.best_estimator_.feature_names_out_
    if O is not None and hasattr(self, 'best_estimator_'):
        self.feature_names_odds_ = self.best_estimator_._get_feature_names_odds(O)
    return self

predict(X)

Predict class labels for multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required

Returns:

Name	Type	Description
Y	BoolData	The positive class labels.

Source code in src/sportsbet/evaluation/_model_selection.py

def predict(self: Self, X: pd.DataFrame) -> BoolData:
    """Predict class labels for multi-output targets.

    Args:
        X:
            The input data.

    Returns:
        Y:
            The positive class labels.
    """
    self._check_attr('predict', False, True)
    return self.best_estimator_.predict(X)

predict_proba(X)

Predict class probabilities for multi-output targets.

Parameters:

Name	Type	Description	Default
X	DataFrame	The input data.	required

Returns:

Name	Type	Description
Y	Data	The positive class probabilities.

Source code in src/sportsbet/evaluation/_model_selection.py

def predict_proba(self: Self, X: pd.DataFrame) -> Data:
    """Predict class probabilities for multi-output targets.

    Args:
        X:
            The input data.

    Returns:
        Y:
            The positive class probabilities.
    """
    self._check_attr('predict_proba', False, True)
    return self.best_estimator_.predict_proba(X)

ClassifierBettor(classifier, betting_markets=None, init_cash=None, stake=None)

Bases: MetaEstimatorMixin, BaseBettor

Bettor based on a Scikit-Learn classifier.

Read more in the user guide.

Parameters:

Name	Type	Description	Default
classifier	BaseEstimator	A scikit-learn classifier object implementing fit, score and predict_proba.	required
betting_markets	list[str] | None	Select the betting markets from the ones included in the data.	None
init_cash	float | None	The initial cash to use when betting.	None
stake	float | None	The stake of each bet.	None

Attributes:

Name	Type	Description
classifier_	BaseEstimator	The fitted clone of classifier.
init_cash_	float	The checked initial cash.

Examples:

>>> from sklearn.tree import DecisionTreeClassifier
>>> from sklearn.preprocessing import OneHotEncoder
>>> from sklearn.impute import SimpleImputer
>>> from sklearn.pipeline import make_pipeline
>>> from sklearn.compose import make_column_transformer
>>> from sportsbet.evaluation import ClassifierBettor, backtest
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> # Create a pipeline to handle categorical features and missing values
>>> clf_pipeline = make_pipeline(
...     make_column_transformer(
...         (OneHotEncoder(handle_unknown='ignore'), ['league', 'home_team', 'away_team']),
...         remainder='passthrough',
...     ),
...     SimpleImputer(),
...     DecisionTreeClassifier(random_state=0),
... )
>>> bettor = ClassifierBettor(clf_pipeline)
>>> results = backtest(bettor, X, Y, O)
>>> 'Number of bets' in results.columns
True

Source code in src/sportsbet/evaluation/_classifier.py

def __init__(
    self: Self,
    classifier: BaseEstimator,
    betting_markets: list[str] | None = None,
    init_cash: float | None = None,
    stake: float | None = None,
) -> None:
    super().__init__(betting_markets, init_cash, stake)
    self.classifier = classifier

OddsComparisonBettor(odds_types=None, alpha=0.05, betting_markets=None, init_cash=None, stake=None)

Bases: BaseBettor

Bettor based on comparison of odds.

It compares each market's odds to a consensus probability. The consensus probability is the average of the selected odds types, adjusted by alpha. The method follows Beating the bookies with their own numbers.

Read more in the user guide.

Parameters:

Name	Type	Description	Default
odds_types	list[str] | None	The odds types to use for the calculation of consensus probabilities. The default value corresponds to 'market_average' if this odds type exists or the average of all the other odds columns if 'market_average' is missing.	None
alpha	float	An adjustment term that corresponds to the difference between the consensus and real probabilities.	0.05
betting_markets	list[str] | None	Select the betting markets from the ones included in the data.	None
init_cash	float | None	The initial cash to use when betting.	None
stake	float | None	The stake of each bet.	None

Attributes:

Name	Type	Description
odds_types_	list[str]	The checked value of the odds types.
alpha_	float	The checked value of the alpha parameter.
output_keys_	list[str]	The market base names of the output columns.

Examples:

>>> from sportsbet.evaluation import OddsComparisonBettor, backtest
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> bettor = OddsComparisonBettor(alpha=0.03)
>>> results = backtest(bettor, X, Y, O)
>>> 'Number of bets' in results.columns
True

Source code in src/sportsbet/evaluation/_rules.py

def __init__(
    self: Self,
    odds_types: list[str] | None = None,
    alpha: float = 0.05,
    betting_markets: list[str] | None = None,
    init_cash: float | None = None,
    stake: float | None = None,
) -> None:
    super().__init__(betting_markets, init_cash, stake)
    self.odds_types = odds_types
    self.alpha = alpha

backtest(bettor, X, Y, O, cv=None, n_jobs=-1, verbose=0)

Backtest the bettor.

Parameters:

Name	Type	Description	Default
bettor	BaseBettor	The bettor object.	required
X	DataFrame	The input data. Each row of X represents information that is available before the start of a specific match. The index should be of type datetime, named as 'date'.	required
Y	DataFrame	The multi-output targets. Each row of Y represents information that is available after the end of a specific event. The column names follow the convention for the output data Y of the method extract_train_data of dataloaders.	required
O	DataFrame	The odds data. The column names follow the convention for the odds data O of the method extract_train_data of dataloaders.	required
cv	TimeSeriesSplit | None	Provides train/test indices to split time series data samples that are observed at fixed time intervals, in train/test sets. The default value of the parameter is None, corresponding to the default TimeSeriesSplit object.	None
n_jobs	int	Number of CPU cores to use when parallelizing the backtesting runs. The default value of -1 means using all processors.	-1
verbose	int	The verbosity level.	0

Returns:

Name	Type	Description
results	DataFrame	The backtesting results.

Raises:

Type	Description
TypeError	If X, Y or O are not pandas dataframes, X has no date index, or cv is not a TimeSeriesSplit cross-validator object.

Examples:

>>> from sklearn.model_selection import TimeSeriesSplit
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> from sportsbet.evaluation import OddsComparisonBettor, backtest
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> bettor = OddsComparisonBettor(betting_markets=['home_win', 'draw', 'away_win'])
>>> results = backtest(bettor, X, Y, O, cv=TimeSeriesSplit(2))
>>> # The folds run forward in time, so a model is never tested on a match it was trained on.
>>> list(results.index.names)
['Training start', 'Training end', 'Testing start', 'Testing end']
>>> 'Yield percentage per bet' in results.columns
True

Source code in src/sportsbet/evaluation/_model_selection.py

def backtest(
    bettor: BaseBettor,
    X: pd.DataFrame,
    Y: pd.DataFrame,
    O: pd.DataFrame,
    cv: TimeSeriesSplit | None = None,
    n_jobs: int = -1,
    verbose: int = 0,
) -> pd.DataFrame:
    """Backtest the bettor.

    Args:
        bettor:
            The bettor object.

        X:
            The input data. Each row of `X` represents information that is available
            before the start of a specific match. The index should be of type
            `datetime`, named as `'date'`.

        Y:
            The multi-output targets. Each row of `Y` represents information
            that is available after the end of a specific event. The column
            names follow the convention for the output data `Y` of the method
            `extract_train_data` of dataloaders.

        O:
            The odds data. The column names follow the convention for the odds
            data `O` of the method `extract_train_data` of dataloaders.

        cv:
            Provides train/test indices to split time series data samples
            that are observed at fixed time intervals, in train/test sets. The
            default value of the parameter is `None`, corresponding to the default
            `TimeSeriesSplit` object.

        n_jobs:
            Number of CPU cores to use when parallelizing the backtesting runs.
            The default value of `-1` means using all processors.

        verbose:
            The verbosity level.

    Returns:
        results:
            The backtesting results.

    Raises:
        TypeError:
            If `X`, `Y` or `O` are not pandas dataframes, `X` has no date index, or
            `cv` is not a `TimeSeriesSplit` cross-validator object.

    Examples:
        >>> from sklearn.model_selection import TimeSeriesSplit
        >>> from sportsbet.dataloaders import DataLoader
        >>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
        >>> from sportsbet.evaluation import OddsComparisonBettor, backtest
        >>> dataloader = DataLoader(
        ...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
        ... )
        >>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
        >>> bettor = OddsComparisonBettor(betting_markets=['home_win', 'draw', 'away_win'])
        >>> results = backtest(bettor, X, Y, O, cv=TimeSeriesSplit(2))
        >>> # The folds run forward in time, so a model is never tested on a match it was trained on.
        >>> list(results.index.names)
        ['Training start', 'Training end', 'Testing start', 'Testing end']
        >>> 'Yield percentage per bet' in results.columns
        True
    """
    check_consistent_length(X, Y, O)
    _check_is_dataframe(X, 'X', date_index=True)
    _check_is_dataframe(Y, 'Y')
    _check_is_dataframe(O, 'O')

    indices = np.argsort(X.index)
    X, Y, O = X.iloc[indices], Y.iloc[indices], O.iloc[indices]

    if cv is None:
        cv = TimeSeriesSplit()
    _check_time_series_cv(cv)

    results = Parallel(n_jobs=n_jobs, verbose=verbose)(
        delayed(_fit_bet)(train_ind, test_ind, bettor, X, Y, O) for train_ind, test_ind in cv.split(X)
    )
    results = pd.DataFrame(results).set_index(['Training start', 'Training end', 'Testing start', 'Testing end'])

    return results

build_bettor(model)

Build a betting model from a scikit-learn expression or a reference to your own.

Parameters:

Name	Type	Description	Default
model	str	A scikit-learn estimator written as a Python expression, with the library's bettors and every scikit-learn estimator already in scope, as in ClassifierBettor(LogisticRegression(C=1.0)). Or a bettor you built in a file, named by where it lives, as in models.py:BETTOR.	required

Returns:

Name	Type	Description
bettor	BaseBettor	The betting model, ready to fit.

Raises:

Type	Description
BuildError	When the expression or the reference does not describe a bettor.

Source code in src/sportsbet/evaluation/_factory.py

def build_bettor(model: str) -> BaseBettor:
    """Build a betting model from a scikit-learn expression or a reference to your own.

    Args:
        model:
            A scikit-learn estimator written as a Python expression, with the library's bettors and every
            scikit-learn estimator already in scope, as in `ClassifierBettor(LogisticRegression(C=1.0))`. Or a
            bettor you built in a file, named by where it lives, as in `models.py:BETTOR`.

    Returns:
        bettor:
            The betting model, ready to fit.

    Raises:
        BuildError:
            When the expression or the reference does not describe a bettor.
    """
    if ':' in model and '(' not in model:
        built = load_object(model)
    else:
        try:
            built = eval(model, _build_bettor_namespace())  # noqa: S307  # nosec B307  # a trusted model expression
        except Exception as error:
            msg = (
                f'`{model}` is not a model. Write it as a scikit-learn expression, as in '
                '`OddsComparisonBettor(alpha=0.05)`, or point to one with `models.py:BETTOR`.'
            )
            raise BuildError(msg) from error
    if not isinstance(built, BaseBettor):
        msg = f'`{model}` is not a bettor.'
        raise BuildError(msg)
    return built

derive_complementary_events(markets)

Return the groups of markets that are mutually exclusive and exhaustive.

Parameters:

Name	Type	Description	Default
markets	list[str]	The betting markets the data carries.	required

Returns:

Name	Type	Description
events	list[list[str]]	The groups of markets whose probabilities sum to one.

Examples:

>>> from sportsbet.evaluation import derive_complementary_events
>>> derive_complementary_events(['home_win', 'draw', 'away_win', 'over_2.5', 'under_2.5'])
[['home_win', 'draw', 'away_win'], ['over_2.5', 'under_2.5']]
>>> derive_complementary_events(['home_win', 'away_win'])
[['home_win', 'away_win']]
>>> derive_complementary_events(['draw'])
[]

Source code in src/sportsbet/evaluation/_base.py

def derive_complementary_events(markets: list[str]) -> list[list[str]]:
    """Return the groups of markets that are mutually exclusive and exhaustive.

    Args:
        markets:
            The betting markets the data carries.

    Returns:
        events:
            The groups of markets whose probabilities sum to one.

    Examples:
        >>> from sportsbet.evaluation import derive_complementary_events
        >>> derive_complementary_events(['home_win', 'draw', 'away_win', 'over_2.5', 'under_2.5'])
        [['home_win', 'draw', 'away_win'], ['over_2.5', 'under_2.5']]
        >>> derive_complementary_events(['home_win', 'away_win'])
        [['home_win', 'away_win']]
        >>> derive_complementary_events(['draw'])
        []
    """
    groups = []
    outcomes = [market for market in OUTCOME_MARKETS if market in markets]
    if len(outcomes) > 1:
        groups.append(outcomes)
    lines: dict[str, list[str]] = {}
    for market in markets:
        for side in ('over', 'under'):
            if market.startswith(f'{side}_'):
                lines.setdefault(market.removeprefix(f'{side}_'), []).append(market)
    groups.extend(sorted(group) for group in lines.values() if len(group) > 1)
    return groups

derive_market_base(market)

Return the base market name (drop the __status__time suffix).

Parameters:

Name	Type	Description	Default
market	str	The market column name to reduce to its base.	required

Returns:

Type	Description
str	The base market name (e.g. home_win).

Source code in src/sportsbet/evaluation/_base.py

def derive_market_base(market: str) -> str:
    """Return the base market name (drop the ``__status__time`` suffix).

    Args:
        market:
            The market column name to reduce to its base.

    Returns:
        The base market name (e.g. `home_win`).
    """
    return market.split('__', maxsplit=1)[0]

find_latest_odds_column(columns, base, provider=None)

Return the odds column for a market base at the latest snapshot.

Parameters:

Name	Type	Description	Default
columns	list[str]	Candidate odds column names ({provider}__{base}__{status}__{time}).	required
base	str	The market base to match (e.g. home_win).	required
provider	str | None	If given, only match this provider.	None

Returns:

Type	Description
str | None	The matching column at the latest (status, time), or None.

Source code in src/sportsbet/evaluation/_base.py

def find_latest_odds_column(columns: list[str], base: str, provider: str | None = None) -> str | None:
    """Return the odds column for a market base at the latest snapshot.

    Args:
        columns:
            Candidate odds column names (`{provider}__{base}__{status}__{time}`).
        base:
            The market base to match (e.g. `home_win`).
        provider:
            If given, only match this provider.

    Returns:
        The matching column at the latest ``(status, time)``, or `None`.
    """
    best: str | None = None
    best_key: tuple[int, pd.Timedelta] | None = None
    for col in columns:
        if not _is_odds_column(col):
            continue
        col_provider, col_base, status, time = col.split('__')
        if col_base != base or (provider is not None and col_provider != provider):
            continue
        key = (STATUS_RANK.get(status, -1), parse_event_time(time))
        if best_key is None or key > best_key:
            best_key, best = key, col
    return best

load_bettor(path)

Load the bettor object.

Parameters:

Name	Type	Description	Default
path	str	The path of the bettor pickled file.	required

Returns:

Name	Type	Description
bettor	BaseBettor	The bettor object.

Examples:

>>> import tempfile
>>> from pathlib import Path
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> from sportsbet.evaluation import OddsComparisonBettor, load_bettor, save_bettor
>>> path = str(Path(tempfile.mkdtemp()) / 'bettor.pkl')
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> save_bettor(OddsComparisonBettor(betting_markets=['home_win']).fit(X, Y, O), path)
>>> bettor = load_bettor(path)
>>> # It is ready to bet without being fitted again: one row per match, one column per market.
>>> bettor.bet(X, O).shape
(380, 1)

Source code in src/sportsbet/evaluation/_base.py

def load_bettor(path: str) -> BaseBettor:
    """Load the bettor object.

    Args:
        path:
            The path of the bettor pickled file.

    Returns:
        bettor:
            The bettor object.

    Examples:
        >>> import tempfile
        >>> from pathlib import Path
        >>> from sportsbet.dataloaders import DataLoader
        >>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
        >>> from sportsbet.evaluation import OddsComparisonBettor, load_bettor, save_bettor
        >>> path = str(Path(tempfile.mkdtemp()) / 'bettor.pkl')
        >>> dataloader = DataLoader(
        ...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
        ... )
        >>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
        >>> save_bettor(OddsComparisonBettor(betting_markets=['home_win']).fit(X, Y, O), path)
        >>> bettor = load_bettor(path)
        >>> # It is ready to bet without being fitted again: one row per match, one column per market.
        >>> bettor.bet(X, O).shape
        (380, 1)
    """
    with Path(path).open('rb') as file:
        bettor = cloudpickle.load(file)
    return bettor

save_bettor(bettor, path)

Save the bettor object.

Parameters:

Name	Type	Description	Default
bettor	BaseBettor	The bettor object.	required
path	str	The path to save the object.	required

Examples:

>>> import tempfile
>>> from pathlib import Path
>>> from sportsbet.dataloaders import DataLoader
>>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
>>> from sportsbet.evaluation import OddsComparisonBettor, load_bettor, save_bettor
>>> path = str(Path(tempfile.mkdtemp()) / 'bettor.pkl')
>>> dataloader = DataLoader(
...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
... )
>>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
>>> bettor = OddsComparisonBettor(betting_markets=['home_win', 'draw', 'away_win']).fit(X, Y, O)
>>> save_bettor(bettor, path)
>>> # A fitted bettor comes back fitted.
>>> load_bettor(path).betting_markets_.tolist()
['home_win', 'draw', 'away_win']

Source code in src/sportsbet/evaluation/_base.py

def save_bettor(bettor: BaseBettor, path: str) -> None:
    """Save the bettor object.

    Args:
        bettor:
            The bettor object.

        path:
            The path to save the object.

    Examples:
        >>> import tempfile
        >>> from pathlib import Path
        >>> from sportsbet.dataloaders import DataLoader
        >>> from sportsbet.sources import SampleSoccerOdds, SampleSoccerStats
        >>> from sportsbet.evaluation import OddsComparisonBettor, load_bettor, save_bettor
        >>> path = str(Path(tempfile.mkdtemp()) / 'bettor.pkl')
        >>> dataloader = DataLoader(
        ...     param_grid={'league': ['England']}, stats=SampleSoccerStats(), odds=SampleSoccerOdds()
        ... )
        >>> X, Y, O = dataloader.extract_train_data(odds_type='market_average')
        >>> bettor = OddsComparisonBettor(betting_markets=['home_win', 'draw', 'away_win']).fit(X, Y, O)
        >>> save_bettor(bettor, path)
        >>> # A fitted bettor comes back fitted.
        >>> load_bettor(path).betting_markets_.tolist()
        ['home_win', 'draw', 'away_win']
    """
    with Path(path).open('wb') as file:
        cloudpickle.dump(bettor, file)