biologyOrderPredictor/bop_scripts/preprocessing.py

"""
    This script contains the preprocessing functions
"""

import sqlite3
import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

def get_Xy_df (X, y):
    """
        Merge together the X and y dataframe on stay_id basis

        Parameters
        ----------
        X: pandas dataframe of features
        y: pandas dataframe of labeles

        Output
        ------
        Xy : merged pandas dataframe
    """

    Xy = pd.merge(
        X,
        y,
        left_on="stay_id",
        right_on="stay_id"
    )

    return Xy

def generate_features_dataset(database, get_drugs=True, get_diseases=True):

    """
        Generate features dataset according to the data

        Parameters
        ----------
        database: str, path of the database sqlite file
        get_drugs: boolean, if true the drug history is returned,
        get_diseases: boolean, if true the disease history is returned
    """

    to_merge = []
    
    # Sqlite connection
    conn = sqlite3.connect("./data/mimic-iv.sqlite")

    ## Getting the features
    features = pd.read_sql(f"""
        SELECT 
            s.stay_id,
            s.intime intime,
            p.gender gender,
            p.anchor_age age,
            t.temperature,
            t.heartrate,
            t.resprate,
            t.o2sat,
            t.sbp,
            t.dbp,
            t.pain,
            t.chiefcomplaint
        FROM edstays s
        LEFT JOIN patients p
            ON p.subject_id = s.subject_id
        LEFT Join triage t
            ON t.stay_id = s.stay_id
    """, conn)

    ## Additional features
    ### Last visit
    last_visit = pd.read_sql(f"""
        SELECT DISTINCT
            s1.stay_id,
            CAST(MAX((julianday(s1.intime)-julianday(s2.intime))) <= 7 AS INT) last_7,
            CAST(MAX((julianday(s1.intime)-julianday(s2.intime))) <= 30 AS INT) last_30
        FROM edstays s1
        INNER JOIN edstays s2
            ON s1.subject_id = s2.subject_id
                AND s1.stay_id != s2.stay_id
                AND s1.intime >= s2.intime
        WHERE (julianday(s1.intime)-julianday(s2.intime)) <= 30
        GROUP BY s1.stay_id 
    """, conn)
    to_merge.append(last_visit)

    ### Past diagnosis
    if get_diseases:
        past_diagnosis = pd.read_sql(f"""
            SELECT 
                s1.stay_id,
                d.icd_code,
                d.icd_version,
                COUNT(1) n
            FROM edstays s1
            INNER JOIN diagnosis d
                ON d.subject_id = s1.subject_id
            INNER JOIN edstays s2
                ON d.stay_id = s2.stay_id
            WHERE 
                s1.intime >= s2.intime
                AND s1.stay_id != s2.stay_id
            GROUP BY 
                s1.stay_id,
                d.icd_code,
                d.icd_version
        """, conn)
        past_diagnosis = pd.pivot_table(
            past_diagnosis.groupby(["stay_id","icd_version"])["icd_code"].agg(lambda x: x.tolist()) \
                    .reset_index(),
                index="stay_id",
                columns="icd_version",
                values="icd_code",
                aggfunc=lambda x: x
        ).reset_index().rename(columns={
            9:"icd9",
            10:"icd10"
        })
        to_merge.append(past_diagnosis)

    ### Drugs
    if get_drugs:
        drugs = pd.read_sql(f"""
            SELECT stay_id, gsn, 1 n
            FROM medrecon
        """, conn)
        drugs = drugs.groupby("stay_id")["gsn"].agg(lambda x: x.tolist()).reset_index()
        to_merge.append(drugs)

    ### Merging all together
    for df_to_merge in to_merge:
        features = pd.merge(
            features,
            df_to_merge,
            left_on="stay_id",
            right_on="stay_id",
            how="left"
        )

    features = features.sort_values("stay_id").reset_index(drop=True)

    return features
    

def generate_labels_dataset(database, lab_dictionnary):
    """
        Generate features dataset according to the data

        Parameters
        ----------
        database: str, path of the database sqlite file
        lab_dictionnary: dictionnary containing the id (keys) and label (value) of the biological exams to predict
    """

    to_merge = []
    
    # Sqlite connection
    conn = sqlite3.connect("./data/mimic-iv.sqlite")

    # Getting biological values
    lab_dictionnary_pd = pd.DataFrame.from_dict(lab_dictionnary, orient="index").reset_index()
    lab_dictionnary_list = [str(x) for x in lab_dictionnary.keys()]

    ## Let's create an index to speed up queries
    curs = conn.cursor()
    curs.execute("CREATE INDEX IF NOT EXISTS biological_index ON labevents (stay_id, itemid)")
    curs.close()

    # 1. Generating features

    ## Getting list of stay_id
    stays = pd.read_sql(
        "SELECT DISTINCT stay_id FROM edstays",
        conn
    )

    ## Getting the features
    labs = pd.read_sql(f"""
        SELECT 
            le.stay_id,
            le.itemid item_id
        FROM labevents le
        WHERE le.itemid IN ('{"','".join(lab_dictionnary_list)}')
        GROUP BY
            le.stay_id,
            le.itemid
    """, conn)

    labs_deduplicate = pd.merge(
        lab_dictionnary_pd.rename(columns={0:"label"}),
        labs,
        left_on="index",
        right_on="item_id"
    ) \
    .drop_duplicates(["stay_id", "label"])[["stay_id","label"]] \
    .reset_index(drop=True)

    labs_deduplicate_pivot = pd.pivot_table(
        labs_deduplicate.assign(value=1),
        index="stay_id",
        columns="label",
        values="value"
    ).fillna(0)

    labs_deduplicate_pivot_final = labs_deduplicate_pivot.join(
        stays[["stay_id"]].set_index("stay_id"),
        how="right"
    ).fillna(0).astype("int8").reset_index()

    labels = labs_deduplicate_pivot_final.sort_values("stay_id").reset_index(drop=True)

    return labels

def remove_outliers (X, variables_ranges):
    """
        This function remove the outliers and replace them by an NA according to the variable_ranges rules

        Parameters
        ----------
        X: pandas Dataframe
        variables_ranges: Dict(variable:[range_inf, range_sup], ...), dictionnary containing for each variable the inferior and superior range

        Outputs
        -------
        Tuple containing :
        - Processing dataframe
        - A Dataframe containing the number and percentage of processed outliers per variable
    """

    outliers = {}
    X_copy = X.copy()

    for key, value in variables_ranges.items():
        outliers_mask = ((X[key] < value[0]) | (X[key] > value[1]))
        outliers[key] = outliers_mask.sum() # Storing the number of outliers
        X_copy.loc[outliers_mask, key] = np.NaN # Setting outliers to NA

    outlier_report = pd.DataFrame.from_dict(outliers, orient="index") \
        .rename(columns={0:"n"}) \
        .assign(total=X[outliers.keys()].count().values,
                pourcentage=lambda x: (x["n"]/x["total"])*100
        )

    return X_copy, outlier_report

class OutlierRemover(BaseEstimator, TransformerMixin):
    """
        Sklearn-like class for removing outliers
        To be included in the pipeline
    """

    def __init__ (self, variables_ranges):
        """
            Parameters:
            ----------
            variables_ranges: Dict(variable:[range_inf, range_sup], ...), dictionnary containing for each variable the inferior and superior range
        """

        super().__init__()

        # Storing ranges rules
        self.variables_ranges = variables_ranges

    def fit(self, X=None, y=None):
        return self

    def transform(self, X):
        X_copy, _ = remove_outliers(X, self.variables_ranges)
        return X_copy

class TextPreprocessing(BaseEstimator, TransformerMixin):
    """
        Sklearn-like class for text preprocessing
        To be included in the pipeline

        What does it do :
        - It fills na with empty string
        - It lower string
        - It replace comma by space
    """

    def __init__ (self):

        """
            Parameters:
            ----------
        """

        super().__init__()

    def fit(self, X=None, y=None):
        return self

    def transform(self, X):
        colname = X.name
        X = X \
            .fillna("") \
            .replace(",", " ").str.lower()

        return X