Classifier Models - Teacher vs. Parent vs. Self-Report

import os
import pandas as pd
import seaborn as sns

from hbn.constants import Defaults
from hbn.visualization import visualize

import warnings
warnings.filterwarnings("ignore")

%load_ext autoreload
%autoreload 2

%matplotlib inline

import plotly.io as pio
pio.renderers.default='notebook'

import ipywidgets as widgets       # interactive display
%config InlineBackend.figure_format = 'svg' # other available formats are: 'retina', 'png', 'jpeg', 'pdf'

# check models
from hbn.models.predictive_modeling import check_models

check_models(filter='2023-01-26_16')

2023-01-26_16:52:55.576236: ['No Diagnosis Given', 'Generalized Anxiety Disorder', 'Other Specified Anxiety Disorder', 'Social Anxiety (Social Phobia)', 'Separation Anxiety', 'Specific Phobia', 'Selective Mutism', 'Unspecified Anxiety Disorder', 'Panic Disorder', 'Agoraphobia']: ['DX_01_Cat_new_binarize']
2023-01-26_16:52:17.468889: ['No Diagnosis Given', 'Autism Spectrum Disorder']: ['DX_01_Cat_new_binarize']
2023-01-26_16:53:09.184825: ['No Diagnosis Given', 'Other Specified Depressive Disorder', 'Major Depressive Disorder', 'Disruptive Mood Dysregulation Disorder', 'Persistent Depressive Disorder (Dysthymia)', 'Depressive Disorder Due to Another Medical Condition']: ['DX_01_Cat_new_binarize']
2023-01-26_16:49:52.804211: ['ADHD-Combined Type', 'ADHD-Inattentive Type', 'Other Specified Attention-Deficit/Hyperactivity Disorder', 'ADHD-Hyperactive/Impulsive Type', 'Unspecified Attention-Deficit/Hyperactivity Disorder']: ['Sex_binarize']
2023-01-26_16:51:30.868341: ['No Diagnosis Given', 'ADHD-Combined Type', 'ADHD-Inattentive Type', 'Other Specified Attention-Deficit/Hyperactivity Disorder', 'ADHD-Hyperactive/Impulsive Type', 'Unspecified Attention-Deficit/Hyperactivity Disorder']: ['DX_01_Cat_new_binarize']
2023-01-26_16:53:33.811355: ['No Diagnosis Given', 'Specific Learning Disorder with Impairment in Reading']: ['DX_01_Cat_new_binarize']
2023-01-26_16:51:53.683959: ['No Diagnosis Given']: ['Sex_binarize']
2023-01-26_16:51:36.783459: ['ADHD-Combined Type', 'ADHD-Inattentive Type']: ['DX_01_binarize']

# load models 

# 2023 models
models_dict = {'Anxiety': '2023-01-26_16:52:55.576236', 
              'Depression': '2023-01-26_16:53:09.184825', 
              'ADHD': '2023-01-26_16:51:30.868341',
              'ASD': '2023-01-26_16:52:17.468889',
              'Learning-Disorder-Reading-Impairment': '2023-01-26_16:53:33.811355',
              'Anxiety2': '2023-01-25_22:10:51.315781', 
              'Depression2': '2023-01-25_22:10:47.981934', 
              'ADHD2': '2023-01-25_18:53:31.426535',
              'ASD2': '2023-01-25_22:11:05.460227',
              'Learning-Disorder-Reading-Impairment2': '2023-01-25_22:10:47.985589',
             }

# loop over models
df_all = pd.DataFrame()
for key, value in models_dict.items():
    MODEL_DIR = os.path.join(Defaults.MODEL_DIR, value)

    df_classify = pd.read_csv(os.path.join(MODEL_DIR, 'classifier-all-phenotypic-models-performance.csv'))
    df_classify['data'] = df_classify['data'].map({'model-data': 'null', 'model-null': 'data'})
    df_classify['participant_group'] = key
    
    df_all = pd.concat([df_all, df_classify])
    
    
df_all['participant_group'] = df_all['participant_group'].replace({'Anxiety2': 'Anxiety', 
                                    'Learning-Disorder-Reading-Impairment2': 'Learning-Disorder-Reading-Impairment',
                                    'Depression2': 'Depression',
                                    'ASD2': 'ASD',
                                    'ADHD2': 'ADHD'})

df_all = df_all.replace({'YSR': 'Youth Self-Report',
                'CBCL': 'Parent Report',
                'CBCL_Pre': 'Parent Report (Preschool)',
                'TRF': 'Teacher Report',
                'TRF_Pre': 'Teacher Report (Preschool)'})

# set plotting style
visualize.plotting_style()

Each model corresponds to a teacher, parent, or self-report (child) measure (questionnaie) - all questionnaires ask approx. the same behavioral questions thus allowing for comparison
Each model is classified as diagnosis (e.g., Anxiety, ADHD) vs. no diagnosis

participants = df_all['participant_group'].unique()
abbrevs = ['Youth Self-Report', 'Parent Report', 'Parent Report (Preschool)', 'Teacher Report', 'Teacher Report (Preschool)']


for participant in participants:
    
    df1 = df_all[(df_all['target']=='DX_01_Cat_new_binarize') & 
            (df_all['participant_group']==participant) &
            (df_all['abbrevs'].isin(abbrevs))
            ]

    visualize.predictive_modeling_group(df=df1, x='abbrevs', y='roc_auc_score', title=participant)