Meta file transfer features

.direnv/bin/nix-direnv-reload

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+if [[ ! -d "/home/zak/work/hbc/boost/beh" ]]; then
+  echo "Cannot find source directory; Did you move it?"
+  echo "(Looking for "/home/zak/work/hbc/boost/beh")"
+  echo 'Cannot force reload with this script - use "direnv reload" manually and then try again'
+  exit 1
+fi
+
+# rebuild the cache forcefully
+_nix_direnv_force_reload=1 direnv exec "/home/zak/work/hbc/boost/beh" true
+
+# Update the mtime for .envrc.
+# This will cause direnv to reload again - but without re-building.
+touch "/home/zak/work/hbc/boost/beh/.envrc"
+
+# Also update the timestamp of whatever profile_rc we have.
+# This makes sure that we know we are up to date.
+touch -r "/home/zak/work/hbc/boost/beh/.envrc" "/home/zak/work/hbc/boost/beh/.direnv"/*.rc

.direnv/flake-inputs/83bbfwhqxddpsxragsspqr4z04ka2yc8-source

@@ -0,0 +1 @@
+/nix/store/83bbfwhqxddpsxragsspqr4z04ka2yc8-source

.direnv/flake-inputs/s1fbk6a410xn8vcaj54iqv22agyn0ria-source

@@ -0,0 +1 @@
+/nix/store/s1fbk6a410xn8vcaj54iqv22agyn0ria-source

.direnv/flake-profile-a5d5b61aa8a61b7d9d765e1daf971a9a578f1cfa

@@ -0,0 +1 @@
+/nix/store/9dawlbrml3g3qcn9ry21xkm6rx45hk0l-nix-shell-env

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -0,0 +1,2 @@
+.venv
+.direnv

code/__init__.py

@@ -0,0 +1 @@
+# code package

code/data_processing/cc_qc.py

@@ -92,7 +92,7 @@ def cc_qc(self, df, threshold, TS=False):
             CATEGORY = 3
             print("Found unreported condition, category set to 3 ")
 
-        return CATEGORY
-
+        # Return both the QC category and the accuracy-by-condition dictionary
+        return CATEGORY, accuracy
 
 

code/data_processing/mem_qc.py

@@ -74,7 +74,7 @@ def fn_sm_qc(self, df, threshold):
             CATEGORY = 3
             print("Found unreported condition, category set to 3 ")
 
-        return CATEGORY
+        return CATEGORY, accuracy
 
 
 

code/data_processing/ps_qc.py

@@ -96,30 +96,6 @@ def ps_qc(self, submission, threshold, DSST=False):
             CATEGORY = 3
             print("Found unreported condition, category set to 3 ")
 
-        return CATEGORY
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+        return CATEGORY, accuracy
 
 

code/main_handler.py

@@ -7,6 +7,7 @@
 from data_processing.plot_utils import CC_PLOTS, MEM_PLOTS, PS_PLOTS
 from data_processing.save_utils import SAVE_EVERYTHING
 import pandas as pd
+from pathlib import Path
 
 
 class Handler:
@@ -28,6 +29,7 @@ def __init__(self):
             "WL": [958, 972, 995, 910, 927, 944]
         }
 
+        self.master_acc = pd.DataFrame(columns=['task', 'subject_id', 'session', 'condition', 'accuracy'])
 
     def pull(self, task):
         pull_instance = Pull(
@@ -85,27 +87,76 @@ def qc_cc_dfs(self, dfs, task):
                 categories.append([subject, category, df])
                 plots.append([subject, plot])
 
-        else:
-            qc_instance = CCqC(task,
-                               MAXRT=1800,
-                               RT_COLUMN_NAME='response_time',
-                               ACC_COLUMN_NAME='correct',
-                               CORRECT_SYMBOL=1,
-                               INCORRECT_SYMBOL=0,
-                               COND_COLUMN_NAME='block_cond')
-            for df in dfs:
-                subject = df['subject_id'][1]
-                print(f"qcing {subject}")
+        categories, plots, master_rows = [], [], []
+        plot_instance = CC_PLOTS()
+
+        # pick the grouping column for accuracy
+        cond_col = "condition" if task in ["AF", "NF", "NNB", "VNB"] else "block_cond"
+
+        qc_instance = CCqC(
+            task,
+            MAXRT=1800,
+            RT_COLUMN_NAME="response_time",
+            ACC_COLUMN_NAME="correct",
+            CORRECT_SYMBOL=1,
+            INCORRECT_SYMBOL=0,
+            COND_COLUMN_NAME=cond_col,
+        )
+
+        for df in dfs:
+            subject = df["subject_id"].iloc[0]
+            # preserve session information from available column
+            if "session" in df.columns:
+                session = df["session"].iloc[0]
+            elif "session_number" in df.columns:
+                session = df["session_number"].iloc[0]
+            else:
+                session = None
+
+            # run QC + choose plot
+            if task in ["AF", "NF"]:
+                category, _ = qc_instance.cc_qc(df, threshold=0.5)
+                plot = plot_instance.af_nf_plot(df)
+            elif task in ["NNB", "VNB"]:
+                category, _ = qc_instance.cc_qc(df, threshold=0.5)
+                plot = plot_instance.nnb_vnb_plot(df)
+            else:
                 category = qc_instance.cc_qc(df, threshold=0.5, TS=True)
                 plot = plot_instance.ats_nts_plot(df)
-                print(f"Category = {category}")
-                categories.append([subject, category, df])
-                plots.append([subject, plot])
+
+            categories.append([subject, category, df])
+            plots.append([subject, plot])
+
+            # compute accuracies per block/condition for THIS df
+            from data_processing.utils import QC_UTILS 
+            qc_util = QC_UTILS()
+            acc_by = qc_util.get_acc_by_block_cond(
+                df, block_cond_column_name=cond_col,
+                acc_column_name="correct",
+                correct_symbol=1,
+                incorrect_symbol=0,
+            )
+            for cond, acc in acc_by.items():
+                master_rows.append([task, subject, session, cond, float(acc)])
+
+        # save artifacts
         save_instance = SAVE_EVERYTHING()
-        save_instance.save_dfs(categories=categories,
-                                task=task)
-        save_instance.save_plots(plots=plots,
-                                     task=task)
+        save_instance.save_dfs(categories=categories, task=task)
+        save_instance.save_plots(plots=plots, task=task)
+
+        # append to master table
+        if master_rows:
+            self.master_acc = pd.concat(
+                [self.master_acc,
+                 pd.DataFrame(master_rows,
+                              columns=["task", "subject_id", "session", "condition", "accuracy"])],
+                ignore_index=True,
+            )
+
+        # save aggregated master accuracy to meta_file.csv at project base directory
+        base_dir = Path(__file__).parents[1]
+        self.master_acc.to_csv(base_dir / "meta_file.csv", index=False)
+
         return categories, plots
 
 
@@ -139,10 +190,42 @@ def qc_ps_dfs(self, dfs, task):
                 plots.append([subject, plot])
 
         save_instance = SAVE_EVERYTHING()
-        save_instance.save_dfs(categories=categories,
-                                task=task)
-        save_instance.save_plots(plots=plots,
-                                    task=task)
+        save_instance.save_dfs(categories=categories, task=task)
+        save_instance.save_plots(plots=plots, task=task)
+
+        # append accuracies by block/condition to master_acc
+        master_rows = []
+        from data_processing.utils import QC_UTILS
+        qc_util = QC_UTILS()
+        cond_col = 'block_c'
+        for subject, _, df in categories:
+            # preserve session information from available column
+            if 'session' in df.columns:
+                session = df['session'].iloc[0]
+            elif 'session_number' in df.columns:
+                session = df['session_number'].iloc[0]
+            else:
+                session = None
+            acc_by = qc_util.get_acc_by_block_cond(
+                df,
+                block_cond_column_name=cond_col,
+                acc_column_name='correct',
+                correct_symbol=1,
+                incorrect_symbol=0,
+            )
+            for cond, acc in acc_by.items():
+                master_rows.append([task, subject, session, cond, float(acc)])
+        if master_rows:
+            self.master_acc = pd.concat(
+                [self.master_acc,
+                 pd.DataFrame(master_rows,
+                              columns=['task','subject_id','session','condition','accuracy'])],
+                ignore_index=True,
+            )
+
+        # save aggregated master accuracy to meta_file.csv at project base directory
+        base_dir = Path(__file__).parents[1]
+        self.master_acc.to_csv(base_dir / "meta_file.csv", index=False)
 
         return categories, plots
 
@@ -172,10 +255,43 @@ def qc_mem_dfs(self, dfs, task):
                 categories.append([subject, category, df])
                 plots.append([subject, plot])
         save_instance = SAVE_EVERYTHING()
-        save_instance.save_dfs(categories=categories,
-                                task=task)
-        save_instance.save_plots(plots=plots,
-                                    task=task)
+        save_instance.save_dfs(categories=categories, task=task)
+        save_instance.save_plots(plots=plots, task=task)
+
+        # append accuracies by block/condition to master_acc
+        master_rows = []
+        from data_processing.utils import QC_UTILS
+        qc_util = QC_UTILS()
+        cond_col = 'block_c'
+        for subject, _, df in categories:
+            # preserve session information from available column
+            if 'session' in df.columns:
+                session = df['session'].iloc[0]
+            elif 'session_number' in df.columns:
+                session = df['session_number'].iloc[0]
+            else:
+                session = None
+            acc_by = qc_util.get_acc_by_block_cond(
+                df,
+                block_cond_column_name=cond_col,
+                acc_column_name='correct',
+                correct_symbol=1,
+                incorrect_symbol=0,
+            )
+            for cond, acc in acc_by.items():
+                master_rows.append([task, subject, session, cond, float(acc)])
+        if master_rows:
+            self.master_acc = pd.concat(
+                [self.master_acc,
+                 pd.DataFrame(master_rows,
+                              columns=['task','subject_id','session','condition','accuracy'])],
+                ignore_index=True,
+            )
+
+        # save aggregated master accuracy to meta_file.csv at project base directory
+        base_dir = Path(__file__).parents[1]
+        self.master_acc.to_csv(base_dir / "meta_file.csv", index=False)
+
         return categories, plots
 
     def qc_wl_dfs(self, dfs, task):
@@ -211,6 +327,7 @@ def qc_wl_dfs(self, dfs, task):
                                     task=task)
         return categories, plots
 
+# mysql 10.6.22
 
 if __name__ == '__main__':
     import os
@@ -224,5 +341,3 @@ def qc_wl_dfs(self, dfs, task):
     elif sys.argv[1] in task_list:
         instance = Handler()
         csv_dfs = instance.pull(task=sys.argv[1])
-
-