feat: build tsv incorporating assembly list json (#5)

Author: hunterckx

data-catalog/.eslintignore

@@ -2,3 +2,5 @@
 **/.next/*
 
 /out
+
+venv

data-catalog/.gitignore

@@ -30,4 +30,7 @@ npm-debug.log*
 /public/favicons/*
 
 # Build Dir
-/out
+/out
+
+# python
+venv

data-catalog/.prettierignore

@@ -6,3 +6,6 @@ node_modules
 
 # build
 /out
+
+# python
+venv

data-catalog/README.md

@@ -0,0 +1,19 @@
+# BRC Analytics Data Catalog
+
+## Building the data source files
+
+In the `data-catalog` directory, create a Python virtual environment and install requirements:
+
+```shell
+python3 -m venv ./venv
+source ./venv/bin/activate
+pip install -r ./files/requirements.txt
+```
+
+Then run the script:
+
+```shell
+python3 ./files/build-genomes-files.py
+```
+
+The environment can be deactivated by running `deactivate`, and re-activated by running `source ./venv/bin/activate` again.

data-catalog/files/build-genomes-files.py

@@ -0,0 +1,26 @@
+import pandas as pd
+import re
+import requests
+
+GENOMES_SOURCE_URL = "https://docs.google.com/spreadsheets/d/1NRfTvebPl6zJ0l9tCqBtq6YCrwV6_XDBlheq3L5HcvQ/gviz/tq?tqx=out:csv&sheet=GenomeDataTypes_Summary.csv"
+ASSEMBLIES_URL = "https://hgdownload.soe.ucsc.edu/hubs/BRC/assembly.list.json"
+
+OUTPUT_PATH = "files/source/genomes.tsv"
+
+def build_genomes_files():
+  print("Building files")
+
+  genomes_source_df = pd.read_csv(GENOMES_SOURCE_URL, keep_default_na=False, usecols=lambda name: re.fullmatch(r"Unnamed: \d+", name) is None)
+  assemblies_df = pd.DataFrame(requests.get(ASSEMBLIES_URL).json()["data"])
+  
+  gen_bank_merge_df = genomes_source_df.merge(assemblies_df, how="left", left_on="Genome Version/Assembly ID", right_on="genBank")
+  ref_seq_merge_df = genomes_source_df.merge(assemblies_df, how="left", left_on="Genome Version/Assembly ID", right_on="refSeq")
+
+  result_df = gen_bank_merge_df.combine_first(ref_seq_merge_df)
+
+  result_df.to_csv(OUTPUT_PATH, index=False, sep="\t")
+
+  print(f"Wrote to {OUTPUT_PATH}")
+
+if __name__ == "__main__":
+  build_genomes_files()

data-catalog/files/requirements.txt

@@ -0,0 +1,11 @@
+certifi==2024.8.30
+charset-normalizer==3.3.2
+idna==3.8
+numpy==2.1.0
+pandas==2.2.2
+python-dateutil==2.9.0.post0
+pytz==2024.1
+requests==2.32.3
+six==1.16.0
+tzdata==2024.1
+urllib3==2.2.2