diff --git a/Exercises/data/Tutorial_Dataset.vcf.gz b/Exercises/data/Tutorial_Dataset.vcf.gz new file mode 100644 index 0000000..75fde40 Binary files /dev/null and b/Exercises/data/Tutorial_Dataset.vcf.gz differ diff --git a/Exercises/data/Tutorial_Dataset.vcf.gz.tbi b/Exercises/data/Tutorial_Dataset.vcf.gz.tbi new file mode 100644 index 0000000..fae65ed Binary files /dev/null and b/Exercises/data/Tutorial_Dataset.vcf.gz.tbi differ diff --git a/Exercises/data/Tutorial_Phenotype.txt b/Exercises/data/Tutorial_Phenotype.txt index 9a867e3..76994e7 100644 --- a/Exercises/data/Tutorial_Phenotype.txt +++ b/Exercises/data/Tutorial_Phenotype.txt @@ -1,159 +1,159 @@ -NA18646 2 -HG02178 2 -HG04063 2 -NA20901 1 -HG03046 2 -HG00457 1 -HG00599 2 -HG01699 1 -NA18517 2 -HG03934 2 -NA19138 1 -HG03105 2 -HG02854 1 -NA20845 1 -NA18988 1 -HG02137 1 -HG01522 2 -NA21122 2 -HG03354 2 -HG03478 1 -NA19786 1 -NA20346 1 -HG03352 1 -HG03074 1 -HG00689 1 -HG02298 2 -HG03267 2 -HG04161 1 -NA12717 2 -NA18591 2 -NA20872 2 -HG01308 1 -HG03953 1 -NA18597 2 -HG02332 1 -NA12006 2 -HG00114 1 -NA12716 1 -NA19652 1 -HG03097 2 -NA18638 1 -NA20881 2 -HG03058 2 -HG03082 2 -HG04209 2 -NA18544 1 -HG03073 2 -HG00445 1 -NA21089 2 -HG04185 1 -NA18630 2 -HG02804 1 -HG03729 1 -NA19377 2 -HG03436 1 -NA18948 1 -HG01253 1 -HG02787 2 -HG00381 2 -HG01631 1 -HG02009 1 -HG02661 2 -NA19072 1 -HG00173 2 -NA20508 2 -NA20850 1 -NA19393 1 -HG03297 2 -HG00449 2 -HG02771 1 -NA20822 2 -HG00566 2 -NA20867 1 -HG01512 1 -HG03511 2 -HG02390 1 -HG01680 1 -NA18941 2 -NA18962 1 -NA18561 1 -HG01537 2 -HG01072 1 -HG01536 1 -NA19324 2 -NA19782 2 -HG01971 2 -HG03730 2 -HG00246 1 -HG01113 2 -HG02651 1 -NA19670 1 -HG02380 1 -NA20542 2 -HG01613 2 -HG00121 2 -HG04107 1 -HG02301 2 -HG03977 2 -HG01351 2 -HG03009 1 -HG02977 1 -HG01080 2 -NA20514 2 -HG01700 1 -HG02040 1 -NA19118 2 -HG01932 1 -NA18639 1 -NA07347 1 -NA20809 1 -HG01130 1 -NA20799 2 -NA18567 2 -NA19023 2 -HG02508 2 -HG02731 2 -HG03829 2 -HG02322 2 -HG02645 1 -HG01804 2 -HG03397 1 -HG04094 1 -HG02697 2 -NA18865 1 -HG03133 1 -HG00109 1 -HG01124 1 -NA20875 2 -NA21102 2 -HG02780 1 -HG04146 1 -NA18560 2 -NA12347 1 -HG01933 2 -NA21133 1 -NA20539 1 -NA20357 2 -HG00583 1 -NA20864 1 -HG01070 2 -HG00258 2 -HG00145 1 -HG01188 2 -NA18923 1 -NA20803 1 -NA20535 2 -NA18963 2 -HG01985 2 -NA18634 2 -HG03479 2 -NA19777 1 -HG00315 2 -HG01095 2 -NA18510 1 -HG01982 1 -NA19087 2 -NA18874 1 -NA19449 2 -NA18574 2 +NA18646 1 +HG02178 1 +HG04063 1 +NA20901 0 +HG03046 1 +HG00457 0 +HG00599 1 +HG01699 0 +NA18517 1 +HG03934 1 +NA19138 0 +HG03105 1 +HG02854 0 +NA20845 0 +NA18988 0 +HG02137 0 +HG01522 1 +NA21122 1 +HG03354 1 +HG03478 0 +NA19786 0 +NA20346 0 +HG03352 0 +HG03074 0 +HG00689 0 +HG02298 1 +HG03267 1 +HG04161 0 +NA12717 1 +NA18591 1 +NA20872 1 +HG01308 0 +HG03953 0 +NA18597 1 +HG02332 0 +NA12006 1 +HG00114 0 +NA12716 0 +NA19652 0 +HG03097 1 +NA18638 0 +NA20881 1 +HG03058 1 +HG03082 1 +HG04209 1 +NA18544 0 +HG03073 1 +HG00445 0 +NA21089 1 +HG04185 0 +NA18630 1 +HG02804 0 +HG03729 0 +NA19377 1 +HG03436 0 +NA18948 0 +HG01253 0 +HG02787 1 +HG00381 1 +HG01631 0 +HG02009 0 +HG02661 1 +NA19072 0 +HG00173 1 +NA20508 1 +NA20850 0 +NA19393 0 +HG03297 1 +HG00449 1 +HG02771 0 +NA20822 1 +HG00566 1 +NA20867 0 +HG01512 0 +HG03511 0 +HG02390 0 +HG01680 0 +NA18941 1 +NA18962 0 +NA18561 0 +HG01537 1 +HG01072 0 +HG01536 0 +NA19324 1 +NA19782 1 +HG01971 1 +HG03730 1 +HG00246 0 +HG01113 1 +HG02651 0 +NA19670 0 +HG02380 0 +NA20542 1 +HG01613 1 +HG00121 1 +HG04107 0 +HG02301 1 +HG03977 1 +HG01351 1 +HG03009 0 +HG02977 0 +HG01080 1 +NA20514 1 +HG01700 0 +HG02040 0 +NA19118 1 +HG01932 0 +NA18639 0 +NA07347 0 +NA20809 0 +HG01130 0 +NA20799 1 +NA18567 1 +NA19023 1 +HG02508 1 +HG02731 1 +HG03829 1 +HG02322 1 +HG02645 0 +HG01804 1 +HG03397 0 +HG04094 0 +HG02697 1 +NA18865 0 +HG03133 0 +HG00109 0 +HG01124 0 +NA20875 1 +NA21102 1 +HG02780 0 +HG04146 0 +NA18560 1 +NA12347 0 +HG01933 1 +NA21133 0 +NA20539 0 +NA20357 1 +HG00583 0 +NA20864 0 +HG01070 1 +HG00258 1 +HG00145 0 +HG01188 1 +NA18923 0 +NA20803 0 +NA20535 1 +NA18963 1 +HG01985 1 +NA18634 1 +HG03479 1 +NA19777 0 +HG00315 1 +HG01095 1 +NA18510 0 +HG01982 0 +NA19087 1 +NA18874 0 +NA19449 1 +NA18574 1 diff --git a/Exercises/scripts/Tutorial_Key.ipynb b/Exercises/scripts/Tutorial_Key.ipynb index f6468f6..3940d50 100644 --- a/Exercises/scripts/Tutorial_Key.ipynb +++ b/Exercises/scripts/Tutorial_Key.ipynb @@ -25,9 +25,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install gdown\n", - "!gdown \"https://drive.google.com/uc?id=1iwJi7QfF3LElnCPen4fEQVoY4Vk4A9_5\"\n", - "!tar -xzvf TUTORIAL.tar.gz" + "!wget -nc https://hgdownload.soe.ucsc.edu/goldenPath/currentGenomes/Homo_sapiens/bigZips/hg38.fa.gz" ] }, { @@ -252,32 +250,35 @@ " lo = LiftOver(chain_file)\n", " n_variants = len(variant_data['CHROM'])\n", " unmapped_mask = np.zeros(n_variants, dtype=bool)\n", - " \n", - " # Create copies to avoid modifying original data\n", + "\n", " new_chroms = variant_data['CHROM'].copy()\n", " new_positions = variant_data['POS'].copy()\n", - " \n", + "\n", " for i, (chrom, pos) in enumerate(zip(variant_data['CHROM'], variant_data['POS'])):\n", " chrom_clean = chrom.replace('chr', '')\n", - " lifted = lo.convert_coordinate(f'chr{chrom_clean}', pos - 1) # Convert to 0-based\n", - " \n", + " lifted = lo.convert_coordinate(f'chr{chrom_clean}', pos - 1)\n", + "\n", " if not lifted:\n", " unmapped_mask[i] = True\n", " else:\n", - " # Update with lifted coordinates\n", " lifted_result = lifted[0]\n", - " lifted_chrom = lifted_result[0]\n", - " lifted_pos = lifted_result[1]\n", - " new_chroms[i] = lifted_chrom\n", - " new_positions[i] = lifted_pos + 1 # Convert back to 1-based\n", - " \n", - " # Update variant_data with new coordinates\n", + " new_chroms[i] = lifted_result[0]\n", + " new_positions[i] = lifted_result[1] + 1\n", + "\n", " variant_data_lifted = variant_data.copy()\n", " variant_data_lifted['CHROM'] = new_chroms\n", " variant_data_lifted['POS'] = new_positions\n", - " \n", - " # Filter out unmapped variants\n", - " return filter_variants(variant_data_lifted, sample_metadata, ~unmapped_mask)\n" + "\n", + " # Keep only successfully mapped variants -- filter directly, don't\n", + " # route through filter_variants' missingness threshold\n", + " keep_mask = ~unmapped_mask\n", + " filtered_variants = {k: v[keep_mask] for k, v in variant_data_lifted.items()}\n", + " filtered_samples = {\n", + " 'samples': sample_metadata['samples'],\n", + " 'sex': sample_metadata['sex'],\n", + " 'genotypes': sample_metadata['genotypes'][keep_mask]\n", + " }\n", + " return filtered_variants, filtered_samples" ] }, { diff --git a/Exercises/scripts/Tutorial_Lesson.ipynb b/Exercises/scripts/Tutorial_Lesson.ipynb index e9053a2..33ee683 100644 --- a/Exercises/scripts/Tutorial_Lesson.ipynb +++ b/Exercises/scripts/Tutorial_Lesson.ipynb @@ -31,9 +31,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install gdown\n", - "!gdown \"https://drive.google.com/uc?id=1iwJi7QfF3LElnCPen4fEQVoY4Vk4A9_5\"\n", - "!tar -xzvf TUTORIAL.tar.gz" + "!wget -nc https://hgdownload.soe.ucsc.edu/goldenPath/currentGenomes/Homo_sapiens/bigZips/hg38.fa.gz" ] }, { @@ -74,45 +72,27 @@ "source": [ "shared_qc_table = []\n", "def count_variants(study_name, variant_data, sample_metadata, step_name):\n", - " # Count variants by chromosome\n", - " chroms = variant_data['variants/CHROM']\n", - " autosomal = sum((chrom.isdigit() and 1 <= int(chrom) <= 22) for chrom in chroms)\n", - " x_chr = sum(chrom in ['X', '23', '25'] for chrom in chroms)\n", - " y_chr = sum(chrom in ['Y', '24'] for chrom in chroms)\n", - " mt_chr = sum(chrom in ['MT', 'M', '26'] for chrom in chroms)\n", - " \n", - " # Count individuals by sex\n", + " chroms = [c.replace('chr', '') for c in variant_data['CHROM']]\n", + " autosomal = sum((c.isdigit() and 1 <= int(c) <= 22) for c in chroms)\n", + " x_chr = sum(c == 'X' for c in chroms)\n", + " y_chr = sum(c == 'Y' for c in chroms)\n", + " mt_chr = sum(c in ['MT', 'M'] for c in chroms)\n", + "\n", " sex_codes = sample_metadata['sex']\n", " males = sum(sex == 1 for sex in sex_codes)\n", " females = sum(sex == 2 for sex in sex_codes)\n", " ambiguous = sum(sex == 0 for sex in sex_codes)\n", " individuals = len(sex_codes)\n", - " \n", - " # Add to shared QC table\n", + "\n", " shared_qc_table.append([\n", - " study_name,\n", - " step_name,\n", - " autosomal,\n", - " x_chr,\n", - " y_chr,\n", - " mt_chr,\n", - " individuals,\n", - " males,\n", - " females,\n", - " ambiguous,\n", + " study_name, step_name, autosomal, x_chr, y_chr, mt_chr,\n", + " individuals, males, females, ambiguous,\n", " ])\n", - " \n", " return {\n", - " \"autosomal\": autosomal,\n", - " \"x_chr\": x_chr,\n", - " \"y_chr\": y_chr,\n", - " \"mt_chr\": mt_chr,\n", - " \"individuals\": individuals,\n", - " \"males\": males,\n", - " \"females\": females,\n", - " \"ambiguous\": ambiguous,\n", - " }\n", - "\n" + " \"autosomal\": autosomal, \"x_chr\": x_chr, \"y_chr\": y_chr,\n", + " \"mt_chr\": mt_chr, \"individuals\": individuals,\n", + " \"males\": males, \"females\": females, \"ambiguous\": ambiguous,\n", + " }\n" ] }, { @@ -288,7 +268,14 @@ " variant_data_lifted['POS'] = new_positions\n", " \n", " # Filter out unmapped variants\n", - " return filter_variants(variant_data_lifted, sample_metadata, ~unmapped_mask)" + " keep_mask = ~unmapped_mask\n", + " filtered_variants = {k: v[keep_mask] for k, v in variant_data_lifted.items()}\n", + " filtered_samples = {\n", + " 'samples': sample_metadata['samples'],\n", + " 'sex': sample_metadata['sex'],\n", + " 'genotypes': sample_metadata['genotypes'][keep_mask]\n", + " }\n", + " return filtered_variants, filtered_samples" ] }, { diff --git a/Outline/MiCM - Workshop PPT.pdf b/Outline/MiCM - Workshop PPT.pdf new file mode 100644 index 0000000..262ce85 Binary files /dev/null and b/Outline/MiCM - Workshop PPT.pdf differ diff --git a/Slides/MiCM - Workshop PPT.pptx b/Slides/MiCM - Workshop PPT.pptx index 175c9cf..cd05b0b 100644 Binary files a/Slides/MiCM - Workshop PPT.pptx and b/Slides/MiCM - Workshop PPT.pptx differ