add multiprocessing

analysis/__init__.py

@@ -0,0 +1,6 @@
+__all__ = ["calculate_accuracy", "calculate_cosine_similarity"]
+
+from move.analysis.metrics import (
+    calculate_accuracy,
+    calculate_cosine_similarity,
+)

analysis/metrics.py

@@ -0,0 +1,99 @@
+__all__ = ["calculate_accuracy", "calculate_cosine_similarity"]
+
+import numpy as np
+
+from move.core.typing import FloatArray
+
+
+def calculate_accuracy(
+    original_input: FloatArray, reconstruction: FloatArray
+) -> FloatArray:
+    """Compute accuracy per sample.
+
+    Args:
+        original_input: Original labels (one-hot encoded as a 3D array).
+        reconstruction: Reconstructed labels (2D array).
+
+    Returns:
+        Array of accuracy scores.
+    """
+    if original_input.ndim != 3:
+        raise ValueError("Expected original input to have three dimensions.")
+    if reconstruction.ndim != 2:
+        raise ValueError("Expected reconstruction to have two dimensions.")
+    if original_input[:, :, 0].shape != reconstruction.shape:
+        raise ValueError(
+            f"Original input {original_input.shape} and reconstruction "
+            f"{reconstruction.shape} shapes do not match."
+        )
+
+    is_nan = original_input.sum(axis=2) == 0
+    original_input = np.argmax(original_input, axis=2)  # 3D => 2D
+    y_true = np.ma.masked_array(original_input, mask=is_nan)
+    y_pred = np.ma.masked_array(reconstruction, mask=is_nan)
+
+    num_features = np.ma.count(y_true, axis=1)
+    scores = np.sum(y_true == y_pred, axis=1) / num_features
+
+    return np.ma.filled(scores, 0)
+
+
+def calculate_cosine_similarity(
+    original_input: FloatArray, reconstruction: FloatArray
+) -> FloatArray:
+    """Compute cosine similarity per sample.
+
+    Args:
+        original_input: Original values (2D array).
+        reconstruction: Reconstructed values (2D array).
+
+    Returns:
+        Array of similarities.
+    """
+    if any((original_input.ndim != 2, reconstruction.ndim != 2)):
+        raise ValueError("Expected both inputs to have two dimensions.")
+    if original_input.shape != reconstruction.shape:
+        raise ValueError(
+            f"Original input {original_input.shape} and reconstruction "
+            f"{reconstruction.shape} shapes do not match."
+        )
+
+    is_nan = original_input == 0
+    x = np.ma.masked_array(original_input, mask=is_nan)
+    y = np.ma.masked_array(reconstruction, mask=is_nan)
+
+    # Equivalent to `np.diag(sklearn.metrics.pairwise.cosine_similarity(x, y))`
+    # But can handle masked arrays
+    scores = np.sum(x * y, axis=1) / (norm(x) * norm(y))
+
+    return np.ma.filled(scores, 0)
+
+
+def norm(x: np.ma.MaskedArray, axis: int = 1) -> np.ma.MaskedArray:
+    """Return Euclidean norm. This function is equivalent to `np.linalg.norm`,
+    but it can handle masked arrays.
+
+    Args:
+        x: 2D masked array
+        axis: Axis along which to the operation is performed. Defaults to 1.
+
+    Returns:
+        1D array with the specified axis removed.
+    """
+    return np.sqrt(np.sum(x**2, axis=axis))
+
+
+def get_2nd_order_polynomial(x_array, y_array, n_points=100):
+    """
+    Given a set of x an y values, find the 2nd oder polynomial fitting best the data.
+
+    Returns:
+        x_pol: x coordinates for the polynomial function evaluation.
+        y_pol: y coordinates for the polynomial function evaluation.
+    """
+    a2, a1, a = np.polyfit(x_array, y_array, deg=2)
+
+    x_pol = np.linspace(np.min(x_array), np.max(x_array), n_points)
+    y_pol = np.array([a2 * x * x + a1 * x + a for x in x_pol])
+
+    return x_pol, y_pol, (a2, a1, a)

isoforms_first_try/config/data/isoforms.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - base_data
+
+raw_data_path: data/
+interim_data_path: interim_data/
+results_path: results/
+
+sample_names: common_samples_500
+
+categorical_inputs:
+  - name: OS
+  - name: DSS
+  - name: PFI
+  - name: DFI
+  - name: ajcc_pathologic_tumor_stage
+  - name: gender
+  - name: cancer_type_abbreviation
+  - name: histological_type
+  - name: vital_status
+
+continuous_inputs:
+  - name: pheno_continuous_float
+  - name: iso_IF_100
+  - name: iso_tpm_100
+  - name: gene_tpm_100

isoforms_first_try/config/task/isoforms__id_assoc_bayes.yaml

@@ -0,0 +1,24 @@
+defaults:
+  - identify_associations_bayes
+
+multiprocess: True
+
+batch_size: 250
+
+num_refits: 2
+
+target_dataset: iso_tpm_100 # Dataset to perturb 
+target_value: maximum # We perturb in all samples for the maximum value of that feature across all samples
+save_refits: True
+
+model:
+  num_hidden:
+    - 150 
+  num_latent: 10 
+  beta: 0.001
+  dropout: .1 
+  cuda: false
+
+training_loop:
+  lr: 1e-4
+  num_epochs: 20

isoforms_first_try/encode_data_iso.py

@@ -0,0 +1,7 @@
+from move.tasks import encode_data
+from move.data import io
+from move.data import preprocessing
+
+config = io.read_config("isoforms", "encode_data")
+encode_data(config.data)
+

isoforms_first_try/encode_data_iso.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# The following are commonly used options for running jobs. Remove one
+# "#" from the "##SBATCH" lines (changing them to "#SBATCH") to enable
+# a given option.
+
+#SBATCH --job-name=encode500
+# The number of CPUs (cores) used by your task. Defaults to 1.
+###SBATCH --cpus-per-task=10
+# The amount of RAM used by your task. Tasks are automatically assigned 15G
+# per CPU (set above) if this option is not set.
+#SBATCH --mem=800G
+# Request a GPU on the GPU code. Use `--gres=gpu:a100:2` to request both GPUs.
+##SBATCH --partition=gpuqueue --gres=gpu:a100:1
+# Send notifications when job ends. Remember to update the email address!
+##SBATCH [email protected] --mail-type=END,FAIL
+# Set an error file
+#SBATCH --error=encode_iso.err
+#SBATCH --out=encode_iso.log
+
+
+module load python/3.11.3
+python encode_data_iso.py

isoforms_first_try/encode_iso.err

@@ -0,0 +1,14 @@
+[INFO  - encode_data]: Beginning task: encode data
+[INFO  - encode_data]: Encoding 'OS'
+[INFO  - encode_data]: Encoding 'DSS'
+[INFO  - encode_data]: Encoding 'PFI'
+[INFO  - encode_data]: Encoding 'DFI'
+[INFO  - encode_data]: Encoding 'ajcc_pathologic_tumor_stage'
+[INFO  - encode_data]: Encoding 'gender'
+[INFO  - encode_data]: Encoding 'cancer_type_abbreviation'
+[INFO  - encode_data]: Encoding 'histological_type'
+[INFO  - encode_data]: Encoding 'vital_status'
+[INFO  - encode_data]: Encoding 'pheno_continuous_float'
+[INFO  - encode_data]: Encoding 'iso_IF_2000_data'
+[INFO  - encode_data]: Encoding 'iso_tpm_2000_data'
+[INFO  - encode_data]: Encoding 'gene_tpm_2000_data'

isoforms_first_try/identify_assoc_bayes_iso_multiprocess.py

@@ -0,0 +1,6 @@
+from move.tasks import identify_associations_multiprocess
+from move.data import io
+
+config = io.read_config("isoforms", "isoforms__id_assoc_bayes")
+identify_associations_multiprocess(config)
+

isoforms_first_try/identify_assoc_bayes_iso_multiprocess.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# The following are commonly used options for running jobs. Remove one
+# "#" from the "##SBATCH" lines (changing them to "#SBATCH") to enable
+# a given option.
+
+#SBATCH --job-name=500multi
+
+# The number of CPUs (cores) used by your task. Defaults to 1.
+#SBATCH --cpus-per-task=50
+# The amount of RAM used by your task. Tasks are automatically assigned 15G
+# per CPU (set above) if this option is not set.
+#SBATCH --mem=1000G
+# Request a GPU on the GPU code. Use `--gres=gpu:a100:2` to request both GPUs.
+##SBATCH --partition=gpuqueue --gres=gpu:a100:1
+# Send notifications when job ends. Remember to update the email address!
+##SBATCH [email protected] --mail-type=END,FAIL
+# Set an error file
+#SBATCH --error=multi_bayes_iso.err
+
+
+module load python/3.11.3
+python identify_assoc_bayes_iso_multiprocess.py

isoforms_first_try/identify_assoc_bayes_iso_notmulti.py

@@ -0,0 +1,6 @@
+from move.tasks import identify_associations
+from move.data import io
+
+config = io.read_config("isoforms", "isoforms__id_assoc_bayes")
+identify_associations(config)
+

isoforms_first_try/identify_assoc_bayes_normal.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# The following are commonly used options for running jobs. Remove one
+# "#" from the "##SBATCH" lines (changing them to "#SBATCH") to enable
+# a given option.
+
+#SBATCH --job-name=500notmulti_bayes
+
+# The number of CPUs (cores) used by your task. Defaults to 1.
+###SBATCH --cpus-per-task=1
+# The amount of RAM used by your task. Tasks are automatically assigned 15G
+# per CPU (set above) if this option is not set.
+#SBATCH --mem=1000G
+# Request a GPU on the GPU code. Use `--gres=gpu:a100:2` to request both GPUs.
+##SBATCH --partition=gpuqueue --gres=gpu:a100:1
+# Send notifications when job ends. Remember to update the email address!
+###SBATCH [email protected] --mail-type=END,FAIL
+# Set an error file
+#SBATCH --error=normal_bayes.err
+
+
+module load python/3.11.3
+python identify_assoc_bayes_iso_notmulti.py

isoforms_first_try/interim_data/DFI.txt

@@ -0,0 +1 @@
+DFI

isoforms_first_try/interim_data/DSS.txt

@@ -0,0 +1 @@
+DSS

isoforms_first_try/interim_data/OS.txt

@@ -0,0 +1 @@
+OS

isoforms_first_try/interim_data/PFI.txt

@@ -0,0 +1 @@
+PFI

isoforms_first_try/interim_data/ajcc_pathologic_tumor_stage.txt

@@ -0,0 +1 @@
+ajcc_pathologic_tumor_stage

isoforms_first_try/interim_data/cancer_type_abbreviation.txt

@@ -0,0 +1 @@
+cancer type abbreviation

isoforms_first_try/interim_data/gender.txt

@@ -0,0 +1 @@
+gender

isoforms_first_try/interim_data/gene_tpm_100.txt

@@ -0,0 +1,99 @@
+ENSG00000005302.17
+ENSG00000006327.13
+ENSG00000006432.15
+ENSG00000017260.19
+ENSG00000018699.11
+ENSG00000026508.16
+ENSG00000047249.16
+ENSG00000051180.16
+ENSG00000052841.14
+ENSG00000060069.16
+ENSG00000064666.14
+ENSG00000065457.10
+ENSG00000066044.13
+ENSG00000066136.19
+ENSG00000066468.20
+ENSG00000069345.11
+ENSG00000073605.18
+ENSG00000074054.17
+ENSG00000075415.12
+ENSG00000075702.16
+ENSG00000078747.12
+ENSG00000080845.17
+ENSG00000084636.17
+ENSG00000090266.12
+ENSG00000099899.14
+ENSG00000099917.17
+ENSG00000100129.17
+ENSG00000100142.14
+ENSG00000100599.15
+ENSG00000100731.15
+ENSG00000102226.9
+ENSG00000104517.12
+ENSG00000104613.11
+ENSG00000104814.12
+ENSG00000104824.16
+ENSG00000105245.9
+ENSG00000105323.16
+ENSG00000106392.10
+ENSG00000106665.15
+ENSG00000110700.6
+ENSG00000110719.9
+ENSG00000117151.12
+ENSG00000117394.19
+ENSG00000117640.17
+ENSG00000122482.20
+ENSG00000122484.8
+ENSG00000123352.17
+ENSG00000124444.15
+ENSG00000125122.14
+ENSG00000125779.21
+ENSG00000125962.14
+ENSG00000125967.16
+ENSG00000125991.18
+ENSG00000130305.16
+ENSG00000132199.18
+ENSG00000132300.18
+ENSG00000132600.16
+ENSG00000133641.17
+ENSG00000134013.15
+ENSG00000135127.11
+ENSG00000135801.9
+ENSG00000135845.9
+ENSG00000137817.16
+ENSG00000137821.11
+ENSG00000138399.17
+ENSG00000139620.12
+ENSG00000140598.13
+ENSG00000140650.11
+ENSG00000141252.19
+ENSG00000143554.13
+ENSG00000149016.15
+ENSG00000149218.4
+ENSG00000152240.12
+ENSG00000153147.5
+ENSG00000156261.12
+ENSG00000156508.17
+ENSG00000157106.16
+ENSG00000157978.11
+ENSG00000158863.21
+ENSG00000159111.12
+ENSG00000160298.17
+ENSG00000160570.13
+ENSG00000160691.18
+ENSG00000161513.11
+ENSG00000163110.14
+ENSG00000163521.15
+ENSG00000165526.8
+ENSG00000165637.13
+ENSG00000166797.10
+ENSG00000166949.15
+ENSG00000166971.16
+ENSG00000168575.9
+ENSG00000168646.12
+ENSG00000169249.12
+ENSG00000169925.16
+ENSG00000169967.16
+ENSG00000170142.11
+ENSG00000170310.14
+ENSG00000171617.13