extract.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author: Irsyad Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GOAL: extract the summary statistics of every gwas study relating to \"heart\", \"cardiomyopathy\", \"cardio\"... etc\n",
    "\n",
    "GWAS Catalog API Documentation: https://www.ebi.ac.uk/gwas/rest/docs/api\n",
    "\n",
    "GWAS API Endpoints Documentation: https://www.ebi.ac.uk/gwas/rest/api\n",
    "\n",
    "API Endpoint: http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/ + 'Valid GWAS ID'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Final Product: \n",
    "\n",
    "# Step 3: run MAGMA using the following command, this takes a GWAS file ${trait}.pval,\n",
    "# which at least has the columns: SNP, P, N, which corresponds to the SNP id\n",
    "# (matched to the ${magma_dir}/g1000_eur.bim), p-value, sample size. For example,\n",
    "# <trait>.pval file looks like\n",
    "#\n",
    "# CHR     BP      SNP             P           N\n",
    "# 1       717587  rs144155419     0.453345    279949\n",
    "# 1       740284  rs61770167      0.921906    282079\n",
    "# 1       769223  rs60320384      0.059349    281744"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/pinglab/Documents/irsyadadam/double_injury_scdrs/gwas-catalog-wrapper'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>First Author</th>\n",
       "      <th>PubMed ID</th>\n",
       "      <th>Study accession</th>\n",
       "      <th>Publication date</th>\n",
       "      <th>Journal</th>\n",
       "      <th>Title</th>\n",
       "      <th>Trait(s)</th>\n",
       "      <th>Reported trait</th>\n",
       "      <th>Data access</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Astle WJ</td>\n",
       "      <td>27863252</td>\n",
       "      <td>GCST004623</td>\n",
       "      <td>2016-11-17</td>\n",
       "      <td>Cell</td>\n",
       "      <td>The Allelic Landscape of Human Blood Cell Trai...</td>\n",
       "      <td>neutrophil percentage of granulocytes</td>\n",
       "      <td>Neutrophil percentage of granulocytes</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Astle WJ</td>\n",
       "      <td>27863252</td>\n",
       "      <td>GCST004624</td>\n",
       "      <td>2016-11-17</td>\n",
       "      <td>Cell</td>\n",
       "      <td>The Allelic Landscape of Human Blood Cell Trai...</td>\n",
       "      <td>sum of eosinophil and basophil counts</td>\n",
       "      <td>Sum eosinophil basophil counts</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Astle WJ</td>\n",
       "      <td>27863252</td>\n",
       "      <td>GCST004625</td>\n",
       "      <td>2016-11-17</td>\n",
       "      <td>Cell</td>\n",
       "      <td>The Allelic Landscape of Human Blood Cell Trai...</td>\n",
       "      <td>monocyte count</td>\n",
       "      <td>Monocyte count</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Astle WJ</td>\n",
       "      <td>27863252</td>\n",
       "      <td>GCST004600</td>\n",
       "      <td>2016-11-17</td>\n",
       "      <td>Cell</td>\n",
       "      <td>The Allelic Landscape of Human Blood Cell Trai...</td>\n",
       "      <td>eosinophil percentage of leukocytes</td>\n",
       "      <td>Eosinophil percentage of white cells</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Astle WJ</td>\n",
       "      <td>27863252</td>\n",
       "      <td>GCST004602</td>\n",
       "      <td>2016-11-17</td>\n",
       "      <td>Cell</td>\n",
       "      <td>The Allelic Landscape of Human Blood Cell Trai...</td>\n",
       "      <td>mean corpuscular volume</td>\n",
       "      <td>Mean corpuscular volume</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30403</th>\n",
       "      <td>de Lange KM</td>\n",
       "      <td>28067908</td>\n",
       "      <td>GCST004133</td>\n",
       "      <td>2017-01-09</td>\n",
       "      <td>Nat Genet</td>\n",
       "      <td>Genome-wide association study implicates immun...</td>\n",
       "      <td>ulcerative colitis</td>\n",
       "      <td>Ulcerative colitis</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30404</th>\n",
       "      <td>de Lange KM</td>\n",
       "      <td>28067908</td>\n",
       "      <td>GCST004131</td>\n",
       "      <td>2017-01-09</td>\n",
       "      <td>Nat Genet</td>\n",
       "      <td>Genome-wide association study implicates immun...</td>\n",
       "      <td>inflammatory bowel disease</td>\n",
       "      <td>Inflammatory bowel disease</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30405</th>\n",
       "      <td>Morris AP</td>\n",
       "      <td>22885922</td>\n",
       "      <td>GCST005047</td>\n",
       "      <td>2012-09-01</td>\n",
       "      <td>Nat Genet</td>\n",
       "      <td>Large-scale association analysis provides insi...</td>\n",
       "      <td>type 2 diabetes mellitus</td>\n",
       "      <td>Type 2 diabetes</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30406</th>\n",
       "      <td>Michailidou K</td>\n",
       "      <td>29059683</td>\n",
       "      <td>GCST004988</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>Nature</td>\n",
       "      <td>Association analysis identifies 65 new breast ...</td>\n",
       "      <td>breast carcinoma</td>\n",
       "      <td>Breast cancer</td>\n",
       "      <td>FTP Download  or  API access</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30407</th>\n",
       "      <td>Verweij N</td>\n",
       "      <td>32916098</td>\n",
       "      <td>GCST010796</td>\n",
       "      <td>2020-09-02</td>\n",
       "      <td>Cell Syst</td>\n",
       "      <td>The Genetic Makeup of the Electrocardiogram.</td>\n",
       "      <td>electrocardiography</td>\n",
       "      <td>Electrocardiogram morphology (amplitude at tem...</td>\n",
       "      <td>FTP Download</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30408 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        First Author  PubMed ID Study accession Publication date    Journal  \\\n",
       "0           Astle WJ   27863252      GCST004623       2016-11-17       Cell   \n",
       "1           Astle WJ   27863252      GCST004624       2016-11-17       Cell   \n",
       "2           Astle WJ   27863252      GCST004625       2016-11-17       Cell   \n",
       "3           Astle WJ   27863252      GCST004600       2016-11-17       Cell   \n",
       "4           Astle WJ   27863252      GCST004602       2016-11-17       Cell   \n",
       "...              ...        ...             ...              ...        ...   \n",
       "30403    de Lange KM   28067908      GCST004133       2017-01-09  Nat Genet   \n",
       "30404    de Lange KM   28067908      GCST004131       2017-01-09  Nat Genet   \n",
       "30405      Morris AP   22885922      GCST005047       2012-09-01  Nat Genet   \n",
       "30406  Michailidou K   29059683      GCST004988       2017-10-23     Nature   \n",
       "30407      Verweij N   32916098      GCST010796       2020-09-02  Cell Syst   \n",
       "\n",
       "                                                   Title  \\\n",
       "0      The Allelic Landscape of Human Blood Cell Trai...   \n",
       "1      The Allelic Landscape of Human Blood Cell Trai...   \n",
       "2      The Allelic Landscape of Human Blood Cell Trai...   \n",
       "3      The Allelic Landscape of Human Blood Cell Trai...   \n",
       "4      The Allelic Landscape of Human Blood Cell Trai...   \n",
       "...                                                  ...   \n",
       "30403  Genome-wide association study implicates immun...   \n",
       "30404  Genome-wide association study implicates immun...   \n",
       "30405  Large-scale association analysis provides insi...   \n",
       "30406  Association analysis identifies 65 new breast ...   \n",
       "30407       The Genetic Makeup of the Electrocardiogram.   \n",
       "\n",
       "                                    Trait(s)  \\\n",
       "0      neutrophil percentage of granulocytes   \n",
       "1      sum of eosinophil and basophil counts   \n",
       "2                             monocyte count   \n",
       "3        eosinophil percentage of leukocytes   \n",
       "4                    mean corpuscular volume   \n",
       "...                                      ...   \n",
       "30403                     ulcerative colitis   \n",
       "30404             inflammatory bowel disease   \n",
       "30405               type 2 diabetes mellitus   \n",
       "30406                       breast carcinoma   \n",
       "30407                    electrocardiography   \n",
       "\n",
       "                                          Reported trait  \\\n",
       "0                  Neutrophil percentage of granulocytes   \n",
       "1                         Sum eosinophil basophil counts   \n",
       "2                                         Monocyte count   \n",
       "3                   Eosinophil percentage of white cells   \n",
       "4                                Mean corpuscular volume   \n",
       "...                                                  ...   \n",
       "30403                                 Ulcerative colitis   \n",
       "30404                         Inflammatory bowel disease   \n",
       "30405                                    Type 2 diabetes   \n",
       "30406                                      Breast cancer   \n",
       "30407  Electrocardiogram morphology (amplitude at tem...   \n",
       "\n",
       "                        Data access  \n",
       "0      FTP Download  or  API access  \n",
       "1      FTP Download  or  API access  \n",
       "2      FTP Download  or  API access  \n",
       "3      FTP Download  or  API access  \n",
       "4      FTP Download  or  API access  \n",
       "...                             ...  \n",
       "30403  FTP Download  or  API access  \n",
       "30404  FTP Download  or  API access  \n",
       "30405  FTP Download  or  API access  \n",
       "30406  FTP Download  or  API access  \n",
       "30407                  FTP Download  \n",
       "\n",
       "[30408 rows x 9 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = \"heart_failure\"\n",
    "\n",
    "folder_name = args\n",
    "query_file = \"%s/%s\" % (args, os.listdir(folder_name)[0])\n",
    "index = pd.read_csv(query_file)\n",
    "index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MAIN FUNCTION\n",
    "def extract_gwas(GWAS_ID) -> None:\n",
    "    \"\"\"\n",
    "    grabs .tsv summary statistics associated with gwas id\n",
    "\n",
    "    ARGS:\n",
    "        GWAS_ID is the gwas id of the research\n",
    "\n",
    "    RETURNS: \n",
    "        None\n",
    "    \"\"\"\n",
    "    #get the url\n",
    "    endpoint = find_endpoint(GWAS_ID)\n",
    "    print(GWAS_ID, endpoint)\n",
    "    summary_stats_link = \"\"\n",
    "\n",
    "    #if endpointis a range of endpoints\n",
    "    if len(endpoint) > len(GWAS_ID):\n",
    "        url = 'http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/' + endpoint + \"/\" + GWAS_ID + \"/\"\n",
    "\n",
    "\n",
    "    #else if single gwas id endpoint\n",
    "    elif endpoint == GWAS_ID:\n",
    "        url = 'http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/' + endpoint + \"/\"\n",
    "    \n",
    "    print(url)\n",
    "    response = requests.get(url=url)\n",
    "    tsv_file = extract_link(response)\n",
    "    if tsv_file == None:\n",
    "            url = url + \"harmonised/\"\n",
    "            response = requests.get(url=url)\n",
    "            tsv_file = extract_link(response)\n",
    "    summary_stats_link = url + tsv_file\n",
    "\n",
    "\n",
    "    #TODO: WGET IS SLOOWWWWWW (~2 hours to download)\n",
    "    cmd = \"wget %s\" % summary_stats_link\n",
    "    print(cmd)\n",
    "    os.system(cmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_link(response: requests.Response) -> str:\n",
    "    \"\"\"\n",
    "    extracts the link from a response from def extract_gwas\n",
    "\n",
    "    ARGS: \n",
    "        response is the http response object\n",
    "\n",
    "    RETURNS:\n",
    "        string associated with the link in the response object\n",
    "    \"\"\"\n",
    "    txts = response.text\n",
    "\n",
    "    #iterate through the lines for the file\n",
    "    #find first occurance of .tsv file\n",
    "    #get the link\n",
    "    for i in txts.splitlines():\n",
    "        if \".tsv\" in i:\n",
    "            start = '>'\n",
    "            end = '<'\n",
    "            link = (i.split(start))[1].split(end)[0]\n",
    "            return link\n",
    "    \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_endpoint(GWAS_ID):\n",
    "    df = pd.read_csv(\"gwas_catalog_index/index.csv\", header = 0)\n",
    "\n",
    "    #get the index\n",
    "    ID = int(GWAS_ID[4:])\n",
    "\n",
    "    endpoint = \"\"\n",
    "\n",
    "    #iterate through the id range column\n",
    "    for i in range(len(df[\"id_range\"])):\n",
    "\n",
    "        range_ids = ast.literal_eval(df[\"id_range\"][i])\n",
    "        # CASE1: if the id is in the range column, and range column only has 1 variable\n",
    "        if len(range_ids) == 1 and range_ids[0] == ID:  \n",
    "            endpoint = df[\"endpoints\"][i]\n",
    "            break\n",
    "\n",
    "        #CASE2 : if id is in between the range, and range column has 2 values\n",
    "        elif len(range_ids) > 1 and range_ids[0] <= ID <= range_ids[1]:\n",
    "            endpoint = df[\"endpoints\"][i]\n",
    "            break\n",
    "    return str(endpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GCST004623 GCST004001-GCST005000\n",
      "http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004623/\n",
      "wget http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004623/harmonised/27863252-GCST004623-EFO_0007994-Build37.f.tsv.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--2022-07-05 15:29:53--  http://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/GCST004001-GCST005000/GCST004623/harmonised/27863252-GCST004623-EFO_0007994-Build37.f.tsv.gz\n",
      "Resolving ftp.ebi.ac.uk (ftp.ebi.ac.uk)... 193.62.193.138\n",
      "Connecting to ftp.ebi.ac.uk (ftp.ebi.ac.uk)|193.62.193.138|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1105907181 (1.0G) [application/octet-stream]\n",
      "Saving to: ‘27863252-GCST004623-EFO_0007994-Build37.f.tsv.gz’\n",
      "\n",
      "     0K .......... .......... .......... .......... ..........  0%  176K 1h42m\n",
      "    50K .......... .......... .......... .......... ..........  0%  170K 1h43m\n",
      "   100K .......... .......... .......... .......... ..........  0%  338K 87m1s\n",
      "   150K .......... .......... .......... .......... ..........  0%  336K 78m40s\n",
      "   200K .......... .......... .......... .......... ..........  0%  336K 73m38s\n",
      "   250K .......... .......... .......... .......... ..........  0%  356K 69m48s\n",
      "   300K .......... .......... .......... .......... ..........  0% 4.53M 60m23s\n",
      "   350K .......... .......... .......... .......... ..........  0%  335K 59m32s\n",
      "   400K .......... .......... .......... .......... ..........  0% 24.9M 53m0s\n",
      "   450K .......... .......... .......... .......... ..........  0%  331K 53m7s\n",
      "   500K .......... .......... .......... .......... ..........  0% 16.6M 48m23s\n",
      "   550K .......... .......... .......... .......... ..........  0%  355K 48m35s\n",
      "   600K .......... .......... .......... .......... ..........  0% 2.78M 45m19s\n",
      "   650K .......... .......... .......... .......... ..........  0% 67.8M 42m6s\n",
      "   700K .......... .......... .......... .......... ..........  0%  332K 42m54s\n",
      "   750K .......... .......... .......... .......... ..........  0% 31.8M 40m16s\n",
      "   800K .......... .......... .......... .......... ..........  0%  355K 40m52s\n",
      "   850K .......... .......... .......... .......... ..........  0% 3.06M 38m55s\n",
      "   900K .......... .......... .......... .......... ..........  0% 53.7M 36m53s\n",
      "   950K .......... .......... .......... .......... ..........  0%  332K 37m45s\n",
      "  1000K .......... .......... .......... .......... ..........  0% 39.4M 35m58s\n",
      "  1050K .......... .......... .......... .......... ..........  0%  357K 36m37s\n",
      "  1100K .......... .......... .......... .......... ..........  0% 2.96M 35m17s\n",
      "  1150K .......... .......... .......... .......... ..........  0% 34.1M 33m50s\n",
      "  1200K .......... .......... .......... .......... ..........  0%  324K 34m42s\n",
      "  1250K .......... .......... .......... .......... ..........  0% 83.9M 33m23s\n",
      "  1300K .......... .......... .......... .......... ..........  0%  324K 34m12s\n",
      "  1350K .......... .......... .......... .......... ..........  0% 34.6M 32m59s\n",
      "  1400K .......... .......... .......... .......... ..........  0% 31.8M 31m52s\n",
      "  1450K .......... .......... .......... .......... ..........  0%  176K 34m13s\n",
      "  1500K .......... .......... .......... .......... ..........  0%  178K 36m22s\n",
      "  1550K .......... .......... .......... .......... ..........  0%  238K 37m35s\n",
      "  1600K .......... .......... .......... .......... ..........  0%  232K 38m48s\n",
      "  1650K .......... .......... .......... .......... ..........  0%  239K 39m52s\n",
      "  1700K .......... .......... .......... .......... ..........  0%  182K 41m33s\n",
      "  1750K .......... .......... .......... .......... ..........  0%  341K 41m52s\n",
      "  1800K .......... .......... .......... .......... ..........  0%  229K 42m51s\n",
      "  1850K .......... .......... .......... .......... ..........  0%  351K 43m4s\n",
      "  1900K .......... .......... .......... .......... ..........  0%  245K 43m50s\n",
      "  1950K .......... .......... .......... .......... ..........  0%  232K 44m40s\n",
      "  2000K .......... .......... .......... .......... ..........  0%  348K 44m50s\n",
      "  2050K .......... .......... .......... .......... ..........  0%  237K 45m34s\n",
      "  2100K .......... .......... .......... .......... ..........  0%  349K 45m42s\n",
      "  2150K .......... .......... .......... .......... ..........  0%  234K 46m25s\n",
      "  2200K .......... .......... .......... .......... ..........  0%  325K 46m36s\n",
      "  2250K .......... .......... .......... .......... ..........  0%  350K 46m42s\n",
      "  2300K .......... .......... .......... .......... ..........  0%  252K 47m14s\n",
      "  2350K .......... .......... .......... .......... ..........  0%  350K 47m19s\n",
      "  2400K .......... .......... .......... .......... ..........  0%  239K 47m53s\n",
      "  2450K .......... .......... .......... .......... ..........  0%  346K 47m57s\n",
      "  2500K .......... .......... .......... .......... ..........  0%  359K 48m0s\n",
      "  2550K .......... .......... .......... .......... ..........  0%  351K 48m3s\n",
      "  2600K .......... .......... .......... .......... ..........  0%  345K 48m7s\n",
      "  2650K .......... .......... .......... .......... ..........  0%  354K 48m10s\n",
      "  2700K .......... .........."
     ]
    }
   ],
   "source": [
    "extract_gwas(\"GCST004623\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('multiomics_integration')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4da1f62f9baebd537c3e678555deb3b178fd350b0a39aa30fd280a12be15ad9e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}