align_unis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from fuzzywuzzy import process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_csr = pd.read_csv('other_ranks/ranks.csv')\n",
    "df_qt = pd.read_csv('other_ranks/qt_uni_ranks.csv',names=['rank','uni'])\n",
    "df_st = pd.read_csv('other_ranks/shanghai_uni_rank.csv',names=['rank','uni'])\n",
    "df_pr = pd.read_csv('other_ranks/prestige_rank.tsv',delimiter='\\t')\n",
    "df_cm = pd.read_csv('other_ranks/csmetrics.csv')\n",
    "df_sr = pd.read_csv('other_ranks/scholar_rank.csv')\n",
    "df_bp = pd.read_csv('other_ranks/best_paper.csv')\n",
    "df_times = pd.read_csv('other_ranks/times_uni_rank.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_names = list(df_csr.uni)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import rankdata\n",
    "\n",
    "df_qt['fixed'] = df_qt.uni\n",
    "df_st['fixed'] = df_st.uni\n",
    "df_pr['fixed'] = df_pr.institution\n",
    "df_cm['fixed'] = df_cm.uni\n",
    "df_sr['fixed'] = df_sr.school\n",
    "df_bp['rank'] = rankdata(df_bp.score, method='max')\n",
    "df_bp['fixed'] = df_bp.school\n",
    "df_times['fixed'] = df_times.uni\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if False:\n",
    "    for i, row in df_times.iterrows():\n",
    "    new_name = process.extractBests(df_times.at[i,'uni'],true_names,limit=1)[0][0]\n",
    "    df_times.at[i,'fixed'] = new_name\n",
    "    print(df_times.at[i,'uni'],'\\t\\t\\t',new_name)\n",
    "    #df_times.to_csv('uni_rank_times.csv',encoding='utf-8-sig')\n",
    "\n",
    "\n",
    "    for i, row in df_bp.iterrows():\n",
    "        new_name = process.extractBests(df_bp.at[i,'school'],true_names,limit=1)[0][0]\n",
    "        df_bp.at[i,'fixed'] = new_name\n",
    "        #print(df_bp.at[i,'school'],'\\t\\t\\t',new_name)\n",
    "    #df_bp.to_csv('uni_rank_bp.csv',encoding='utf-8-sig')\n",
    "\n",
    "    for i, row in df_sr.iterrows():\n",
    "        new_name = process.extractBests(df_sr.at[i,'school'],true_names,limit=1)[0][0]\n",
    "        df_sr.at[i,'fixed'] = new_name\n",
    "        #print(df_sr.at[i,'school'],'\\t\\t\\t',new_name)\n",
    "    #df_sr.to_csv('uni_rank_sr.csv')\n",
    "\n",
    "    for i, row in df_cm.iterrows():\n",
    "        new_name = process.extractBests(df_cm.at[i,'uni'],true_names,limit=1)[0][0]\n",
    "        df_cm.at[i,'fixed'] = new_name\n",
    "        #print(df_cm.at[i,'uni'],'\\t\\t\\t',new_name)\n",
    "    #df_cm.to_csv('uni_rank_cs.csv')\n",
    "\n",
    "    for i, row in df_qt.iterrows():\n",
    "        new_name = process.extractBests(df_qt.at[i,'uni'],true_names,limit=1)[0][0]\n",
    "        df_qt.at[i,'fixed'] = new_name\n",
    "        #print(df_qt.at[i,'uni'],'\\t\\t\\t',new_name)\n",
    "    #df_qt.to_csv('uni_rank_qt.csv')\n",
    "\n",
    "    for i, row in df_st.iterrows():\n",
    "        new_name = process.extractBests(df_st.at[i,'uni'],true_names,limit=1)[0][0]\n",
    "        df_st.at[i,'fixed'] = new_name\n",
    "        #print(df_st.at[i,'uni'],'\\t\\t\\t',new_name)\n",
    "    #df_st.to_csv('uni_rank_st.csv')\n",
    "\n",
    "    for i, row in df_pr.iterrows():\n",
    "        new_name = process.extractBests(df_pr.at[i,'institution'],true_names,limit=1)[0][0]\n",
    "        df_pr.at[i,'fixed'] = new_name\n",
    "        #print(df_pr.at[i,'institution'],'\\t\\t\\t',new_name)\n",
    "    #df_pr.to_csv('uni_rank_pr.csv')    \n",
    "\n",
    "    long_list = list(pd.read_csv('usn2018.csv',names=['uni']).uni)\n",
    "    usn = []\n",
    "    for i in range(0,len(long_list),3):\n",
    "        #print(long_list[i],)\n",
    "        usn.append({'rank':int(long_list[i+2].split(' ')[0]),'uni':long_list[i]})\n",
    "    #pd.DataFrame(usn).to_csv('usn2018.csv')\n",
    "\n",
    "    df_usn = pd.read_csv('usn2018.csv')\n",
    "    df_usn['fixed'] = df_usn.uni\n",
    "    for i, row in df_usn.iterrows():\n",
    "        new_name = process.extractBests(df_usn.at[i,'uni'],true_names,limit=1)[0][0]\n",
    "        df_usn.at[i,'fixed'] = new_name\n",
    "        #print(df_usn.at[i,'uni'],'\\t\\t\\t',new_name)\n",
    "    #df_usn.to_csv('uni_rank_usn.csv')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bp2 = pd.read_csv('other_ranks/uni_rank_bp.csv')\n",
    "srf2 = pd.read_csv('other_ranks/uni_rank_mergedscholar.csv')\n",
    "st2 = pd.read_csv('other_ranks/uni_rank_st.csv')\n",
    "qt2 = pd.read_csv('other_ranks/uni_rank_qt.csv')\n",
    "sr2 = pd.read_csv('other_ranks/uni_rank_sr.csv')\n",
    "pr2 = pd.read_csv('other_ranks/uni_rank_pr.csv')\n",
    "cm2 = pd.read_csv('other_ranks/uni_rank_cs.csv')\n",
    "usn2 = pd.read_csv('other_ranks/uni_rank_usn.csv')\n",
    "pr2.USN2010 = pr2.USN2010.map(lambda x: int(x) if x.isnumeric() else np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pasted_dataset2 = [(10157.00312681101, 'Carnegie Mellon University'),\n",
    " (6984.626155491176, 'University of California - Berkeley'),\n",
    " (6229.332544779517, 'Univ. of Illinois at Urbana-Champaign'),\n",
    " (6152.953178683866, 'Massachusetts Institute of Technology'),\n",
    " (5803.91017684282, 'Technion'),\n",
    " (5310.123641320455, 'Georgia Institute of Technology'),\n",
    " (5195.343574037974, 'University of Michigan'),\n",
    " (5081.8364660051, 'Stanford University'),\n",
    " (4151.711603577301, 'University of Southern California'),\n",
    " (4110.145317884101, 'University of California - San Diego'),\n",
    " (4096.28136938051, 'University of Maryland - College Park'),\n",
    " (3988.4405120002625, 'University of California - Los Angeles'),\n",
    " (3970.1258826433773, 'Cornell University'),\n",
    " (3948.698396934478, 'Tsinghua University'),\n",
    " (3936.7267574573707, 'Princeton University'),\n",
    " (3736.080459090736, 'Columbia University'),\n",
    " (3442.301201388676, 'University of Texas at Austin'),\n",
    " (3371.712223243161, 'University of California - Irvine'),\n",
    " (3370.968574751654, 'Tel Aviv University'),\n",
    " (3299.7608886769463, 'University of Washington'),\n",
    " (3278.0600409501644, 'National University of Singapore'),\n",
    " (3267.20571672267, 'University of Toronto'),\n",
    " (3193.1256092684657, 'University of Waterloo'),\n",
    " (3118.5720169717356, 'Purdue University'),\n",
    " (3112.600779296973, 'HKUST'),\n",
    " (3095.807075684528, 'EPFL'),\n",
    " (3064.7115704428297, 'ETH Zurich'),\n",
    " (3013.7549944662783, 'University of Pennsylvania'),\n",
    " (2946.2071764887464, 'University of Edinburgh'),\n",
    " (2647.0032797010927, 'University of Massachusetts Amherst'),\n",
    " (2630.822588994419, 'University of Wisconsin - Madison'),\n",
    " (2609.3604259297745, 'Rutgers University'),\n",
    " (2500.458509703014, 'Pennsylvania State University'),\n",
    " (2402.8519216623636, 'Hebrew University of Jerusalem'),\n",
    " (2314.178346687512, 'University of Minnesota'),\n",
    " (2312.202927699842, 'University of California - Riverside'),\n",
    " (2235.28460451737, 'University of Tokyo'),\n",
    " (2222.76797383822, 'University of Oxford'),\n",
    " (2216.0691766567547, 'New York University'),\n",
    " (2180.8497380432023, 'University of California - Santa Cruz'),\n",
    " (2142.2643493612686, 'Peking University'),\n",
    " (2056.343921719378, 'Ohio State University'),\n",
    " (2054.455022026539, 'Northwestern University'),\n",
    " (2053.642109146481, 'Northeastern University'),\n",
    " (2012.8913633355783, 'Imperial College London'),\n",
    " (2004.3599060945792, 'University of California - Santa Barbara'),\n",
    " (1903.966325642319, 'Nanyang Technological University'),\n",
    " (1884.0794495468597, 'University of British Columbia'),\n",
    " (1876.065066487012, 'Texas A&M University'),\n",
    " (1874.6112022791472, 'University of Illinois at Chicago'),\n",
    " (1864.3539566769803, 'Harvard University')]\n",
    "ps2 = [(v[1],i+1) for i,v in enumerate(pasted_dataset2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    [(row[2],row[1]) for row in st2.itertuples()],\n",
    "    [(row[2],row[1]) for row in qt2.itertuples()],\n",
    "    [(row[6],row[1]) for row in pr2.itertuples()],\n",
    "    [(row[2],row[1]) for row in cm2.itertuples()],\n",
    "    [(row[2],row[1]) for row in sr2.itertuples()],\n",
    "    [(row[-1],row[2]) for row in srf2.itertuples()],\n",
    "    [(row[2],row[1]) for row in df_csr.itertuples()],\n",
    "    ps2,\n",
    "    [(row[2],row[1]) for row in bp2.itertuples()],\n",
    "    [(row[6],row[4]) for row in pr2.sort_values('NRC95',ascending=True).itertuples() ],\n",
    "    [(row[6],row[3]) for row in pr2.sort_values('USN2010',ascending=True).itertuples() if np.isfinite(row[3]) ],\n",
    "    [(row[2],row[1]) for row in usn2.itertuples()]\n",
    "]\n",
    "dataset_names = ['Shanghai','Times','Prestige','CSMetrics',\n",
    "                 'ScholarRank','ScholarRankFull','CSRankings','Mine','BestPaper','NRC95',\"USN10\",'USN18']\n",
    "n_datasets = len(datasets)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_matrix = np.zeros((n_datasets,n_datasets))\n",
    "count_matrix = np.zeros((n_datasets,n_datasets))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_datasets = [ [v[0] for v in d] for d in datasets ]\n",
    "all_names = sorted(list(set(sum(name_datasets,[]))))\n",
    "all_vec = [sum([name in d for d in name_datasets])>=(len(datasets)-4) for name in all_names]\n",
    "subset_names = [name for name,vec in zip(all_names,all_vec) if vec]\n",
    "subset_names,len(subset_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats\n",
    "for i in range(n_datasets):\n",
    "    inames = [u[0] for u in datasets[i]]\n",
    "    for j in range(i,n_datasets):\n",
    "        jnames = [u[0] for u in datasets[j]]\n",
    "\n",
    "        exist_1 = [((ni in subset_names) and (ni in jnames)) for ni in inames]\n",
    "        exist_2 = [((nj in subset_names) and (nj in inames))for nj in jnames]\n",
    "        #exist_1 = [((True) and (ni in jnames)) for ni in inames]\n",
    "        #exist_2 = [((True) and (nj in inames))for nj in jnames]\n",
    "        \n",
    "        d1 = np.array(datasets[i])[exist_1]\n",
    "        d2 = np.array(datasets[j])[exist_2]\n",
    "        v1 = d1[:,1].astype(np.float)\n",
    "        v2 = np.array([d2[np.where(d2[:,0] == name)[0][0],1] for name in d1[:,0]]).astype(np.float)\n",
    "        c = stats.kendalltau(v1,v2)[0]\n",
    "        corr_matrix[i][j] = c\n",
    "        corr_matrix[j][i] = c\n",
    "        count_matrix[i][j] = len(v1)\n",
    "        count_matrix[j][i] = len(v2)\n",
    "        #print(v1,v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def matprint(mat, fmt=\"g\"):\n",
    "    col_maxes = [max([len((\"{:\"+fmt+\"}\").format(x)) for x in col]) for col in mat.T]\n",
    "    for x in mat:\n",
    "        for i, y in enumerate(x):\n",
    "            print((\"{:\"+str(col_maxes[i])+fmt+\"}\").format(y), end=\"  \")\n",
    "        print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=2)\n",
    "print(corr_matrix)\n",
    "\n",
    "print('mean best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names,corr_matrix.mean(1))],reverse=True):\n",
    "    print('{:30s}\\t{:.2f}'.format(n,s))\n",
    "print('\\n usnews best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names,corr_matrix[-1])],reverse=True):\n",
    "    print('{:30s}\\t{:.2f}'.format(n,s))\n",
    "print('\\n names')\n",
    "\n",
    "for n in dataset_names:\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matprint(count_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_linear = [\n",
    "    [(row[6],1/row[2]) for row in pr2.itertuples()],\n",
    "    [(row[2],row[-1]) for row in cm2.itertuples()],\n",
    "    [(row[2],row[-1]) for row in sr2.itertuples()],\n",
    "\n",
    "    [(row[-1],row[-5]) for row in srf2.itertuples()],\n",
    "    [(row[2],row[-2]) for row in df_csr.itertuples()],\n",
    "    [(n,s) for s,n in pasted_dataset2],\n",
    "    [(row[2],row[-1]) for row in bp2.itertuples()],\n",
    "    [(row[-1],row[-6]) for row in srf2.itertuples()],\n",
    "]\n",
    "dataset_names_linear = ['Prestige','CSMetrics',\n",
    "                 'ScholarRank','ScholarRankFull','CSRankings','Mine','BestPaper','USN18']\n",
    "n_datasets_linear = len(datasets_linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_datasets_linear = [ [v[0] for v in d] for d in datasets_linear ]\n",
    "all_names_linear = sorted(list(set(sum(name_datasets_linear,[]))))\n",
    "all_vec_linear = [sum([name in d for d in name_datasets_linear])>=(len(datasets_linear)-2) for name in all_names_linear]\n",
    "subset_names_linear = [name for name,vec in zip(all_names_linear,all_vec_linear) if vec]\n",
    "scholar_corr_set =  [row[-1] for row in srf2.itertuples() if row[-6] > 2.6]\n",
    "subset_names_linear = scholar_corr_set#[name for name in subset_names_linear if name in scholar_corr_set]\n",
    "sum([row[-6] for row in srf2.itertuples() if row[-6] > 2.6]),len(subset_names_linear),sorted(scholar_corr_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_matrix_linear = np.zeros((n_datasets_linear,n_datasets_linear))\n",
    "count_matrix_linear = np.zeros((n_datasets_linear,n_datasets_linear))\n",
    "import scipy.stats as stats\n",
    "for i in range(n_datasets_linear):\n",
    "    inames = [u[0] for u in datasets_linear[i]]\n",
    "    for j in range(i,n_datasets_linear):\n",
    "        jnames = [u[0] for u in datasets_linear[j]]\n",
    "\n",
    "        exist_1 = [((ni in subset_names_linear) and (ni in jnames)) for ni in inames]\n",
    "        exist_2 = [((nj in subset_names_linear) and (nj in inames)) for nj in jnames]\n",
    "        #exist_1 = [((True) and (ni in jnames)) for ni in inames]\n",
    "        #exist_2 = [((True) and (nj in inames))for nj in jnames]\n",
    "        \n",
    "        d1 = np.array(datasets_linear[i])[exist_1]\n",
    "        d2 = np.array(datasets_linear[j])[exist_2]\n",
    "        v1 = d1[:,1].astype(np.float)\n",
    "        v2 = np.array([d2[np.where(d2[:,0] == name)[0][0],1] for name in d1[:,0]]).astype(np.float)\n",
    "        c = stats.pearsonr(v1,v2)[0]\n",
    "        corr_matrix_linear[i][j] = c\n",
    "        corr_matrix_linear[j][i] = c\n",
    "        count_matrix_linear[i][j] = len(v1)\n",
    "        count_matrix_linear[j][i] = len(v2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=2)\n",
    "print(corr_matrix_linear)\n",
    "print('means best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names_linear,corr_matrix_linear.mean(1))],reverse=True):\n",
    "    print('{:30s}\\t{:.3f}'.format(n,s))\n",
    "print('\\nusnews best')\n",
    "for s,n in sorted([(s,n) for n,s in zip(dataset_names_linear,corr_matrix_linear[-1])],reverse=True):\n",
    "    print('{:30s}\\t{:.3f}'.format(n,s))\n",
    "print()\n",
    "for n in dataset_names_linear:\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix_linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(v1,v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "srf2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(set([name for name in sum(name_datasets,[]) ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}