1-compute-training-job-cloud.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 1: Understand the training code "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "gather": {
          "logged": 1684434354391
        }
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "train = pd.read_parquet('data/training_data.parquet')\n",
        "test = pd.read_parquet('data/testing_data.parquet')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "gather": {
          "logged": 1684434354665
        }
      },
      "outputs": [],
      "source": [
        "# Split train and test data into features X and targets Y.\n",
        "target_column_name = 'readmit_status'\n",
        "Y_train = train[target_column_name]\n",
        "X_train = train.drop([target_column_name], axis = 1)  \n",
        "Y_test = test[target_column_name]\n",
        "X_test = test.drop([target_column_name], axis = 1)  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1684434354800
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Training model...\n",
            "Accuracy score:  0.8390804597701149\n"
          ]
        }
      ],
      "source": [
        "from sklearn.compose import make_column_selector as selector\n",
        "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import make_pipeline\n",
        "import numpy as np\n",
        "\n",
        "# Transform string data to numeric one-hot vectors\n",
        "categorical_selector = selector(dtype_exclude=np.number)\n",
        "categorical_columns = categorical_selector(X_train)\n",
        "categorial_encoder = OneHotEncoder(handle_unknown='ignore')\n",
        "\n",
        "# Standardize numeric data by removing the mean and scaling to unit variance\n",
        "numerical_selector = selector(dtype_include=np.number)\n",
        "numerical_columns = numerical_selector(X_train)\n",
        "numerical_encoder = StandardScaler()\n",
        "\n",
        "# Create a preprocessor that will preprocess both numeric and categorical data\n",
        "preprocessor = ColumnTransformer([\n",
        "('categorical-encoder', categorial_encoder, categorical_columns),\n",
        "('standard_scaler', numerical_encoder, numerical_columns)])\n",
        "\n",
        "clf = make_pipeline(preprocessor, LogisticRegression(max_iter=1000))\n",
        "\n",
        "print('Training model...') \n",
        "model = clf.fit(X_train, Y_train)\n",
        "print('Accuracy score: ', clf.score(X_test,Y_test))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 2: Create a cloud client"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "gather": {
          "logged": 1684434356366
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Found the config file in: /home/bstollnitz/git/config.json\n"
          ]
        }
      ],
      "source": [
        "from azure.ai.ml import MLClient\n",
        "from azure.identity import DefaultAzureCredential\n",
        "\n",
        "credential = DefaultAzureCredential()\n",
        "ml_client = MLClient.from_config(credential=credential)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 3: Register the training and test data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "gather": {
          "logged": 1684434358363
        }
      },
      "outputs": [],
      "source": [
        "from azure.ai.ml.entities import Data\n",
        "from azure.ai.ml.constants import AssetTypes\n",
        "\n",
        "train_data_name = 'hospital_train_parquet'\n",
        "test_data_name = 'hospital_test_parquet'\n",
        "\n",
        "training_data = Data(\n",
        "    name=train_data_name,\n",
        "    path='data/training_data.parquet',\n",
        "    type=AssetTypes.URI_FILE,\n",
        "    description='RAI hospital train data'\n",
        ")\n",
        "tr_data = ml_client.data.create_or_update(training_data)\n",
        "\n",
        "test_data = Data(\n",
        "    name=test_data_name,\n",
        "    path='data/testing_data.parquet',\n",
        "    type=AssetTypes.URI_FILE,\n",
        "    description='RAI hospital test data'\n",
        ")\n",
        "ts_data = ml_client.data.create_or_update(test_data)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 4: Create a compute cluster"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "gather": {
          "logged": 1684434419020
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "AmlCompute({'type': 'amlcompute', 'created_on': None, 'provisioning_state': 'Succeeded', 'provisioning_errors': None, 'name': 'trainingcompute', 'description': None, 'tags': {}, 'properties': {}, 'id': '/subscriptions/92a04f75-fa76-4b98-bc2e-ac45670d4044/resourceGroups/rg-bstollnitz/providers/Microsoft.MachineLearningServices/workspaces/mlws-bstollnitz/computes/trainingcompute', 'Resource__source_path': None, 'base_path': '/home/bstollnitz/git/BUILD-AzureML-workshop', 'creation_context': None, 'serialize': <msrest.serialization.Serializer object at 0x7f52ecd0d880>, 'resource_id': None, 'location': 'westus2', 'size': 'STANDARD_DS2_V2', 'min_instances': 0, 'max_instances': 4, 'idle_time_before_scale_down': 3600.0, 'identity': None, 'ssh_public_access_enabled': True, 'ssh_settings': None, 'network_settings': <azure.ai.ml.entities._compute.compute.NetworkSettings object at 0x7f52ecd0da00>, 'tier': 'dedicated', 'subnet': None})"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from azure.ai.ml.entities import AmlCompute\n",
        "import time\n",
        "\n",
        "compute_name = 'trainingcompute'\n",
        "\n",
        "my_compute = AmlCompute(\n",
        "    name=compute_name,\n",
        "    size='Standard_DS12_v2',\n",
        "    min_instances=0,\n",
        "    max_instances=4,\n",
        "    idle_time_before_scale_down=3600\n",
        ")\n",
        "ml_client.compute.begin_create_or_update(my_compute).result()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 5: Create the job"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "gather": {
          "logged": 1684434758448
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RunId: hungry_feijoa_zld1qsbvn9\n",
            "Web View: https://ml.azure.com/runs/hungry_feijoa_zld1qsbvn9?wsid=/subscriptions/92a04f75-fa76-4b98-bc2e-ac45670d4044/resourcegroups/rg-bstollnitz/workspaces/mlws-bstollnitz\n"
          ]
        }
      ],
      "source": [
        "from azure.ai.ml import command, Input, Output\n",
        "\n",
        "target_column_name = 'readmit_status'\n",
        "\n",
        "# Create the job.\n",
        "job = command(\n",
        "    description='Trains hospital readmission model',\n",
        "    experiment_name='hospital_readmission',\n",
        "    compute=compute_name,\n",
        "    inputs=dict(training_data=Input(type='uri_file', path=f'{train_data_name}@latest'), \n",
        "                target_column_name=target_column_name),\n",
        "    outputs=dict(model_output=Output(type=AssetTypes.MLFLOW_MODEL)),\n",
        "    code='src/',\n",
        "    environment='azureml://registries/azureml/environments/AzureML-responsibleai-0.20-ubuntu20.04-py38-cpu/versions/4',\n",
        "    command='python train.py ' + \n",
        "            '--training_data ${{inputs.training_data}} ' +\n",
        "            '--target_column_name ${{inputs.target_column_name}} ' +\n",
        "            '--model_output ${{outputs.model_output}}'\n",
        ")\n",
        "job = ml_client.jobs.create_or_update(job)\n",
        "ml_client.jobs.stream(job.name)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Task 6: Register the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1684434758713
        }
      },
      "outputs": [],
      "source": [
        "from azure.ai.ml.entities import Model\n",
        "\n",
        "model_name = 'hospital_readmission_model'\n",
        "\n",
        "# Register the model.\n",
        "model_path = f'azureml://jobs/{job.name}/outputs/model_output'\n",
        "model = Model(name=model_name,\n",
        "                path=model_path,\n",
        "                type=AssetTypes.MLFLOW_MODEL)\n",
        "registered_model = ml_client.models.create_or_update(model)"
      ]
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python38-azureml"
    },
    "kernelspec": {
      "display_name": "build-2023",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 1
}