Aws docker support

.github/workflows/conda-cpp-redis.yml

@@ -0,0 +1,77 @@
+name: Conda C++/Python/Redis - gcc,OpenMPI,Redis,UCX/UCC
+
+on:
+  push:
+    branches:
+      - main
+      - 0.**
+  pull_request:
+    branches:
+      - main
+      - 0.**
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      # explicit include-based build matrix, of known valid options
+      matrix:
+        include:
+          # 20.04 supports CUDA 11.0+
+          - os: ubuntu-20.04
+            gcc: 9
+            ucc: "master"
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # Specify the correct host compilers
+      - name: Install/Select gcc and g++
+        run: |
+          sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} git
+          echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV
+          echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: cylon_dev
+          environment-file: conda/environments/cylon.yml
+
+      - name: Activate conda
+        run: conda activate cylon_dev
+
+      - name: Install UCC
+        run: |
+          git clone --single-branch -b ${{ matrix.ucc }} https://github.com/openucx/ucc.git $HOME/ucc
+          cd $HOME/ucc
+          echo "conda ucx: $(conda list | grep ucx)"
+          ./autogen.sh
+          ./configure --prefix=$HOME/ucc/install --with-ucx=$CONDA/envs/cylon_dev
+          make install
+
+      - name: Install Redis
+        run: |
+          git clone https://github.com/redis/hiredis.git $HOME/hiredis
+          cd $HOME/hiredis
+          make 
+          sudo make install
+          git clone https://github.com/sewenew/redis-plus-plus.git $HOME/redis-plus-plus
+          cd $HOME/redis-plus-plus
+          mkdir build
+          cd build
+          cmake -DREDIS_PLUS_PLUS_CXX_STANDARD=11 ..
+          make
+          sudo make install
+
+      - name: Build cylon, pycylon and run cpp test
+        run: python build.py -cmake-flags="-DCYLON_UCX=1 -DCYLON_UCC=1 -DUCC_INSTALL_PREFIX=$HOME/ucc/install -DCYLON_USE_REDIS=1" -ipath="$HOME/cylon/install" --cpp --python --test
+
+      - name: Run pytest
+        run: python build.py -ipath="$HOME/cylon/install" --pytest
+
+      - name: Build Java
+        run: python build.py -ipath="$HOME/cylon/install" --java

aws/README.md

@@ -0,0 +1,5 @@
+# Running Cylon on AWS ECS
+
+Mills Wellons Staylor, III
+
+

aws/scripts/S3_run_script.py

@@ -0,0 +1,64 @@
+import subprocess
+import time
+import argparse
+
+import boto3
+from botocore.exceptions import ClientError
+import os
+
+import logging
+
+def environ_or_required(key):
+    return (
+        {'default': os.environ.get(key)} if os.environ.get(key)
+        else {'required': True}
+    )
+
+def get_file(file_name, bucket, object_name=None):
+    """Upload a file to an S3 bucket
+
+    :param file_name: File to upload
+    :param bucket: Bucket to upload to
+    :param object_name: S3 object name. If not specified then file_name is used
+    :return: True if file was uploaded, else False
+    """
+
+    # If S3 object_name was not specified, use file_name
+    if object_name is None:
+        object_name = os.path.basename(file_name)
+
+    # download the file
+    s3_client = boto3.client('s3')
+    try:
+        with open(file_name, 'wb') as f:
+            s3_client.download_fileobj(bucket, object_name, f)
+        return f
+    except ClientError as e:
+        logging.error(e)
+        return None
+
+
+
+def join(data=None):
+    script = get_file(file_name=data['output_filename'], bucket=data['s3_bucket'], object_name=data['s3_object_name'])
+
+    if script is None:
+        print(f"unable to retrieve file {data['output_filename']} from AWS S3")
+
+    cmd = data['args'].split()
+    subprocess.call(['python'] + [data['output_filename']] + cmd, shell=False)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run S3 script")
+
+    parser.add_argument('-b', dest='s3_bucket', type=str, help="S3 Bucket Name", **environ_or_required('S3_BUCKET'))
+    parser.add_argument('-o', dest='s3_object_name', type=str, help="S3 Object Name", **environ_or_required('S3_OBJECT_NAME'))
+    parser.add_argument('-f', dest='output_filename', type=str, help="Output filename",
+                        **environ_or_required('OUTPUT_FILENAME'))
+    parser.add_argument('-a', dest='args', type=str, help="script exec arguments",
+                        **environ_or_required('EXEC_ARGS'))
+
+
+
+    args = vars(parser.parse_args())
+    join(args)

aws/scripts/cloudformation/cylon-elasticache.yaml

@@ -0,0 +1,71 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  AvailabilityZone1:
+    Type: String
+
+  AvailabilityZone2:
+    Type: String
+
+  CacheEngine:
+    Type: String
+
+  CacheEngineVersion:
+    Type: String
+
+  CacheNodeType:
+    Type: String
+
+  CacheParameterGroupName:
+    Type: String
+
+  CacheSecurityGroup:
+    Type: String
+
+  CacheSubnet1:
+    Type: String
+
+  CacheSubnet2:
+    Type: String
+
+  Prefix:
+    Type: String
+
+  RedisPort:
+    Type: Number
+
+  ReplicaCount:
+    Type: Number
+
+
+Resources:
+  SubnetGroup:
+    Type: AWS::ElastiCache::SubnetGroup
+    Properties:
+      CacheSubnetGroupName: !Sub "${Prefix}-subnetgroup"
+      Description: !Sub "${Prefix}-SubnetGroup"
+      SubnetIds:
+        - !Ref CacheSubnet1
+        - !Ref CacheSubnet2
+      Tags:
+        - Key: "name"
+          Value: !Sub "${Prefix}-Redis SubnetGroup"
+
+
+
+
+  CacheCluster:
+    Type: AWS::ElastiCache::CacheCluster
+    Properties:
+      ClusterName: !Sub "${Prefix}-Redis"
+      CacheNodeType: !Ref CacheNodeType
+      CacheSubnetGroupName: !Ref SubnetGroup
+      Engine: !Ref CacheEngine
+      EngineVersion: 7.0
+      NumCacheNodes: 1 #has to be 1 for redis
+      VpcSecurityGroupIds:
+        - !Ref CacheSecurityGroup
+      Tags:
+        - Key: "name"
+          Value: !Sub "${Prefix}-Redis Cluster"
+    DependsOn: SubnetGroup

aws/scripts/cloudformation/cylon-redis.yaml

@@ -0,0 +1,90 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Parameters:
+  TemplateBucketName:
+    Type: String
+    Default: staylor.dev2
+
+  Prefix:
+    Type: String
+    Default: cylon
+
+  Architecture:
+    Type: String
+    Default: arm64
+
+  AvailabilityZone1:
+    Type: String
+    Default: us-east-1c
+
+  AvailabilityZone2:
+    Type: String
+    Default: us-east-1d
+
+  CacheEngine:
+    Type: String
+    Default: redis
+
+  CacheEngineVersion:
+    Type: String
+    Default: 6.2
+
+  CacheNodeType:
+    Type: String
+    Default: cache.t4g.micro
+
+  CacheParameterGroupName:
+    Type: String
+    Default: default.redis7.cluster.on
+
+  CacheSecurityGroupName:
+    Type: String
+    Default: sg-0da3e3dcebe706315
+
+  CacheSubnet1:
+    Type: String
+    Default: subnet-07995eea6c462cd73
+
+  CacheSubnet2:
+    Type: String
+    Default: subnet-039df5ab7fd94f516
+
+  ImageId:
+    Type: String
+    Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-arm64-gp2
+
+  InstanceType:
+    Type: String
+    Default: t4g.nano
+
+  ReplicaCount:
+    Type: Number
+    Default: 1
+
+  Runtime:
+    Type: String
+    Default: python3.8
+
+  RedisPort:
+    Type: Number
+    Default: 6379
+
+
+Resources:
+  ElastiCacheStack:
+    Type: AWS::CloudFormation::Stack
+    "DeletionPolicy" : "Delete"
+    Properties:
+      TemplateURL: !Sub "https://s3.amazonaws.com/${TemplateBucketName}/${Prefix}/${Prefix}-elasticache.yaml"
+      Parameters:
+        AvailabilityZone1: !Ref AvailabilityZone1
+        AvailabilityZone2: !Ref AvailabilityZone2
+        CacheEngine: !Ref CacheEngine
+        CacheEngineVersion: !Ref CacheEngineVersion
+        CacheNodeType: !Ref CacheNodeType
+        CacheParameterGroupName: !Ref CacheParameterGroupName
+        CacheSecurityGroup: !Ref CacheSecurityGroupName
+        CacheSubnet1: !Ref CacheSubnet1
+        CacheSubnet2: !Ref CacheSubnet2
+        Prefix: !Ref Prefix
+        RedisPort: !Ref RedisPort
+        ReplicaCount: !Ref ReplicaCount