From b954035cc99eb25e6be708fb1fa6aa5b2e791d73 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Fri, 6 Nov 2020 22:39:09 -0700 Subject: [PATCH 01/12] Added Django session tracking --- apps/web/views.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/web/views.py b/apps/web/views.py index d18a808..11f8d94 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -40,15 +40,25 @@ "unauthenticated_review_search": 3, } +import uuid + +def getSessionID(request): + if 'userID' not in request.session: + if not request.user.is_authenticated(): + request.session['userID'] = uuid.uuid4().hex + else: + request.session['userID'] = request.user.username + return request.session['userID'] @require_safe def landing(request): return render(request, 'landing.html', { 'page_javascript': 'LayupList.Web.Landing()', 'review_count': Review.objects.count(), + 'request_user': str(request.user.username) if request.user.is_authenticated() else 'not auth', + 'session': getSessionID(request) }) - def signup(request): if request.method == 'POST': form = SignupForm(request.POST) @@ -72,6 +82,7 @@ def auth_login(request): if user is not None: if user.is_active: login(request, user) + request.session['userID'] = user.username return redirect(next_url) else: return render(request, 'login.html', { @@ -92,6 +103,7 @@ def auth_login(request): @login_required def auth_logout(request): logout(request) + request.session['userID'] = uuid.uuid4().hex return render(request, 'logout.html') From 47720bf5f6cb8d271dbb98a635593d50a60519f4 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Fri, 6 Nov 2020 22:47:41 -0700 Subject: [PATCH 02/12] Setup Pub/Sub publisher client --- apps/web/views.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/web/views.py b/apps/web/views.py index 11f8d94..329bd80 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -34,15 +34,21 @@ from lib.departments import get_department_name from lib import constants -LIMITS = { - "courses": 20, - "reviews": 5, - "unauthenticated_review_search": 3, +import uuid +from google.cloud import pubsub_v1 + +pub_sub_publisher = pubsub_v1.PublisherClient() +topic_paths = { + 'course-views': publisher.topic_path(os.environ['GCLOUD_PROJECT_ID'], 'course-views'). } -import uuid +LIMITS = { + 'courses': 20, + 'reviews': 5, + 'unauthenticated_review_search': 3, +} -def getSessionID(request): +def get_session_id(request): if 'userID' not in request.session: if not request.user.is_authenticated(): request.session['userID'] = uuid.uuid4().hex @@ -50,15 +56,15 @@ def getSessionID(request): request.session['userID'] = request.user.username return request.session['userID'] + @require_safe def landing(request): return render(request, 'landing.html', { 'page_javascript': 'LayupList.Web.Landing()', - 'review_count': Review.objects.count(), - 'request_user': str(request.user.username) if request.user.is_authenticated() else 'not auth', - 'session': getSessionID(request) + 'review_count': Review.objects.count() }) + def signup(request): if request.method == 'POST': form = SignupForm(request.POST) @@ -66,10 +72,10 @@ def signup(request): form.save_and_send_confirmation(request) return render(request, 'instructions.html') else: - return render(request, 'signup.html', {"form": form}) + return render(request, 'signup.html', {'form': form}) else: - return render(request, 'signup.html', {"form": SignupForm()}) + return render(request, 'signup.html', {'form': SignupForm()}) def auth_login(request): @@ -191,6 +197,7 @@ def current_term(request, sort): def course_detail(request, course_id): try: course = Course.objects.get(pk=course_id) + except Course.DoesNotExist: return HttpResponseNotFound('

Page not found

') From 156ee260b8ba86a9fca9493da66fe5146ed860b0 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Fri, 6 Nov 2020 22:57:30 -0700 Subject: [PATCH 03/12] Added gcloud pubsub_v1 and jwt packages --- requirements.txt | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bfda14b..8197220 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,15 @@ amqp==2.1.3 appnope==0.1.0 +astroid==1.6.6 +backports.functools-lru-cache==1.6.1 backports.shutil-get-terminal-size==1.0.0 beautifulsoup4==4.5.1 billiard==3.5.0.2 +cachetools==3.1.1 celery==4.0.1 +certifi==2020.6.20 +chardet==3.0.4 +configparser==4.0.2 decorator==4.0.10 dj-database-url==0.4.1 Django==1.11.29 @@ -19,34 +25,53 @@ django-postgrespool==0.3.0 enum34==1.1.6 factory-boy==2.7.0 fake-factory==0.7.2 -futures==3.0.5 +futures==3.3.0 +google-api-core==1.23.0 +google-auth==1.23.0 +google-cloud-pubsub==1.7.0 +googleapis-common-protos==1.52.0 +grpc-google-iam-v1==0.12.3 +grpcio==1.33.2 gunicorn==19.6.0 +idna==2.10 ipaddress==1.0.17 ipdb==0.10.1 ipython==5.1.0 ipython-genutils==0.1.0 +isort==4.3.21 kombu==4.0.1 +lazy-object-proxy==1.5.1 MarkupSafe==0.23 +mccabe==0.6.1 numpy==1.11.2 pathlib2==2.1.0 pexpect==4.2.1 pickleshare==0.7.4 prompt-toolkit==1.0.9 +protobuf==3.13.0 psycopg2==2.8.5 ptyprocess==0.5.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 PyExecJS==1.4.0 Pygments==2.1.3 +pylint==1.9.5 PyReact==0.6.0 python-dateutil==2.6.0 pytz==2016.10 redis==2.10.5 +requests==2.24.0 +rsa==4.5 scikit-learn==0.18.1 scipy==0.18.1 simplegeneric==0.8.1 -six==1.10.0 +singledispatch==3.4.0.3 +six==1.15.0 SQLAlchemy==1.1.4 sqlparse==0.2.2 traitlets==4.3.1 +urllib3==1.25.11 vine==1.1.3 wcwidth==0.1.7 whitenoise==3.2.2 +wrapt==1.12.1 From 41ba99d21722ecd79924b80c536c251e3547c98f Mon Sep 17 00:00:00 2001 From: ziruihao Date: Fri, 6 Nov 2020 22:57:54 -0700 Subject: [PATCH 04/12] Added gcloud service account credentials to Pub/Sub publisher client --- apps/web/views.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/web/views.py b/apps/web/views.py index 329bd80..9be87f7 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -36,8 +36,15 @@ import uuid from google.cloud import pubsub_v1 +from google.auth import jwt -pub_sub_publisher = pubsub_v1.PublisherClient() +serviceAccount = json.load(open('course-activity-service-account.json')) +audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher' +credentials = jwt.Credentials.from_service_account_info( + serviceAccount, + audience = audience +) +pub_sub_publisher = pubsub_v1.PublisherClient(credentials = credentials) topic_paths = { 'course-views': publisher.topic_path(os.environ['GCLOUD_PROJECT_ID'], 'course-views'). } @@ -197,7 +204,7 @@ def current_term(request, sort): def course_detail(request, course_id): try: course = Course.objects.get(pk=course_id) - + publisher.publish(topic_paths['course-views'], courseID=course_id, previousCourseID='', userID=get_session_id(request)) except Course.DoesNotExist: return HttpResponseNotFound('

Page not found

') From a7b0e7f37c88bccbf8df18112cccf2e60fdde1ff Mon Sep 17 00:00:00 2001 From: ziruihao Date: Fri, 6 Nov 2020 22:58:02 -0700 Subject: [PATCH 05/12] Added run scripts --- run.zsh | 1 + shell.zsh | 1 + 2 files changed, 2 insertions(+) create mode 100644 run.zsh create mode 100644 shell.zsh diff --git a/run.zsh b/run.zsh new file mode 100644 index 0000000..a119045 --- /dev/null +++ b/run.zsh @@ -0,0 +1 @@ +echo 'yes' | python manage.py collectstatic; forego start; \ No newline at end of file diff --git a/shell.zsh b/shell.zsh new file mode 100644 index 0000000..7017094 --- /dev/null +++ b/shell.zsh @@ -0,0 +1 @@ +python manage.py shell \ No newline at end of file From 68639610b5a910107bdb298c0eddb0cea00a98f9 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Sat, 7 Nov 2020 09:56:53 -0700 Subject: [PATCH 06/12] Added publishing logic to course view --- apps/web/views.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/web/views.py b/apps/web/views.py index 9be87f7..eab4d1a 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -1,4 +1,5 @@ import sys +import datetime from django.shortcuts import render, redirect from django.conf import settings from django.views.decorators.http import require_safe, require_POST @@ -36,15 +37,8 @@ import uuid from google.cloud import pubsub_v1 -from google.auth import jwt -serviceAccount = json.load(open('course-activity-service-account.json')) -audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher' -credentials = jwt.Credentials.from_service_account_info( - serviceAccount, - audience = audience -) -pub_sub_publisher = pubsub_v1.PublisherClient(credentials = credentials) +pub_sub_publisher = pubsub_v1.PublisherClient() topic_paths = { 'course-views': publisher.topic_path(os.environ['GCLOUD_PROJECT_ID'], 'course-views'). } @@ -64,6 +58,15 @@ def get_session_id(request): return request.session['userID'] +def get_prior_course_id(request, current_course_id): + if 'priorCourseID' not in request.session: + request.session['priorCourseID'] = current_course_id + return 'null' + else: + prior_course_id = request.session['priorCourseID'] + request.session['priorCourseID'] = current_course_id + return prior_course_id + @require_safe def landing(request): return render(request, 'landing.html', { @@ -204,7 +207,11 @@ def current_term(request, sort): def course_detail(request, course_id): try: course = Course.objects.get(pk=course_id) - publisher.publish(topic_paths['course-views'], courseID=course_id, previousCourseID='', userID=get_session_id(request)) + result = publisher.publish(topic_paths['course-views'], courseID=course_id, priorCourseID=get_prior_course_id(request), userID=get_session_id(request), timestamp=datetime.datetime.utcnow().isoformat()) + try: + result.result() + except: + print('Error publishing view activity for ', course_id) except Course.DoesNotExist: return HttpResponseNotFound('

Page not found

') From df735751cfe94f2b837aec3ba051ebcbe1ccb0ce Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 00:00:38 -0700 Subject: [PATCH 07/12] Added timeout for prior couse relation --- apps/web/views.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/web/views.py b/apps/web/views.py index eab4d1a..4013689 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -1,5 +1,7 @@ +import os import sys import datetime +import dateutil.parser from django.shortcuts import render, redirect from django.conf import settings from django.views.decorators.http import require_safe, require_POST @@ -40,7 +42,7 @@ pub_sub_publisher = pubsub_v1.PublisherClient() topic_paths = { - 'course-views': publisher.topic_path(os.environ['GCLOUD_PROJECT_ID'], 'course-views'). + 'course-views': pub_sub_publisher.topic_path(os.environ['GCLOUD_PROJECT_ID'], 'course-views') } LIMITS = { @@ -50,22 +52,23 @@ } def get_session_id(request): - if 'userID' not in request.session: + if 'user_id' not in request.session: if not request.user.is_authenticated(): - request.session['userID'] = uuid.uuid4().hex + request.session['user_id'] = uuid.uuid4().hex else: - request.session['userID'] = request.user.username - return request.session['userID'] + request.session['user_id'] = request.user.username + return request.session['user_id'] def get_prior_course_id(request, current_course_id): - if 'priorCourseID' not in request.session: - request.session['priorCourseID'] = current_course_id - return 'null' - else: - prior_course_id = request.session['priorCourseID'] - request.session['priorCourseID'] = current_course_id - return prior_course_id + prior_course_id = None + if 'prior_course_id' in request.session and 'prior_course_timestamp' in request.session: + prior_course_timestamp = request.session['prior_course_timestamp'] + if dateutil.parser.parse(prior_course_timestamp) + datetime.timedelta(seconds=15) >= datetime.datetime.now(): + prior_course_id = request.session['prior_course_id'] + request.session['prior_course_id'] = current_course_id + request.session['prior_course_timestamp'] = datetime.datetime.now().isoformat() + return prior_course_id @require_safe def landing(request): @@ -207,7 +210,11 @@ def current_term(request, sort): def course_detail(request, course_id): try: course = Course.objects.get(pk=course_id) - result = publisher.publish(topic_paths['course-views'], courseID=course_id, priorCourseID=get_prior_course_id(request), userID=get_session_id(request), timestamp=datetime.datetime.utcnow().isoformat()) + prior_course_id = get_prior_course_id(request, course_id) + if prior_course_id is not None: + result = pub_sub_publisher.publish(topic_paths['course-views'], b'', courseID=course_id, priorCourseID=prior_course_id, userID=get_session_id(request), timestamp=datetime.datetime.utcnow().isoformat()) + else: + result = pub_sub_publisher.publish(topic_paths['course-views'], b'', courseID=course_id, userID=get_session_id(request), timestamp=datetime.datetime.utcnow().isoformat()) try: result.result() except: From 5b451a40aaee9207d2524968d487e87312eff412 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 00:00:53 -0700 Subject: [PATCH 08/12] Updated auth script --- auth.zsh | 1 + 1 file changed, 1 insertion(+) create mode 100644 auth.zsh diff --git a/auth.zsh b/auth.zsh new file mode 100644 index 0000000..6d5628e --- /dev/null +++ b/auth.zsh @@ -0,0 +1 @@ +export GOOGLE_APPLICATION_CREDENTIALS="master-service-account.json" \ No newline at end of file From 6064bd595e1e963b92ca05822b9dd98364a3da54 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 00:01:03 -0700 Subject: [PATCH 09/12] Added gcloud service account credentials --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c851d1a..e0f145c 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ node_modules # node version compabilities .nvmrc +master-service-account.json From 00c7416b4bccf106e1d6506001eeb7d4eaa14369 Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 14:37:30 -0700 Subject: [PATCH 10/12] Syncing unauthed session IDs to users --- apps/web/models/student.py | 3 +++ apps/web/views.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/web/models/student.py b/apps/web/models/student.py index fdd28ae..2b6074e 100644 --- a/apps/web/models/student.py +++ b/apps/web/models/student.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals from django.db import models +from django.contrib.postgres.fields import ArrayField from django.contrib.auth.models import User from django.conf import settings from django.core.mail import send_mail @@ -35,6 +36,8 @@ class Student(models.Model): created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + unauth_session_ids = ArrayField(base_field=models.CharField(max_length=32, unique=True), default=list()) + def send_confirmation_link(self, request): full_link = request.build_absolute_uri( reverse('confirmation')) + '?link=' + self.confirmation_link diff --git a/apps/web/views.py b/apps/web/views.py index 4013689..a68a390 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -101,7 +101,12 @@ def auth_login(request): if user is not None: if user.is_active: login(request, user) - request.session['userID'] = user.username + if 'user_id' in request.session: + student = Student.objects.get(user=user) + student.unauth_session_ids.append(request.session['user_id']) + student.save() + request.session['user_id'] = user.username + return redirect(next_url) else: return render(request, 'login.html', { From de5c4b443c79e71f424e64ceaf29576d3e8c07dc Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 14:37:49 -0700 Subject: [PATCH 11/12] Created migration script to add unauth_session_id to Student model --- .../0023_student_unauth_session_ids.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 apps/web/migrations/0023_student_unauth_session_ids.py diff --git a/apps/web/migrations/0023_student_unauth_session_ids.py b/apps/web/migrations/0023_student_unauth_session_ids.py new file mode 100644 index 0000000..46ea960 --- /dev/null +++ b/apps/web/migrations/0023_student_unauth_session_ids.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-11-09 21:02 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0022_increase_period_length'), + ] + + operations = [ + migrations.AddField( + model_name='student', + name='unauth_session_ids', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=32, unique=True), default=[], size=None), + ), + ] From e536f98ab2c12f8688d4459137b7dbcff1f9856e Mon Sep 17 00:00:00 2001 From: ziruihao Date: Mon, 9 Nov 2020 14:37:55 -0700 Subject: [PATCH 12/12] Script updates --- build.zsh | 1 + init.sh | 2 ++ run.zsh | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 build.zsh create mode 100644 init.sh diff --git a/build.zsh b/build.zsh new file mode 100644 index 0000000..c29e482 --- /dev/null +++ b/build.zsh @@ -0,0 +1 @@ +echo 'yes' | python manage.py collectstatic \ No newline at end of file diff --git a/init.sh b/init.sh new file mode 100644 index 0000000..aa1ade3 --- /dev/null +++ b/init.sh @@ -0,0 +1,2 @@ +source ./scripts/dev/environment.sh +source ./scripts/dev/virtualize.sh \ No newline at end of file diff --git a/run.zsh b/run.zsh index a119045..8dbedbb 100644 --- a/run.zsh +++ b/run.zsh @@ -1 +1 @@ -echo 'yes' | python manage.py collectstatic; forego start; \ No newline at end of file +forego start \ No newline at end of file