From ca79c056b9c221e70ddd194692e9a3296b0f9328 Mon Sep 17 00:00:00 2001 From: Illya Moskvin Date: Thu, 14 Apr 2016 15:36:30 -0400 Subject: [PATCH] Upgrade to Django 1.9 This is a fairly extensive commit, since most of the requirements had to be upgraded as well. Dependencies have been locked down to specific versions for consistency and troubleshooting. Notable changes: - Moved wsgi.py to a more standard location, updated apache config. Now referenced in roundware.settings.WSGI_APPLICATION - For roundware.api2, implemented the new Application Configuration structure. See http://stackoverflow.com/q/32795227/1943591 - django-guardian now creates its own AnonymousUser in the database. Removed AnonymousUser references from fixtures to avoid conflicts. http://django-guardian.readthedocs.io/en/stable/configuration.html - django-guardian assign is being depricated for assign_perm See django-guardian/django-guardian@1419048 - ManyToManyField no longer accepts null=True - django.contrib.auth.models.User should not be accessed directly. Use django.conf.settings.AUTH_USER_MODEL instead - Django changed the way it loads models, which broke streaming. Moving django.setup() in roundwared.rwstreamd fixed the issue. - django_chartit is no longer maintained. Moved to django_chartit2 This may require us to install jQuery and Highcharts manually - django.forms.models.save_instance has been removed. We are now calling save() instead, but it might need more testing. https://github.com/django/django/commit/8656cf https://github.com/django/django/commit/b11564 - Fixed errors in Travis install script (thanks @hburgund) - Fixed jsocol/django-adminplus/issues/42 - Fixed django-admin-bootstrapped dependency See IMAmuseum/django-admin-bootstrapped - Fixed queryset filtering in ProjectProtectedThrough classes in admin.py Added explicit permission-based queryset filtering to ProjectAdmin - Fixed duplicate rows in auth_permission, removed default_auth fixtures. This closes #229, fixes #291, and supersedes #282. --- .travis.yml | 3 +- UPGRADING.md | 53 ++ deploy.sh | 2 +- files/etc-apache2-sites-available-roundware | 2 +- requirements/common.txt | 24 +- requirements/dev.txt | 16 +- roundware/api2/__init__.py | 18 +- roundware/api2/apps.py | 10 + roundware/api2/signals.py | 20 + roundware/rw/admin.py | 35 +- roundware/rw/filters.py | 2 +- roundware/rw/fixtures/default_auth.json | 837 ------------------ roundware/rw/fixtures/sample_project.json | 11 +- roundware/rw/forms.py | 2 +- .../migrations/0015_remove_null_from_mtm.py | 65 ++ roundware/rw/models.py | 25 +- roundware/rw/views.py | 7 +- roundware/settings/common.py | 35 +- roundware/urls.py | 1 + files/roundware.wsgi => roundware/wsgi.py | 0 roundwared/rwstreamd.py | 5 +- scripts/create-new-default-fixtures.sh | 2 +- scripts/runserver.sh | 1 + tests/roundware/rw/common.py | 6 +- tests/roundware/rw/test_admin.py | 6 +- 25 files changed, 255 insertions(+), 933 deletions(-) create mode 100644 roundware/api2/apps.py create mode 100644 roundware/api2/signals.py create mode 100644 roundware/rw/migrations/0015_remove_null_from_mtm.py rename files/roundware.wsgi => roundware/wsgi.py (100%) diff --git a/.travis.yml b/.travis.yml index 20362b92..54dc879d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: - sudo su - postgres -c "psql -c \"alter user round password 'round'\"" - sudo su - postgres -c "psql roundware -c 'create extension postgis'" - pip install -r requirements/dev.txt + - sudo apt-get install --reinstall python-setuptools before_script: # Setup Roundware log file. - sudo touch /var/log/roundware @@ -24,4 +25,4 @@ before_script: - sudo mkdir /var/www - sudo chmod 777 /var/www - export PYTHONPATH=.:/usr/lib/python2.7/dist-packages/ -script: python roundware/manage.py test --settings=roundware.settings.testing +script: python roundware/manage.py test --settings=roundware.settings.testing \ No newline at end of file diff --git a/UPGRADING.md b/UPGRADING.md index f0765ea0..7bd85372 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -19,6 +19,59 @@ Done! The following instructions describe modifications to the standard upgrade process required due to specific changes. Items are listed in reverse chronological order. +### 6/13/16 - Upgrade Django from 1.7 to 1.9 +Related Github issue: https://github.com/roundware/roundware-server/pull/283 + +The time has come to upgrade Django and other required apps to their newest versions. If you are +installing Roundware from scratch, there is no need to take extra steps. However, if your +installation is based on a commit prior to the Django 1.9 migration, some manual setup is needed. + +#### Detailed Steps (for production machines, not vagrant) + +1. Pull the relevant post-upgrade `roundware-server` commit (or newer) +2. Remove obsolete `auth_permission` rows (see issue #291): + +``` +sudo su - postgres -c 'psql -c "DELETE FROM auth_permission WHERE id IN (SELECT id FROM auth_permission EXCEPT ((SELECT permission_id FROM auth_group_permissions) UNION (SELECT permission_id FROM auth_user_user_permissions)))" roundware' +``` + +3. Initial deploy: `sudo ./deploy.sh` (this will error on `django-guardian` but don't worry, + we're about to fix that) +4. Run migrations + + ``` + sudo su - roundware -c "/var/www/roundware/source/roundware/manage.py migrate guardian --fake-initial" + sudo su - roundware -c "/var/www/roundware/source/roundware/manage.py migrate" + ``` +5. Replace apache config (note this will over-write any customizations you may have made) + + ``` + sudo rm -f /etc/apache2/sites-available/roundware.conf + sudo su - -c "sed s/USERNAME/roundware/g /var/www/roundware/source/files/etc-apache2-sites-available-roundware > /etc/apache2/sites-available/roundware.conf" + ``` +6. Run `sudo ./deploy.sh` + +#### Notes / Troubleshooting + +* As part of the upgrade process, [django-guardian](http://django-guardian.readthedocs.io/en/stable/) +must be updated from 1.2.4 to 1.4.4. While the old versions of guardian used `syncdb` to populate +the database, newer versions use migrations. The database structures are identical; however, these +migrations do not check if the tables and relationships are already in place. Therefore, you must +suppress guardian's initial migration. +* You might have to manually uninstall `django-chartit`. `django-chartit2` is meant +to be a drop-in replacement, but if `django-chartit` is still installed, it will not work. Ensure +that the `roundware` user can access all new `site-packages`. +* You might also need to uninstall `django-admin-bootstrapped` manually. Symptomatically, if the +Django admin panel has no theme, it's likely that an old version of `django-admin-bootstrapped` was installed globally +(`/usr/local/lib`) and is now interfering with the new `django-admin-bootstrapped` in the `virtualenv`. Old versions +required `django_admin_bootstrapped.bootstrap3` to be in `INSTALLED_APPS` to render the theme. +Otherwise, it would fail silently, and no theme would be rendered. Try running `pip freeze`; +if `django-admin-bootstrapped` is `v2.0.4`, run `pip uninstall django-admin-bootstrapped`. +* Roundware's `wsgi.py` was moved in this commit, and the Apache conf file must be updated. If you +get 404 errors after upgrading, chances are you skipped this step. +* Check the log to ensure that all of the required apps are located in `/var/www/roundware/lib/`. +It isn't necessary, but it might save you some headache with permissions. + ### 3/7/16 - Convert from MySQL to Postgresql for GIS speaker upgrades Related Github issue: https://github.com/roundware/roundware-server/pull/270 diff --git a/deploy.sh b/deploy.sh index 60c687e1..4d393b19 100755 --- a/deploy.sh +++ b/deploy.sh @@ -48,7 +48,7 @@ export PYTHONPATH=$CODE_PATH # Install upgrade pip pip install -U pip -# Install RoundWare requirements +# Install Roundware requirements pip install -r $CODE_PATH/requirements.txt --upgrade # Apply patch to fix M2M field deserializing for Tag relationships, force command to return true. diff --git a/files/etc-apache2-sites-available-roundware b/files/etc-apache2-sites-available-roundware index 59895143..d4ce28ba 100644 --- a/files/etc-apache2-sites-available-roundware +++ b/files/etc-apache2-sites-available-roundware @@ -36,7 +36,7 @@ WSGIDaemonProcess roundware user=USERNAME group=USERNAME umask=002 WSGIApplicationGroup %{GLOBAL} WSGIProcessGroup roundware - WSGIScriptAlias / /var/www/roundware/source/files/roundware.wsgi + WSGIScriptAlias / /var/www/roundware/source/roundware/wsgi.py WSGIPassAuthorization On # allow CORS for listen map, session map etc diff --git a/requirements/common.txt b/requirements/common.txt index a10e4b74..36b9097f 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,28 +1,26 @@ -Django<1.8 +Django==1.9 # Creates REST APIs -djangorestframework==3.2.2 +djangorestframework==3.3.3 # Used for DRF filtering -django-filter==0.9 +django-filter==0.13 # Used in roundware/rw/chart_functions.py -django-chartit==0.1 +django_chartit2 # Used in roundware/rw/admin.py, roundware/rw/forms.py, roundware/rw/views, and more. -django-guardian==1.2.4 +django-guardian==1.4.4 # Used by roundware/api1/commands.py -psutil==2.1.3 +psutil==3.4.2 # Used by roundwared/db.py django-cache-utils==0.7.2 # Used by roundware/rw/fields.py django-validated-file==2.0.1 -# Loaded in roundware/settings/common.py -django-admin-bootstrapped==2.0.4 # Used in roundware/rw/views.py django-braces==1.4.0 # Loaded in roundware/urls.py django-adminplus==0.2.1 # Used in roundware/rw/forms.py -django-crispy-forms==1.4.0 +django-crispy-forms==1.6.0 # Used in roundware/rw/widgets.py -django-floppyforms==1.2 +django-floppyforms==1.6.1 # Used in roundware/rw/views.py: django-extra-views==0.6.5 # Used in roundware/rw/fields.py and roundware/rw/widgets.py @@ -42,6 +40,8 @@ django-cors-headers # fiona is a useful tool for processing geographic files (ETL) fiona # geographic extensions for djangorestframework-gis -djangorestframework-gis +djangorestframework-gis==0.10.1 # leaflet map utilities for django admin -django-leaflet \ No newline at end of file +django-leaflet==0.18.0 +# Bootstrap admin theme for Django 1.9 +https://github.com/IMAmuseum/django-admin-bootstrapped/zipball/3.0.0 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 457e08a0..6f60fb8f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,15 +1,15 @@ -r ./common.txt -django-debug-toolbar +django-debug-toolbar==1.4 django-profiler==2.0 -#testing. -coverage==3.7.1 +#testing. +coverage==4.0.3 webtest==2.0.9 -django-webtest==1.7.5 -model_mommy==1.2 -mock==1.0.1 -python-dbusmock==0.8 +django-webtest==1.7.9 +model_mommy==1.2.6 +mock==2.0.0 +python-dbusmock==0.16.3 #interactive interpreter for debugging -ipython \ No newline at end of file +ipython==4.0.3 \ No newline at end of file diff --git a/roundware/api2/__init__.py b/roundware/api2/__init__.py index e7fa6f9c..37e2426a 100644 --- a/roundware/api2/__init__.py +++ b/roundware/api2/__init__.py @@ -1,20 +1,4 @@ # Roundware Server is released under the GNU Affero General Public License v3. # See COPYRIGHT.txt, AUTHORS.txt, and LICENSE.txt in the project root directory. -# Contains Roundware DRF REST API V2 signals. -from __future__ import unicode_literals -from django.contrib.auth import get_user_model -from django.db.models.signals import post_save -from rest_framework.authtoken.models import Token -import logging - -logger = logging.getLogger(__name__) - -def create_auth_token(sender, instance=None, created=False, **kwargs): - """ - Create an access Token for every new user - """ - if created: - Token.objects.create(user=instance) - -post_save.connect(create_auth_token, get_user_model) +default_app_config = 'api2.apps.RoundwareApi2Config' diff --git a/roundware/api2/apps.py b/roundware/api2/apps.py new file mode 100644 index 00000000..af63a0ba --- /dev/null +++ b/roundware/api2/apps.py @@ -0,0 +1,10 @@ +# Roundware Server is released under the GNU Affero General Public License v3. +# See COPYRIGHT.txt, AUTHORS.txt, and LICENSE.txt in the project root directory. + +from django.apps import AppConfig + +class RoundwareApi2Config(AppConfig): + name = 'api2' + + def ready(self): + import roundware.api2.signals \ No newline at end of file diff --git a/roundware/api2/signals.py b/roundware/api2/signals.py new file mode 100644 index 00000000..e7fa6f9c --- /dev/null +++ b/roundware/api2/signals.py @@ -0,0 +1,20 @@ +# Roundware Server is released under the GNU Affero General Public License v3. +# See COPYRIGHT.txt, AUTHORS.txt, and LICENSE.txt in the project root directory. + +# Contains Roundware DRF REST API V2 signals. +from __future__ import unicode_literals +from django.contrib.auth import get_user_model +from django.db.models.signals import post_save +from rest_framework.authtoken.models import Token +import logging + +logger = logging.getLogger(__name__) + +def create_auth_token(sender, instance=None, created=False, **kwargs): + """ + Create an access Token for every new user + """ + if created: + Token.objects.create(user=instance) + +post_save.connect(create_auth_token, get_user_model) diff --git a/roundware/rw/admin.py b/roundware/rw/admin.py index 0c246c11..a07150c4 100644 --- a/roundware/rw/admin.py +++ b/roundware/rw/admin.py @@ -62,23 +62,30 @@ def get_queryset(self, request): return qset accessible_projects = get_objects_for_user( - request.user, 'rw.access_project') + request.user, 'rw.access_project', Project) authorized_objects = model_class.objects.filter( project__in=accessible_projects) return qset.filter(**{field_name + "__in": authorized_objects}) + return get_queryset class ProjectProtectedThroughAssetModelAdmin(admin.ModelAdmin): - queryset = project_restricted_queryset_through(Asset, 'asset') + + def get_queryset(self, request): + return project_restricted_queryset_through(Asset, 'asset')(self, request) class ProjectProtectedThroughSessionModelAdmin(admin.ModelAdmin): - queryset = project_restricted_queryset_through(Session, 'session') + + def get_queryset(self, request): + return project_restricted_queryset_through(Session, 'session')(self, request) class ProjectProtectedThroughUIModelAdmin(admin.ModelAdmin): - queryset = project_restricted_queryset_through(MasterUI, 'master_ui') + + def get_queryset(self, request): + return project_restricted_queryset_through(MasterUI, 'master_ui')(self, request) class ProjectProtectedModelAdmin(admin.ModelAdmin): @@ -89,7 +96,22 @@ def get_queryset(self, request): if request.user.is_superuser: return qset - return qset.filter(project__in=get_objects_for_user(request.user, 'rw.access_project')) + # TODO: Consider using an M2M field instead, if more complex permissions not needed: + # models.py: user = models.ManyToManyField(settings.AUTH_USER_MODEL, blank=True) + # admin.py: accessible_projects = Project.objects.filter(user=request.user) + + accessible_projects = get_objects_for_user(request.user, 'rw.access_project', Project) + + return qset.filter(project__in=accessible_projects) + +class ProjectModelAdmin(GuardedModelAdmin): + + def get_queryset(self, request): + + if request.user.is_superuser: + return super(admin.ModelAdmin, self).get_queryset(request) + + return get_objects_for_user(request.user, 'rw.access_project', Project) def copy_asset(modeladmin, request, queryset): @@ -267,7 +289,7 @@ class VoteAdmin(ProjectProtectedThroughAssetModelAdmin): ordering = ['id'] -class ProjectAdmin(GuardedModelAdmin): +class ProjectAdmin(ProjectModelAdmin): list_display = ('id', 'name', 'latitude', 'longitude', 'max_recording_length', 'recording_radius') ordering = ['id'] @@ -295,7 +317,6 @@ class ProjectAdmin(GuardedModelAdmin): }), ) - class SessionAdmin(ProjectProtectedModelAdmin): list_display = ('id', 'project', 'starttime', 'device_id', 'language') list_filter = ('project', 'language', 'starttime') diff --git a/roundware/rw/filters.py b/roundware/rw/filters.py index 6a23441e..48b0b579 100644 --- a/roundware/rw/filters.py +++ b/roundware/rw/filters.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals from django.contrib.admin import DateFieldListFilter, RelatedFieldListFilter -from django.contrib.admin.util import (get_model_from_relation, +from django.contrib.admin.utils import (get_model_from_relation, prepare_lookup_value) from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_unicode diff --git a/roundware/rw/fixtures/default_auth.json b/roundware/rw/fixtures/default_auth.json index 934eb577..7117a19e 100644 --- a/roundware/rw/fixtures/default_auth.json +++ b/roundware/rw/fixtures/default_auth.json @@ -1,841 +1,4 @@ [ -{ - "fields": { - "codename": "add_permission", - "name": "Can add permission", - "content_type": 1 - }, - "model": "auth.permission", - "pk": 1 -}, -{ - "fields": { - "codename": "change_permission", - "name": "Can change permission", - "content_type": 1 - }, - "model": "auth.permission", - "pk": 2 -}, -{ - "fields": { - "codename": "delete_permission", - "name": "Can delete permission", - "content_type": 1 - }, - "model": "auth.permission", - "pk": 3 -}, -{ - "fields": { - "codename": "add_group", - "name": "Can add group", - "content_type": 2 - }, - "model": "auth.permission", - "pk": 4 -}, -{ - "fields": { - "codename": "change_group", - "name": "Can change group", - "content_type": 2 - }, - "model": "auth.permission", - "pk": 5 -}, -{ - "fields": { - "codename": "delete_group", - "name": "Can delete group", - "content_type": 2 - }, - "model": "auth.permission", - "pk": 6 -}, -{ - "fields": { - "codename": "add_user", - "name": "Can add user", - "content_type": 3 - }, - "model": "auth.permission", - "pk": 7 -}, -{ - "fields": { - "codename": "change_user", - "name": "Can change user", - "content_type": 3 - }, - "model": "auth.permission", - "pk": 8 -}, -{ - "fields": { - "codename": "delete_user", - "name": "Can delete user", - "content_type": 3 - }, - "model": "auth.permission", - "pk": 9 -}, -{ - "fields": { - "codename": "add_contenttype", - "name": "Can add content type", - "content_type": 4 - }, - "model": "auth.permission", - "pk": 10 -}, -{ - "fields": { - "codename": "change_contenttype", - "name": "Can change content type", - "content_type": 4 - }, - "model": "auth.permission", - "pk": 11 -}, -{ - "fields": { - "codename": "delete_contenttype", - "name": "Can delete content type", - "content_type": 4 - }, - "model": "auth.permission", - "pk": 12 -}, -{ - "fields": { - "codename": "add_session", - "name": "Can add session", - "content_type": 5 - }, - "model": "auth.permission", - "pk": 13 -}, -{ - "fields": { - "codename": "change_session", - "name": "Can change session", - "content_type": 5 - }, - "model": "auth.permission", - "pk": 14 -}, -{ - "fields": { - "codename": "delete_session", - "name": "Can delete session", - "content_type": 5 - }, - "model": "auth.permission", - "pk": 15 -}, -{ - "fields": { - "codename": "add_site", - "name": "Can add site", - "content_type": 6 - }, - "model": "auth.permission", - "pk": 16 -}, -{ - "fields": { - "codename": "change_site", - "name": "Can change site", - "content_type": 6 - }, - "model": "auth.permission", - "pk": 17 -}, -{ - "fields": { - "codename": "delete_site", - "name": "Can delete site", - "content_type": 6 - }, - "model": "auth.permission", - "pk": 18 -}, -{ - "fields": { - "codename": "add_logentry", - "name": "Can add log entry", - "content_type": 7 - }, - "model": "auth.permission", - "pk": 19 -}, -{ - "fields": { - "codename": "change_logentry", - "name": "Can change log entry", - "content_type": 7 - }, - "model": "auth.permission", - "pk": 20 -}, -{ - "fields": { - "codename": "delete_logentry", - "name": "Can delete log entry", - "content_type": 7 - }, - "model": "auth.permission", - "pk": 21 -}, -{ - "fields": { - "codename": "add_migrationhistory", - "name": "Can add migration history", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 22 -}, -{ - "fields": { - "codename": "change_migrationhistory", - "name": "Can change migration history", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 23 -}, -{ - "fields": { - "codename": "delete_migrationhistory", - "name": "Can delete migration history", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 24 -}, -{ - "fields": { - "codename": "add_groupobjectpermission", - "name": "Can add group object permission", - "content_type": 9 - }, - "model": "auth.permission", - "pk": 25 -}, -{ - "fields": { - "codename": "change_groupobjectpermission", - "name": "Can change group object permission", - "content_type": 9 - }, - "model": "auth.permission", - "pk": 26 -}, -{ - "fields": { - "codename": "delete_groupobjectpermission", - "name": "Can delete group object permission", - "content_type": 9 - }, - "model": "auth.permission", - "pk": 27 -}, -{ - "fields": { - "codename": "add_token", - "name": "Can add token", - "content_type": 10 - }, - "model": "auth.permission", - "pk": 28 -}, -{ - "fields": { - "codename": "change_token", - "name": "Can change token", - "content_type": 10 - }, - "model": "auth.permission", - "pk": 29 -}, -{ - "fields": { - "codename": "delete_token", - "name": "Can delete token", - "content_type": 10 - }, - "model": "auth.permission", - "pk": 30 -}, -{ - "fields": { - "codename": "add_language", - "name": "Can add language", - "content_type": 11 - }, - "model": "auth.permission", - "pk": 31 -}, -{ - "fields": { - "codename": "change_language", - "name": "Can change language", - "content_type": 11 - }, - "model": "auth.permission", - "pk": 32 -}, -{ - "fields": { - "codename": "delete_language", - "name": "Can delete language", - "content_type": 11 - }, - "model": "auth.permission", - "pk": 33 -}, -{ - "fields": { - "codename": "add_localizedstring", - "name": "Can add localized string", - "content_type": 12 - }, - "model": "auth.permission", - "pk": 34 -}, -{ - "fields": { - "codename": "change_localizedstring", - "name": "Can change localized string", - "content_type": 12 - }, - "model": "auth.permission", - "pk": 35 -}, -{ - "fields": { - "codename": "delete_localizedstring", - "name": "Can delete localized string", - "content_type": 12 - }, - "model": "auth.permission", - "pk": 36 -}, -{ - "fields": { - "codename": "add_project", - "name": "Can add project", - "content_type": 13 - }, - "model": "auth.permission", - "pk": 37 -}, -{ - "fields": { - "codename": "change_project", - "name": "Can change project", - "content_type": 13 - }, - "model": "auth.permission", - "pk": 38 -}, -{ - "fields": { - "codename": "delete_project", - "name": "Can delete project", - "content_type": 13 - }, - "model": "auth.permission", - "pk": 39 -}, -{ - "fields": { - "codename": "access_project", - "name": "Access Project", - "content_type": 13 - }, - "model": "auth.permission", - "pk": 40 -}, -{ - "fields": { - "codename": "add_session", - "name": "Can add session", - "content_type": 14 - }, - "model": "auth.permission", - "pk": 41 -}, -{ - "fields": { - "codename": "change_session", - "name": "Can change session", - "content_type": 14 - }, - "model": "auth.permission", - "pk": 42 -}, -{ - "fields": { - "codename": "delete_session", - "name": "Can delete session", - "content_type": 14 - }, - "model": "auth.permission", - "pk": 43 -}, -{ - "fields": { - "codename": "add_tagcategory", - "name": "Can add tag category", - "content_type": 15 - }, - "model": "auth.permission", - "pk": 44 -}, -{ - "fields": { - "codename": "change_tagcategory", - "name": "Can change tag category", - "content_type": 15 - }, - "model": "auth.permission", - "pk": 45 -}, -{ - "fields": { - "codename": "delete_tagcategory", - "name": "Can delete tag category", - "content_type": 15 - }, - "model": "auth.permission", - "pk": 46 -}, -{ - "fields": { - "codename": "add_tag", - "name": "Can add tag", - "content_type": 16 - }, - "model": "auth.permission", - "pk": 47 -}, -{ - "fields": { - "codename": "change_tag", - "name": "Can change tag", - "content_type": 16 - }, - "model": "auth.permission", - "pk": 48 -}, -{ - "fields": { - "codename": "delete_tag", - "name": "Can delete tag", - "content_type": 16 - }, - "model": "auth.permission", - "pk": 49 -}, -{ - "fields": { - "codename": "add_masterui", - "name": "Can add master ui", - "content_type": 17 - }, - "model": "auth.permission", - "pk": 50 -}, -{ - "fields": { - "codename": "change_masterui", - "name": "Can change master ui", - "content_type": 17 - }, - "model": "auth.permission", - "pk": 51 -}, -{ - "fields": { - "codename": "delete_masterui", - "name": "Can delete master ui", - "content_type": 17 - }, - "model": "auth.permission", - "pk": 52 -}, -{ - "fields": { - "codename": "add_uimapping", - "name": "Can add ui mapping", - "content_type": 18 - }, - "model": "auth.permission", - "pk": 53 -}, -{ - "fields": { - "codename": "change_uimapping", - "name": "Can change ui mapping", - "content_type": 18 - }, - "model": "auth.permission", - "pk": 54 -}, -{ - "fields": { - "codename": "delete_uimapping", - "name": "Can delete ui mapping", - "content_type": 18 - }, - "model": "auth.permission", - "pk": 55 -}, -{ - "fields": { - "codename": "add_audiotrack", - "name": "Can add audiotrack", - "content_type": 19 - }, - "model": "auth.permission", - "pk": 56 -}, -{ - "fields": { - "codename": "change_audiotrack", - "name": "Can change audiotrack", - "content_type": 19 - }, - "model": "auth.permission", - "pk": 57 -}, -{ - "fields": { - "codename": "delete_audiotrack", - "name": "Can delete audiotrack", - "content_type": 19 - }, - "model": "auth.permission", - "pk": 58 -}, -{ - "fields": { - "codename": "add_event", - "name": "Can add event", - "content_type": 20 - }, - "model": "auth.permission", - "pk": 59 -}, -{ - "fields": { - "codename": "change_event", - "name": "Can change event", - "content_type": 20 - }, - "model": "auth.permission", - "pk": 60 -}, -{ - "fields": { - "codename": "delete_event", - "name": "Can delete event", - "content_type": 20 - }, - "model": "auth.permission", - "pk": 61 -}, -{ - "fields": { - "codename": "add_asset", - "name": "Can add asset", - "content_type": 21 - }, - "model": "auth.permission", - "pk": 62 -}, -{ - "fields": { - "codename": "change_asset", - "name": "Can change asset", - "content_type": 21 - }, - "model": "auth.permission", - "pk": 63 -}, -{ - "fields": { - "codename": "delete_asset", - "name": "Can delete asset", - "content_type": 21 - }, - "model": "auth.permission", - "pk": 64 -}, -{ - "fields": { - "codename": "add_envelope", - "name": "Can add envelope", - "content_type": 22 - }, - "model": "auth.permission", - "pk": 65 -}, -{ - "fields": { - "codename": "change_envelope", - "name": "Can change envelope", - "content_type": 22 - }, - "model": "auth.permission", - "pk": 66 -}, -{ - "fields": { - "codename": "delete_envelope", - "name": "Can delete envelope", - "content_type": 22 - }, - "model": "auth.permission", - "pk": 67 -}, -{ - "fields": { - "codename": "add_speaker", - "name": "Can add speaker", - "content_type": 23 - }, - "model": "auth.permission", - "pk": 68 -}, -{ - "fields": { - "codename": "change_speaker", - "name": "Can change speaker", - "content_type": 23 - }, - "model": "auth.permission", - "pk": 69 -}, -{ - "fields": { - "codename": "delete_speaker", - "name": "Can delete speaker", - "content_type": 23 - }, - "model": "auth.permission", - "pk": 70 -}, -{ - "fields": { - "codename": "add_listeninghistoryitem", - "name": "Can add listening history item", - "content_type": 24 - }, - "model": "auth.permission", - "pk": 71 -}, -{ - "fields": { - "codename": "change_listeninghistoryitem", - "name": "Can change listening history item", - "content_type": 24 - }, - "model": "auth.permission", - "pk": 72 -}, -{ - "fields": { - "codename": "delete_listeninghistoryitem", - "name": "Can delete listening history item", - "content_type": 24 - }, - "model": "auth.permission", - "pk": 73 -}, -{ - "fields": { - "codename": "add_timedasset", - "name": "Can add timed asset", - "content_type": 25 - }, - "model": "auth.permission", - "pk": 74 -}, -{ - "fields": { - "codename": "change_timedasset", - "name": "Can change timed asset", - "content_type": 25 - }, - "model": "auth.permission", - "pk": 75 -}, -{ - "fields": { - "codename": "delete_timedasset", - "name": "Can delete timed asset", - "content_type": 25 - }, - "model": "auth.permission", - "pk": 76 -}, -{ - "fields": { - "codename": "add_userprofile", - "name": "Can add user profile", - "content_type": 26 - }, - "model": "auth.permission", - "pk": 77 -}, -{ - "fields": { - "codename": "change_userprofile", - "name": "Can change user profile", - "content_type": 26 - }, - "model": "auth.permission", - "pk": 78 -}, -{ - "fields": { - "codename": "delete_userprofile", - "name": "Can delete user profile", - "content_type": 26 - }, - "model": "auth.permission", - "pk": 79 -}, -{ - "fields": { - "codename": "add_vote", - "name": "Can add vote", - "content_type": 27 - }, - "model": "auth.permission", - "pk": 80 -}, -{ - "fields": { - "codename": "change_vote", - "name": "Can change vote", - "content_type": 27 - }, - "model": "auth.permission", - "pk": 81 -}, -{ - "fields": { - "codename": "delete_vote", - "name": "Can delete vote", - "content_type": 27 - }, - "model": "auth.permission", - "pk": 82 -}, -{ - "fields": { - "codename": "add_modelnotification", - "name": "Can add model notification", - "content_type": 28 - }, - "model": "auth.permission", - "pk": 83 -}, -{ - "fields": { - "codename": "change_modelnotification", - "name": "Can change model notification", - "content_type": 28 - }, - "model": "auth.permission", - "pk": 84 -}, -{ - "fields": { - "codename": "delete_modelnotification", - "name": "Can delete model notification", - "content_type": 28 - }, - "model": "auth.permission", - "pk": 85 -}, -{ - "fields": { - "codename": "add_actionnotification", - "name": "Can add action notification", - "content_type": 29 - }, - "model": "auth.permission", - "pk": 86 -}, -{ - "fields": { - "codename": "change_actionnotification", - "name": "Can change action notification", - "content_type": 29 - }, - "model": "auth.permission", - "pk": 87 -}, -{ - "fields": { - "codename": "delete_actionnotification", - "name": "Can delete action notification", - "content_type": 29 - }, - "model": "auth.permission", - "pk": 88 -}, -{ - "fields": { - "codename": "add_userobjectpermission", - "name": "Can add user object permission", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 89 -}, -{ - "fields": { - "codename": "change_userobjectpermission", - "name": "Can change user object permission", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 90 -}, -{ - "fields": { - "codename": "delete_userobjectpermission", - "name": "Can delete user object permission", - "content_type": 8 - }, - "model": "auth.permission", - "pk": 91 -}, -{ - "fields": { - "username": "AnonymousUser", - "first_name": "", - "last_name": "", - "is_active": true, - "is_superuser": false, - "is_staff": false, - "last_login": "2015-10-08T13:02:44", - "groups": [], - "user_permissions": [], - "password": "", - "email": "", - "date_joined": "2015-10-08T13:02:44" - }, - "model": "auth.user", - "pk": -1 -}, { "fields": { "username": "round", diff --git a/roundware/rw/fixtures/sample_project.json b/roundware/rw/fixtures/sample_project.json index 63809d63..8aa3ce0b 100644 --- a/roundware/rw/fixtures/sample_project.json +++ b/roundware/rw/fixtures/sample_project.json @@ -908,15 +908,6 @@ "model": "rw.speaker", "pk": 1 }, -{ - "fields": { - "client_type": null, - "user": -1, - "device_id": null - }, - "model": "rw.userprofile", - "pk": 1 -}, { "fields": { "client_type": null, @@ -924,6 +915,6 @@ "device_id": null }, "model": "rw.userprofile", - "pk": 2 + "pk": 1 } ] diff --git a/roundware/rw/forms.py b/roundware/rw/forms.py index 66260d3c..f20c258f 100644 --- a/roundware/rw/forms.py +++ b/roundware/rw/forms.py @@ -68,7 +68,7 @@ class Media: class BatchTagFormset(BaseModelFormSet): def save(self): - """ saving is handled by view. May need to revisit this if we + """ saving is handled by view. May need to revisit this if we add inlines. """ pass diff --git a/roundware/rw/migrations/0015_remove_null_from_mtm.py b/roundware/rw/migrations/0015_remove_null_from_mtm.py new file mode 100644 index 00000000..b3fc8d11 --- /dev/null +++ b/roundware/rw/migrations/0015_remove_null_from_mtm.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-20 15:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rw', '0014__data__speaker_shape_creation'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='loc_description', + field=models.ManyToManyField(blank=True, to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='asset', + name='tags', + field=models.ManyToManyField(blank=True, to='rw.Tag'), + ), + migrations.AlterField( + model_name='masterui', + name='header_text_loc', + field=models.ManyToManyField(blank=True, to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='project', + name='demo_stream_message_loc', + field=models.ManyToManyField(blank=True, related_name='demo_stream_msg_string', to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='project', + name='legal_agreement_loc', + field=models.ManyToManyField(blank=True, related_name='legal_agreement_string', to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='project', + name='out_of_range_message_loc', + field=models.ManyToManyField(blank=True, related_name='out_of_range_msg_string', to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='project', + name='sharing_message_loc', + field=models.ManyToManyField(blank=True, related_name='sharing_msg_string', to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='tag', + name='loc_description', + field=models.ManyToManyField(blank=True, related_name='tag_desc', to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='tag', + name='loc_msg', + field=models.ManyToManyField(blank=True, to='rw.LocalizedString'), + ), + migrations.AlterField( + model_name='tag', + name='relationships', + field=models.ManyToManyField(blank=True, related_name='_tag_relationships_+', to='rw.Tag'), + ), + ] diff --git a/roundware/rw/models.py b/roundware/rw/models.py index 59a95b39..aaefe17f 100644 --- a/roundware/rw/models.py +++ b/roundware/rw/models.py @@ -16,7 +16,6 @@ from django.conf import settings from datetime import datetime from cache_utils.decorators import cached -from django.contrib.auth.models import User from django.db.models.signals import post_save import logging from geopy.distance import vincenty @@ -68,9 +67,9 @@ class Project(models.Model): speak_questions_dynamic = models.BooleanField(default=False) sharing_url = models.CharField(max_length=512) sharing_message_loc = models.ManyToManyField( - LocalizedString, related_name='sharing_msg_string', null=True, blank=True) + LocalizedString, related_name='sharing_msg_string', blank=True) out_of_range_message_loc = models.ManyToManyField( - LocalizedString, related_name='out_of_range_msg_string', null=True, blank=True) + LocalizedString, related_name='out_of_range_msg_string', blank=True) out_of_range_url = models.CharField(max_length=512) recording_radius = models.IntegerField(null=True) listen_enabled = models.BooleanField(default=False) @@ -80,7 +79,7 @@ class Project(models.Model): reset_tag_defaults_on_startup = models.BooleanField(default=False) timed_asset_priority = models.BooleanField(default=True) legal_agreement_loc = models.ManyToManyField( - LocalizedString, related_name='legal_agreement_string', null=True, blank=True) + LocalizedString, related_name='legal_agreement_string', blank=True) repeat_mode = models.CharField(default=STOP, max_length=10, blank=False, choices=REPEAT_MODES) @@ -97,7 +96,7 @@ class Project(models.Model): demo_stream_enabled = models.BooleanField(default=False) demo_stream_url = models.CharField(max_length=512, blank=True) demo_stream_message_loc = models.ManyToManyField( - LocalizedString, related_name='demo_stream_msg_string', null=True, blank=True) + LocalizedString, related_name='demo_stream_msg_string', blank=True) out_of_range_distance = models.FloatField(default=1000) @@ -156,11 +155,11 @@ class Tag(models.Model): value = models.TextField() description = models.TextField(null=True, blank=True) loc_description = models.ManyToManyField( - LocalizedString, null=True, blank=True, related_name='tag_desc') - loc_msg = models.ManyToManyField(LocalizedString, null=True, blank=True) + LocalizedString, blank=True, related_name='tag_desc') + loc_msg = models.ManyToManyField(LocalizedString, blank=True) data = models.TextField(null=True, blank=True) relationships = models.ManyToManyField( - 'self', symmetrical=True, related_name='related_to', null=True, blank=True) + 'self', symmetrical=True, related_name='related_to', blank=True) filter = models.CharField( max_length=255, default="", null=False, blank=True, choices=FILTERS) @@ -199,7 +198,7 @@ class MasterUI(models.Model): ) name = models.CharField(max_length=50) header_text_loc = models.ManyToManyField( - LocalizedString, null=True, blank=True) + LocalizedString, blank=True) ui_mode = models.CharField(default=LISTEN, max_length=6, blank=False, choices=UI_MODES) tag_category = models.ForeignKey(TagCategory) @@ -340,7 +339,7 @@ class Meta: created = models.DateTimeField(default=datetime.now) audiolength = models.BigIntegerField(null=True, blank=True) - tags = models.ManyToManyField(Tag, null=True, blank=True) + tags = models.ManyToManyField(Tag, blank=True) language = models.ForeignKey(Language, null=True) weight = models.IntegerField( choices=[(i, i) for i in range(0, 100)], default=50) @@ -348,7 +347,7 @@ class Meta: max_length=16, choices=ASSET_MEDIA_TYPES, default='audio') description = models.TextField(max_length=2048, blank=True) loc_description = models.ManyToManyField( - LocalizedString, null=True, blank=True) + LocalizedString, blank=True) # enables inline adding/editing of Assets in Envelope Admin. # creates a relationship of an Asset to the Envelope, in which it was @@ -632,7 +631,7 @@ def __unicode__(self): return "%s: Asset id: %s: Start: %s: End: %s" % (self.id, self.asset.id, self.start, self.end) class UserProfile(models.Model): - user = models.OneToOneField(User) + user = models.OneToOneField(settings.AUTH_USER_MODEL) device_id = models.CharField(max_length=255, null=True) client_type = models.CharField(max_length=255, null=True) @@ -641,7 +640,7 @@ def create_user_profile(sender, instance, created, **kwargs): if created: UserProfile.objects.get_or_create(user=instance) -post_save.connect(create_user_profile, sender=User) +post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL) def get_field_names_from_model(model): diff --git a/roundware/rw/views.py b/roundware/rw/views.py index a1088783..a09cfbd6 100644 --- a/roundware/rw/views.py +++ b/roundware/rw/views.py @@ -7,7 +7,6 @@ from django.shortcuts import get_object_or_404 from django.template.loader import render_to_string from django.template.context import RequestContext -from django.forms.models import save_instance from django.views.generic.base import TemplateView from django.utils.safestring import mark_safe from django.shortcuts import render_to_response @@ -217,8 +216,10 @@ def valid_all(self, valid_forms): 'tag').filter(master_ui=mui) # instance isn't constructed yet with data from form so we can't # use form.save() but have to do the following with construct=True - save_instance(form, mui, form._meta.fields, 'form changed', True, - form._meta.exclude, True) + + save(form, mui, form._meta.fields, 'form changed', True, + form._meta.exclude, True) + self.update_ui_mappings(uimaps, formtags, defaults, indexes, mui) else: diff --git a/roundware/settings/common.py b/roundware/settings/common.py index 0120f61c..cc6d4613 100644 --- a/roundware/settings/common.py +++ b/roundware/settings/common.py @@ -19,6 +19,7 @@ # here() gives us file paths from the root of the system to the directory # holding the current file. +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # 1.8 here = lambda * x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x) # Root of roundware Django project @@ -61,7 +62,7 @@ # session_id assigned to files that are uploaded through the admin # MUST correspond to session_id that exists in session table -DEFAULT_SESSION_ID = "-10" +DEFAULT_SESSION_ID = "1" MANAGERS = ADMINS # change this to the proper id for AnonymousUser in database for Guardian ANONYMOUS_USER_ID = -1 @@ -124,6 +125,8 @@ # calendars according to the current locale USE_L10N = True +USE_TZ = False # Django 1.9 (timezone) + # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/home/media/media.lawrence.com/media/" MEDIA_ROOT = '/var/www/roundware/rwmedia/' @@ -187,6 +190,8 @@ ROOT_URLCONF = 'roundware.urls' +WSGI_APPLICATION = 'roundware.wsgi.application' + TEMPLATE_DIRS = ( # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". # Always use forward slashes, even on Windows. @@ -196,18 +201,18 @@ PROJECT_PATH + '/../rw/templates/rw', ) +# The numbers in comments indicate rough loading order for troubleshooting INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', + 'django.contrib.auth', #1 + 'django.contrib.contenttypes', #2 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.gis', - 'django_admin_bootstrapped.bootstrap3', 'django_admin_bootstrapped', - 'django.contrib.admin.apps.SimpleAdminConfig', - 'guardian', + 'django.contrib.admin.apps.SimpleAdminConfig', #5 + 'guardian', #3 'chartit', 'validatedfile', 'adminplus', @@ -221,7 +226,7 @@ 'leaflet', 'corsheaders', 'roundware.lib', - 'roundware.rw', + 'roundware.rw', #4 'roundware.notifications', 'roundware.api1', 'roundware.api2', @@ -241,8 +246,8 @@ 'rest_framework.throttling.UserRateThrottle' ), 'DEFAULT_THROTTLE_RATES': { - 'anon': '100/day', - 'user': '1000/day' + 'anon': '100000/day', # previously 100 + 'user': '1000000/day' # previously 1000 } } @@ -272,21 +277,22 @@ 'formatter': 'verbose', }, }, + # consider adding mail_admins to the handlers below as needed 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['file'], 'level': 'ERROR', 'propagate': True, }, # The roundware system logger. 'roundware': { 'level': 'DEBUG', - 'handlers': ['file', 'mail_admins'], + 'handlers': ['file'], }, # The roundwared stream manager logger. 'roundwared': { 'level': 'DEBUG', - 'handlers': ['file', 'mail_admins'], + 'handlers': ['file'], }, }, 'formatters': { @@ -321,3 +327,8 @@ # use Twitter Bootstrap template pack for django-crispy-forms CRISPY_TEMPLATE_PACK = 'bootstrap' + +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'guardian.backends.ObjectPermissionBackend' +) diff --git a/roundware/urls.py b/roundware/urls.py index 47a443ad..674d343c 100644 --- a/roundware/urls.py +++ b/roundware/urls.py @@ -16,6 +16,7 @@ from roundware.rw import urls as rw_urls admin.site = AdminSitePlus() +admin.sites.site = admin.site admin.autodiscover() urlpatterns = patterns( diff --git a/files/roundware.wsgi b/roundware/wsgi.py similarity index 100% rename from files/roundware.wsgi rename to roundware/wsgi.py diff --git a/roundwared/rwstreamd.py b/roundwared/rwstreamd.py index b2ca2f0c..dec7b965 100755 --- a/roundwared/rwstreamd.py +++ b/roundwared/rwstreamd.py @@ -7,14 +7,15 @@ import logging import traceback +import django +django.setup() + from roundwared.stream import RoundStream from roundwared import dbus_receive import getopt import sys import re import os -import django -django.setup() def listofint(s): return map(int, s.split(',')) diff --git a/scripts/create-new-default-fixtures.sh b/scripts/create-new-default-fixtures.sh index 9e8920f9..d3fc6ff1 100755 --- a/scripts/create-new-default-fixtures.sh +++ b/scripts/create-new-default-fixtures.sh @@ -2,5 +2,5 @@ # Generate new rw and auth fixtures # Necessary when schema is changed or default data updates are needed # WARNING: over-writes current default fixtures! -~/roundware-server/roundware/manage.py dumpdata auth.permission auth.user --indent=4 > ~/roundware-server/roundware/rw/fixtures/default_auth.json +~/roundware-server/roundware/manage.py dumpdata auth.user --indent=4 > ~/roundware-server/roundware/rw/fixtures/default_auth.json ~/roundware-server/roundware/manage.py dumpdata rw --indent=4 > ~/roundware-server/roundware/rw/fixtures/sample_project.json diff --git a/scripts/runserver.sh b/scripts/runserver.sh index d1ce4e1c..e5e8c012 100755 --- a/scripts/runserver.sh +++ b/scripts/runserver.sh @@ -1,6 +1,7 @@ #!/bin/bash # Run the Django development server for Roundware on port 8888 pip install -r ~/roundware-server/requirements/dev.txt + ~/roundware-server/roundware/manage.py runserver 0.0.0.0:8888 --settings=roundware.settings.dev # Kill off the rwstreamd.py scripts killall python diff --git a/tests/roundware/rw/common.py b/tests/roundware/rw/common.py index 30f290c8..0843685a 100644 --- a/tests/roundware/rw/common.py +++ b/tests/roundware/rw/common.py @@ -11,8 +11,8 @@ def use_locmemcache(module, varname): - locmem_cache = cache.get_cache( - 'django.core.cache.backends.locmem.LocMemCache') + locmem_cache = cache.caches[ + 'locmemcache'] locmem_cache.clear() return patch.object(module, varname, locmem_cache) @@ -27,7 +27,7 @@ def validated_file_field_gen(): class RWTestCase(TestCase): - """ provide common testcase data for roundware.rw test cases + """ provide common testcase data for roundware.rw test cases """ def setUp(self): diff --git a/tests/roundware/rw/test_admin.py b/tests/roundware/rw/test_admin.py index 15796434..70ee3f05 100644 --- a/tests/roundware/rw/test_admin.py +++ b/tests/roundware/rw/test_admin.py @@ -12,7 +12,7 @@ from roundware.rw.admin import * from roundware.rw.models import Project, Asset, Session, MasterUI from roundware.settings import DEFAULT_SESSION_ID -from guardian.shortcuts import assign +from guardian.shortcuts import assign_perm from model_mommy import mommy from tests.roundware.api1.test_commands import TEST_POLYGONS @@ -56,7 +56,7 @@ def _login(self, username, password): return form.submit() def need_updating_for_boostrapped_test_func_filter_by_alpha_tag_cat_show_correct_filters(self): - """ test that custom filter providing alphabetized filters by + """ test that custom filter providing alphabetized filters by tag category are available, based on tags on assets """ self._login(username=self.username, password='foo') @@ -149,7 +149,7 @@ def setUp(self): # pub_date="1111-11-11", max_recording_length=1) # self.excluded_project = Project.objects.create(name="excluded", latitude=1, longitude=1, # pub_date="1111-11-11", max_recording_length=1) - assign("access_project", self.user, self.permitted_project) + assign_perm("access_project", self.user, self.permitted_project) self.user.save() self.request = FakeRequest() self.request.user = self.user