From af86281104c7b46c6f261017fdc1b8604ad7a819 Mon Sep 17 00:00:00 2001 From: Anton Belonovich Date: Thu, 10 Oct 2019 01:08:53 +0400 Subject: [PATCH 1/5] Add suggestions prototype for username field --- djangoql/admin.py | 17 +++++++++++++++++ djangoql/schema.py | 1 - djangoql/static/djangoql/js/completion.js | 23 +++++++++++++++++++++-- test_project/core/admin.py | 10 ++++++++-- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/djangoql/admin.py b/djangoql/admin.py index 1f42608..1a20945 100644 --- a/djangoql/admin.py +++ b/djangoql/admin.py @@ -123,6 +123,11 @@ def get_urls(self): )), name='djangoql_syntax_help', ), + url( + r'^suggestions/$', + self.admin_site.admin_view(self.suggestions), + name='djangoql_field_value_suggestions', + ), ] return custom_urls + super(DjangoQLSearchMixin, self).get_urls() @@ -132,3 +137,15 @@ def introspect(self, request): content=json.dumps(response, indent=2), content_type='application/json; charset=utf-8', ) + + def suggestions(self, request): + # TODO move fields list generation to DjangoQLSchema + suggestions = list( + # TODO replace "username" with real field name + # TODO order by field? + self.model.objects.filter(username__startswith=request.GET['text']).values_list('username', flat=True)[:50] # TODO limit from field settings + default limit + ) + return HttpResponse( + content=json.dumps(suggestions, indent=2), + content_type='application/json; charset=utf-8', + ) diff --git a/djangoql/schema.py b/djangoql/schema.py index 8d164b2..3c44f81 100644 --- a/djangoql/schema.py +++ b/djangoql/schema.py @@ -41,7 +41,6 @@ def as_dict(self): return { 'type': self.type, 'nullable': self.nullable, - 'options': list(self.get_options()) if self.suggest_options else [], } def _field_choices(self): diff --git a/djangoql/static/djangoql/js/completion.js b/djangoql/static/djangoql/js/completion.js index 6742ef7..0864c9b 100644 --- a/djangoql/static/djangoql/js/completion.js +++ b/djangoql/static/djangoql/js/completion.js @@ -110,6 +110,22 @@ }; } + // TODO docstring + // TODO move to DjangoQL object? + function loadSuggestions(text, callback) { + var xhr = new XMLHttpRequest(); + var params = new URLSearchParams(); + params.set('text', text); + + xhr.open('GET', 'suggestions/?' + params.toString(), true); // TODO move request URL to options + xhr.onload = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + callback(JSON.parse(xhr.responseText)); + } + }; + xhr.send(); + } + // Main DjangoQL object var DjangoQL = function (options) { this.options = options; @@ -807,8 +823,11 @@ }.bind(this); } this.highlightCaseSensitive = this.valuesCaseSensitive; - this.suggestions = field.options.map(function (f) { - return suggestion(f, snippetBefore, snippetAfter); + var self = this; // TODO do it in a more elegant way + loadSuggestions(context.prefix,function (suggestions) { + self.suggestions = suggestions.map(function (f) { + return suggestion(f, snippetBefore, snippetAfter); + }); }); } else if (field.type === 'bool') { this.suggestions = [ diff --git a/test_project/core/admin.py b/test_project/core/admin.py index f0fffa2..18d201d 100644 --- a/test_project/core/admin.py +++ b/test_project/core/admin.py @@ -5,7 +5,7 @@ from django.utils.timezone import now from djangoql.admin import DjangoQLSearchMixin -from djangoql.schema import DjangoQLSchema, IntField +from djangoql.schema import DjangoQLSchema, IntField, StrField from .models import Book @@ -86,6 +86,12 @@ def years_ago(self, n): return timestamp.replace(month=2, day=28, year=timestamp.year - n) +class UserNameField(StrField): + model = User + name = 'username' + suggest_options = True + + class UserQLSchema(DjangoQLSchema): exclude = (Book,) suggest_options = { @@ -95,7 +101,7 @@ class UserQLSchema(DjangoQLSchema): def get_fields(self, model): fields = super(UserQLSchema, self).get_fields(model) if model == User: - fields = [UserAgeField(), IntField(name='groups_count')] + fields + fields = [UserNameField(), UserAgeField(), IntField(name='groups_count')]# + fields return fields From 2ed4254099e8b009e30d19f8b8f1f4ea2ad4a4a8 Mon Sep 17 00:00:00 2001 From: Anton Belonovich Date: Thu, 10 Oct 2019 01:11:38 +0400 Subject: [PATCH 2/5] Add fake user generator for testing value suggestions --- .../core/management/commands/generate_users.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test_project/core/management/commands/generate_users.py diff --git a/test_project/core/management/commands/generate_users.py b/test_project/core/management/commands/generate_users.py new file mode 100644 index 0000000..07877a0 --- /dev/null +++ b/test_project/core/management/commands/generate_users.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth.models import User + +ALPHABET = 'abcdefghijklmnopqrstuvwxyz' + + +class Command(BaseCommand): + help = 'Generate fake users for testing value suggestion feature on username field' + + def handle(self, *args, **options): + for letter in ALPHABET: + for i in range(2, 150): + user = User(username=letter * i) + user.save() From b4a16e6708af8f17005958d7220cda3ad2c26c98 Mon Sep 17 00:00:00 2001 From: Anton Belonovich Date: Thu, 10 Oct 2019 15:44:57 +0400 Subject: [PATCH 3/5] Add field value suggestions limit --- README.rst | 17 +++++++ djangoql/admin.py | 15 +++--- djangoql/schema.py | 32 ++++++++++--- djangoql/static/djangoql/js/completion.js | 46 ++++++++++--------- .../static/djangoql/js/completion_admin.js | 1 + test_project/core/admin.py | 11 ++--- 6 files changed, 77 insertions(+), 45 deletions(-) diff --git a/README.rst b/README.rst index 5b2c177..1d82172 100644 --- a/README.rst +++ b/README.rst @@ -236,6 +236,23 @@ In this example we've defined a custom GroupNameField that sorts suggestions for group names by popularity (no. of users in a group) instead of default alphabetical sorting. +**Set value suggestions limit** + +By default, field value suggestions number is limited to 50 entries. +If you want to change this, override ``suggestions_limit`` field in your custom schema. + +.. code:: python + + from djangoql.schema import DjangoQLSchema, StrField + + + class UserQLSchema(DjangoQLSchema): + exclude = (Book,) + suggest_options = { + Group: ['name'], + } + suggestions_limit = 30 # Set a desired limit here + **Custom search lookup** DjangoQL base fields provide two basic methods that you can override to diff --git a/djangoql/admin.py b/djangoql/admin.py index 1a20945..1c2a098 100644 --- a/djangoql/admin.py +++ b/djangoql/admin.py @@ -125,7 +125,7 @@ def get_urls(self): ), url( r'^suggestions/$', - self.admin_site.admin_view(self.suggestions), + self.admin_site.admin_view(self.field_value_suggestions), name='djangoql_field_value_suggestions', ), ] @@ -138,14 +138,11 @@ def introspect(self, request): content_type='application/json; charset=utf-8', ) - def suggestions(self, request): - # TODO move fields list generation to DjangoQLSchema - suggestions = list( - # TODO replace "username" with real field name - # TODO order by field? - self.model.objects.filter(username__startswith=request.GET['text']).values_list('username', flat=True)[:50] # TODO limit from field settings + default limit - ) + def field_value_suggestions(self, request): + suggestions = self.djangoql_schema(self.model)\ + .get_field_instance(self.model, request.GET['field_name'])\ + .get_sugestions(request.GET['text']) return HttpResponse( - content=json.dumps(suggestions, indent=2), + content=json.dumps(list(suggestions), indent=2), content_type='application/json; charset=utf-8', ) diff --git a/djangoql/schema.py b/djangoql/schema.py index 3c44f81..2ffb4ca 100644 --- a/djangoql/schema.py +++ b/djangoql/schema.py @@ -27,7 +27,7 @@ class DjangoQLField(object): value_types_description = '' def __init__(self, model=None, name=None, nullable=None, - suggest_options=None): + suggest_options=None, suggestions_limit=None): if model is not None: self.model = model if name is not None: @@ -36,6 +36,11 @@ def __init__(self, model=None, name=None, nullable=None, self.nullable = nullable if suggest_options is not None: self.suggest_options = suggest_options + if suggestions_limit is not None: + self.suggestions_limit = suggestions_limit + + def _get_options_queryset(self): + return self.model.objects.order_by(self.name) def as_dict(self): return { @@ -53,15 +58,27 @@ def _field_choices(self): def get_options(self): """ - Override this method to provide custom suggestion options + DEPRECATED: field value suggestions are now using get_sugestions() method """ choices = self._field_choices() if choices: return [c[1] for c in choices] else: - return self.model.objects.\ - order_by(self.name).\ - values_list(self.name, flat=True) + return self._get_options_queryset().values_list(self.name, flat=True) + + def get_sugestions(self, text): + """ + Override this method to provide custom suggestion options + """ + choices = self._field_choices() + if choices: + return [c[1] for c in choices] + + kwargs = {'{}__icontains'.format(self.name): text} + + return self._get_options_queryset()\ + .filter(**kwargs)[:self.suggestions_limit]\ + .values_list(self.name, flat=True) def get_lookup_name(self): """ @@ -266,12 +283,13 @@ class RelationField(DjangoQLField): type = 'relation' def __init__(self, model, name, related_model, nullable=False, - suggest_options=False): + suggest_options=False, suggestions_limit=None): super(RelationField, self).__init__( model=model, name=name, nullable=nullable, suggest_options=suggest_options, + suggestions_limit=suggestions_limit, ) self.related_model = related_model @@ -289,6 +307,7 @@ class DjangoQLSchema(object): include = () # models to include into introspection exclude = () # models to exclude from introspection suggest_options = None + suggestions_limit = 50 def __init__(self, model): if not inspect.isclass(model) or not issubclass(model, models.Model): @@ -396,6 +415,7 @@ def get_field_instance(self, model, field_name): field_kwargs['suggest_options'] = ( field.name in self.suggest_options.get(model, []) ) + field_kwargs['suggestions_limit'] = self.suggestions_limit field_instance = field_cls(**field_kwargs) # Check if suggested options conflict with field type if field_cls != StrField and field_instance.suggest_options: diff --git a/djangoql/static/djangoql/js/completion.js b/djangoql/static/djangoql/js/completion.js index 0864c9b..3d14135 100644 --- a/djangoql/static/djangoql/js/completion.js +++ b/djangoql/static/djangoql/js/completion.js @@ -110,22 +110,6 @@ }; } - // TODO docstring - // TODO move to DjangoQL object? - function loadSuggestions(text, callback) { - var xhr = new XMLHttpRequest(); - var params = new URLSearchParams(); - params.set('text', text); - - xhr.open('GET', 'suggestions/?' + params.toString(), true); // TODO move request URL to options - xhr.onload = function () { - if (xhr.readyState === 4 && xhr.status === 200) { - callback(JSON.parse(xhr.responseText)); - } - }; - xhr.send(); - } - // Main DjangoQL object var DjangoQL = function (options) { this.options = options; @@ -823,12 +807,12 @@ }.bind(this); } this.highlightCaseSensitive = this.valuesCaseSensitive; - var self = this; // TODO do it in a more elegant way - loadSuggestions(context.prefix,function (suggestions) { - self.suggestions = suggestions.map(function (f) { - return suggestion(f, snippetBefore, snippetAfter); - }); - }); + this.loadSuggestions( + context.prefix, context.field, function (responseData) { + this.suggestions = responseData.map(function (f) { + return suggestion(f, snippetBefore, snippetAfter); + }); + }.bind(this)); } else if (field.type === 'bool') { this.suggestions = [ suggestion('True', '', ' '), @@ -861,6 +845,24 @@ } else { this.selected = null; } + }, + // Load suggestions from backend + loadSuggestions: function (text, fieldName, callback) { + var xhr = new XMLHttpRequest(); + var params = new URLSearchParams(); + params.set('text', text); + params.set('field_name', fieldName); + xhr.open( + 'GET', + this.options.suggestionsUrl + params.toString(), + true + ); + xhr.onload = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + callback(JSON.parse(xhr.responseText)); + } + }; + xhr.send(); } }; diff --git a/djangoql/static/djangoql/js/completion_admin.js b/djangoql/static/djangoql/js/completion_admin.js index f354360..8e855a9 100644 --- a/djangoql/static/djangoql/js/completion_admin.js +++ b/djangoql/static/djangoql/js/completion_admin.js @@ -95,6 +95,7 @@ djangoQL = new DjangoQL({ completionEnabled: QLEnabled, introspections: 'introspect/', + suggestionsUrl: 'suggestions/?', syntaxHelp: 'djangoql-syntax/', selector: 'textarea[name=q]', autoResize: true diff --git a/test_project/core/admin.py b/test_project/core/admin.py index 18d201d..c577163 100644 --- a/test_project/core/admin.py +++ b/test_project/core/admin.py @@ -86,22 +86,17 @@ def years_ago(self, n): return timestamp.replace(month=2, day=28, year=timestamp.year - n) -class UserNameField(StrField): - model = User - name = 'username' - suggest_options = True - - class UserQLSchema(DjangoQLSchema): exclude = (Book,) suggest_options = { - Group: ['name'], + Group: ['name'], User: ['username'] } + suggestions_limit = 30 def get_fields(self, model): fields = super(UserQLSchema, self).get_fields(model) if model == User: - fields = [UserNameField(), UserAgeField(), IntField(name='groups_count')]# + fields + fields = [UserAgeField(), IntField(name='groups_count')] + fields return fields From a31c5619be15055dd541212bd651cc4717d002ac Mon Sep 17 00:00:00 2001 From: Anton Belonovich Date: Thu, 10 Oct 2019 15:58:31 +0400 Subject: [PATCH 4/5] Add field value suggestions to the demo page --- test_project/core/templates/completion_demo.html | 5 ++++- test_project/core/views.py | 12 ++++++++++++ test_project/test_project/urls.py | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/test_project/core/templates/completion_demo.html b/test_project/core/templates/completion_demo.html index 1cfb27e..ca76a44 100644 --- a/test_project/core/templates/completion_demo.html +++ b/test_project/core/templates/completion_demo.html @@ -40,7 +40,10 @@ // doesn't fit, and shrink back when text is removed. The purpose // of this is to see full search query without scrolling, could be // helpful for really long queries. - autoResize: true + autoResize: true, + + // An URL to load field value suggestions + suggestionsUrl: 'suggestions/?' }); }); diff --git a/test_project/core/views.py b/test_project/core/views.py index 90fccee..2b0d506 100644 --- a/test_project/core/views.py +++ b/test_project/core/views.py @@ -3,6 +3,7 @@ from django.contrib.auth.models import Group, User from django.shortcuts import render from django.views.decorators.http import require_GET +from django.http.response import HttpResponse from djangoql.exceptions import DjangoQLError from djangoql.queryset import apply_search @@ -30,3 +31,14 @@ def completion_demo(request): 'search_results': query, 'introspections': json.dumps(UserQLSchema(query.model).as_dict()), }) + + +@require_GET +def suggestions(request): + payload = UserQLSchema(User) \ + .get_field_instance(User, request.GET['field_name']) \ + .get_sugestions(request.GET['text']) + return HttpResponse( + content=json.dumps(list(payload), indent=2), + content_type='application/json; charset=utf-8', + ) diff --git a/test_project/test_project/urls.py b/test_project/test_project/urls.py index e189557..734dbb3 100644 --- a/test_project/test_project/urls.py +++ b/test_project/test_project/urls.py @@ -17,12 +17,13 @@ from django.conf.urls import url, include from django.contrib import admin -from core.views import completion_demo +from core.views import completion_demo, suggestions urlpatterns = [ url(r'^admin/', admin.site.urls), url(r'^$', completion_demo), + url(r'^suggestions/', suggestions), ] if settings.DEBUG and settings.DJDT: From f0a9f498187efd23c7a22c7512b807ecfca20941 Mon Sep 17 00:00:00 2001 From: Anton Belonovich Date: Thu, 10 Oct 2019 17:26:51 +0400 Subject: [PATCH 5/5] Add suggestion tests for DjangoQLField --- .../management/commands/generate_users.py | 14 -------- test_project/core/tests/test_schema.py | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) delete mode 100644 test_project/core/management/commands/generate_users.py diff --git a/test_project/core/management/commands/generate_users.py b/test_project/core/management/commands/generate_users.py deleted file mode 100644 index 07877a0..0000000 --- a/test_project/core/management/commands/generate_users.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.core.management.base import BaseCommand -from django.contrib.auth.models import User - -ALPHABET = 'abcdefghijklmnopqrstuvwxyz' - - -class Command(BaseCommand): - help = 'Generate fake users for testing value suggestion feature on username field' - - def handle(self, *args, **options): - for letter in ALPHABET: - for i in range(2, 150): - user = User(username=letter * i) - user.save() diff --git a/test_project/core/tests/test_schema.py b/test_project/core/tests/test_schema.py index 2bb587f..d3168bc 100644 --- a/test_project/core/tests/test_schema.py +++ b/test_project/core/tests/test_schema.py @@ -166,3 +166,37 @@ def test_validation_fail(self): self.fail('This query should\'t pass validation: %s' % query) except DjangoQLSchemaError as e: pass + + +class UserQLSchema(DjangoQLSchema): + include = (User,) + suggestions_limit = 30 + + +class BookQLSchema(DjangoQLSchema): + include = (Book,) + suggestions_limit = 2 + + +class DjangoQLFieldTest(TestCase): + def setUp(self): + for i in range(2, 70): + User.objects.create(username='a' * i) + + def test_get_suggestions_limit_doesnt_affect_choices(self): + result = BookQLSchema(Book).get_field_instance(Book, 'genre').get_sugestions('blah') + self.assertEqual(len(result), len(Book.GENRES)) + + def test_get_suggestions_with_string_field_should_be_limited(self): + suggestions = IncludeUserGroupSchema(User).get_field_instance(User, 'username').get_sugestions('aaa') + self.assertEqual(len(suggestions), 50) + + def test_get_suggestions_with_custom_limit(self): + suggestions = UserQLSchema(User).get_field_instance(User, 'username').get_sugestions('aaa') + self.assertEqual(len(suggestions), UserQLSchema.suggestions_limit) + + def test_suggestions_should_contain_query_string(self): + query_string = 'aaa' + suggestions = UserQLSchema(User).get_field_instance(User, 'username').get_sugestions(query_string) + for suggestion in suggestions: + self.assertIn(query_string, suggestion)