feat(query): introduce fts_term() function for preparing tokens for F…

…TS5 token matching
pelias · Dec 13, 2021 · bfcdc8b · bfcdc8b
1 parent a60df73
commit bfcdc8b
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 9 deletions.
diff --git a/lib/Queries.js b/lib/Queries.js
@@ -148,16 +148,16 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
   if( 0 > RTREE_THRESHOLD ){ return cb( null, [] ); }
 
   if( isPartialToken ){
-    object = object.replace(/ /g, '_').replace(REMOVE_PARTIAL_TOKEN_REGEX, '');
+    object = object.replace(REMOVE_PARTIAL_TOKEN_REGEX, '');
     if( '' === object.trim() ){ return cb( null, [] ); }
 
     this._queryAll(
       this.prepare( query.match_subject_object_geom_intersects_autocomplete ),
       {
         subject,
         object,
-        subject_quoted: `"${subject}"`,
-        object_quoted: `"${object}"`,
+        subject_fts: fts_term(subject),
+        object_fts: fts_term(object),
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },
@@ -169,12 +169,20 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
       {
         subject,
         object,
-        subject_quoted: `"${subject}"`,
-        object_quoted: `"${object}"`,
+        subject_fts: fts_term(subject),
+        object_fts: fts_term(object),
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },
       cb
     );
   }
 };
+
+// fts_term() converts a regular term (of one or more tokens)
+// into a format which can be used by the FTS5 table.
+//  1. terms should be enclosed in double-quotes
+//  2. spaces should be replaced with underscores (as per the analyzer)
+function fts_term(term) {
+  return `"${term.replace(/ /g, '_')}"`;
+}
diff --git a/query/match_subject_object_geom_intersects.sql b/query/match_subject_object_geom_intersects.sql
@@ -11,7 +11,7 @@ FROM fulltext f1
         (r1.minY - $threshold) < r2.maxY AND
         (r1.maxY + $threshold) > r2.minY
       )
-        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_fts
           JOIN tokens t2 ON (
             f2.rowid = t2.rowid
             AND r2.id = t2.id
@@ -22,7 +22,7 @@ FROM fulltext f1
               t2.lang IN ('eng', 'und')
             )
           )
-WHERE f1.fulltext MATCH $subject_quoted
+WHERE f1.fulltext MATCH $subject_fts
 AND t1.token = $subject
 GROUP BY t1.id, t2.id
 ORDER BY t1.id ASC, t2.id ASC

diff --git a/query/match_subject_object_geom_intersects_autocomplete.sql b/query/match_subject_object_geom_intersects_autocomplete.sql
@@ -11,7 +11,7 @@ FROM fulltext f1
         (r1.minY - $threshold) < r2.maxY AND
         (r1.maxY + $threshold) > r2.minY
       )
-        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted OR $object_quoted*
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_fts OR $object_fts*
           JOIN tokens t2 ON (
             f2.rowid = t2.rowid
             AND r2.id = t2.id
@@ -22,7 +22,7 @@ FROM fulltext f1
               t2.lang IN ('eng', 'und')
             )
           )
-WHERE f1.fulltext MATCH $subject_quoted
+WHERE f1.fulltext MATCH $subject_fts
 AND t1.token = $subject
 GROUP BY t1.id, t2.id
 ORDER BY t1.id ASC, t2.id ASC