Add REST API query normalization

hedera-mirror-rest/__tests__/middleware/requestNormalizer.test.js

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {normalizeRequestQueryParams} from '../../middleware/requestNormalizer.js';
+
+describe('request normalizer', () => {
+  const testSpecs = [
+    {
+      input: {
+        path: '/api/v1/blocks',
+        query: {},
+      },
+      expected: '/api/v1/blocks?limit=25&order=desc',
+    },
+    {
+      input: {
+        path: '/api/v1/blocks',
+        query: {
+          order: 'asc',
+          unknown: '3',
+          limit: '5',
+        },
+      },
+      expected: '/api/v1/blocks?limit=5&order=asc&unknown=3',
+    },
+    {
+      input: {
+        path: '/api/v1/accounts',
+        query: {
+          'account.id': ['gt:0.0.20', 'lt:0.0.21'],
+          limit: '3',
+        },
+      },
+      expected: '/api/v1/accounts?account.id=gt:0.0.20&account.id=lt:0.0.21&balance=true&limit=3&order=asc',
+    },
+    {
+      input: {
+        path: undefined,
+        query: {
+          'outOfOrder:': '1',
+          'account.id': ['lt:0.0.21', 'gt:0.0.20'],
+        },
+      },
+      // Query parameters that are arrays are not sorted if no path is provided
+      expected: 'undefined?account.id=lt:0.0.21&account.id=gt:0.0.20&outOfOrder:=1',
+    },
+    {
+      input: {
+        path: undefined,
+        query: undefined,
+      },
+      expected: undefined,
+    },
+    {
+      input: {
+        path: '/api/v1/accounts',
+        query: {
+          limit: ['2', '4'],
+        },
+      },
+      expected: '/api/v1/accounts?balance=true&limit=2&limit=4&order=asc',
+    },
+    {
+      input: {
+        path: '/api/v1/contracts/results/logs',
+        query: {
+          topic0: 'A',
+          timestamp: '1639010141.000000000',
+          index: 'lt:1',
+        },
+      },
+      expected: '/api/v1/contracts/results/logs?index=lt:1&limit=25&order=desc&timestamp=1639010141.000000000&topic0=A',
+    },
+    {
+      input: {
+        path: '/api/v1/accounts/0.0.1001/nfts',
+        query: {
+          'token.id': 'gte:1500',
+          serialnumber: 'gte:2',
+          'spender.id': 'gte:2004',
+          order: 'asc',
+        },
+      },
+      expected:
+        '/api/v1/accounts/0.0.1001/nfts?limit=25&order=asc&serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500',
+    },
+    {
+      input: {
+        path: '/api/v1/tokens/1500/nfts/2/transactions',
+        query: {
+          timestamp: 'gte:1234567890.000000005',
+          order: 'asc',
+        },
+      },
+      expected: '/api/v1/tokens/1500/nfts/2/transactions?limit=25&order=asc&timestamp=gte:1234567890.000000005',
+    },
+    {
+      input: {
+        path: '/api/v1/tokens/1500/nfts/2/transactions',
+        query: {},
+      },
+      expected: '/api/v1/tokens/1500/nfts/2/transactions?limit=25&order=desc',
+    },
+    {
+      input: {
+        path: '/api/v1/tokens/0.0.1500/nfts/2/transactions',
+        query: {},
+      },
+      expected: '/api/v1/tokens/0.0.1500/nfts/2/transactions?limit=25&order=desc',
+    },
+    {
+      input: {
+        path: '/api/v1/accounts/0.0.1001/nfts',
+        query: {
+          'token.id': 'gte:1500',
+          serialnumber: 'gte:2',
+          'spender.id': 'gte:2004',
+        },
+      },
+      expected:
+        '/api/v1/accounts/0.0.1001/nfts?limit=25&order=desc&serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500',
+    },
+    {
+      input: {
+        path: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd',
+        query: {},
+      },
+      expected: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd',
+    },
+    {
+      input: {
+        path: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd/results',
+        query: {
+          limit: '3',
+        },
+      },
+      expected: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd/results?internal=false&limit=3&order=desc',
+    },
+    {
+      input: {
+        path: '/api/v1/topics/7/messages',
+        query: {},
+      },
+      expected: '/api/v1/topics/7/messages?limit=25&order=desc',
+    },
+  ];
+
+  testSpecs.forEach((spec) => {
+    test(spec.input.path, () => {
+      expect(normalizeRequestQueryParams(spec.input.path, spec.input.query)).toEqual(spec.expected);
+    });
+  });
+});

hedera-mirror-rest/api/v1/openapi.yml

@@ -2452,7 +2452,7 @@ components:
     EntityId:
       type: string
       description: "Network entity ID in the format of `shard.realm.num`"
-      pattern: '^\d{1,10}\.\d{1,10}\.\d{1,10}$'
+      pattern: '^(\d{1,10}\.\d{1,10}\.\d{1,10}|\d{1,10})$'
       example: "0.0.2"
       nullable: true
     EntityIdQuery:

hedera-mirror-rest/middleware/openapiHandler.js

@@ -26,6 +26,10 @@ import config from '../config';
 
 let v1OpenApiDocument;
 let v1OpenApiFile;
+let openApiMap;
+
+const pathParameterRegex = /{([^}]*)}/g;
+const integerRegexPattern = '\\d{1,10}';
 
 /**
  * Check if apiVersion is currently supported
@@ -80,6 +84,116 @@ const getV1OpenApiObject = () => {
   return v1OpenApiDocument;
 };
 
+/**
+ * Get the path to parameter properties map for the OpenApi Spec
+ *
+ * @returns {Map<string, {parameterName, defaultValue, pattern}, regex: RegExp>}
+ */
+const getOpenApiMap = () => {
+  if (_.isUndefined(openApiMap)) {
+    const openApiObject = getV1OpenApiObject();
+    const patternMap = getPathParametersPatterns(openApiObject);
+    openApiMap = new Map();
+    Object.keys(openApiObject.paths).forEach((path) => {
+      const parameters = getOpenApiParameters(path, openApiObject);
+      const regex = pathToRegexConverter(path, patternMap);
+      openApiMap.set(path, {
+        parameters,
+        regex,
+      });
+    });
+  }
+
+  return openApiMap;
+};
+
+/**
+ * Given a path, gets the query parameters and their default values
+ * @param path
+ * @param openApiObject
+ */
+const getOpenApiParameters = (path, openApiObject) => {
+  const pathObject = openApiObject.paths[path];
+  const parameters = pathObject?.get?.parameters;
+  if (parameters === undefined) {
+    return {};
+  }
+
+  return (
+    parameters
+      // Each open api parameter is prefixed by #/components/parameters/, which is 24 characters long
+      .map((p) => p.$ref?.substring(24))
+      .filter((p) => p !== undefined)
+      .map((p) => openApiObject.components.parameters[p])
+      .filter((p) => p.in !== 'path')
+      .map((p) => {
+        const defaultValue = p.schema?.default;
+        const parameterName = p.name;
+        return {parameterName, defaultValue};
+      })
+  );
+};
+
+/**
+ * Converts an OpenApi path to a regex using the OpenApi regex patterns
+ * @param path
+ * @param patternMap
+ * @returns {RegExp}
+ */
+const pathToRegexConverter = (path, patternMap) => {
+  const splitPath = path.split('/');
+  for (let i = 0; i < splitPath.length; i++) {
+    const value = splitPath[i];
+    if (pathParameterRegex.test(value)) {
+      let pattern = patternMap.get(value);
+      if (!pattern) {
+        // When no pattern is present default to regex for an integer
+        pattern = integerRegexPattern;
+      } else {
+        // Remove beginning and ending of string regex characters
+        if (pattern.charAt(0) === '^') {
+          pattern = pattern.substring(1);
+        }
+        if (pattern.charAt(pattern.length - 1) === '$') {
+          pattern = pattern.substring(0, pattern.length - 1);
+        }
+      }
+
+      splitPath[i] = pattern;
+    }
+  }
+
+  // Add beginning and ending of string regex characters to the entire path
+  path = '^' + splitPath.join('/') + '$';
+  return new RegExp(path);
+};
+
+/**
+ * Gets the regex patterns for each of the path parameters
+ * @return {Map<string, string>}
+ */
+const getPathParametersPatterns = (openApiObject) => {
+  const pathParameters = new Map();
+  const openApiParameters = openApiObject.components.parameters;
+  Object.keys(openApiParameters)
+    .map((p) => openApiParameters[p])
+    .filter((p) => p.in === 'path')
+    .forEach((p) => {
+      // Path parameters are denoted by brackets within the OpenApi paths, such as: /api/v1/accounts/{idOrAliasOrEvmAddress}
+      const key = '{' + p.name + '}';
+
+      // A schema may be nested within the parameter directly or it may be a reference to a schema in the components/schema object
+      // Remove the prefix: #/components/schemas/
+      const schemaReference = p.schema.$ref?.substring(21);
+      const schema = schemaReference ? openApiObject.components.schemas[schemaReference] : p.schema;
+
+      const pattern = schema?.pattern;
+      pathParameters.set(key, pattern);
+    });
+
+  return pathParameters;
+};
+
 const serveSpec = (req, res) => res.type('text/yaml').send(getV1OpenApiFile());
 
 /**
@@ -113,4 +227,4 @@ const openApiValidator = (app) => {
   );
 };
 
-export {getV1OpenApiObject, openApiValidator, serveSwaggerDocs};
+export {getV1OpenApiObject, getOpenApiMap, openApiValidator, serveSwaggerDocs};

hedera-mirror-rest/middleware/requestNormalizer.js

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+import qs from 'qs';
+
+import {getOpenApiMap} from './openapiHandler.js';
+
+const openApiMap = getOpenApiMap();
+
+/**
+ * Normalizes a request by adding any missing default values and sorting the query parameters.
+ * @param path {string}
+ * @param query
+ * @returns {string}
+ */
+const normalizeRequestQueryParams = (path, query) => {
+  const openApiPathMap = getOpenApiPathMap(path);
+  if (_.isEmpty(openApiPathMap)) {
+    return formatPathQuery(path, query);
+  }
+
+  for (const openApiParam of openApiPathMap.parameters) {
+    if (query[openApiParam?.parameterName] === undefined && openApiParam?.defaultValue !== undefined) {
+      // Add default value to query parameter
+      query[openApiParam.parameterName] = openApiParam.defaultValue;
+    } else if (Array.isArray(query[openApiParam?.parameterName])) {
+      // Sort any listed query parameters
+      query[openApiParam.parameterName].sort();
+    }
+  }
+
+  return formatPathQuery(path, query);
+};
+
+/**
+ * @param path
+ * @returns {parameters [{parameterName, defaultValue}]}
+ */
+const getOpenApiPathMap = (path) => {
+  let openApiPathMap = openApiMap.get(path);
+
+  // If the path doesn't match a key from the map, iterate through the map to find a regex match
+  if (openApiPathMap === undefined) {
+    for (let [key, value] of openApiMap) {
+      if (value.regex.test(path)) {
+        openApiPathMap = value;
+        break;
+      }
+    }
+  }
+
+  return openApiPathMap;
+};
+
+const alphabeticalSort = (a, b) => {
+  return a.localeCompare(b);
+};
+
+const formatPathQuery = (path, query) => {
+  return _.isEmpty(query) ? path : path + '?' + sortQueryProperties(query);
+};
+
+const sortQueryProperties = (query) => {
+  return qs.stringify(query, stringifyOptions);
+};
+
+// alphabeticalSort is used to sort the properties of the Object
+const stringifyOptions = {encode: false, indices: false, sort: alphabeticalSort};
+
+export {normalizeRequestQueryParams};