add support for paginated monitors (#167)

datadog_sync/model/monitors.py

@@ -8,6 +8,7 @@
 from typing import TYPE_CHECKING, Optional, List, Dict, cast
 
 from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
+from datadog_sync.utils.custom_client import PaginationConfig
 
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
@@ -36,9 +37,18 @@ class Monitors(BaseResource):
         ],
     )
     # Additional Monitors specific attributes
+    pagination_config = PaginationConfig(
+        page_size=100,
+        page_number_param="page",
+        page_size_param="page_size",
+        remaining_func=lambda *args: 1,
+        response_list_accessor=None,
+    )
 
     def get_resources(self, client: CustomClient) -> List[Dict]:
-        resp = client.get(self.resource_config.base_path).json()
+        resp = client.paginated_request(client.get)(
+            self.resource_config.base_path, pagination_config=self.pagination_config
+        )
 
         return resp
 

datadog_sync/utils/custom_client.py

@@ -111,10 +111,17 @@ def wrapper(*args, **kwargs):
                 resp.raise_for_status()
 
                 resp_json = resp.json()
-                resources.extend(resp_json["data"])
-                if len(resp_json["data"]) < page_size:
-                    remaining = 0
-                    continue
+                resp_len = 0
+                if pagination_config.response_list_accessor:
+                    resources.extend(resp_json[pagination_config.response_list_accessor])
+                    resp_len = len(resp_json[pagination_config.response_list_accessor])
+                else:
+                    resources.extend(resp_json)
+                    resp_len = len(resp_json)
+
+                if resp_len < page_size:
+                    break
+
                 remaining = pagination_config.remaining_func(idx, resp_json, page_size, page_number)
                 page_number = pagination_config.page_number_func(idx, page_size, page_number)
                 idx += 1
@@ -163,3 +170,4 @@ class PaginationConfig(object):
     page_number_param: Optional[str] = "page[number]"
     remaining_func: Optional[Callable] = remaining_func
     page_number_func: Optional[Callable] = page_number_func
+    response_list_accessor: Optional[str] = "data"

tests/conftest.py

@@ -19,11 +19,12 @@
 
 tracer = None
 try:
-    from ddtrace import config, patch
+    if os.getenv("DD_AGENT_HOST"):
+        from ddtrace import config, patch
 
-    config.httplib["distributed_tracing"] = True
-    config.requests["distributed_tracing"] = True
-    patch(httplib=True, requests=True)
+        config.httplib["distributed_tracing"] = True
+        config.requests["distributed_tracing"] = True
+        patch(httplib=True, requests=True)
 except ImportError:
     pass
 

tests/integration/cassettes/test_cli/TestCli.test_cleanup.frozen

@@ -1 +1 @@
-2023-07-13T11:45:13.357791-04:00
+2023-08-24T10:06:43.745795-04:00

tests/integration/cassettes/test_cli/TestCli.test_import.frozen

@@ -1 +1 @@
-2023-07-13T11:44:26.267672-04:00
+2023-08-24T10:05:52.402421-04:00

tests/integration/cassettes/test_cli/TestCli.test_sync.frozen

@@ -1 +1 @@
-2023-07-13T11:44:30.260569-04:00
+2023-08-24T10:05:55.640028-04:00

tests/integration/resources/cassettes/test_monitors/TestMonitorsResources.test_no_resource_diffs.frozen

@@ -1 +1 @@
-2023-01-04T16:34:43.927845-05:00
+2023-08-24T10:09:49.346349-04:00

tests/integration/resources/cassettes/test_monitors/TestMonitorsResources.test_resource_cleanup.frozen

@@ -1 +1 @@
-2023-01-04T16:34:43.945229-05:00
+2023-08-24T10:09:49.354636-04:00

tests/integration/resources/cassettes/test_monitors/TestMonitorsResources.test_resource_cleanup.yaml

@@ -8,11 +8,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-00000000000000009508ae4e0b6b527f-19c057d2dd41a74e-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '1855579609643460430'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '10739024961687212671'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380053?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742272?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380053}'
+      string: '{"deleted_monitor_id": 12742272}'
     headers: {}
     status:
       code: 200
@@ -26,11 +38,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-0000000000000000e136065bacb87bbb-3fb554e5a7c2d43b-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '4590668740512896059'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '16228165298085723067'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380047?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742271?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380047}'
+      string: '{"deleted_monitor_id": 12742271}'
     headers: {}
     status:
       code: 200
@@ -44,11 +68,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-0000000000000000643a0edc743014ee-ffa2030a825c4c33-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '18420288769570393139'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '7222101292446913774'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380049?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742274?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380049}'
+      string: '{"deleted_monitor_id": 12742274}'
     headers: {}
     status:
       code: 200
@@ -62,11 +98,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-000000000000000043fa3a09cfef7e95-aa4c066c97edab4e-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '12271190148152929102'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '4898291358536531605'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380051?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742275?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380051}'
+      string: '{"deleted_monitor_id": 12742275}'
     headers: {}
     status:
       code: 200
@@ -80,11 +128,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-0000000000000000378e81de1d16febc-d457418428aa4478-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '15300770294888285304'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '4003279909749718716'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380054?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742273?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380054}'
+      string: '{"deleted_monitor_id": 12742273}'
     headers: {}
     status:
       code: 200
@@ -98,11 +158,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-000000000000000099a7d55763f9b1dd-ef3b7d8247e0381e-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '17238509997195802654'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '11072052780229833181'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380050?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742269?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380050}'
+      string: '{"deleted_monitor_id": 12742269}'
     headers: {}
     status:
       code: 200
@@ -116,11 +188,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-0000000000000000e52fb0ce368f4567-b10aa7a25a76b11c-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '12757193210224619804'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '16514612758316008807'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380048?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742270?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380048}'
+      string: '{"deleted_monitor_id": 12742270}'
     headers: {}
     status:
       code: 200
@@ -134,29 +218,23 @@ interactions:
       - gzip, deflate
       Content-Type:
       - application/json
+      traceparent:
+      - 00-0000000000000000667832788d872316-724b1ae6138983bf-01
+      tracestate:
+      - dd=s:1;t.dm:-0
+      x-datadog-parent-id:
+      - '8235705919049663423'
+      x-datadog-sampling-priority:
+      - '1'
+      x-datadog-tags:
+      - _dd.p.dm=-0
+      x-datadog-trace-id:
+      - '7383707082425836310'
     method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380046?force=True
+    uri: https://api.datadoghq.eu/api/v1/monitor/12742268?force=True
   response:
     body:
-      string: '{"deleted_monitor_id": 9380046}'
-    headers: {}
-    status:
-      code: 200
-      message: OK
-- request:
-    body: null
-    headers:
-      Accept:
-      - '*/*'
-      Accept-Encoding:
-      - gzip, deflate
-      Content-Type:
-      - application/json
-    method: DELETE
-    uri: https://api.datadoghq.eu/api/v1/monitor/9380052?force=True
-  response:
-    body:
-      string: '{"deleted_monitor_id": 9380052}'
+      string: '{"deleted_monitor_id": 12742268}'
     headers: {}
     status:
       code: 200

tests/integration/resources/cassettes/test_monitors/TestMonitorsResources.test_resource_import.frozen

@@ -1 +1 @@
-2023-01-04T16:34:39.476177-05:00
+2023-08-24T10:09:42.606412-04:00