-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for logs_archives resource (#270)
* add support for logs_archives resources
- Loading branch information
Showing
22 changed files
with
611 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| # Unless explicitly stated otherwise all files in this repository are licensed | ||
| # under the 3-clause BSD style license (see LICENSE). | ||
| # This product includes software developed at Datadog (https://www.datadoghq.com/). | ||
| # Copyright 2019 Datadog, Inc. | ||
|
|
||
| from __future__ import annotations | ||
| from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, cast | ||
|
|
||
| from datadog_sync.utils.base_resource import BaseResource, ResourceConfig | ||
|
|
||
| if TYPE_CHECKING: | ||
| from datadog_sync.utils.custom_client import CustomClient | ||
|
|
||
|
|
||
| class LogsArchives(BaseResource): | ||
| resource_type = "logs_archives" | ||
| resource_config = ResourceConfig( | ||
| base_path="/api/v2/logs/config/archives", | ||
| excluded_attributes=["id", "attributes.state"], | ||
| ) | ||
| # Additional LogsArchives specific attributes | ||
|
|
||
| async def get_resources(self, client: CustomClient) -> List[Dict]: | ||
| resp = await client.get(self.resource_config.base_path) | ||
|
|
||
| return resp["data"] | ||
|
|
||
| async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: | ||
| if _id: | ||
| source_client = self.config.source_client | ||
| resource = (await source_client.get(self.resource_config.base_path + f"/{_id}"))["data"] | ||
| resource = cast(dict, resource) | ||
|
|
||
| return resource["id"], resource | ||
|
|
||
| async def pre_resource_action_hook(self, _id, resource: Dict) -> None: | ||
| pass | ||
|
|
||
| async def pre_apply_hook(self) -> None: | ||
| pass | ||
|
|
||
| async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: | ||
| destination_client = self.config.destination_client | ||
| payload = {"data": resource} | ||
| resp = await destination_client.post(self.resource_config.base_path, payload) | ||
|
|
||
| return _id, resp["data"] | ||
|
|
||
| async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: | ||
| destination_client = self.config.destination_client | ||
| payload = {"data": resource} | ||
| resp = await destination_client.put( | ||
| self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}", | ||
| payload, | ||
| ) | ||
|
|
||
| self.config.state.destination[self.resource_type][_id] = resp["data"] | ||
| return _id, resp["data"] | ||
|
|
||
| async def delete_resource(self, _id: str) -> None: | ||
| destination_client = self.config.destination_client | ||
| await destination_client.delete( | ||
| self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}" | ||
| ) | ||
|
|
||
| def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]: | ||
| pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| # Unless explicitly stated otherwise all files in this repository are licensed | ||
| # under the 3-clause BSD style license (see LICENSE). | ||
| # This product includes software developed at Datadog (https://www.datadoghq.com/). | ||
| # Copyright 2019 Datadog, Inc. | ||
|
|
||
| from __future__ import annotations | ||
| from typing import TYPE_CHECKING, Optional, List, Dict, Tuple | ||
| from copy import deepcopy | ||
|
|
||
| from deepdiff.operator import BaseOperator | ||
|
|
||
| from datadog_sync.utils.base_resource import BaseResource, ResourceConfig | ||
|
|
||
| if TYPE_CHECKING: | ||
| from datadog_sync.utils.custom_client import CustomClient | ||
|
|
||
|
|
||
| class LogsArchivesOrderIdsComparator(BaseOperator): | ||
| def match(self, level): | ||
| if "archive_ids" in level.t1 and "archive_ids" in level.t2: | ||
| # make copy so we do not mutate the original | ||
| level.t1 = deepcopy(level.t1) | ||
| level.t2 = deepcopy(level.t2) | ||
|
|
||
| # If we are at the top level, modify the list to exclude extra archives in destination. | ||
| t1 = set(level.t1["archive_ids"]) | ||
| t2 = set(level.t2["archive_ids"]) | ||
| d_ignore = t1 - t2 | ||
|
|
||
| level.t1["archive_ids"] = [_id for _id in level.t1["archive_ids"] if _id not in d_ignore] | ||
| return True | ||
|
|
||
| def give_up_diffing(self, level, diff_instance) -> bool: | ||
| return False | ||
|
|
||
|
|
||
| class LogsArchivesOrder(BaseResource): | ||
| resource_type = "logs_archives_order" | ||
| resource_config = ResourceConfig( | ||
| concurrent=False, | ||
| base_path="/api/v2/logs/config/archive-order", | ||
| resource_connections={ | ||
| "logs_archives": ["data.attributes.archive_ids"], | ||
| }, | ||
| deep_diff_config={ | ||
| "ignore_order": False, | ||
| "custom_operators": [LogsArchivesOrderIdsComparator()], | ||
| }, | ||
| ) | ||
| # Additional LogsArchivesOrder specific attributes | ||
| destination_archives_order: Dict[str, Dict] = dict() | ||
| default_id: str = "logs-archives-order" | ||
|
|
||
| async def get_resources(self, client: CustomClient) -> List[Dict]: | ||
| resp = await client.get(self.resource_config.base_path) | ||
|
|
||
| return [resp] | ||
|
|
||
| async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: | ||
| if _id: | ||
| source_client = self.config.source_client | ||
| resource = await source_client.get(self.resource_config.base_path) | ||
|
|
||
| return self.default_id, resource | ||
|
|
||
| async def pre_resource_action_hook(self, _id, resource: Dict) -> None: | ||
| self.destination_archives_order = await self.get_destination_archives_order() | ||
|
|
||
| async def pre_apply_hook(self) -> None: | ||
| pass | ||
|
|
||
| async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: | ||
| if not self.destination_archives_order: | ||
| raise Exception("Failed to retrieve destination orgs logs archive order") | ||
|
|
||
| self.config.state.destination[self.resource_type][_id] = self.destination_archives_order | ||
| return await self.update_resource(_id, resource) | ||
|
|
||
| async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: | ||
| destination_resources = ( | ||
| self.destination_archives_order or self.config.state.destination[self.resource_type][_id] | ||
| ) | ||
| ids_to_omit = set(resource["data"]["attributes"]["archive_ids"]) - set( | ||
| destination_resources["data"]["attributes"]["archive_ids"] | ||
| ) | ||
|
|
||
| extra_ids_to_include = [ | ||
| _id | ||
| for _id in destination_resources["data"]["attributes"]["archive_ids"] | ||
| if _id not in resource["data"]["attributes"]["archive_ids"] | ||
| ] | ||
|
|
||
| resource["data"]["attributes"]["archive_ids"] = [ | ||
| _id for _id in resource["data"]["attributes"]["archive_ids"] if _id not in ids_to_omit | ||
| ] | ||
| resource["data"]["attributes"]["archive_ids"] = ( | ||
| resource["data"]["attributes"]["archive_ids"] + extra_ids_to_include | ||
| ) | ||
|
|
||
| destination_client = self.config.destination_client | ||
| resp = await destination_client.put(self.resource_config.base_path, resource) | ||
|
|
||
| return _id, resp | ||
|
|
||
| async def delete_resource(self, _id: str) -> None: | ||
| self.config.logger.warning("logs_archives_order cannot deleted. Removing resource from config only.") | ||
|
|
||
| def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]: | ||
| return super(LogsArchivesOrder, self).connect_id(key, r_obj, resource_to_connect) | ||
|
|
||
| async def get_destination_archives_order(self): | ||
| destination_client = self.config.destination_client | ||
| resp = await self.get_resources(destination_client) | ||
|
|
||
| return resp[0] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
1 change: 1 addition & 0 deletions
1
...rces/cassettes/test_logs_archives/TestLogsArchivesResources.test_no_resource_diffs.frozen
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 2024-09-04T13:11:22.480442-04:00 |
1 change: 1 addition & 0 deletions
1
...urces/cassettes/test_logs_archives/TestLogsArchivesResources.test_resource_cleanup.frozen
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 2024-09-04T13:11:22.498371-04:00 |
30 changes: 30 additions & 0 deletions
30
...sources/cassettes/test_logs_archives/TestLogsArchivesResources.test_resource_cleanup.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| interactions: | ||
| - request: | ||
| body: null | ||
| headers: | ||
| Content-Type: | ||
| - application/json | ||
| method: DELETE | ||
| uri: https://api.datadoghq.eu/api/v2/logs/config/archives/Zulo48JVQXSyM97uKjGHpA | ||
| response: | ||
| body: | ||
| string: '' | ||
| headers: {} | ||
| status: | ||
| code: 204 | ||
| message: No Content | ||
| - request: | ||
| body: null | ||
| headers: | ||
| Content-Type: | ||
| - application/json | ||
| method: DELETE | ||
| uri: https://api.datadoghq.eu/api/v2/logs/config/archives/3kkgEN2qQ7i_qfP0yFZx5g | ||
| response: | ||
| body: | ||
| string: '' | ||
| headers: {} | ||
| status: | ||
| code: 204 | ||
| message: No Content | ||
| version: 1 |
1 change: 1 addition & 0 deletions
1
...ources/cassettes/test_logs_archives/TestLogsArchivesResources.test_resource_import.frozen
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 2024-09-04T13:11:21.025343-04:00 |
29 changes: 29 additions & 0 deletions
29
...esources/cassettes/test_logs_archives/TestLogsArchivesResources.test_resource_import.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| interactions: | ||
| - request: | ||
| body: null | ||
| headers: | ||
| Content-Type: | ||
| - application/json | ||
| method: GET | ||
| uri: https://api.datadoghq.com/api/v2/logs/config/archives | ||
| response: | ||
| body: | ||
| string: '{"data": [{"type": "archives", "id": "V49TnL93R0C3QADZQllO5Q", "attributes": | ||
| {"name": "my first s3 archive", "query": "service:tutu", "state": "UNKNOWN", | ||
| "destination": {"bucket": "my-bucket", "path": "/path/foo", "type": "s3", | ||
| "integration": {"role_name": "testacc-datadog-integration-role", "account_id": | ||
| "123456789112"}}, "rehydration_tags": ["team:intake", "team:app"], "include_tags": | ||
| true, "rehydration_max_scan_size_in_gb": 123}}, {"type": "archives", "id": | ||
| "RK1PeXaNRButwKNMn_dRJQ", "attributes": {"name": "my first azure archive", | ||
| "query": "service:toto", "state": "UNKNOWN", "destination": {"container": | ||
| "my-container", "storage_account": "storageaccount", "path": "/path/blou", | ||
| "type": "azure", "integration": {"tenant_id": "92f7df25-f9d7-4e76-a3b6-4011e64958ea", | ||
| "client_id": "a75fbdd2-ade6-43d0-a810-4d886c53871e"}}, "rehydration_tags": | ||
| [], "include_tags": false, "rehydration_max_scan_size_in_gb": null}}]}' | ||
| headers: | ||
| Content-Type: | ||
| - application/json | ||
| status: | ||
| code: 200 | ||
| message: OK | ||
| version: 1 |
1 change: 1 addition & 0 deletions
1
...esources/cassettes/test_logs_archives/TestLogsArchivesResources.test_resource_sync.frozen
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 2024-09-04T13:11:21.284161-04:00 |
Oops, something went wrong.