Skip to content

Commit

Permalink
Tdl 9685 (#227)
Browse files Browse the repository at this point in the history 
* All fields Test using new framework

* Add method to exclude the non-upsert fields

* Implemented PR Review comments

* excluded the problematic streams
  • Loading branch information
@bhuvana-talend
bhuvana-talend committed Oct 18, 2023
1 parent 37274c3 commit 7a94090
Show file tree
Hide file tree
Showing 2 changed files with 249 additions and 1 deletion.
219 changes: 218 additions & 1 deletion tests/base_new_frmwrk.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def get_properties(self):
'end_date': '2021-04-09T00:00:00Z',
'insights_buffer_days': '1',
}

@staticmethod
def get_credentials():
"""Authentication information for the test account"""
Expand Down Expand Up @@ -150,6 +150,223 @@ def set_replication_methods(self, conn_id, catalogs, replication_methods):
connections.set_non_discoverable_metadata(
conn_id, catalog, menagerie.get_annotated_schema(conn_id, catalog['stream_id']), replication_md)

### Method to return the fields that are upsert only -
### exclude the non-upsert fields from the all fields test as these non-upsert fields are not replicated
def get_upsert_only_fields(self, selected_fields, stream=None):
non_upsert_fields = {
"ads_insights" : {
'outbound_clicks',
'conversions',
'cost_per_action_type',
'video_p100_watched_actions',
'video_p75_watched_actions',
'action_values',
'video_30_sec_watched_actions',
'canvas_avg_view_percent',
'video_p50_watched_actions',
'video_p25_watched_actions',
'video_play_curve_actions',
'conversion_values',
'unique_outbound_clicks',
'canvas_avg_view_time',
'cost_per_unique_action_type'
},
"campaigns" : {
'adlabels'
},
"adsets" : {
'adlabels',
'bid_info'
},
"adcreative" : {
'image_crops',
'product_set_id',
'url_tags',
'applink_treatment',
'object_id',
'link_og_id',
'template_url',
'template_url_spec',
'object_url',
'link_url',
'adlabels',
'instagram_story_id'
},
"ads_insights_country": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_age_and_gender": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_dma": {
'video_p75_watched_actions',
'conversions',
'cost_per_unique_click',
'inline_link_click_ctr',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_ctr',
'unique_outbound_clicks',
'unique_inline_link_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'unique_link_clicks_ctr',
'video_p50_watched_actions',
'engagement_rate_ranking',
'unique_inline_link_click_ctr',
'video_p25_watched_actions',
'quality_ranking',
'cpp',
'video_play_curve_actions',
'canvas_avg_view_time',
'video_30_sec_watched_actions',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_region": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_hourly_advertiser": {
'conversions',
'cost_per_estimated_ad_recallers',
'cost_per_unique_click',
'cost_per_unique_outbound_click',
'frequency',
'conversion_values',
'canvas_avg_view_percent',
'cost_per_conversion',
'cost_per_thruplay',
'action_values',
'full_view_impressions',
'place_page_name',
'instant_experience_outbound_clicks',
'cost_per_unique_action_type',
'estimated_ad_recallers',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'instant_experience_clicks_to_start',
'attribution_setting',
'engagement_rate_ranking',
'purchase_roas',
'reach',
'cost_per_outbound_click',
'estimated_ad_recall_rate',
'quality_ranking',
'cpp',
'catalog_segment_value',
'canvas_avg_view_time',
'cost_per_action_type',
'outbound_clicks_ctr',
'qualifying_question_qualify_answer_rate',
'converted_product_quantity',
'converted_product_value',
'instant_experience_clicks_to_open',
'conversion_rate_ranking'
},
"ads": {
'bid_amount',
'bid_info',
'recommendations',
'adlabels'
},
"ads_insights_platform_and_device": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'conversion_rate_ranking',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'placement'
}
}
actual_expected = {}
non_upsert_streams = non_upsert_fields.keys()

if stream:
if stream in non_upsert_streams:
actual_expected[stream] = selected_fields.get(stream).difference(non_upsert_fields.get(stream))
else:
actual_expected[stream] = selected_fields.get(stream)
else:
for key, fields in selected_fields.items():
if key in non_upsert_streams:
actual_expected[key] = fields.difference(non_upsert_fields.get(key))
else:
actual_expected[key] = fields
return actual_expected

@classmethod
def setUpClass(cls,logging="Ensuring environment variables are sourced."):
super().setUpClass(logging=logging)
Expand Down
31 changes: 31 additions & 0 deletions tests/test_facebook_all_fields.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
"""
Test that with no fields selected for a stream all fields are still replicated
"""

from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest
from base_new_frmwrk import FacebookBaseTest
import base


class FacebookAllFieldsTest(AllFieldsTest, FacebookBaseTest):
"""Test that with no fields selected for a stream all fields are still replicated"""
is_done = None

@staticmethod
def name():
return "tt_facebook_all_fields_test"

def streams_to_test(self):
#return set(self.expected_metadata().keys())
# Fail the test when the JIRA card is done to allow stream to be re-added and tested
if self.is_done is None:
self.is_done = base.JIRA_CLIENT.get_status_category("TDL-24312") == 'done'
self.assert_message = ("JIRA ticket has moved to done, re-add the "
"ads_insights_hourly_advertiser stream to the test.")
assert self.is_done != True, self.assert_message

return self.expected_metadata().keys() - {'ads_insights_hourly_advertiser', 'ads_insights'}

def test_all_fields_for_streams_are_replicated(self):
self.selected_fields = self.get_upsert_only_fields(AllFieldsTest.selected_fields)
super().test_all_fields_for_streams_are_replicated()

0 comments on commit 7a94090

Please sign in to comment.