Skip to content

Commit

Permalink
all_fields_test, update streams to test, move upsert method to missin…
Browse files Browse the repository at this point in the history 
…g fields, document in jira (#233)
  • Loading branch information
@bhtowles
bhtowles committed Oct 26, 2023
1 parent 7a94090 commit d9f361e
Show file tree
Hide file tree
Showing 2 changed files with 199 additions and 223 deletions.
218 changes: 0 additions & 218 deletions tests/base_new_frmwrk.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

import os
from datetime import timedelta
from tap_tester import connections, menagerie, runner, LOGGER
Expand Down Expand Up @@ -150,223 +149,6 @@ def set_replication_methods(self, conn_id, catalogs, replication_methods):
connections.set_non_discoverable_metadata(
conn_id, catalog, menagerie.get_annotated_schema(conn_id, catalog['stream_id']), replication_md)

### Method to return the fields that are upsert only -
### exclude the non-upsert fields from the all fields test as these non-upsert fields are not replicated
def get_upsert_only_fields(self, selected_fields, stream=None):
non_upsert_fields = {
"ads_insights" : {
'outbound_clicks',
'conversions',
'cost_per_action_type',
'video_p100_watched_actions',
'video_p75_watched_actions',
'action_values',
'video_30_sec_watched_actions',
'canvas_avg_view_percent',
'video_p50_watched_actions',
'video_p25_watched_actions',
'video_play_curve_actions',
'conversion_values',
'unique_outbound_clicks',
'canvas_avg_view_time',
'cost_per_unique_action_type'
},
"campaigns" : {
'adlabels'
},
"adsets" : {
'adlabels',
'bid_info'
},
"adcreative" : {
'image_crops',
'product_set_id',
'url_tags',
'applink_treatment',
'object_id',
'link_og_id',
'template_url',
'template_url_spec',
'object_url',
'link_url',
'adlabels',
'instagram_story_id'
},
"ads_insights_country": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_age_and_gender": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_dma": {
'video_p75_watched_actions',
'conversions',
'cost_per_unique_click',
'inline_link_click_ctr',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_ctr',
'unique_outbound_clicks',
'unique_inline_link_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'unique_link_clicks_ctr',
'video_p50_watched_actions',
'engagement_rate_ranking',
'unique_inline_link_click_ctr',
'video_p25_watched_actions',
'quality_ranking',
'cpp',
'video_play_curve_actions',
'canvas_avg_view_time',
'video_30_sec_watched_actions',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_region": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'conversion_rate_ranking'
},
"ads_insights_hourly_advertiser": {
'conversions',
'cost_per_estimated_ad_recallers',
'cost_per_unique_click',
'cost_per_unique_outbound_click',
'frequency',
'conversion_values',
'canvas_avg_view_percent',
'cost_per_conversion',
'cost_per_thruplay',
'action_values',
'full_view_impressions',
'place_page_name',
'instant_experience_outbound_clicks',
'cost_per_unique_action_type',
'estimated_ad_recallers',
'outbound_clicks',
'social_spend',
'cost_per_unique_inline_link_click',
'instant_experience_clicks_to_start',
'attribution_setting',
'engagement_rate_ranking',
'purchase_roas',
'reach',
'cost_per_outbound_click',
'estimated_ad_recall_rate',
'quality_ranking',
'cpp',
'catalog_segment_value',
'canvas_avg_view_time',
'cost_per_action_type',
'outbound_clicks_ctr',
'qualifying_question_qualify_answer_rate',
'converted_product_quantity',
'converted_product_value',
'instant_experience_clicks_to_open',
'conversion_rate_ranking'
},
"ads": {
'bid_amount',
'bid_info',
'recommendations',
'adlabels'
},
"ads_insights_platform_and_device": {
'video_p75_watched_actions',
'conversions',
'conversion_values',
'canvas_avg_view_percent',
'action_values',
'unique_outbound_clicks',
'cost_per_unique_action_type',
'outbound_clicks',
'social_spend',
'video_p50_watched_actions',
'engagement_rate_ranking',
'video_p25_watched_actions',
'quality_ranking',
'video_play_curve_actions',
'conversion_rate_ranking',
'video_30_sec_watched_actions',
'canvas_avg_view_time',
'cost_per_action_type',
'video_p100_watched_actions',
'placement'
}
}
actual_expected = {}
non_upsert_streams = non_upsert_fields.keys()

if stream:
if stream in non_upsert_streams:
actual_expected[stream] = selected_fields.get(stream).difference(non_upsert_fields.get(stream))
else:
actual_expected[stream] = selected_fields.get(stream)
else:
for key, fields in selected_fields.items():
if key in non_upsert_streams:
actual_expected[key] = fields.difference(non_upsert_fields.get(key))
else:
actual_expected[key] = fields
return actual_expected

@classmethod
def setUpClass(cls,logging="Ensuring environment variables are sourced."):
super().setUpClass(logging=logging)
Expand Down
Loading

0 comments on commit d9f361e

Please sign in to comment.