Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

When catalog is missing, discover the catalog, and auto enable selected streams from the config file selected option list #77

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

When catalog is missing, discover the catalog, and auto enable selected streams from the config file selected option list #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d26c96
When catalog is missing, discover the catalog, and auto enable select…
ome9ax May 7, 2022
5957631
Singer Tap tap-google-sheets Auto-selected ingestion: config default …
ome9ax May 10, 2022
85d9ebb
Singer Tap tap-google-sheets Auto-selected ingestion: no catalog requ…
ome9ax May 13, 2022
38fb30d
Singer Tap tap-google-sheets Auto-selected ingestion: fully automated
ome9ax May 13, 2022
00c9550
Singer Tap tap-google-sheets Auto-selected do_discover returns None
ome9ax May 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions tap_google_sheets/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
'user_agent'
]

def do_discover(client, spreadsheet_id):
def do_discover(client, spreadsheet_id, config = None):

LOGGER.info('Starting discover')
catalog = discover(client, spreadsheet_id)
catalog = discover(client, spreadsheet_id, config)
json.dump(catalog.to_dict(), sys.stdout, indent=2)
LOGGER.info('Finished discover')

Expand All @@ -48,11 +48,16 @@ def main():
spreadsheet_id = config.get('spreadsheet_id')

if parsed_args.discover:
do_discover(client, spreadsheet_id)
do_discover(client, spreadsheet_id, config)
elif parsed_args.catalog:
sync(client=client,
config=config,
catalog=parsed_args.catalog,
catalog=parsed_args.catalog or discover(client, spreadsheet_id, config),
state=state)
else:
sync(client=client,
config=config,
catalog=discover(client, spreadsheet_id, config),
state=state)

if __name__ == '__main__':
Expand Down
6 changes: 4 additions & 2 deletions tap_google_sheets/discover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from singer.catalog import Catalog, CatalogEntry, Schema
from singer.metadata import to_map, write, to_list
from tap_google_sheets.schema import STREAMS


def discover(client, spreadsheet_id):
def discover(client, spreadsheet_id, config = None):
catalog = Catalog([])

for stream, stream_obj in STREAMS.items():
Expand Down Expand Up @@ -34,7 +35,8 @@ def discover(client, spreadsheet_id):
tap_stream_id=stream_name,
key_properties=key_props,
schema=schema,
metadata=mdata
metadata=to_list(write(to_map(mdata), (), 'selected', True)) \
if stream_name in (config or {}).get('selected', []) else mdata
))

return catalog