Added implementation to exclude unauthorized streams from discovery

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 
+# 1.2.0
+  * Streams returning 403 Forbidden during discovery are now excluded from the catalog; discovery fails only if none are accessible. [#52](https://github.com/singer-io/tap-gitlab/pull/52)
 # 1.1.0
   * Add optional `api_url` config field to support on-premises GitLab instances; defaults to `https://gitlab.com`. [#51](https://github.com/singer-io/tap-gitlab/pull/51)
   * Bump singer-python to 6.8.0 and requests to 2.34.2.

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='tap-gitlab',
-    version='1.1.0',
+    version='1.2.0',
     description='Singer.io tap for extracting data from the GitLab API',
     author='Stitch',
     url='https://singer.io',

tap_gitlab/__init__.py

@@ -11,12 +11,12 @@
 REQUIRED_CONFIG_KEYS = ["private_token", "start_date", "groups", "projects"]
 
 
-def do_discover():
+def do_discover(client):
     """
     Discover and emit the catalog to stdout
     """
     LOGGER.info("Starting discover")
-    catalog = discover()
+    catalog = discover(client)
     json.dump(catalog.to_dict(), sys.stdout, indent=2)
     LOGGER.info("Finished discover")
 
@@ -33,7 +33,7 @@ def main():
 
     with Client(parsed_args.config) as client:
         if parsed_args.discover:
-            do_discover()
+            do_discover(client)
         elif parsed_args.catalog:
             sync(
                 client=client,

tap_gitlab/discover.py

@@ -2,15 +2,67 @@
 from singer import metadata
 from singer.catalog import Catalog, CatalogEntry, Schema
 from tap_gitlab.schema import get_schemas
+from tap_gitlab.streams import STREAMS
+from tap_gitlab.exceptions import ForbiddenError
 
 LOGGER = singer.get_logger()
 
 
-def discover() -> Catalog:
+def _prune_inaccessible_children(schemas: dict, field_metadata: dict) -> None:
+    """
+    Remove child streams from the catalog whose parent stream was excluded.
+    Mutates schemas and field_metadata in place.
+    """
+    for name, stream_cls in list(STREAMS.items()):
+        parent = getattr(stream_cls, "parent", None)
+        if name in schemas and parent and parent not in schemas:
+            LOGGER.warning(
+                "Stream '%s' excluded from catalog because its parent stream '%s' is not accessible.",
+                name, parent,
+            )
+            schemas.pop(name, None)
+            field_metadata.pop(name, None)
+
+
+def _apply_access_checks(client, schemas: dict, field_metadata: dict) -> None:
+    """
+    Probe each stream for read access and remove inaccessible streams
+    (and their children) from schemas and field_metadata in place.
+    Raises ForbiddenError if no parent streams are accessible.
+    """
+    inaccessible_streams = [
+        stream_name
+        for stream_name, stream_cls in STREAMS.items()
+        if stream_name in schemas
+        and not stream_cls(client=client).check_access()
+    ]
+
+    for stream_name in inaccessible_streams:
+        schemas.pop(stream_name, None)
+        field_metadata.pop(stream_name, None)
+
+    _prune_inaccessible_children(schemas, field_metadata)
+
+    if not schemas:
+        raise ForbiddenError(
+            "No streams are accessible. Ensure the credentials have read permission for at least one stream."
+        )
+    elif inaccessible_streams:
+        LOGGER.warning(
+            "These streams have been excluded due to HTTP-Error-Code:403 Forbidden: %s",
+            ", ".join(inaccessible_streams),
+        )
+
+
+def discover(client) -> Catalog:
     """
     Run the discovery mode, prepare the catalog file and return the catalog.
+    Access to each stream is verified using the provided client and streams
+    the credentials cannot read are excluded from the returned catalog.
     """
     schemas, field_metadata = get_schemas()
+    _apply_access_checks(client, schemas, field_metadata)
+
     catalog = Catalog([])
 
     for stream_name, schema_dict in schemas.items():

tap_gitlab/streams/abstracts.py

@@ -13,6 +13,8 @@
 from dateutil import parser
 from datetime import datetime, timezone
 
+from tap_gitlab.exceptions import ForbiddenError
+
 LOGGER = get_logger()
 
 
@@ -31,8 +33,8 @@ class BaseStream(ABC):
     def __init__(self, client=None, catalog=None) -> None:
         self.client = client
         self.catalog = catalog
-        self.schema = self.catalog.schema.to_dict() if self.catalog else None
-        self.metadata = metadata.to_map(self.catalog.metadata) if self.catalog else None
+        self.schema = self.catalog.schema.to_dict() if self.catalog else {}
+        self.metadata = metadata.to_map(self.catalog.metadata) if self.catalog else {}
         self.child_to_sync = []
         self.params = {}
 
@@ -116,6 +118,29 @@ def modify_object(self, record: Dict, parent_record: Dict = None) -> Dict:
     def get_url_endpoint(self, parent_obj: Dict = None) -> str:
         return self.url_endpoint or f"{self.client.base_url}/{self.path}"
 
+    def check_access(self) -> bool:
+        """
+        Verify that the API credentials have read access to this stream.
+        Returns True if accessible, False if a 403 Forbidden error is raised.
+        Child streams always return True (access is governed by the parent check).
+        """
+        if self.parent:
+            return True
+
+        url = self.get_url_endpoint()
+        params = {"per_page": 1}
+
+        try:
+            self.client.get(url, params, self.headers, None)
+            return True
+        except ForbiddenError as exc:
+            LOGGER.warning(
+                "Unauthorized Stream: %s, excluding from catalog. HTTP-Error-Message:'%s'",
+                self.tap_stream_id,
+                str(exc),
+            )
+            return False
+
 
 class IncrementalStream(BaseStream):
     def get_bookmark(self, state: dict, stream: str, key: Any = None) -> int:

tests/base.py

@@ -18,6 +18,7 @@ class BaseTest(BaseCase):
     in tap-tester tests. Shared tap-specific methods (as needed).
     """
     start_date = "2019-01-01T00:00:00Z"
+    IS_FORBIDDEN_STREAM = "is-forbidden-stream"
 
     @staticmethod
     def tap_name():
@@ -91,6 +92,14 @@ def expected_metadata(cls):
             }
         }
 
+    def expected_stream_names(self):
+        """The expected stream names and exclude forbidden streams."""
+        return {
+            stream_name
+            for stream_name, metadata in self.expected_metadata().items()
+            if not metadata.get(self.IS_FORBIDDEN_STREAM, False)
+        }
+
     @staticmethod
     def get_credentials():
         """Authentication information for the test account."""