SAC-31205 | Add on-premises GitLab instance support via api_url config field

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+
+# 1.1.0
+  * Add optional `api_url` config field to support on-premises GitLab instances; defaults to `https://gitlab.com`. [#51](https://github.com/singer-io/tap-gitlab/pull/51)
+  * Bump singer-python to 6.8.0 and requests to 2.34.2.
+
 # 1.0.1
   * Bump requests to 2.33.0 for security updates [#50](https://github.com/singer-io/tap-gitlab/pull/50)
 

README.md

@@ -35,7 +35,7 @@ This tap:
 
     Create a JSON file called `config.json` containing:
     - Access token you just created
-    - API URL for your GitLab account. If you are using the public gitlab.com this will be `https://gitlab.com/api/v3`
+    - *(Optional)* `api_url` — the base URL of your GitLab instance, **without** the `/api/v4` path (e.g. `https://gitlab.mycompany.com`). Defaults to `https://gitlab.com`. Set this only when connecting to a **self-hosted / on-premises** GitLab server. The tap appends `/api/v4` automatically.
     - Groups to track (space separated)
     - Projects to track (space separated)
 
@@ -48,6 +48,7 @@ This tap:
     - 'groups' contains space separated list of groups id.
     - 'projects' contains space separated list of projects id.
 
+    GitLab Cloud (default):
     ```json
     {
         "private_token": "your-access-token",
@@ -57,6 +58,17 @@ This tap:
     }
     ```
 
+    Self-hosted / on-premises GitLab:
+    ```json
+    {
+        "private_token": "your-access-token",
+        "api_url": "https://gitlab.mycompany.com",
+        "groups": "myorg mygroup",
+        "projects": "myorg/repo-a myorg/repo-b",
+        "start_date": "2018-01-01T00:00:00Z"
+    }
+    ```
+
 4. [Optional] Create the initial state file
 
     You can provide JSON file that contains a date for the API endpoints

setup.py

@@ -5,14 +5,14 @@
 
 setup(
     name='tap-gitlab',
-    version='1.0.1',
+    version='1.1.0',
     description='Singer.io tap for extracting data from the GitLab API',
     author='Stitch',
     url='https://singer.io',
     classifiers=['Programming Language :: Python :: 3 :: Only'],
     install_requires=[
-        'singer-python==6.1.1',
-        'requests==2.33.0',
+        'singer-python==6.8.0',
+        'requests==2.34.2',
         'backoff==2.2.1'
     ],
     entry_points='''

tap_gitlab/client.py

@@ -1,4 +1,5 @@
 from typing import Any, Dict, Mapping, Optional, Tuple
+import re
 
 import backoff
 import requests
@@ -75,7 +76,15 @@ class Client:
     def __init__(self, config: Mapping[str, Any]) -> None:
         self.config = config
         self._session = session()
-        self.base_url = "https://gitlab.com/api/v4"
+
+        api_url = (config.get("api_url") or "").strip() or "https://gitlab.com"
+        api_url = api_url.rstrip("/")
+        if not api_url.startswith(("http://", "https://")):
+            api_url = f"https://{api_url}"
+        # Strip any existing /api/vN suffix so both 'https://gitlab.com' and
+        # 'https://gitlab.com/api/v4' are accepted without producing a double suffix.
+        api_url = re.sub(r"/api/v\d+$", "", api_url)
+        self.base_url = f"{api_url}/api/v4"
 
         config_request_timeout = config.get("request_timeout")
         self.request_timeout = float(config_request_timeout) if config_request_timeout else REQUEST_TIMEOUT
@@ -93,12 +102,20 @@ def check_api_credentials(self) -> None:
         params = {'private_token': self.config.get("private_token")}
         endpoint = f"{self.base_url}/user"
 
-        response = self._session.get(
-            endpoint,
-            params=params,
-            headers=headers,
-            timeout=self.request_timeout
-        )
+        try:
+            response = self._session.get(
+                endpoint,
+                params=params,
+                headers=headers,
+                timeout=self.request_timeout
+            )
+        except ConnectionError as e:
+            raise ConnectionError(
+                f"Unable to reach GitLab at '{self.base_url}'. "
+                "Please verify that 'api_url' in your config is correct and the host is reachable. "
+                f"({type(e).__name__})"
+            ) from e
+
         raise_for_error(response)
 
         user_data = response.json()

tests/test_all_fields.py

@@ -9,7 +9,6 @@ class AllFields(AllFieldsTest, BaseTest):
     MISSING_FIELDS = {
         "groups": {
             "repository_storage",
-            "ip_restriction_ranges"
         },
         "issues": {
             'assignee_id',
@@ -29,11 +28,6 @@ class AllFields(AllFieldsTest, BaseTest):
             'owner',
             'repository_storage',
             'mirror_trigger_builds',
-            'ci_restrict_pipeline_cancellation_role',
-            'approvals_before_merge',
-            'mirror',
-            'allow_pipeline_trigger_approve_deployment',
-            'secret_push_protection_enabled',
         }
     }
 

tests/unittests/test_client.py

@@ -100,3 +100,88 @@ def test_generic_http_error(self, mock_request, mock_auth, mock_check_creds, moc
         with self.assertRaises(Error):
             with Client(config) as client:
                 client.get(endpoint="dummy", params={}, headers={})
+
+
+class TestClientBaseUrl(unittest.TestCase):
+
+    def test_default_base_url_uses_gitlab_cloud(self):
+        """When api_url is absent the client targets gitlab.com."""
+        config = {"private_token": "dummy_token"}
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.com/api/v4")
+
+    def test_empty_api_url_falls_back_to_gitlab_cloud(self):
+        """An empty or whitespace-only api_url must fall back to gitlab.com."""
+        for value in ["", "   ", None]:
+            with self.subTest(api_url=value):
+                config = {"private_token": "dummy_token", "api_url": value}
+                client = Client(config)
+                self.assertEqual(client.base_url, "https://gitlab.com/api/v4")
+
+    def test_custom_api_url_sets_base_url(self):
+        """When api_url is set the client targets the on-prem instance."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "https://gitlab.mycompany.com",
+        }
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4")
+
+    def test_trailing_slash_in_api_url_is_stripped(self):
+        """Trailing slashes in api_url must not produce a double-slash in base_url."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "https://gitlab.mycompany.com/",
+        }
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4")
+
+    def test_api_url_without_scheme_defaults_to_https(self):
+        """An api_url with no scheme (e.g. 'gitlab.mycompany.com') must get https:// prepended."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "gitlab.mycompany.com",
+        }
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4")
+
+    def test_api_url_with_port_and_no_scheme_gets_https(self):
+        """host:port without a scheme must not be mis-detected by urlparse and must get https://."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "gitlab.mycompany.com:8443",
+        }
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.mycompany.com:8443/api/v4")
+
+    def test_api_url_with_existing_api_v4_suffix_is_not_doubled(self):
+        """Providing a full API base URL must not produce /api/v4/api/v4."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "https://gitlab.mycompany.com/api/v4",
+        }
+        client = Client(config)
+        self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4")
+
+
+class TestCheckApiCredentials(unittest.TestCase):
+
+    @patch("requests.Session.get", side_effect=ConnectionError("Failed to resolve 'test.gitlab.com'"))
+    def test_unreachable_host_raises_friendly_connection_error(self, mock_get):
+        """A DNS/network failure in check_api_credentials raises a descriptive ConnectionError
+        that does not leak the raw urllib3 message (which may contain the private_token URL)."""
+        config = {
+            "private_token": "dummy_token",
+            "api_url": "https://test.gitlab.com",
+        }
+        client = Client(config)
+        with self.assertRaises(ConnectionError) as ctx:
+            client.check_api_credentials()
+        msg = str(ctx.exception)
+        self.assertIn("Unable to reach GitLab", msg)
+        # base_url (no token) is shown, not the raw original error
+        self.assertIn("test.gitlab.com", msg)
+        self.assertIn("api_url", msg)
+        # Raw urllib3 message must not be included to avoid token leakage
+        self.assertNotIn("Failed to resolve", msg)
+        self.assertNotIn("private_token", msg)