diff --git a/CHANGELOG.md b/CHANGELOG.md index 401a509..522064b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog + +# 1.1.0 + * Add optional `api_url` config field to support on-premises GitLab instances; defaults to `https://gitlab.com`. [#51](https://github.com/singer-io/tap-gitlab/pull/51) + * Bump singer-python to 6.8.0 and requests to 2.34.2. + # 1.0.1 * Bump requests to 2.33.0 for security updates [#50](https://github.com/singer-io/tap-gitlab/pull/50) diff --git a/README.md b/README.md index eed8982..103cdc8 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ This tap: Create a JSON file called `config.json` containing: - Access token you just created - - API URL for your GitLab account. If you are using the public gitlab.com this will be `https://gitlab.com/api/v3` + - *(Optional)* `api_url` — the base URL of your GitLab instance, **without** the `/api/v4` path (e.g. `https://gitlab.mycompany.com`). Defaults to `https://gitlab.com`. Set this only when connecting to a **self-hosted / on-premises** GitLab server. The tap appends `/api/v4` automatically. - Groups to track (space separated) - Projects to track (space separated) @@ -48,6 +48,7 @@ This tap: - 'groups' contains space separated list of groups id. - 'projects' contains space separated list of projects id. + GitLab Cloud (default): ```json { "private_token": "your-access-token", @@ -57,6 +58,17 @@ This tap: } ``` + Self-hosted / on-premises GitLab: + ```json + { + "private_token": "your-access-token", + "api_url": "https://gitlab.mycompany.com", + "groups": "myorg mygroup", + "projects": "myorg/repo-a myorg/repo-b", + "start_date": "2018-01-01T00:00:00Z" + } + ``` + 4. [Optional] Create the initial state file You can provide JSON file that contains a date for the API endpoints diff --git a/setup.py b/setup.py index 65afcb4..543e125 100644 --- a/setup.py +++ b/setup.py @@ -5,14 +5,14 @@ setup( name='tap-gitlab', - version='1.0.1', + version='1.1.0', description='Singer.io tap for extracting data from the GitLab API', author='Stitch', url='https://singer.io', classifiers=['Programming Language :: Python :: 3 :: Only'], install_requires=[ - 'singer-python==6.1.1', - 'requests==2.33.0', + 'singer-python==6.8.0', + 'requests==2.34.2', 'backoff==2.2.1' ], entry_points=''' diff --git a/tap_gitlab/client.py b/tap_gitlab/client.py index 7d46b1b..eb6b8e3 100644 --- a/tap_gitlab/client.py +++ b/tap_gitlab/client.py @@ -1,4 +1,5 @@ from typing import Any, Dict, Mapping, Optional, Tuple +import re import backoff import requests @@ -75,7 +76,15 @@ class Client: def __init__(self, config: Mapping[str, Any]) -> None: self.config = config self._session = session() - self.base_url = "https://gitlab.com/api/v4" + + api_url = (config.get("api_url") or "").strip() or "https://gitlab.com" + api_url = api_url.rstrip("/") + if not api_url.startswith(("http://", "https://")): + api_url = f"https://{api_url}" + # Strip any existing /api/vN suffix so both 'https://gitlab.com' and + # 'https://gitlab.com/api/v4' are accepted without producing a double suffix. + api_url = re.sub(r"/api/v\d+$", "", api_url) + self.base_url = f"{api_url}/api/v4" config_request_timeout = config.get("request_timeout") self.request_timeout = float(config_request_timeout) if config_request_timeout else REQUEST_TIMEOUT @@ -93,12 +102,20 @@ def check_api_credentials(self) -> None: params = {'private_token': self.config.get("private_token")} endpoint = f"{self.base_url}/user" - response = self._session.get( - endpoint, - params=params, - headers=headers, - timeout=self.request_timeout - ) + try: + response = self._session.get( + endpoint, + params=params, + headers=headers, + timeout=self.request_timeout + ) + except ConnectionError as e: + raise ConnectionError( + f"Unable to reach GitLab at '{self.base_url}'. " + "Please verify that 'api_url' in your config is correct and the host is reachable. " + f"({type(e).__name__})" + ) from e + raise_for_error(response) user_data = response.json() diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 769bf4e..4d3d1bc 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -9,7 +9,6 @@ class AllFields(AllFieldsTest, BaseTest): MISSING_FIELDS = { "groups": { "repository_storage", - "ip_restriction_ranges" }, "issues": { 'assignee_id', @@ -29,11 +28,6 @@ class AllFields(AllFieldsTest, BaseTest): 'owner', 'repository_storage', 'mirror_trigger_builds', - 'ci_restrict_pipeline_cancellation_role', - 'approvals_before_merge', - 'mirror', - 'allow_pipeline_trigger_approve_deployment', - 'secret_push_protection_enabled', } } diff --git a/tests/unittests/test_client.py b/tests/unittests/test_client.py index 6e613c5..72ee810 100644 --- a/tests/unittests/test_client.py +++ b/tests/unittests/test_client.py @@ -100,3 +100,88 @@ def test_generic_http_error(self, mock_request, mock_auth, mock_check_creds, moc with self.assertRaises(Error): with Client(config) as client: client.get(endpoint="dummy", params={}, headers={}) + + +class TestClientBaseUrl(unittest.TestCase): + + def test_default_base_url_uses_gitlab_cloud(self): + """When api_url is absent the client targets gitlab.com.""" + config = {"private_token": "dummy_token"} + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.com/api/v4") + + def test_empty_api_url_falls_back_to_gitlab_cloud(self): + """An empty or whitespace-only api_url must fall back to gitlab.com.""" + for value in ["", " ", None]: + with self.subTest(api_url=value): + config = {"private_token": "dummy_token", "api_url": value} + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.com/api/v4") + + def test_custom_api_url_sets_base_url(self): + """When api_url is set the client targets the on-prem instance.""" + config = { + "private_token": "dummy_token", + "api_url": "https://gitlab.mycompany.com", + } + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4") + + def test_trailing_slash_in_api_url_is_stripped(self): + """Trailing slashes in api_url must not produce a double-slash in base_url.""" + config = { + "private_token": "dummy_token", + "api_url": "https://gitlab.mycompany.com/", + } + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4") + + def test_api_url_without_scheme_defaults_to_https(self): + """An api_url with no scheme (e.g. 'gitlab.mycompany.com') must get https:// prepended.""" + config = { + "private_token": "dummy_token", + "api_url": "gitlab.mycompany.com", + } + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4") + + def test_api_url_with_port_and_no_scheme_gets_https(self): + """host:port without a scheme must not be mis-detected by urlparse and must get https://.""" + config = { + "private_token": "dummy_token", + "api_url": "gitlab.mycompany.com:8443", + } + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.mycompany.com:8443/api/v4") + + def test_api_url_with_existing_api_v4_suffix_is_not_doubled(self): + """Providing a full API base URL must not produce /api/v4/api/v4.""" + config = { + "private_token": "dummy_token", + "api_url": "https://gitlab.mycompany.com/api/v4", + } + client = Client(config) + self.assertEqual(client.base_url, "https://gitlab.mycompany.com/api/v4") + + +class TestCheckApiCredentials(unittest.TestCase): + + @patch("requests.Session.get", side_effect=ConnectionError("Failed to resolve 'test.gitlab.com'")) + def test_unreachable_host_raises_friendly_connection_error(self, mock_get): + """A DNS/network failure in check_api_credentials raises a descriptive ConnectionError + that does not leak the raw urllib3 message (which may contain the private_token URL).""" + config = { + "private_token": "dummy_token", + "api_url": "https://test.gitlab.com", + } + client = Client(config) + with self.assertRaises(ConnectionError) as ctx: + client.check_api_credentials() + msg = str(ctx.exception) + self.assertIn("Unable to reach GitLab", msg) + # base_url (no token) is shown, not the raw original error + self.assertIn("test.gitlab.com", msg) + self.assertIn("api_url", msg) + # Raw urllib3 message must not be included to avoid token leakage + self.assertNotIn("Failed to resolve", msg) + self.assertNotIn("private_token", msg)