104 raise errors when appropriate in is public method

src/osfexport/cli.py

@@ -26,6 +26,10 @@ def prompt_pat(project_id='', usetest=False):
     -----------------
         pat: str
             Personal Access Token to use to authorise a user.
+
+    Raises
+    -------------------
+        HTTPError, URLError - passed on from is_public method.
     """
 
     if usetest:
@@ -81,11 +85,12 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False):
     else:
         click.echo('No project ID provided, extracting all projects.')
 
-    if not pat and not dryrun:
-        pat = prompt_pat(project_id=project_id, usetest=usetest)
-
-    click.echo('Downloading project data...')
     try:
+        if not pat and not dryrun:
+            pat = prompt_pat(project_id=project_id, usetest=usetest)
+
+        click.echo('Downloading project data...')
+
         projects, root_nodes = exporter.get_nodes(
             pat, dryrun=dryrun, project_id=project_id, usetest=usetest
         )

src/osfexport/exporter.py

@@ -2,7 +2,7 @@
 import json
 import os
 import datetime
-from urllib.error import HTTPError
+from urllib.error import HTTPError, URLError
 import urllib.request as webhelper
 import importlib.metadata
 import time
@@ -176,20 +176,37 @@ def get_host(is_test):
 
 def is_public(url):
     """Return boolean to indicate if a URL is public (True) or not (False).
+    This is mainly used for checking if a project is publicly accessible.
 
     Parameters
     ------------
     url: str
         The URL to test.
+
+    Returns
+    ----------------
+        is_public: bool
+            Whether we can access the URL without a PAT (i.e. status code 200)
+
+    Raises
+    -------------------
+        HTTPError, URLError
+            If we get a HTTP error code that isn't 401/403, or a connection error.
     """
 
     try:
         result = call_api(
             url, pat='', method='GET'
         ).status
-    except webhelper.HTTPError as e:
-        result = e
-    return result == 200
+    except (HTTPError, URLError) as e:
+        # Don't raise error if we get a HTTP error with certain codes
+        valid_error_codes = [401, 403]
+        if isinstance(e, HTTPError) and e.code in valid_error_codes:
+            result = e.code
+        else:
+            raise e
+    is_public = result == 200
+    return is_public
 
 
 def call_api(
@@ -703,7 +720,7 @@ def get_project_data(nodes, **kwargs):
                             parent['data']['attributes']['title'],
                             parent['data']['links']['html']
                         )
-                    except (webhelper.HTTPError, ValueError):
+                    except (HTTPError, ValueError):
                         click.echo(f"Failed to load parent for {project_data['metadata']['title']}")
                         click.echo("Try to give a PAT beforehand using the --pat flag.", "\n")
 

tests/test_clitool.py

@@ -262,27 +262,40 @@ def test_call_api_handle_429_errors(self, mock_request_class, mock_urlopen):
         assert len(mock_urlopen.call_args_list) == 5
 
     def test_get_public_status(self):
-        # Public url
         mock_response = MagicMock()
-        mock_response.status = 200
-        with patch('osfexport.exporter.call_api') as mock_call_api:
-            mock_call_api.return_value = mock_response
-            result = is_public('url')
-            mock_call_api.assert_called_once_with(
-                'url', pat='', method='GET'
-            )
-            assert result is True
 
-        # Private url
-        mock_response = MagicMock()
-        mock_response.status = 401
-        with patch('osfexport.exporter.call_api') as mock_call_api:
-            mock_call_api.return_value = mock_response
-            result = is_public('url')
-            mock_call_api.assert_called_once_with(
-                'url', pat='', method='GET'
-            )
-            assert result is False
+        valid_codes = [(200, True), (401, False), (403, False)]
+        for pair in valid_codes:
+            mock_response.status = pair[0]
+            with patch('osfexport.exporter.call_api') as mock_call_api:
+                mock_call_api.return_value = mock_response
+                result = is_public('url')
+                mock_call_api.assert_called_once_with(
+                    'url', pat='', method='GET'
+                )
+                assert result == pair[1], (
+                    f"Expected: {pair}"
+                    f"Actual code: {result}"
+                )
+
+        bad_codes = [400, 500, -1]
+        for code in bad_codes:
+            with patch('osfexport.exporter.call_api') as mock_call_api:
+                if code != -1:
+                    mock_call_api.side_effect = urllib.error.HTTPError(
+                        url='https://test.osf.io',
+                        code=code,
+                        msg='HTTP Error',
+                        hdrs={},
+                        fp=None
+                    )
+                else:
+                    mock_call_api.side_effect = urllib.error.URLError(
+                        reason="URL Error"
+                    )
+                with self.assertRaises(type(mock_call_api.side_effect)):
+                    result = is_public('url')
+                    print(f"Code {code} should have failed by now!")
 
     def test_explore_mock_file_tree(self):
         files = explore_file_tree(
@@ -1039,8 +1052,9 @@ def test_prompt_pat_if_exporting_all_projects(self, mock_obj):
     @patch('osfexport.cli.prompt_pat')
     @patch('osfexport.exporter.get_nodes')
     def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt):
-        codes = [401, 402, 403, 404, 429, 500, -1]
-        for code in codes:
+        # Handle errors from exporting nodes
+        export_codes = [401, 402, 403, 404, 429, 500, -1]
+        for code in export_codes:
             mock_prompt.return_value = '-'
             if code != -1:
                 mock_func.side_effect = urllib.error.HTTPError(
@@ -1062,7 +1076,35 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt):
                 ],
                 terminal_width=60
             )
-            assert "Exporting failed as an error occurred:" in result.output
+            assert "Exporting failed as an error occurred:" in result.output, (
+                result.output
+            )
+
+        # Handle errors from is_public call when prompting for PAT
+        for code in export_codes:
+            if code != -1:
+                mock_prompt.side_effect = urllib.error.HTTPError(
+                    url='https://test.osf.io',
+                    code=code,
+                    msg='HTTP Error',
+                    hdrs={},
+                    fp=None
+                )
+            else:
+                mock_prompt.side_effect = urllib.error.URLError(
+                    reason="URL Error"
+                )
+            runner = CliRunner()
+            result = runner.invoke(
+                cli, [
+                    'export-projects',
+                    '--usetest'
+                ],
+                terminal_width=60
+            )
+            assert "Exporting failed as an error occurred:" in result.output, (
+                result.output
+            )
 
     def test_pull_projects_command_on_mocks(self):
         """Test generating a PDF from parsed project data.