diff --git a/src/osfexport/cli.py b/src/osfexport/cli.py index 8147262..bb50f97 100644 --- a/src/osfexport/cli.py +++ b/src/osfexport/cli.py @@ -26,6 +26,10 @@ def prompt_pat(project_id='', usetest=False): ----------------- pat: str Personal Access Token to use to authorise a user. + + Raises + ------------------- + HTTPError, URLError - passed on from is_public method. """ if usetest: @@ -81,11 +85,12 @@ def export_projects(folder, pat='', dryrun=False, url='', usetest=False): else: click.echo('No project ID provided, extracting all projects.') - if not pat and not dryrun: - pat = prompt_pat(project_id=project_id, usetest=usetest) - - click.echo('Downloading project data...') try: + if not pat and not dryrun: + pat = prompt_pat(project_id=project_id, usetest=usetest) + + click.echo('Downloading project data...') + projects, root_nodes = exporter.get_nodes( pat, dryrun=dryrun, project_id=project_id, usetest=usetest ) diff --git a/src/osfexport/exporter.py b/src/osfexport/exporter.py index 8bd6b9f..87ee0f0 100644 --- a/src/osfexport/exporter.py +++ b/src/osfexport/exporter.py @@ -2,7 +2,7 @@ import json import os import datetime -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError import urllib.request as webhelper import importlib.metadata import time @@ -176,20 +176,37 @@ def get_host(is_test): def is_public(url): """Return boolean to indicate if a URL is public (True) or not (False). + This is mainly used for checking if a project is publicly accessible. Parameters ------------ url: str The URL to test. + + Returns + ---------------- + is_public: bool + Whether we can access the URL without a PAT (i.e. status code 200) + + Raises + ------------------- + HTTPError, URLError + If we get a HTTP error code that isn't 401/403, or a connection error. """ try: result = call_api( url, pat='', method='GET' ).status - except webhelper.HTTPError as e: - result = e - return result == 200 + except (HTTPError, URLError) as e: + # Don't raise error if we get a HTTP error with certain codes + valid_error_codes = [401, 403] + if isinstance(e, HTTPError) and e.code in valid_error_codes: + result = e.code + else: + raise e + is_public = result == 200 + return is_public def call_api( @@ -703,7 +720,7 @@ def get_project_data(nodes, **kwargs): parent['data']['attributes']['title'], parent['data']['links']['html'] ) - except (webhelper.HTTPError, ValueError): + except (HTTPError, ValueError): click.echo(f"Failed to load parent for {project_data['metadata']['title']}") click.echo("Try to give a PAT beforehand using the --pat flag.", "\n") diff --git a/tests/test_clitool.py b/tests/test_clitool.py index 5c87575..ba6eeaa 100644 --- a/tests/test_clitool.py +++ b/tests/test_clitool.py @@ -262,27 +262,40 @@ def test_call_api_handle_429_errors(self, mock_request_class, mock_urlopen): assert len(mock_urlopen.call_args_list) == 5 def test_get_public_status(self): - # Public url mock_response = MagicMock() - mock_response.status = 200 - with patch('osfexport.exporter.call_api') as mock_call_api: - mock_call_api.return_value = mock_response - result = is_public('url') - mock_call_api.assert_called_once_with( - 'url', pat='', method='GET' - ) - assert result is True - # Private url - mock_response = MagicMock() - mock_response.status = 401 - with patch('osfexport.exporter.call_api') as mock_call_api: - mock_call_api.return_value = mock_response - result = is_public('url') - mock_call_api.assert_called_once_with( - 'url', pat='', method='GET' - ) - assert result is False + valid_codes = [(200, True), (401, False), (403, False)] + for pair in valid_codes: + mock_response.status = pair[0] + with patch('osfexport.exporter.call_api') as mock_call_api: + mock_call_api.return_value = mock_response + result = is_public('url') + mock_call_api.assert_called_once_with( + 'url', pat='', method='GET' + ) + assert result == pair[1], ( + f"Expected: {pair}" + f"Actual code: {result}" + ) + + bad_codes = [400, 500, -1] + for code in bad_codes: + with patch('osfexport.exporter.call_api') as mock_call_api: + if code != -1: + mock_call_api.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=code, + msg='HTTP Error', + hdrs={}, + fp=None + ) + else: + mock_call_api.side_effect = urllib.error.URLError( + reason="URL Error" + ) + with self.assertRaises(type(mock_call_api.side_effect)): + result = is_public('url') + print(f"Code {code} should have failed by now!") def test_explore_mock_file_tree(self): files = explore_file_tree( @@ -1039,8 +1052,9 @@ def test_prompt_pat_if_exporting_all_projects(self, mock_obj): @patch('osfexport.cli.prompt_pat') @patch('osfexport.exporter.get_nodes') def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): - codes = [401, 402, 403, 404, 429, 500, -1] - for code in codes: + # Handle errors from exporting nodes + export_codes = [401, 402, 403, 404, 429, 500, -1] + for code in export_codes: mock_prompt.return_value = '-' if code != -1: mock_func.side_effect = urllib.error.HTTPError( @@ -1062,7 +1076,35 @@ def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): ], terminal_width=60 ) - assert "Exporting failed as an error occurred:" in result.output + assert "Exporting failed as an error occurred:" in result.output, ( + result.output + ) + + # Handle errors from is_public call when prompting for PAT + for code in export_codes: + if code != -1: + mock_prompt.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=code, + msg='HTTP Error', + hdrs={}, + fp=None + ) + else: + mock_prompt.side_effect = urllib.error.URLError( + reason="URL Error" + ) + runner = CliRunner() + result = runner.invoke( + cli, [ + 'export-projects', + '--usetest' + ], + terminal_width=60 + ) + assert "Exporting failed as an error occurred:" in result.output, ( + result.output + ) def test_pull_projects_command_on_mocks(self): """Test generating a PDF from parsed project data.