diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..3694eb9 --- /dev/null +++ b/.env.template @@ -0,0 +1,6 @@ +# Bases of host URL to the test and production API servers +# Defaults are https://api.test.osf.io/v2/ (test) and +# https://api.osf.io/v2/ (production) +# See https://developer.osf.io/#tag/General-Usage +API_HOST_TEST=https://api.test.osf.io/v2 +API_HOST_PROD=https://api.osf.io/v2 \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..b1f3978 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Test on Pull Request + +on: + pull_request: + branches: [ "development", "main" ] + +permissions: + contents: read + +jobs: + test: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v3 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements.core.txt ]; then pip install -r requirements.core.txt; fi + pip install --editable . + - name: Check linting quality + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 100 chars wide + flake8 . --max-line-length=100 --statistics + - name: Run local tests + run: | + python -m unittest tests.test_clitool.TestExporter + python -m unittest tests.test_clitool.TestFormatter + python -m unittest tests.test_clitool.TestCLI + - name: Run API tests + run: | + python -m unittest -f tests.test_clitool.TestAPI \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8fc0763 --- /dev/null +++ b/.gitignore @@ -0,0 +1,198 @@ +# PDFs +*.pdf + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +.vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 49fb0aa..e8b9d32 100644 --- a/README.md +++ b/README.md @@ -1 +1,29 @@ -# OSF Project Exporter \ No newline at end of file +# OSF Project Exporter + +This is a CLI tool for exporting research project data and files from the [OSF website](https://osf.io/). This is to prototype tool to export projects from the OSF website to a PDF, allowing users to back up, share or document their OSF projects in an offline medium. + +## Development Setup + +### Virtual Environment + +1. Clone this repository onto your local machine. +2. Create a virtual environment to install dependencies. For `virtualenv` this is done with ``virtualenv ``. Make sure your virtual environment is setup to use Python 3.12 or above (e.g., ``virtualenv --python="/usr/bin/python3.12"`` on Linux.) +3. From local Git repo: Activate your virtual environment and run ``pip install -e osfexport`` to install this repository as a modifiable package. Then install other requirements separately via `pip install -r requirements.txt`. +4. On the OSF website, create or log in to your account. Set up a personal access token (PAT) by going into your account settings, select `Personal access tokens` in the left side menu, and clicking `Create token`. You should give the token a name that helps you remember why you made it, like "PDF export", and choose the `osf.full_read` scope - this allows this token to read all public and private projects on your account. You can delete this token once you have finished exporting your projects. + +## Installation + +### From PyPI: releases 0.1.4 and onwards + +Activate your virtual environment: for example, using `virtualenv` this is done by: + +- `source /bin/activate` on Linux +- `\Scripts\activate` on Windows/Mac + +Next, run `python -m pip install osfexport`. This will download and install this package and other dependencies from the PyPI index. + +## Usage + +- Run `osfexport` to get a list of basic commands you can use. +- To see what a command needs as input, type `--help` after the command name (e.g. `osfexport welcome --help`; `osfexport --help`) +- To export all your projects from the OSF into a PDF, run `osfexport projects`. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..70d5922 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +[project] +name = "osfexport" +version = "1.0.0" +authors = [ + {name = "Benito Matischen", email = "benito.matischen@manchester.ac.uk"}, + {name = "Ramiro Bravo", email = "ramiro.bravo@manchester.ac.uk"}, + {name = "Sarah Jaffa", email = "sarah.jaffa@manchester.ac.uk"}, + {name = "Center for Open Science", email = "contact@cos.io"} +] +description = "A package for exporting projects from the Open Science Framework web app. Comes with a command-line interface for exporting projects." +readme = "README.md" +requires-python = ">=3.12" +classifiers = [ + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Topic :: System :: Archiving :: Backup" +] +license = "Apache-2.0" +license-files = ["LICEN[CS]E*"] +dependencies = [ + "click==8.2.1", + "fpdf2==2.8.3", + "pypdf==5.7.0", + "mistletoe==1.4.0", + "qrcode==8.2" +] +keywords = [ + "OSF", "osf", "backup", "Open Science Framework", "export", + "Centre for Open Science", "COS", "open science", "archive" +] + +[project.urls] +Repository = "https://github.com/CenterForOpenScience/osf-project-exporter/tree/development" +Issues = "https://github.com/CenterForOpenScience/osf-project-exporter/issues" +ReleaseNotes = "https://github.com/CenterForOpenScience/osf-project-exporter/releases" + +[project.scripts] +# Executable scripts, names are commands +osfexport = "osfexport.cli:cli" + +[build-system] +requires = ["flit_core<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "osfexport" + +[tool.black] +line-length=100 +preview=true + +[tool.flake8] +max-line-length=100 + +[tool.ruff] +line-length=100 +exclude = ["tests/*"] + +[tool.ruff.lint] +select = ["E", "F"] \ No newline at end of file diff --git a/src/osfexport/__init__.py b/src/osfexport/__init__.py new file mode 100644 index 0000000..c94b2df --- /dev/null +++ b/src/osfexport/__init__.py @@ -0,0 +1,28 @@ +from osfexport.exporter import ( + call_api, is_public, + extract_project_id, + MockAPIResponse, get_nodes, + paginate_json_result +) + +from osfexport.cli import ( + prompt_pat, cli +) + +from osfexport.formatter import ( + write_pdf +) + +__all__ = [ + 'call_api', + 'get_nodes', + 'write_pdf', + 'is_public', + 'extract_project_id', + 'MockAPIResponse', + 'get_nodes', + 'paginate_json_result', + 'extract_project_id', + 'prompt_pat', + 'cli' +] diff --git a/src/osfexport/cli.py b/src/osfexport/cli.py new file mode 100644 index 0000000..7fabbdf --- /dev/null +++ b/src/osfexport/cli.py @@ -0,0 +1,174 @@ +import os +from urllib.error import HTTPError, URLError + +import click + +import osfexport.exporter as exporter +import osfexport.formatter as formatter + +API_HOST_TEST = os.getenv('API_HOST_TEST', 'https://api.test.osf.io/v2') +API_HOST_PROD = os.getenv('API_HOST_PROD', 'https://api.osf.io/v2') + + +def prompt_pat(project_id='', usetest=False): + """ + Ask for a PAT if exporting a single project or all projects a user has. + + Parameters + ------------- + project_id: str + ID of a single project to export. + If one provided then ask for a PAT. + usetest: bool + Flag to indicate whether to use the test/production API server. + + Returns + ----------------- + pat: str + Personal Access Token to use to authorise a user. + + Raises + ------------------- + HTTPError, URLError - passed on from is_public method. + """ + + if usetest: + api_host = API_HOST_TEST + else: + api_host = API_HOST_PROD + + if not project_id: + pat = click.prompt( + 'Please enter your PAT to export all your projects', + type=str, + hide_input=True + ) + elif not exporter.is_public(f'{api_host}/nodes/{project_id}/'): + pat = click.prompt( + 'Please enter your PAT to export this private project', + type=str, + hide_input=True + ) + else: + pat = '' + + return pat + + +@click.command(name='projects') +@click.option('--pat', type=str, default='', + prompt='Enter your PAT', prompt_required=False, hide_input=True, + help='Personal Access Token to authorise OSF account access.') +@click.option('--dryrun', is_flag=True, default=False, + help='If enabled, use mock responses in place of the API.') +@click.option('--usetest', is_flag=True, default=False, + help=f"""If passed, set {API_HOST_TEST} as the API hostname. + Otherwise, {API_HOST_PROD} is the default hostname.""") +@click.option('--folder', type=str, default='', + help='The folder path to export PDFs to.') +@click.option('--url', type=str, default='', + help="""A link to one project you want to export. + The project ID should be at the end. + + For example: https://osf.io/dry9j/ + + Leave blank to export all projects you have access to.""") +def export_projects(folder, pat='', dryrun=False, url='', usetest=False): + """Pull and export OSF projects to a PDF file. + You can export all projects you have access to, or one specific one + with the --url option.""" + + project_id = '' + if url: + project_id = exporter.extract_project_id(url) + click.echo(f'Extracting project with ID: {project_id}') + else: + click.echo('No project ID provided, extracting all projects.') + + try: + if not pat and not dryrun: + pat = prompt_pat(project_id=project_id, usetest=usetest) + + click.echo('Downloading project data...') + + projects, root_nodes = exporter.get_nodes( + pat, dryrun=dryrun, project_id=project_id, usetest=usetest + ) + click.echo(f'Found {len(root_nodes)} projects.') + for idx in root_nodes: + title = projects[idx]['metadata']['title'] + click.echo(f'Exporting project {title}...') + pdf, path = formatter.write_pdf(projects, idx, folder) + click.echo(f'Project exported to {path}') + except (HTTPError, URLError) as e: + click.echo("Exporting failed as an error occurred: ") + if isinstance(e, HTTPError): + if e.code == 401: + click.echo( + "We couldn't authenticate you with the personal access token." + ) + click.echo( + "If you already have access to the OSF, please check the token is correct." + ) + elif e.code == 404: + click.echo( + "The project couldn't be found. Please check the URL/project ID is correct." + ) + elif e.code == 403: + if project_id: + click.echo( + "Please check you are a contributor for this private project." + ) + click.echo( + "If you are, does your token have the \"osf.full_read\" permission?" + ) + else: + click.echo( + "Does your personal access token have the \"osf.full_read\" permission?" + ) + click.echo( + "This is needed to allow access to your projects with this token." + ) + elif e.code == 429: + click.echo( + """Too many requests to the API, please try again in a few minutes.""" + ) + else: + click.echo( + f"Unexpected error: HTTP {e.code}. Please try again later." + ) + else: + click.echo( + f"Unexpected error connecting to the OSF: {e.reason}. Please try again later." + ) + + +@click.command(name='welcome') +@click.option('--pat', type=str, default='', + prompt=True, hide_input=True, + help='Personal Access Token to authorise OSF account access.') +@click.option('--usetest', is_flag=True, default=False, + help="""Use this to connect to the test API environment. + Otherwise, the production environment will be used.""") +def show_welcome(pat, usetest): + """Get a welcome message from the OSF site. + You can use this to test if the OSF server is running.""" + + if usetest: + api_host = API_HOST_TEST + else: + api_host = API_HOST_PROD + + result = exporter.call_api(f'{api_host}/', pat=pat, method='GET') + click.echo(result.read()) + click.echo(result.status) + + +# Group will be used as entry point for CLI +@click.group() +def cli(): + pass + + +cli.add_command(export_projects) +cli.add_command(show_welcome) diff --git a/src/osfexport/exporter.py b/src/osfexport/exporter.py new file mode 100644 index 0000000..bd29ca2 --- /dev/null +++ b/src/osfexport/exporter.py @@ -0,0 +1,930 @@ +from collections import deque +import json +import os +import datetime +from urllib.error import HTTPError, URLError +import urllib.request as webhelper +import importlib.metadata +import time +import random +import logging + +logging.basicConfig( + level=logging.WARNING, format='%(message)s' +) + + +API_HOST_TEST = os.getenv('API_HOST_TEST', 'https://api.test.osf.io/v2') +API_HOST_PROD = os.getenv('API_HOST_PROD', 'https://api.osf.io/v2') + +STUBS_DIR = os.path.join( + os.path.dirname(__file__), 'stubs' +) + +# Reduce response size by applying filters on fields +URL_FILTERS = { + 'identifiers': { + 'category': 'doi' + } +} + + +class MockAPIResponse: + """Simulate OSF API response for testing purposes.""" + + JSON_FILES = { + 'nodes': os.path.join( + STUBS_DIR, 'nodestubs.json'), + 'nodes2': os.path.join( + STUBS_DIR, 'nodestubs2.json'), + 'x': os.path.join( + STUBS_DIR, 'singlenode.json'), + 'a': os.path.join( + STUBS_DIR, 'asingle.json'), + 'affiliated_institutions': os.path.join( + STUBS_DIR, 'institutionstubs.json'), + 'contributors': os.path.join( + STUBS_DIR, 'contributorstubs.json'), + 'identifiers': os.path.join( + STUBS_DIR, 'doistubs.json'), + 'custom_metadata': os.path.join( + STUBS_DIR, 'custommetadatastub.json'), + 'root_folder': os.path.join( + STUBS_DIR, 'files', 'rootfolders.json'), + 'root_files': os.path.join( + STUBS_DIR, 'files', 'rootfiles.json'), + 'tf1_folder': os.path.join( + STUBS_DIR, 'files', 'tf1folders.json'), + 'tf1-2_folder': os.path.join( + STUBS_DIR, 'files', 'tf1folders-2.json'), + 'tf1-2_files': os.path.join( + STUBS_DIR, 'files', 'tf2-second-folders.json'), + 'tf1_files': os.path.join( + STUBS_DIR, 'files', 'tf1files.json'), + 'tf2_folder': os.path.join( + STUBS_DIR, 'files', 'tf2folders.json'), + 'tf2-second_folder': os.path.join( + STUBS_DIR, 'files', 'tf2-second-folders.json'), + 'tf2_files': os.path.join( + STUBS_DIR, 'files', 'tf2files.json'), + 'tf2-second_files': os.path.join( + STUBS_DIR, 'files', 'tf2-second-files.json'), + 'tf2-second-2_files': os.path.join( + STUBS_DIR, 'files', 'tf2-second-files-2.json'), + 'license': os.path.join( + STUBS_DIR, 'licensestub.json'), + 'subjects': os.path.join( + STUBS_DIR, 'subjectsstub.json'), + 'wikis': os.path.join( + STUBS_DIR, 'wikis', 'wikistubs.json'), + 'wikis2': os.path.join( + STUBS_DIR, 'wikis', 'wikis2stubs.json'), + 'x-child-1': os.path.join( + STUBS_DIR, 'components', 'x-child-1.json'), + 'x-child-2': os.path.join( + STUBS_DIR, 'components', 'x-child-2.json'), + 'empty-children': os.path.join( + STUBS_DIR, 'components', 'empty-children.json'), + } + + MARKDOWN_FILES = { + 'helloworld': os.path.join( + STUBS_DIR, 'wikis', 'helloworld.md'), + 'home': os.path.join( + STUBS_DIR, 'wikis', 'home.md'), + 'anotherone': os.path.join( + STUBS_DIR, 'wikis', 'anotherone.md'), + } + + @staticmethod + def read(field): + """Get mock response for a field. + + Parameters + ----------- + field: str + ID associated to a JSON or Markdown mock file. + Available fields to mock are listed in class-level + JSON_FILES and MARKDOWN_FILES attributes. + + Returns + ------------ + Parsed JSON dictionary or Markdown.""" + + if field in MockAPIResponse.JSON_FILES.keys(): + with open(MockAPIResponse.JSON_FILES[field], 'r') as file: + return json.load(file) + elif field in MockAPIResponse.MARKDOWN_FILES.keys(): + with open(MockAPIResponse.MARKDOWN_FILES[field], 'r') as file: + return file.read() + else: + return {'data': {}} + + +def extract_project_id(url): + """Extract project ID from a given OSF project URL. + + Parameters + ---------- + url: str + URL of the OSF project which should contain the project ID. E.g.: + - Full URL with parameters (https://osf.io/xyz/?param=value) + - API URL (https://api.test.osf.io/v2/nodes/xyz) + - Just the ID (xyz) + - Empty string + + Returns + ------- + str + Project ID extracted from the URL. + """ + + if not url: + return '' + + parts = url.strip("/").split("/") + # Handle case of just ID provided + if len(parts) == 1: + return parts[0] + + # API URLs are of form /nodes/id/... + if 'nodes' in parts: + idx = parts.index('nodes') + if idx + 1 < len(parts): + return parts[idx + 1] + + # For regular URLs, extract ID from last path component before query params + if '?' in parts[-1]: + return parts[-2] + else: + return parts[-1] + + +def get_host(is_test): + """Get API host based on flag. + + Parameters + ---------- + is_test: bool + If True, return test API host, otherwise return production host. + + Returns + ------- + str + API host URL for the test site or production site. + """ + + return API_HOST_TEST if is_test else API_HOST_PROD + + +def is_public(url): + """Return boolean to indicate if a URL is public (True) or not (False). + This is mainly used for checking if a project is publicly accessible. + + Parameters + ------------ + url: str + The URL to test. + + Returns + ---------------- + is_public: bool + Whether we can access the URL without a PAT (i.e. status code 200) + + Raises + ------------------- + HTTPError, URLError + If we get a HTTP error code that isn't 401/403, or a connection error. + """ + + try: + result = call_api( + url, pat='', method='GET' + ).status + except (HTTPError, URLError) as e: + # Don't raise error if we get a HTTP error with certain codes + valid_error_codes = [401, 403] + if isinstance(e, HTTPError) and e.code in valid_error_codes: + result = e.code + else: + raise e + is_public = result == 200 + return is_public + + +def call_api( + url, pat, method='GET', per_page=100, filters={}, is_json=True, + usetest=False, max_tries=5): + """Call OSF v2 API methods. + + Parameters + ---------- + url: str + URL to API method/resource/query. + method: str + HTTP method for the request. + pat: str + Personal Access Token to authorise a user with. + per_page: int + Number of items to include in a JSON page for API responses. + The maximum is 100. + filters: dict + Dictionary of query parameters to filter results with. + + Example Input: {'category': 'project', 'title': 'ttt'} + Example Query String: ?filter[category]=project&filter[title]=ttt + is_json: bool + If true, set API version to get correct API responses. + usetest: bool + If True, use fixed delay of 0.1 seconds for tests. + If False, use a random delay between [1, 60] seconds between requests. + This spaces out requests over time to give the API chance to recover. + max_tries: int + Number of attempts to make before raising a 429 error. Default is 5, Limit is 7. + + Throws + ------------- + HTTPError - 429 error if we can't connect to the API after retries. + + Returns + ---------- + result: HTTPResponse + Response to the request from the API. + """ + if (filters or per_page) and method == 'GET': + query_string = '&'.join([f'filter[{key}]={value}' + for key, value in filters.items() + if not isinstance(value, dict)]) + if per_page: + query_string += f'&page[size]={per_page}' + url = f'{url}?{query_string}' + + request = webhelper.Request(url, method=method) + request.add_header('Authorization', f'Bearer {pat}') + + version = importlib.metadata.version("osfexport") + request.add_header('User-Agent', f'osfexport/{version} (Python)') + + # Pin API version so that JSON has correct format + API_VERSION = '2.20' + if is_json: + request.add_header( + 'Accept', + f'application/vnd.api+json;version={API_VERSION}' + ) + + if max_tries > 7: + max_tries = 7 # Cap retries to reduce requests sent and max delay time + + # Retry requests if we get 429 errors + try_count = 0 + result = None + while try_count < max_tries and result is None: + try: + result = webhelper.urlopen(request) + except HTTPError as e: + # Other error codes tell us directly something is wrong + if e.code == 429: + if not usetest: + # Wait longer between requests to give API more recovery time + # Wait random periods to avoid hammering requests all at once + min_wait = (try_count+1)**2 + time.sleep(random.uniform(min_wait, 60)) + else: + time.sleep(0.5) # Wait constant time for tests + try_count += 1 + else: + raise e + if result is None: + raise HTTPError( + url=url, + code=429, + msg="Too many requests to the OSF API.", + hdrs=request.headers, + fp=None + ) + return result + + +def paginate_json_result(start, action, fail_on_first=True, **kwargs): + """Loop through paginated JSON responses and perform action on each. + + Parameters + ------------- + start: str + Link to start looping from + action: func + Takes in found JSON page and returns a result + per_page: int + How many items to include on one page. Default is 100. + Valid range is from 1-1000. + filters: dict + Optional key-value dict to filter queries by. + is_json: + If JSON response expected, add header to specify JSON format. + pat: str + Personal Access Token to authorise a user. + dryrun: bool + Flag for whether mock JSON or real API will be used. + **kwargs + Extra keyword args to pass down to action and call_api. + + Returns + ------------------ + results: deque + Queue of results per page + """ + + next_link = start + is_last_page = False + is_first_item = True # Want to throw error if very first item fails + results = deque() + per_page = kwargs.pop('per_page', 100) + filters = kwargs.pop('filters', {}) + is_json = kwargs.pop('is_json', True) + pat = kwargs.get('pat', '') + dryrun = kwargs.get('dryrun', False) + while not is_last_page: + try: + if not dryrun: + curr_page = call_api( + next_link, pat, per_page=per_page, filters=filters, + is_json=is_json) + # Catch error if call_api is replaced with mock in tests + try: + curr_page = curr_page.read() + if is_json: + curr_page = json.loads(curr_page) + except AttributeError: + pass + else: + curr_page = MockAPIResponse.read(next_link) + results.append(action(curr_page, **kwargs)) + except HTTPError as e: + if fail_on_first and is_first_item or e.code == 429: + raise e + else: + logging.warning("Warning: Couldn't parse JSON page, skipping to next page...") + # Stop if no next link found + try: + next_link = curr_page['links']['next'] + is_last_page = not next_link + except (KeyError, UnboundLocalError): + is_last_page = True + return results + + +def explore_file_tree(curr_link, pat, dryrun=True): + """Explore and get names of files stored in OSF. + + Parameters + ---------- + curr_link: string + URL/name to use to get real/mock files and folders. + pat: string + Personal Access Token to authorise a user. + dryrun: bool + Flag to indicate whether to use mock JSON files or real API calls. + + Returns + ---------- + files_found: list[str] + List of file paths found in the project.""" + + FILE_FILTER = { + 'kind': 'file' + } + FOLDER_FILTER = { + 'kind': 'folder' + } + per_page = 100 + + files_found = [] + + is_last_page_folders = False + while not is_last_page_folders: + if dryrun: + folders = MockAPIResponse.read(f"{curr_link}_folder") + else: + folders = json.loads( + call_api( + curr_link, pat, + per_page=per_page, filters=FOLDER_FILTER + ).read() + ) + + # Find deepest subfolders first to avoid missing files + try: + for folder in folders['data']: + links = folder['relationships']['files']['links'] + link = links['related']['href'] + files_found += explore_file_tree(link, pat, dryrun=dryrun) + except KeyError: + pass + + # For each folder, loop through pages for its files + is_last_page_files = False + while not is_last_page_files: + if dryrun: + files = MockAPIResponse.read(f"{curr_link}_files") + else: + files = json.loads( + call_api( + curr_link, pat, + per_page=per_page, filters=FILE_FILTER + ).read() + ) + try: + for file in files['data']: + size = file['attributes']['size'] + size_mb = size / (1024 ** 2) # Convert bytes to MB + data = ( + file['attributes']['materialized_path'], + str(round(size_mb, 2)), + file['links']['download'] + ) + files_found.append(data) + except KeyError: + pass + curr_link = files['links']['next'] + if curr_link is None: + is_last_page_files = True + + curr_link = folders['links']['next'] + if curr_link is None: + is_last_page_folders = True + + return files_found + + +def explore_wikis(link, pat, dryrun=True): + """Get wiki contents for a particular project. + + Parameters: + ------------- + link: str + URL to project wikis or name of wikis field to access mock JSON. + pat: str + Personal Access Token to authenticate a user with. + dryrun: bool + Flag to indicate whether to use mock JSON files or real API calls. + + Returns + --------------- + wikis: List of JSON representing wikis for a project.""" + + wiki_content = {} + is_last_page = False + if dryrun: + wikis = MockAPIResponse.read('wikis') + else: + wikis = json.loads( + call_api(link, pat).read() + ) + + while not is_last_page: + for wiki in wikis['data']: + if dryrun: + content = MockAPIResponse.read(wiki['attributes']['name']) + else: + # Decode Markdown content to allow parsing later on + content = call_api( + wiki['links']['download'], pat=pat, is_json=False + ).read().decode('utf-8') + wiki_content[wiki['attributes']['name']] = content + + # Go to next page of wikis if pagination applied + # so that we don't miss wikis + link = wikis['links']['next'] + if not link: + is_last_page = True + else: + if dryrun: + wikis = MockAPIResponse.read(link) + else: + wikis = json.loads( + call_api(link, pat).read() + ) + + return wiki_content + + +def get_nodes(pat, page_size=100, dryrun=False, project_id='', usetest=False): + """Pull and list projects for a user from the OSF. + + Parameters + ---------- + pat: str + Personal Access Token to authorise a user with. + page_size: int + How many nodes to put onto a page. Default is 100. + Possible range is 1-1000 + dryrun: bool + If True, use test data from JSON stubs to mock API calls. + project_id: str + Optional ID for a specific OSF project to export. + usetest: bool + If True, use test API host, otherwise use production host. + + Returns + ---------- + projects: list[dict] + List of all project objects found + root_nodes: list[int] + List of indexes for root nodes in projects list. + These are the nodes to make PDFs for and start from in PDFs. + """ + + # Set start link and page size filter based on flags + api_host = get_host(usetest) + node_filter = {} + if not dryrun: + if project_id: + start = f'{api_host}/nodes/{project_id}/' + else: + start = f'{api_host}/users/me/nodes/' + node_filter = { + 'parent': '' + } + else: + page_size = 4 # Nodes found are hardcoded for --dryrun + if project_id: + start = project_id + else: + start = 'nodes' + + results = paginate_json_result( + start, get_project_data, dryrun=dryrun, usetest=usetest, + pat=pat, filters=node_filter, project_id=project_id, per_page=page_size + ) + if len(results) > 0: + l1, l2 = zip(*list(results)) + else: + l1, l2 = (), () + projects = [item for sublist in l1 for item in sublist] + + # After pagination we get indexes of root nodes local to each page + # We need to convert these to global indexes before merging the list + page_idx = -1 + for page in l2: + page_idx += 1 + for idx, n in enumerate(page): + global_node_idx = page_size*page_idx + n + page[idx] = global_node_idx + root_nodes = [item for sublist in l2 for item in sublist] + + return projects, root_nodes + + +def get_project_data(nodes, **kwargs): + """Pull and list projects for a user from the OSF. + + Parameters + ---------- + pat: str + Personal Access Token to authorise a user with. + dryrun: bool + If True, use test data from JSON stubs to mock API calls. + project_id: str + Optional ID for a specific OSF project to export. + usetest: bool + If True, use test API host, otherwise use production host. + + Returns + ---------- + projects: list[dict] + List of dictionaries representing projects. + """ + + pat = kwargs.pop('pat', '') + dryrun = kwargs.pop('dryrun', False) + usetest = kwargs.pop('usetest', False) + project_id = kwargs.pop('project_id', '') + + api_host = get_host(usetest) + + if not dryrun and project_id: + nodes = {'data': [nodes['data']]} + elif project_id: + # Put data into same format as if multiple nodes found + nodes = {'data': [MockAPIResponse.read(project_id)['data']]} + + projects = [] + root_nodes = [] # Track indexes of start nodes for PDFs + added_node_ids = set() # Track added node IDs to avoid duplicates + + # Dispatch table used to define how to process JSON + # Add new field by giving name and function + fields = { + 'metadata': { + 'title': lambda project, **kwargs: project['attributes']['title'], + 'id': lambda project, **kwargs: project['id'], + 'url': lambda project, **kwargs: project['links']['html'], + 'description': lambda project, **kwargs: project['attributes']['description'], + # timestamps are rendered as yyyy-mm-dd hour:minute UTC (24hr) + 'date_created': lambda project, **kwargs: datetime.datetime.fromisoformat( + project['attributes']['date_created'] + ).astimezone( + datetime.timezone.utc + ).strftime( + '%Y-%m-%d %H:%M %Z' + ), + 'date_modified': lambda project, **kwargs: datetime.datetime.fromisoformat( + project['attributes']['date_modified'] + ).astimezone( + datetime.timezone.utc + ).strftime( + '%Y-%m-%d %H:%M %Z' + ), + 'public': lambda project, **kwargs: project['attributes']['public'], + 'category': get_category, + 'tags': get_tags, + 'resource_type': lambda project, **kwargs: 'NA', + 'resource_lang': lambda project, **kwargs: 'NA', + 'affiliated_institutions': get_affiliated_institutions, + 'identifiers': get_identifiers, + 'license': get_license, + 'subjects': get_subjects, + 'funders': lambda project, **kwargs: [], + }, + 'contributors': get_contributors + } + + for idx, project in enumerate(nodes['data']): + try: + if project['id'] in added_node_ids: + continue + else: + added_node_ids.add(project['id']) + + project_data = { + 'metadata': {} + } + for field in fields['metadata']: + project_data['metadata'][field] = fields['metadata'][field]( + project, dryrun=dryrun, key=field, pat=pat + ) + project_data['contributors'] = fields['contributors']( + project, dryrun=dryrun, key='contributors', pat=pat + ) + + # TODO: split into function + # Resource type/lang/funding info share specific endpoint + # that isn't linked to in user nodes' responses + if dryrun: + metadata = MockAPIResponse.read('custom_metadata') + else: + metadata = json.loads(call_api( + f"{api_host}/custom_item_metadata_records/{project['id']}/", + pat + ).read()) + metadata = metadata['data']['attributes'] + resource_type = metadata['resource_type_general'] + resource_lang = metadata['language'] + project_data['metadata']['resource_type'] = resource_type + project_data['metadata']['resource_lang'] = resource_lang + for funder in metadata['funders']: + project_data['metadata']['funders'].append(funder) + # ========= + + relations = project['relationships'] + + # Get list of files in project + if dryrun: + link = 'root' + use_mocks = True + else: + link = relations['files']['links']['related']['href'] + link += 'osfstorage/' # ID for OSF Storage + use_mocks = False + project_data['files'] = explore_file_tree(link, pat, dryrun=use_mocks) + + project_data['wikis'] = explore_wikis( + f'{api_host}/nodes/{project['id']}/wikis/', + pat=pat, dryrun=dryrun + ) + + # Check if parent info has been passed down to save effort + # If not then search for links to parent + try: + project_data['parent'] = project['parent'] + except KeyError: + project_data['parent'] = None + + # In general, start nodes for PDFs have no parents + if 'links' not in project['relationships']['parent']: + root_nodes.append(idx) + elif project_data['parent'] is None: + parent_link = project['relationships']['parent'][ + 'links']['related']['href'] + try: + if not dryrun: + parent = json.loads( + call_api( + parent_link, + pat=pat, + is_json=True + ).read() + ) + else: + parent = MockAPIResponse.read(parent_link) + project_data['parent'] = ( + parent['data']['attributes']['title'], + parent['data']['links']['html'] + ) + except (HTTPError, ValueError): + logging.warning( + f"Warning: Parent of {project_data['metadata']['title']} is private." + ) + logging.warning( + "Try to give a PAT beforehand using the --pat flag." + ) + + # Projects specified by ID to export also count as start nodes for PDFs + # This will be the first node in list of root nodes + if project_data['metadata']['id'] == project_id and 0 not in root_nodes: + root_nodes.append(idx) + + def get_children(json_page, **kwargs): + children = [] + for child in json_page['data']: + child['parent'] = [ + project_data['metadata']['title'], + project_data['metadata']['url'] + ] + children.append(child['id']) + nodes['data'].append(child) # Add to list of nodes to search + return children + + children_link = relations['children']['links']['related']['href'] + children = list(paginate_json_result( + children_link, dryrun=dryrun, pat=pat, action=get_children + )) + newlist = [item for sublist in children for item in sublist] + project_data['children'] = newlist + + projects.append(project_data) + except (HTTPError, KeyError) as e: + if isinstance(e, HTTPError): + if e.code == 429: + raise e + logging.warning(f"Warning: A project failed to export: {e.code}") + else: + logging.warning("Warning: A project failed to export: Unexpected API response.") + logging.warning("Continuing with exporting other projects...") + + return projects, root_nodes + + +def get_category(project, **kwargs): + # Define nice representations of categories if needed + CATEGORY_STRS = { + '': 'Uncategorized', + 'methods and measures': 'Methods and Measures' + } + if project['attributes']['category'] in CATEGORY_STRS: + return CATEGORY_STRS[project['attributes']['category']] + else: + return project['attributes']['category'].title() + + +def get_tags(project, **kwargs): + if project['attributes']['tags']: + return ', '.join(project['attributes']['tags']) + else: + return 'NA' + + +def get_contributors(project, **kwargs): + dryrun = kwargs.pop('dryrun', True) + key = kwargs.pop('key', 'contributors') + pat = kwargs.pop('pat', '') + if not dryrun: + # Check relationship exists and can get link to linked data + # Otherwise just pass a placeholder dict + try: + link = project['relationships'][key]['links']['related']['href'] + json_data = json.loads( + call_api( + link, pat, + filters=URL_FILTERS.get(key, {}) + ).read() + ) + except KeyError: + json_data = {'data': None} + else: + json_data = MockAPIResponse.read(key) + values = [] + for item in json_data['data']: + values.append(( + item['embeds']['users']['data'] + ['attributes']['full_name'], + item['attributes']['bibliographic'], + item['embeds']['users']['data']['links']['html'] + )) + return values + + +def get_affiliated_institutions(project, **kwargs): + dryrun = kwargs.pop('dryrun', True) + key = kwargs.pop('key', 'affiliated_institutions') + pat = kwargs.pop('pat', '') + if not dryrun: + # Check relationship exists and can get link to linked data + # Otherwise just pass a placeholder dict + try: + link = project['relationships'][key]['links']['related']['href'] + json_data = json.loads( + call_api( + link, pat, + filters=URL_FILTERS.get(key, {}) + ).read() + ) + except KeyError: + json_data = {'data': None} + else: + json_data = MockAPIResponse.read(key) + values = [] + for item in json_data['data']: + values.append(item['attributes']['name']) + values = ', '.join(values) + if not values: + values = 'NA' + return values + + +def get_identifiers(project, **kwargs): + dryrun = kwargs.pop('dryrun', True) + key = kwargs.pop('key', 'identifiers') + pat = kwargs.pop('pat', '') + if not dryrun: + # Check relationship exists and can get link to linked data + # Otherwise just pass a placeholder dict + try: + link = project['relationships'][key]['links']['related']['href'] + json_data = json.loads( + call_api( + link, pat, + filters=URL_FILTERS.get(key, {}) + ).read() + ) + except KeyError: + json_data = {'data': None} + else: + json_data = MockAPIResponse.read(key) + values = [] + for item in json_data['data']: + values.append(item['attributes']['value']) + values = ', '.join(values) + return values + + +def get_license(project, **kwargs): + dryrun = kwargs.pop('dryrun', True) + key = kwargs.pop('key', 'license') + pat = kwargs.pop('pat', '') + if not dryrun: + # Check relationship exists and can get link to linked data + # Otherwise just pass a placeholder dict + try: + link = project['relationships'][key]['links']['related']['href'] + json_data = json.loads( + call_api( + link, pat, + filters=URL_FILTERS.get(key, {}) + ).read() + ) + except KeyError: + json_data = {'data': None} + else: + json_data = MockAPIResponse.read(key) + if json_data['data'] is not None: + return json_data['data']['attributes']['name'] + else: + return None + + +def get_subjects(project, **kwargs): + dryrun = kwargs.pop('dryrun', True) + key = kwargs.pop('key', 'subjects') + pat = kwargs.pop('pat', '') + if not dryrun: + # Check relationship exists and can get link to linked data + # Otherwise just pass a placeholder dict + try: + link = project['relationships'][key]['links']['related']['href'] + json_data = json.loads( + call_api( + link, pat, + filters=URL_FILTERS.get(key, {}) + ).read() + ) + except KeyError: + raise KeyError() # Subjects should have a href link + else: + json_data = MockAPIResponse.read(key) + values = [] + for item in json_data['data']: + values.append(item['attributes']['text']) + values = ', '.join(values) + return values diff --git a/src/osfexport/font/DejaVuSans-Bold.ttf b/src/osfexport/font/DejaVuSans-Bold.ttf new file mode 100644 index 0000000..0f4d5e9 Binary files /dev/null and b/src/osfexport/font/DejaVuSans-Bold.ttf differ diff --git a/src/osfexport/font/DejaVuSans-BoldOblique.ttf b/src/osfexport/font/DejaVuSans-BoldOblique.ttf new file mode 100644 index 0000000..48f06a8 Binary files /dev/null and b/src/osfexport/font/DejaVuSans-BoldOblique.ttf differ diff --git a/src/osfexport/font/DejaVuSans-Oblique.ttf b/src/osfexport/font/DejaVuSans-Oblique.ttf new file mode 100644 index 0000000..4485638 Binary files /dev/null and b/src/osfexport/font/DejaVuSans-Oblique.ttf differ diff --git a/src/osfexport/font/DejaVuSans.ttf b/src/osfexport/font/DejaVuSans.ttf new file mode 100644 index 0000000..27cff47 Binary files /dev/null and b/src/osfexport/font/DejaVuSans.ttf differ diff --git a/src/osfexport/font/DejaVu_LICENSE.txt b/src/osfexport/font/DejaVu_LICENSE.txt new file mode 100644 index 0000000..254e2cc --- /dev/null +++ b/src/osfexport/font/DejaVu_LICENSE.txt @@ -0,0 +1,99 @@ +Fonts are (c) Bitstream (see below). DejaVu changes are in public domain. +Glyphs imported from Arev fonts are (c) Tavmjong Bah (see below) + +Bitstream Vera Fonts Copyright +------------------------------ + +Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Bitstream Vera is +a trademark of Bitstream, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of the fonts accompanying this license ("Fonts") and associated +documentation files (the "Font Software"), to reproduce and distribute the +Font Software, including without limitation the rights to use, copy, merge, +publish, distribute, and/or sell copies of the Font Software, and to permit +persons to whom the Font Software is furnished to do so, subject to the +following conditions: + +The above copyright and trademark notices and this permission notice shall +be included in all copies of one or more of the Font Software typefaces. + +The Font Software may be modified, altered, or added to, and in particular +the designs of glyphs or characters in the Fonts may be modified and +additional glyphs or characters may be added to the Fonts, only if the fonts +are renamed to names not containing either the words "Bitstream" or the word +"Vera". + +This License becomes null and void to the extent applicable to Fonts or Font +Software that has been modified and is distributed under the "Bitstream +Vera" names. + +The Font Software may be sold as part of a larger software package but no +copy of one or more of the Font Software typefaces may be sold by itself. + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, +TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL BITSTREAM OR THE GNOME +FOUNDATION BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, INCLUDING +ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE +FONT SOFTWARE. + +Except as contained in this notice, the names of Gnome, the Gnome +Foundation, and Bitstream Inc., shall not be used in advertising or +otherwise to promote the sale, use or other dealings in this Font Software +without prior written authorization from the Gnome Foundation or Bitstream +Inc., respectively. For further information, contact: fonts at gnome dot +org. + +Arev Fonts Copyright +------------------------------ + +Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the fonts accompanying this license ("Fonts") and +associated documentation files (the "Font Software"), to reproduce +and distribute the modifications to the Bitstream Vera Font Software, +including without limitation the rights to use, copy, merge, publish, +distribute, and/or sell copies of the Font Software, and to permit +persons to whom the Font Software is furnished to do so, subject to +the following conditions: + +The above copyright and trademark notices and this permission notice +shall be included in all copies of one or more of the Font Software +typefaces. + +The Font Software may be modified, altered, or added to, and in +particular the designs of glyphs or characters in the Fonts may be +modified and additional glyphs or characters may be added to the +Fonts, only if the fonts are renamed to names not containing either +the words "Tavmjong Bah" or the word "Arev". + +This License becomes null and void to the extent applicable to Fonts +or Font Software that has been modified and is distributed under the +"Tavmjong Bah Arev" names. + +The Font Software may be sold as part of a larger software package but +no copy of one or more of the Font Software typefaces may be sold by +itself. + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL +TAVMJONG BAH BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. + +Except as contained in this notice, the name of Tavmjong Bah shall not +be used in advertising or otherwise to promote the sale, use or other +dealings in this Font Software without prior written authorization +from Tavmjong Bah. For further information, contact: tavmjong @ free +. fr. + +$Id: LICENSE 2133 2007-11-28 02:46:28Z lechimp $ diff --git a/src/osfexport/formatter.py b/src/osfexport/formatter.py new file mode 100644 index 0000000..811ddbe --- /dev/null +++ b/src/osfexport/formatter.py @@ -0,0 +1,420 @@ +import datetime +import os +import io +import html + +import PIL +from fpdf import FPDF, Align +from fpdf.fonts import FontFace +from fpdf.image_parsing import get_img_info +from mistletoe import markdown, HTMLRenderer +import qrcode +import urllib + + +class HTMLImageSizeCapRenderer(HTMLRenderer): + """Custom Markdown to HTML renderer which caps image size.""" + + max_width = 300 + max_height = 300 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def render_image(self, token): + """Render an image with a specified size.""" + + template = '{}' + + # Cap image size if needed so they can fit on the page + try: + img_info = get_img_info(token.src) + except (urllib.error.HTTPError, PIL.UnidentifiedImageError): + return f'{token.src}' + + if img_info['w'] > HTMLImageSizeCapRenderer.max_width: + new_width = HTMLImageSizeCapRenderer.max_width + else: + new_width = img_info['w'] + width = ' width="{}"'.format(html.escape(str(new_width))) + + if img_info['h'] > HTMLImageSizeCapRenderer.max_height: + new_height = HTMLImageSizeCapRenderer.max_height + else: + new_height = img_info['h'] + height = ' height="{}"'.format(html.escape(str(new_height))) + + if token.title: + title = ' title="{}"'.format(html.escape(token.title)) + else: + title = "" + return template.format(token.src, self.render_to_plain(token), title, width, height) + + +class PDF(FPDF): + """Custom PDF class to implement extra customisation. + Attributes: + date_printed: datetime + Date and time when the project was exported. + url: str + Current URL to include in QR codes. + parent_url: str + URL of root project to use in component sections. + parent_title: str + Title of root project to use in component sections. + """ + + # Global styles for PDF + BLUE = (173, 216, 230) + HEADINGS_STYLE = FontFace(emphasis="BOLD", fill_color=BLUE) + FONT_SIZES = { + 'h1': 16, # Project titles + 'h2': 12, # Section titles + 'h3': 10, # Section sub-titles + 'h4': 9, # Body + 'h5': 8 # Footer + } + LINK_STYLE = FontFace(emphasis="UNDERLINE", size_pt=FONT_SIZES['h5']) + LINE_PADDING = -1 # Gaps between lines + TITLE_CELL_WIDTH = 150 # Shorter width to avoid QR code clipping + CELL_WIDTH = 180 # Width of text cells + + def __init__(self, url=''): + super().__init__() + self.date_printed = datetime.datetime.now( + datetime.timezone.utc + ) + self.url = url + # Setup unicode font for use. Can have 4 styles + self.font = 'dejavu-sans' + self.add_font(self.font, style="", fname=os.path.join( + os.path.dirname(__file__), 'font', 'DejaVuSans.ttf')) + self.add_font(self.font, style="b", fname=os.path.join( + os.path.dirname(__file__), 'font', 'DejaVuSans-Bold.ttf')) + self.add_font(self.font, style="i", fname=os.path.join( + os.path.dirname(__file__), 'font', 'DejaVuSans-Oblique.ttf')) + self.add_font(self.font, style="bi", fname=os.path.join( + os.path.dirname(__file__), 'font', 'DejaVuSans-BoldOblique.ttf')) + + def generate_qr_code(self): + qr = qrcode.make(self.url) + img_byte_arr = io.BytesIO() + qr.save(img_byte_arr, format='PNG') + img_byte_arr.seek(0) + return img_byte_arr + + def footer(self): + self.set_y(-15) + self.set_x(-30) + self.set_font(self.font, size=PDF.FONT_SIZES['h5']) + self.cell(0, 10, f"Page: {self.page_no()}", align="C") + self.set_x(10) + timestamp = self.date_printed.strftime( + '%Y-%m-%d %H:%M:%S %Z' + ) + self.cell(0, 10, f"Exported: {timestamp}", align="L") + self.set_x(10) + self.set_y(-15) + qr_img = self.generate_qr_code() + self.image(qr_img, w=15, h=15, x=Align.C) + + def _write_list_section(self, key, fielddict): + """Handle writing fields of different types inplace to a PDF. + Possible types are lists, strings or dictionaries. + + Parameters + ----------- + key: str + Name of the field to write. + fielddict: dict + Dictionary containing the field data. + """ + + # Set nicer display names for certain PDF fields + pdf_display_names = { + 'identifiers': 'DOI', + 'funders': 'Support/Funding Information' + } + if key in pdf_display_names: + field_name = pdf_display_names[key] + else: + field_name = key.replace('_', ' ').title() + + if isinstance(fielddict[key], list): + # Create separate paragraphs for more complex attributes + self.write(0, '\n') + self.set_font(self.font, size=PDF.FONT_SIZES['h3']) + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, + text=f'**{field_name}**\n', + align='L', markdown=True, padding=PDF.LINE_PADDING + ) + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + if len(fielddict[key]) > 0: + for idx, item in enumerate(fielddict[key]): + for subkey in item.keys(): + if subkey in pdf_display_names: + field_name = pdf_display_names[subkey] + else: + field_name = subkey.replace('_', ' ').title() + + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, + text=f'**{field_name}:** {item[subkey]}\n', + align='L', markdown=True, padding=PDF.LINE_PADDING + ) + if idx < len(fielddict[key])-1: + self.ln() + self.set_x(9) + else: + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, + text='NA', + align='L', markdown=True, padding=PDF.LINE_PADDING + ) + else: + # Simple key-value attributes can go on one-line + self.multi_cell( + w=PDF.CELL_WIDTH, + h=None, + text=f'**{field_name}:** {fielddict[key]}\n', + align='L', + markdown=True, + padding=PDF.LINE_PADDING + ) + + def _write_project_body(self, project): + """Write inplace the body of a project to the PDF. + + Parameters + ----------- + project: dict + Dictionary containing project data to write. + parent_title: str + Title of the parent project. + """ + + self.add_page() + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + wikis = project['wikis'] + + # Start with parent, project headers and links + parent = project['parent'] + if parent: + self.set_font(self.font, size=PDF.FONT_SIZES['h1'], style='B') + self.write(h=0, text=f'Parent: {parent[0]}\n') + self.set_font(self.font, size=PDF.FONT_SIZES['h3'], style='U') + self.write(h=0, text=f'{parent[1]}\n', link=parent[1]) + self.ln(h=5) + + # Pop URL field to avoid printing it out in Metadata section + url = project['metadata'].pop('url', '') + self.url = url # Set current URL to use in QR codes + qr_img = self.generate_qr_code() + self.image(qr_img, w=30, x=180, y=5) + title = project['metadata']['title'] + self.set_font(self.font, size=PDF.FONT_SIZES['h1'], style='B') + self.write( + h=0, + text=f"{'Component: ' if parent else ''}{title} \n" + ) + self.set_font(self.font, size=PDF.FONT_SIZES['h3'], style='U') + self.write(h=0, text=f'{url}\n', link=url) + self.ln(h=5) + + # Write title for metadata section, then actual fields + self.set_font(self.font, size=PDF.FONT_SIZES['h2'], style='B') + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, text='1. Project Metadata\n', + align='L', padding=PDF.LINE_PADDING) + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + for key in project['metadata']: + self._write_list_section(key, project['metadata']) + self.ln(h=7) + + # Write Contributors in table + self.set_x(8) + self.set_font(self.font, size=PDF.FONT_SIZES['h2'], style='B') + self.multi_cell(w=PDF.CELL_WIDTH, h=None, text='2. Contributors\n', align='L') + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + with self.table( + headings_style=PDF.HEADINGS_STYLE, + col_widths=(0.8, 0.5, 1.2), align="LEFT" + ) as table: + row = table.row() + row.cell('Name') + row.cell('Bibliographic?') + row.cell('Profile Link') + self.set_font(self.font, size=PDF.FONT_SIZES['h5']) + for data_row in project['contributors']: + row = table.row() + for idx, datum in enumerate(data_row): + if datum is True: + datum = 'Yes' + if datum is False: + datum = 'No' + if idx == 2: + row.cell(text=datum, link=datum, style=self.LINK_STYLE) + else: + row.cell(datum) + self.ln(h=7) + + # List files stored in storage providers + # For now only OSF Storage is involved + self.set_x(8) + self.set_font(self.font, size=PDF.FONT_SIZES['h2'], style='B') + self.multi_cell(w=PDF.CELL_WIDTH, h=None, text='3. Files in Main Project\n', align='L') + self.set_font(self.font, size=PDF.FONT_SIZES['h3'], style='B') + self.multi_cell(w=PDF.CELL_WIDTH, h=None, text='OSF Storage\n', align='L') + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + if len(project['files']) > 0: + with self.table( + headings_style=PDF.HEADINGS_STYLE, + col_widths=(1, 0.3, 1.2), align="LEFT" + ) as table: + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + row = table.row() + row.cell('File Name') + row.cell('Size (MB)') + row.cell('Download Link') + self.set_font(self.font, size=PDF.FONT_SIZES['h5']) + for data_row in project['files']: + row = table.row() + for idx, datum in enumerate(data_row): + if datum is True: + datum = 'Yes' + if datum is False or datum is None: + datum = 'N/A' + if idx == 2: + row.cell(text=datum, link=datum, style=self.LINK_STYLE) + else: + row.cell(datum) + else: + self.write(0, '\n') + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, text='No files found for this project.\n', align='L' + ) + self.write(0, '\n') + self.ln(h=10) + + # Write wikis separately to more easily handle Markdown parsing + self._write_wiki_pages( + wikis, parent=parent, title=project['metadata']['title'] + ) + + def _write_wiki_pages(self, wikis, title, parent=None): + """Write inplace the wiki pages to the PDF. + + Parameters + ----------- + wikis: dict + Dictionary containing wiki data to write. + """ + + for i, wiki in enumerate(wikis.keys()): + self.add_page() + if i == 0: + self.set_font(self.font, size=PDF.FONT_SIZES['h2'], style='B') + title_template = '4. Wiki ({}{}{})' + if parent: + header = title_template.format(f'Parent: {parent[0]}', ' | ', title) + else: + header = title_template.format('', '', title) + self.multi_cell( + w=PDF.CELL_WIDTH, h=None, text=f'{header}\n', + align='L') + self.ln() + self.set_font(self.font, size=PDF.FONT_SIZES['h2'], style='B') + self.multi_cell(w=PDF.CELL_WIDTH, h=None, text=f'{wiki}\n') + self.set_font(self.font, size=PDF.FONT_SIZES['h4']) + html = markdown( + wikis[wiki], + renderer=HTMLImageSizeCapRenderer + ) + self.write_html(html) + + +def explore_project_tree(project, projects, pdf=None): + """Recursively find child projects and write them to a PDF. + + Parameters + ----------- + project: dict + Dictionary containing project data to write. + projects: list[dict] + List of all projects to explore. + pdf: PDF + PDF object to write to. If None, a new PDF will be created. + parent_title: str + Title of the parent project. + + Returns + ----------- + pdf: PDF + PDF object with the project and its children written to it.""" + + # Start with no PDF at root projects + if not pdf: + pdf = PDF() + + pdf.set_line_width(0.05) + pdf.set_left_margin(10) + pdf.set_right_margin(30) + + # Add current project to PDF + pdf._write_project_body(project) + + # Do children last so that they come at end of the PDF + children = project['children'] + for child_id in children: + child_project = next( + (p for p in projects if p['metadata']['id'] == child_id), None + ) + if child_project: + pdf = explore_project_tree( + child_project, projects, pdf=pdf + ) + + return pdf + + +def write_pdf(projects, root_idx, folder=''): + """Make PDF for each project. + + Parameters + ------------ + projects: dict[str, str|tuple] + Projects found to export into the PDF. + root_idx: int + Position of root node (no parent) in the projects list. + This is used for accessing root projects without sorting the list. + folder: str + The path to the folder to output the project PDFs in. + Default is the current working directory. + + Returns + ------------ + pdfs: list + List of created PDF files. + """ + + curr_project = projects[root_idx] + title = curr_project['metadata']['title'] + pdf = explore_project_tree(curr_project, projects) + + # Remove spaces in file name for better behaviour on Linux + # Add timestamp to allow distinguishing between PDFs at a glance + timestamp = pdf.date_printed.strftime( + '%Y-%m-%d %H-%M-%S %Z' + ).replace(' ', '-') + filename = f'{title.replace(' ', '-')}-{timestamp}.pdf' + + if folder: + if not os.path.exists(folder): + os.mkdir(folder) + path = os.path.join(os.getcwd(), folder, filename) + else: + path = os.path.join(os.getcwd(), filename) + pdf.output(path) + + return pdf, path diff --git a/src/osfexport/stubs/asingle.json b/src/osfexport/stubs/asingle.json new file mode 100644 index 0000000..97bc9f9 --- /dev/null +++ b/src/osfexport/stubs/asingle.json @@ -0,0 +1,322 @@ +{ + "data": { + "id": "a", + "type": "nodes", + "attributes": { + "title": "Component1", + "description": "Component1", + "category": "", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "links": { + "related": { + "href": "x", + "meta": {} + } + }, + "data": { + "id": "puhnd", + "type": "nodes" + } + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/a/", + "self": "https://api.test.osf.io/v2/nodes/a/", + "iri": "https://test.osf.io/a" + } + }, + "links": { + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/components/empty-children.json b/src/osfexport/stubs/components/empty-children.json new file mode 100644 index 0000000..e83ed86 --- /dev/null +++ b/src/osfexport/stubs/components/empty-children.json @@ -0,0 +1,15 @@ +{ + "data": [], + "meta": { + "total": 0, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/empty/children/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/components/x-child-1.json b/src/osfexport/stubs/components/x-child-1.json new file mode 100644 index 0000000..e946c93 --- /dev/null +++ b/src/osfexport/stubs/components/x-child-1.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "id": "a", + "type": "nodes", + "attributes": { + "title": "Component1", + "description": "Component1", + "category": "", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "links": { + "related": { + "href": "x", + "meta": {} + } + }, + "data": { + "id": "puhnd", + "type": "nodes" + } + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/a/", + "self": "https://api.test.osf.io/v2/nodes/a/", + "iri": "https://test.osf.io/x" + } + } + ], + "links": { + "next": "x-child-2" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/components/x-child-2.json b/src/osfexport/stubs/components/x-child-2.json new file mode 100644 index 0000000..ec5c598 --- /dev/null +++ b/src/osfexport/stubs/components/x-child-2.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "id": "b", + "type": "nodes", + "attributes": { + "title": "secondComp", + "description": "secondComp", + "category": "Data", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "links": { + "related": { + "href": "a", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "nodes" + } + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/b/", + "self": "https://api.test.osf.io/v2/nodes/b/", + "iri": "https://test.osf.io/b" + } + } + ], + "links": { + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/contributorstubs.json b/src/osfexport/stubs/contributorstubs.json new file mode 100644 index 0000000..1cc8231 --- /dev/null +++ b/src/osfexport/stubs/contributorstubs.json @@ -0,0 +1,105 @@ +{ + "data": [ + { + "id": "x-userid", + "type": "contributors", + "attributes": { + "index": 0, + "bibliographic": false, + "permission": "admin", + "unregistered_contributor": null, + "is_curator": false + }, + "relationships": {}, + "embeds": { + "users": { + "data": { + "id": "userid", + "type": "users", + "attributes": { + "full_name": "Test User 1", + "given_name": "", + "middle_names": "", + "family_name": "", + "suffix": "", + "date_registered": "", + "active": true, + "timezone": "Etc/UTC", + "locale": "en_US", + "social": {}, + "employment": [], + "education": [] + }, + "relationships": {}, + "links": { + "html": "https://test.osf.io/userid/", + "profile_image": "https://secure.gravatar.com/avatar/id?d=identicon", + "self": "https://api.test.osf.io/v2/users/userid/", + "iri": "https://test.osf.io/userid" + } + } + } + }, + "links": { + "html": "https://test.osf.io/userid/", + "profile_image": "https://secure.gravatar.com/avatar/id?d=identicon", + "self": "https://api.test.osf.io/v2/users/userid/", + "iri": "https://test.osf.io/userid" + } + }, + { + "id": "x-userid2", + "type": "contributors", + "attributes": { + "index": 1, + "bibliographic": true, + "permission": "admin", + "unregistered_contributor": null, + "is_curator": false + }, + "relationships": {}, + "embeds": { + "users": { + "data": { + "id": "userid2", + "type": "users", + "attributes": { + "full_name": "Test User 2", + "given_name": "", + "middle_names": "", + "family_name": "", + "suffix": "", + "date_registered": "", + "active": true, + "timezone": "Etc/UTC", + "locale": "en_US" + }, + "relationships": {}, + "links": { + "html": "https://test.osf.io/userid2/", + "profile_image": "https://secure.gravatar.com/avatar/id?d=identicon", + "self": "https://api.test.osf.io/v2/users/userid2/", + "iri": "https://test.osf.io/userid2" + } + } + } + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/contributors/userid2/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "total_bibliographic": 2, + "version": "2.20" + }, + "links": { + "self": null, + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/custommetadatastub.json b/src/osfexport/stubs/custommetadatastub.json new file mode 100644 index 0000000..28d7cbf --- /dev/null +++ b/src/osfexport/stubs/custommetadatastub.json @@ -0,0 +1,44 @@ +{ + "data": { + "id": "x", + "type": "custom-item-metadata-records", + "attributes": { + "language": "eng", + "resource_type_general": "Other", + "funders": [ + { + "funder_name": "funder1", + "funder_identifier": "123456", + "funder_identifier_type": "ROR", + "award_number": "fund-1234", + "award_uri": "https://click.palletsprojects.com/en/stable/changes/", + "award_title": "A Research Grant" + }, + { + "funder_name": "funder1", + "funder_identifier": "https://doi.org/12,918eufusdduds", + "funder_identifier_type": "Crossref Funder ID", + "award_number": "123", + "award_uri": "", + "award_title": "title2" + } + ] + }, + "relationships": { + "guid": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/guids/x/", + "meta": {} + } + } + } + }, + "links": { + "self": "https://api.test.osf.io/v2/custom_item_metadata_records/x/" + } + }, + "meta": { + "version": "2.20" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/doistubs.json b/src/osfexport/stubs/doistubs.json new file mode 100644 index 0000000..3b64b95 --- /dev/null +++ b/src/osfexport/stubs/doistubs.json @@ -0,0 +1,34 @@ +{ + "data": [ + { + "relationships": { + "referent": { + "links": { + "related": { + "href": "https://api.osf.io/v2/nodes/x/", + "meta": {} + } + } + } + }, + "links": { + "self": "https://api.osf.io/v2/identifiers/x/" + }, + "attributes": { + "category": "doi", + "value": "10.4-2-6-25/OSF.IO/74PAD" + }, + "type": "identifiers", + "id": "x" + } + ], + "links": { + "first": null, + "last": null, + "prev": null, + "next": null, + "meta": null, + "total": 2, + "per_page": 10 + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/rootfiles.json b/src/osfexport/stubs/files/rootfiles.json new file mode 100644 index 0000000..b652a4b --- /dev/null +++ b/src/osfexport/stubs/files/rootfiles.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "id": "x", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "helloworld.txt.txt", + "kind": "file", + "path": "/x", + "size": 12, + "provider": "osfstorage", + "materialized_path": "/helloworld.txt.txt", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": "z", + "sha256": "z" + }, + "downloads": 0 + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/versions/", + "meta": {} + } + } + }, + "comments": { + "data": null + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "download": "https://test.osf.io/download/z/", + "render": "https://mfr.de-1.test.osf.io/render?url=https%3A%2F%2Ftest.osf.io%2Fdownload%2Fz%2F%3Fdirect%26mode%3Drender", + "html": "https://test.osf.io/x/files/osfstorage/z", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/rootfolders.json b/src/osfexport/stubs/files/rootfolders.json new file mode 100644 index 0000000..868ed62 --- /dev/null +++ b/src/osfexport/stubs/files/rootfolders.json @@ -0,0 +1,95 @@ +{ + "data": [ + { + "id": "z", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "tf1", + "kind": "folder", + "path": "/z/", + "size": null, + "provider": "osfstorage", + "materialized_path": "/tf1/", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": null, + "sha256": null + } + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "files": { + "links": { + "related": { + "href": "tf1", + "meta": {} + } + } + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "new_folder": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/?kind=folder", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf1files.json b/src/osfexport/stubs/files/tf1files.json new file mode 100644 index 0000000..b25991e --- /dev/null +++ b/src/osfexport/stubs/files/tf1files.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "id": "x", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "helloworld.txt.txt", + "kind": "file", + "path": "/x", + "size": 2202010, + "provider": "osfstorage", + "materialized_path": "/tf1/helloworld.txt.txt", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": "z", + "sha256": "z" + }, + "downloads": 0 + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/versions/", + "meta": {} + } + } + }, + "comments": { + "data": null + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "download": "https://test.osf.io/download/z/", + "render": "https://mfr.de-1.test.osf.io/render?url=https%3A%2F%2Ftest.osf.io%2Fdownload%2Fz%2F%3Fdirect%26mode%3Drender", + "html": "https://test.osf.io/x/files/osfstorage/z", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf1folders-2.json b/src/osfexport/stubs/files/tf1folders-2.json new file mode 100644 index 0000000..7a5e5a4 --- /dev/null +++ b/src/osfexport/stubs/files/tf1folders-2.json @@ -0,0 +1,95 @@ +{ + "data": [ + { + "id": "z", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "tf3", + "kind": "folder", + "path": "/z/", + "size": null, + "provider": "osfstorage", + "materialized_path": "/tf1/tf3/", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": null, + "sha256": null + } + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "files": { + "links": { + "related": { + "href": "tf2-second", + "meta": {} + } + } + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "new_folder": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/?kind=folder", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": "tf1-1", + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf1folders.json b/src/osfexport/stubs/files/tf1folders.json new file mode 100644 index 0000000..f22b232 --- /dev/null +++ b/src/osfexport/stubs/files/tf1folders.json @@ -0,0 +1,95 @@ +{ + "data": [ + { + "id": "z", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "tf2", + "kind": "folder", + "path": "/z/", + "size": null, + "provider": "osfstorage", + "materialized_path": "/tf1/tf2/", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": null, + "sha256": null + } + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "files": { + "links": { + "related": { + "href": "tf2", + "meta": {} + } + } + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/", + "new_folder": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z/?kind=folder", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": "tf1-2" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf2-second-files-2.json b/src/osfexport/stubs/files/tf2-second-files-2.json new file mode 100644 index 0000000..8155bcf --- /dev/null +++ b/src/osfexport/stubs/files/tf2-second-files-2.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "id": "x", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "file.txt", + "kind": "file", + "path": "/x", + "size": 2202010, + "provider": "osfstorage", + "materialized_path": "/tf1/tf2-second/thirdpage.txt", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": "z", + "sha256": "z" + }, + "downloads": 0 + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/versions/", + "meta": {} + } + } + }, + "comments": { + "data": null + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "download": "https://test.osf.io/download/z/", + "render": "https://mfr.de-1.test.osf.io/render?url=https%3A%2F%2Ftest.osf.io%2Fdownload%2Fz%2F%3Fdirect%26mode%3Drender", + "html": "https://test.osf.io/x/files/osfstorage/z", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": "tf2-second", + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf2-second-files.json b/src/osfexport/stubs/files/tf2-second-files.json new file mode 100644 index 0000000..9fa4c5b --- /dev/null +++ b/src/osfexport/stubs/files/tf2-second-files.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "id": "x", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "file.txt", + "kind": "file", + "path": "/x", + "size": 2202010, + "provider": "osfstorage", + "materialized_path": "/tf1/tf2-second/secondpage.txt", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": "z", + "sha256": "z" + }, + "downloads": 0 + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/versions/", + "meta": {} + } + } + }, + "comments": { + "data": null + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "download": "https://test.osf.io/download/z/", + "render": "https://mfr.de-1.test.osf.io/render?url=https%3A%2F%2Ftest.osf.io%2Fdownload%2Fz%2F%3Fdirect%26mode%3Drender", + "html": "https://test.osf.io/x/files/osfstorage/z", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": "tf2-second-2" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf2-second-folders.json b/src/osfexport/stubs/files/tf2-second-folders.json new file mode 100644 index 0000000..bdf890a --- /dev/null +++ b/src/osfexport/stubs/files/tf2-second-folders.json @@ -0,0 +1,17 @@ +{ + "data": [ + {} + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf2files.json b/src/osfexport/stubs/files/tf2files.json new file mode 100644 index 0000000..875dfb7 --- /dev/null +++ b/src/osfexport/stubs/files/tf2files.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "id": "x", + "type": "files", + "attributes": { + "guid": null, + "checkout": null, + "name": "file.txt", + "kind": "file", + "path": "/x", + "size": 2202010, + "provider": "osfstorage", + "materialized_path": "/tf1/tf2/file.txt", + "last_touched": null, + "date_modified": null, + "date_created": null, + "extra": { + "hashes": { + "md5": "z", + "sha256": "z" + }, + "downloads": 0 + }, + "tags": [], + "current_user_can_comment": true, + "current_version": 1, + "show_as_unviewed": false + }, + "relationships": { + "parent_folder": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/", + "meta": {} + } + }, + "data": { + "id": "z", + "type": "files" + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/versions/", + "meta": {} + } + } + }, + "comments": { + "data": null + }, + "target": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": { + "type": "nodes" + } + } + }, + "data": { + "type": "nodes", + "id": "x" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/files/z/cedar_metadata_records/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/files/z/", + "move": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "upload": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "delete": "https://files.de-1.test.osf.io/v1/resources/x/providers/osfstorage/z", + "download": "https://test.osf.io/download/z/", + "render": "https://mfr.de-1.test.osf.io/render?url=https%3A%2F%2Ftest.osf.io%2Fdownload%2Fz%2F%3Fdirect%26mode%3Drender", + "html": "https://test.osf.io/x/files/osfstorage/z", + "self": "https://api.test.osf.io/v2/files/z/" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/files/tf2folders.json b/src/osfexport/stubs/files/tf2folders.json new file mode 100644 index 0000000..bdf890a --- /dev/null +++ b/src/osfexport/stubs/files/tf2folders.json @@ -0,0 +1,17 @@ +{ + "data": [ + {} + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/files/osfstorage/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/institutionstubs.json b/src/osfexport/stubs/institutionstubs.json new file mode 100644 index 0000000..56c1c14 --- /dev/null +++ b/src/osfexport/stubs/institutionstubs.json @@ -0,0 +1,11 @@ +{ + "data": [ + { + "id": "cos", + "type": "institutions", + "attributes": { + "name": "Center For Open Science [Test]" + } + } + ] +} \ No newline at end of file diff --git a/src/osfexport/stubs/licensestub.json b/src/osfexport/stubs/licensestub.json new file mode 100644 index 0000000..6ea54b4 --- /dev/null +++ b/src/osfexport/stubs/licensestub.json @@ -0,0 +1,18 @@ +{ + "data": { + "id": "x", + "type": "licenses", + "attributes": { + "name": "mynewlicense", + "text": "this is a fancy license", + "url": "url", + "required_fields": [] + }, + "links": { + "self": "link" + } + }, + "meta": { + "version": "2.20" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/nodestubs.json b/src/osfexport/stubs/nodestubs.json new file mode 100644 index 0000000..a9691b5 --- /dev/null +++ b/src/osfexport/stubs/nodestubs.json @@ -0,0 +1,1591 @@ +{ + "data": [ + { + "id": "x", + "type": "nodes", + "attributes": { + "title": "Test1", + "description": "Test1 Description", + "category": "methods and measures", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [ + "test1", + "test2", + "test3" + ], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": true + }, + "relationships": { + "license": { + "links": { + "related": { + "href": "license", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "licenses" + } + }, + "children": { + "links": { + "related": { + "href": "x-child-1", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "data": null + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/x/", + "self": "https://api.test.osf.io/v2/nodes/x/", + "iri": "https://test.osf.io/x/" + } + }, + { + "id": "y", + "type": "nodes", + "attributes": { + "title": "Test2", + "description": "Test2 Description", + "category": "", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "data": null + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/y/", + "self": "https://api.test.osf.io/v2/nodes/y/", + "iri": "https://test.osf.io/y" + } + }, + { + "id": "a", + "type": "nodes", + "attributes": { + "title": "Component1", + "description": "Component1", + "category": "", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "links": { + "related": { + "href": "x", + "meta": {} + } + }, + "data": { + "id": "puhnd", + "type": "nodes" + } + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/a/", + "self": "https://api.test.osf.io/v2/nodes/a/", + "iri": "https://test.osf.io/a" + } + }, + { + "id": "b", + "type": "nodes", + "attributes": { + "title": "secondComp", + "description": "secondComp", + "category": "Data", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "links": { + "related": { + "href": "a", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "nodes" + } + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/b/", + "self": "https://api.test.osf.io/v2/nodes/b/", + "iri": "https://test.osf.io/b" + } + }, + { + "id": "x", + "type": "nodes", + "attributes": { + "title": "Test1", + "description": "Test1 Description", + "category": "project", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [ + "test1", + "test2", + "test3" + ], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "links": { + "related": { + "href": "license", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "licenses" + } + }, + "children": { + "links": { + "related": { + "href": "x-child-1", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "data": null + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/x/", + "self": "https://api.test.osf.io/v2/nodes/x/", + "iri": "https://test.osf.io/x" + } + } + ], + "links": { + "next": "nodes2" + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/nodestubs2.json b/src/osfexport/stubs/nodestubs2.json new file mode 100644 index 0000000..e94753d --- /dev/null +++ b/src/osfexport/stubs/nodestubs2.json @@ -0,0 +1,328 @@ +{ + "data": [ + { + "id": "x2", + "type": "nodes", + "attributes": { + "title": "Testx2", + "description": "Node on second page", + "category": "methods and measures", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [ + "test1", + "test2", + "test3" + ], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": true + }, + "relationships": { + "license": { + "links": { + "related": { + "href": "license", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "licenses" + } + }, + "children": { + "links": { + "related": { + "href": "empty-children", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "contributors", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "data": null + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/x/", + "self": "https://api.test.osf.io/v2/nodes/x/", + "iri": "https://test.osf.io/x" + } + } + ], + "links": { + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/singlenode.json b/src/osfexport/stubs/singlenode.json new file mode 100644 index 0000000..45770d4 --- /dev/null +++ b/src/osfexport/stubs/singlenode.json @@ -0,0 +1,329 @@ +{ + "data": { + "id": "x", + "type": "nodes", + "attributes": { + "title": "Test1", + "description": "Test1 Description", + "category": "project", + "custom_citation": null, + "date_created": "2000-01-01T14:18:00.376705Z", + "date_modified": "2000-01-01T14:18:00.376705Z", + "registration": false, + "preprint": false, + "fork": false, + "collection": false, + "tags": [ + "test1", + "test2", + "test3" + ], + "node_license": null, + "analytics_key": "", + "current_user_can_comment": true, + "current_user_permissions": [ + "admin", + "write", + "read" + ], + "current_user_is_contributor": true, + "current_user_is_contributor_or_group_member": true, + "wiki_enabled": true, + "public": false + }, + "relationships": { + "license": { + "links": { + "related": { + "href": "license", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "licenses" + } + }, + "children": { + "links": { + "related": { + "href": "x-child-1", + "meta": {} + } + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/comments/?filter%5Btarget%5D=x", + "meta": {} + } + } + }, + "contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/contributors/", + "meta": {} + } + } + }, + "bibliographic_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/bibliographic_contributors/", + "meta": {} + } + } + }, + "implicit_contributors": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/implicit_contributors/", + "meta": {} + } + } + }, + "files": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/files/", + "meta": {} + } + } + }, + "settings": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/settings/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-setting" + } + }, + "wikis": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/wikis/", + "meta": {} + } + } + }, + "forked_from": { + "data": null + }, + "template_node": { + "data": null + }, + "forks": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/forks/", + "meta": {} + } + } + }, + "groups": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/groups/", + "meta": {} + } + } + }, + "linked_by_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_nodes/", + "meta": {} + } + } + }, + "linked_by_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_by_registrations/", + "meta": {} + } + } + }, + "parent": { + "data": null + }, + "identifiers": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/identifiers/", + "meta": {} + } + } + }, + "affiliated_institutions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/institutions/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/institutions/", + "meta": {} + } + } + }, + "draft_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/draft_registrations/", + "meta": {} + } + } + }, + "registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/registrations/", + "meta": {} + } + } + }, + "region": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/regions/de-1/", + "meta": {} + } + }, + "data": { + "id": "de-1", + "type": "regions" + } + }, + "root": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "nodes" + } + }, + "logs": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/logs/", + "meta": {} + } + } + }, + "linked_nodes": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_nodes/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_nodes/", + "meta": {} + } + } + }, + "linked_registrations": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/linked_registrations/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/linked_registrations/", + "meta": {} + } + } + }, + "view_only_links": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/view_only_links/", + "meta": {} + } + } + }, + "citation": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/citation/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "citation" + } + }, + "preprints": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/preprints/", + "meta": {} + } + } + }, + "storage": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/storage/", + "meta": {} + } + }, + "data": { + "id": "x", + "type": "node-storage" + } + }, + "cedar_metadata_records": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/cedar_metadata_records/", + "meta": {} + } + } + }, + "subjects_acceptable": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/", + "meta": {} + } + } + }, + "subjects": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/x/subjects/", + "meta": {} + }, + "self": { + "href": "https://api.test.osf.io/v2/nodes/x/relationships/subjects/", + "meta": {} + } + } + } + }, + "links": { + "html": "https://test.osf.io/x/", + "self": "https://api.test.osf.io/v2/nodes/x/", + "iri": "https://test.osf.io/x" + } + }, + "meta": { + "version": "2.20" + }, + "links": { + "next": null + } +} diff --git a/src/osfexport/stubs/subjectsstub.json b/src/osfexport/stubs/subjectsstub.json new file mode 100644 index 0000000..75c04f1 --- /dev/null +++ b/src/osfexport/stubs/subjectsstub.json @@ -0,0 +1,118 @@ +{ + "data": [ + { + "id": "6284093a3747220011135b99", + "type": "subjects", + "attributes": { + "text": "Education", + "taxonomy_name": "SocArXiv" + }, + "relationships": { + "parent": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/6284093a3747220011135b99/children/", + "meta": {} + } + } + } + }, + "embeds": { + "parent": { + "errors": [ + { + "detail": "Not found." + } + ] + } + }, + "links": { + "self": "https://api.test.osf.io/v2/subjects/6284093a3747220011135b99/", + "iri": "https://api.test.osf.io/v2/subjects/6284093a3747220011135b99" + } + }, + { + "id": "59552883da3e240081ba32ab", + "type": "subjects", + "attributes": { + "text": "Literature", + "taxonomy_name": "bepress" + }, + "relationships": { + "parent": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba32ab/children/", + "meta": {} + } + } + } + }, + "embeds": { + "parent": { + "errors": [ + { + "detail": "Not found." + } + ] + } + }, + "links": { + "self": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba32ab/", + "iri": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba32ab" + } + }, + { + "id": "59552883da3e240081ba3289", + "type": "subjects", + "attributes": { + "text": "Geography", + "taxonomy_name": "bepress" + }, + "relationships": { + "parent": { + "data": null + }, + "children": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba3289/children/", + "meta": {} + } + } + } + }, + "embeds": { + "parent": { + "errors": [ + { + "detail": "Not found." + } + ] + } + }, + "links": { + "self": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba3289/", + "iri": "https://api.test.osf.io/v2/subjects/59552883da3e240081ba3289" + } + } + ], + "meta": { + "total": 3, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/x/subjects/", + "first": null, + "last": null, + "prev": null, + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/wikis/anotherone.md b/src/osfexport/stubs/wikis/anotherone.md new file mode 100644 index 0000000..582f433 --- /dev/null +++ b/src/osfexport/stubs/wikis/anotherone.md @@ -0,0 +1 @@ +This wiki was on page 2 of paginated JSON. \ No newline at end of file diff --git a/src/osfexport/stubs/wikis/helloworld.md b/src/osfexport/stubs/wikis/helloworld.md new file mode 100644 index 0000000..bc7774a --- /dev/null +++ b/src/osfexport/stubs/wikis/helloworld.md @@ -0,0 +1 @@ +hello world! \ No newline at end of file diff --git a/src/osfexport/stubs/wikis/home.md b/src/osfexport/stubs/wikis/home.md new file mode 100644 index 0000000..e06102d --- /dev/null +++ b/src/osfexport/stubs/wikis/home.md @@ -0,0 +1,44 @@ +hello world! + +# h1 +## h2 +### h3 +#### h4 +##### h5 +###### h6 + +**bold** +*italics* +***bold italics*** +~~strikethrough~~ + + +---------- +#### hr rules +---------- + + +> here's a quote + +> fancy + + here's some code + another line of code; + +[link to project board][1] + + 1. List item + 2. item + 3. item + + + +item + + - List item + - more + - more + + [1]: https://github.com/orgs/CenterForOpenScience/projects/10 + + \ No newline at end of file diff --git a/src/osfexport/stubs/wikis/wikis2stubs.json b/src/osfexport/stubs/wikis/wikis2stubs.json new file mode 100644 index 0000000..3dbdaa9 --- /dev/null +++ b/src/osfexport/stubs/wikis/wikis2stubs.json @@ -0,0 +1,81 @@ +{ + "data": [ + { + "id": "apaaoalm", + "type": "wikis", + "attributes": { + "name": "anotherone", + "kind": "file", + "size": 53, + "path": "/a", + "materialized_path": "/a", + "date_modified": "2025-07-10T14:30:16.009373Z", + "content_type": "text/markdown", + "current_user_can_comment": true, + "extra": { + "version": 2 + } + }, + "relationships": { + "user": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/users/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "users" + } + }, + "node": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "nodes" + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/comments/?filter%5Btarget%5D=a", + "meta": {} + } + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/wikis/a/versions/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/wikis/a/", + "download": "helloworld", + "self": "https://api.test.osf.io/v2/wikis/a/", + "iri": "https://test.osf.io/a" + } + } + ], + "meta": { + "total": 1, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/a/wikis/", + "first": null, + "last": null, + "prev": "wikis", + "next": null + } +} \ No newline at end of file diff --git a/src/osfexport/stubs/wikis/wikistubs.json b/src/osfexport/stubs/wikis/wikistubs.json new file mode 100644 index 0000000..89c9dcd --- /dev/null +++ b/src/osfexport/stubs/wikis/wikistubs.json @@ -0,0 +1,146 @@ +{ + "data": [ + { + "id": "dsdsbjhds", + "type": "wikis", + "attributes": { + "name": "helloworld", + "kind": "file", + "size": 53, + "path": "/a", + "materialized_path": "/a", + "date_modified": "2025-07-10T14:30:16.009373Z", + "content_type": "text/markdown", + "current_user_can_comment": true, + "extra": { + "version": 2 + } + }, + "relationships": { + "user": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/users/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "users" + } + }, + "node": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "nodes" + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/comments/?filter%5Btarget%5D=a", + "meta": {} + } + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/wikis/a/versions/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/wikis/a/", + "download": "helloworld", + "self": "https://api.test.osf.io/v2/wikis/a/", + "iri": "https://test.osf.io/a" + } + }, + { + "id": "dbkjsdamnbs", + "type": "wikis", + "attributes": { + "name": "home", + "kind": "file", + "size": 477, + "path": "/a", + "materialized_path": "/a", + "date_modified": "2025-07-10T14:29:29.899826Z", + "content_type": "text/markdown", + "current_user_can_comment": true, + "extra": { + "version": 1 + } + }, + "relationships": { + "user": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/users/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "users" + } + }, + "node": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/", + "meta": {} + } + }, + "data": { + "id": "a", + "type": "nodes" + } + }, + "comments": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/nodes/a/comments/?filter%5Btarget%5D=a", + "meta": {} + } + } + }, + "versions": { + "links": { + "related": { + "href": "https://api.test.osf.io/v2/wikis/a/versions/", + "meta": {} + } + } + } + }, + "links": { + "info": "https://api.test.osf.io/v2/wikis/a/", + "download": "home", + "self": "https://api.test.osf.io/v2/wikis/a/", + "iri": "https://test.osf.io/a" + } + } + ], + "meta": { + "total": 2, + "per_page": 10, + "version": "2.20" + }, + "links": { + "self": "https://api.test.osf.io/v2/nodes/a/wikis/", + "first": null, + "last": null, + "prev": null, + "next": "wikis2" + } +} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_clitool.py b/tests/test_clitool.py new file mode 100644 index 0000000..6ac1d38 --- /dev/null +++ b/tests/test_clitool.py @@ -0,0 +1,1134 @@ +from collections import deque +import datetime +from unittest import TestCase +import os +import shutil +import json +import traceback +import urllib.error +from unittest.mock import patch, MagicMock, call +import importlib.metadata + +import PIL +from click.testing import CliRunner +from pypdf import PdfReader +from mistletoe import markdown + +from osfexport.exporter import ( + MockAPIResponse, + call_api, + get_project_data, + get_nodes, + explore_file_tree, + explore_wikis, + is_public, + extract_project_id, + paginate_json_result +) +from osfexport.cli import ( + cli, prompt_pat +) +from osfexport.formatter import ( + HTMLImageSizeCapRenderer, + write_pdf +) + +TEST_PDF_FOLDER = 'good-pdfs' +TEST_INPUT = 'test_pdf.pdf' +FOLDER_OUT = os.path.join('tests', 'outfolder') + + +class TestAPI(TestCase): + """Tests for interacting with the OSF API.""" + + API_HOST = 'https://api.test.osf.io/v2' + + def test_basic_api_call_works(self): + """Test for if JSON for user's projects are loaded correctly""" + + data = call_api( + f'{TestAPI.API_HOST}/', + pat='' + ) + assert data.status == 200 + + data = json.loads(data.read()) + assert isinstance(data, dict) + # All mocked data assumes API version 2.20 is used + assert data['meta']['version'] == '2.20', ( + 'Expected API version 2.20, actual version: ', + data['meta']['version'] + ) + + def test_get_public_status_on_code(self): + assert not is_public(f'{TestAPI.API_HOST}/users/me') + assert is_public(f'{TestAPI.API_HOST}') + + def test_get_public_projects_if_no_pat(self): + public_node_id = json.loads( + call_api( + f'{TestAPI.API_HOST}/nodes', pat='', + per_page=1, + filters={ + 'parent': '' + } + ).read() + )['data'][0]['id'] + + result = call_api( + f'{TestAPI.API_HOST}/nodes/{public_node_id}/', pat='', + ) + assert result.status == 200 + + def test_parse_single_project_json_as_expected(self): + # Use first public project available for this test + # TODO: allow choosing individual components to start export from + # Currently using a component will cause a fail + data = call_api( + f'{TestAPI.API_HOST}/nodes/', + pat='', + per_page=1, + filters={ + 'parent': '' + } + ) + node = json.loads(data.read())['data'][0] + id = extract_project_id(node['links']['html']) + projects, root_projects = get_nodes( + pat='', dryrun=False, + usetest=True, project_id=id + ) + + expected_child_count = len( + json.loads( + call_api( + f'{TestAPI.API_HOST}/nodes/{node["id"]}/children/', + pat='' + ).read() + )['data'] + ) + assert len(projects) == expected_child_count + 1 + assert len(root_projects) == 1, (root_projects) + assert projects[0]['metadata']['title'] == node['attributes']['title'] + assert isinstance(projects[0]['files'], list) + + def test_write_image_html_with_new_size(self): + url = 'https://osf.io/download/x/' + text = f"""This has an image in the wiki page. +![Someone taking a pic on their phone camera][1]This is an image above this text. +Another paragraph. + + [1]: {url}""" + + # Mock requests to simulate errors when trying to download images + with patch('urllib.request.urlopen') as mock_get: + mock_get.side_effect = urllib.error.HTTPError( + url='https://osf.io/download/x/', + code=401, + msg='Unauthorized', + hdrs={}, + fp=None + ) + html = markdown(text, renderer=HTMLImageSizeCapRenderer) + assert f'{url}' in html, ( + f'{url}', + html + ) + with patch('urllib.request.urlopen') as mock_get: + mock_get.side_effect = PIL.UnidentifiedImageError() + html = markdown(text, renderer=HTMLImageSizeCapRenderer) + assert f'{url}' in html, ( + f'{url}', + html + ) + + +class TestExporter(TestCase): + """Tests for the exporter without real API usage.""" + + @patch('osfexport.exporter.get_affiliated_institutions') + def test_get_project_data_handles_HTTP_errors(self, mock_get_inst): + mock_get_inst.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=401, + msg=' HTTP Error 401: Unauthorized', + hdrs={}, + fp=None + ) + nodes = MockAPIResponse.read('nodes') + projects, root_nodes = get_project_data( + nodes, + pat='', + dryrun=False, + usetest=True + ) + assert isinstance(projects, list) + assert isinstance(root_nodes, list) + # There are 4 real nodes in the mock JSON data + assert mock_get_inst.call_count == 4, ( + f'Wrong num of calls: {mock_get_inst.call_count}' + ) + + mock_get_inst.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=429, + msg='Too many requests', + hdrs={}, + fp=None + ) + nodes = MockAPIResponse.read('nodes') + with self.assertRaises(urllib.error.HTTPError): + projects, root_nodes = get_project_data( + nodes, + pat='', + dryrun=False, + usetest=True + ) + + @patch('osfexport.exporter.get_project_data') + def test_paginate_json_result_gets_next_page_despite_function_errors(self, mock_get_data): + mock_get_data.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=401, + msg=' HTTP Error 401: Unauthorized', + hdrs={}, + fp=None + ) + results = paginate_json_result( + start='nodes', action=mock_get_data, dryrun=True, usetest=False, + pat='', filters={}, project_id='', per_page=20, fail_on_first=False + ) + # There are 2 pages of nodes to read in the mock JSON data + assert mock_get_data.call_count == 2, ( + f'Wrong num of calls: {mock_get_data.call_count}' + ) + assert results is not None + + # Raise error on first error + with self.assertRaises(urllib.error.HTTPError): + results = paginate_json_result( + start='nodes', action=mock_get_data, dryrun=True, usetest=False, + pat='', filters={}, project_id='', per_page=20 + ) + + # Raise error if it's HTTP 429 error code + mock_get_data.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=429, + msg='Too many requests', + hdrs={}, + fp=None + ) + with self.assertRaises(urllib.error.HTTPError): + results = paginate_json_result( + start='nodes', action=mock_get_data, dryrun=True, usetest=False, + pat='', filters={}, project_id='', per_page=20, fail_on_first=False + ) + + @patch('urllib.request.urlopen') + @patch('urllib.request.Request') + def test_call_api_add_headers(self, mock_request_class, mock_urlopen): + # Mock Request instances to check headers + # Mock urlopen to avoid real HTTP calls + mock_request_instance = MagicMock() + mock_request_class.return_value = mock_request_instance + call_api('https://test.osf.io', pat='pat', is_json=True) + version = importlib.metadata.version("osfexport") + expected_calls = [ + call().add_header('Authorization', 'Bearer pat'), + call().add_header('User-Agent', f'osfexport/{version} (Python)'), + call().add_header('Accept', 'application/vnd.api+json;version=2.20') + ] + mock_request_class.assert_has_calls(expected_calls, any_order=False) + + @patch('urllib.request.urlopen') + @patch('urllib.request.Request') + def test_call_api_handle_429_errors(self, mock_request_class, mock_urlopen): + # Mock Request instances to check headers + # Mock urlopen to avoid real HTTP calls + mock_request_instance = MagicMock() + mock_request_class.return_value = mock_request_instance + mock_urlopen.side_effect = urllib.error.HTTPError( + code=429, + msg="error", + url="", + hdrs={}, + fp=None + ) + + with self.assertRaises(urllib.error.HTTPError): + # Use constant time delays instead of random for quick test + call_api('https://test.osf.io', pat='pat', is_json=True, usetest=True) + assert len(mock_urlopen.call_args_list) == 5 + + def test_get_public_status(self): + mock_response = MagicMock() + + valid_codes = [(200, True), (401, False), (403, False)] + for pair in valid_codes: + mock_response.status = pair[0] + with patch('osfexport.exporter.call_api') as mock_call_api: + mock_call_api.return_value = mock_response + result = is_public('url') + mock_call_api.assert_called_once_with( + 'url', pat='', method='GET' + ) + assert result == pair[1], ( + f"Expected: {pair}" + f"Actual code: {result}" + ) + + bad_codes = [400, 500, -1] + for code in bad_codes: + with patch('osfexport.exporter.call_api') as mock_call_api: + if code != -1: + mock_call_api.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=code, + msg='HTTP Error', + hdrs={}, + fp=None + ) + else: + mock_call_api.side_effect = urllib.error.URLError( + reason="URL Error" + ) + with self.assertRaises(type(mock_call_api.side_effect)): + result = is_public('url') + print(f"Code {code} should have failed by now!") + + def test_explore_mock_file_tree(self): + files = explore_file_tree( + 'root', pat='', dryrun=True + ) + + assert '/helloworld.txt.txt' == files[4][0] + assert '/tf1/helloworld.txt.txt' == files[1][0] + assert '/tf1/tf2/file.txt' == files[0][0] + assert '/tf1/tf2-second/secondpage.txt' == files[2][0] + assert '/tf1/tf2-second/thirdpage.txt' == files[3][0] + assert files[0][1] == "2.1", (files[0][1]) + assert isinstance(files[0][2], str) + + def test_get_latest_mock_wiki_version(self): + link = 'wiki' + wikis = explore_wikis( + link, pat='', dryrun=True + ) + assert len(wikis) == 3 + assert 'helloworld' in wikis.keys(), ( + 'Missing wiki IDs' + ) + assert 'home' in wikis.keys(), ( + 'Missing wiki IDs' + ) + assert 'anotherone' in wikis.keys(), ( + 'Missing wiki IDs' + ) + + assert 'hello world' in wikis['helloworld'], ( + wikis['helloworld'] + ) + assert '~~strikethrough~~' in wikis['home'], ( + wikis['home'] + ) + + def test_get_project_data_for_json_mocks(self): + nodes = MockAPIResponse.read('nodes') + projects, root_nodes = get_project_data( + nodes, + pat='', + dryrun=True, + usetest=True + ) + + assert len(projects) == 4, ( + f'Expected 4 projects in the stub data, got {len(projects)}' + ) + assert len(root_nodes) == 2, ( + f'Expected 2 root nodes in the stub data, got {len(root_nodes)}' + ) + assert root_nodes[0] == 0 + assert root_nodes[1] == 1 + assert projects[root_nodes[0]]['metadata']['id'] == 'x', ( + 'Expected ID x, got: ', + projects[root_nodes[0]]['metadata']['id'] + ) + assert projects[root_nodes[1]]['metadata']['id'] == 'y', ( + 'Expected ID y, got: ', + projects[root_nodes[1]]['metadata']['id'] + ) + + assert projects[0]['metadata']['title'] == 'Test1', ( + 'Expected title Test1, got: ', + projects[0]['metadata']['title'] + ) + assert projects[0]['metadata']['id'] == 'x', ( + 'Expected ID x, got: ', + projects[0]['metadata']['id'] + ) + assert projects[1]['metadata']['title'] == 'Test2', ( + 'Expected title Test2, got: ', + projects[1]['metadata']['title'] + ) + assert projects[0]['metadata']['license'] == 'mynewlicense', ( + 'Expected mynewlicense, got: ', + projects[0]['metadata']['license'] + ) + assert projects[0]['metadata']['description'] == 'Test1 Description', ( + 'Expected description Test1 Description, got: ', + projects[0]['metadata']['description'] + ) + assert projects[1]['metadata']['description'] == 'Test2 Description', ( + 'Expected description Test2 Description, got: ', + projects[1]['description'] + ) + expected_date = '2000-01-01 14:18 UTC' + assert str( + projects[0]['metadata']['date_created'] + ) == expected_date, ( + f'Expected date_created {expected_date}, got: ', + projects[0]['metadata']['date_created'] + ) + assert str( + projects[0]['metadata']['date_modified'] + ) == expected_date, ( + f'Expected date_modified {expected_date}, got: ', + projects[0]['metadata']['date_modified'] + ) + assert projects[0]['metadata']['tags'] == 'test1, test2, test3', ( + 'Expected tags test1, test2, test3, got: ', + projects[0]['metadata']['tags'] + ) + assert projects[1]['metadata']['tags'] == 'NA', ( + 'Expected tags NA, got: ', + projects[1]['metadata']['tags'] + ) + + assert projects[0]['contributors'][0][0] == 'Test User 1', ( + "Expected contributor Test User 1, got: ", + projects[0]['contributors'][0][0] + ) + assert not projects[0]['contributors'][0][1], ( + "Expected contributor status False, got: ", + projects[0]['contributors'][0][1] + ) + link = 'https://test.osf.io/userid/' + link_two = 'https://test.osf.io/userid2/' + assert projects[0]['contributors'][0][2] == link, ( + f"Expected contributor link {link}, got: ", + projects[0]['contributors'][0][2] + ) + assert projects[0]['contributors'][1][0] == 'Test User 2', ( + "Expected contributor Test User 2, got: ", + projects[0]['contributors'][1][0] + ) + assert projects[0]['contributors'][1][1], ( + "Expected contributor status True, got: ", + projects[0]['contributors'][1][1] + ) + assert projects[0]['contributors'][1][2] == link_two, ( + f"Expected contributor link {link_two}, got: ", + projects[0]['contributors'][1][2] + ) + + doi = projects[0]['metadata']['identifiers'] + assert doi == '10.4-2-6-25/OSF.IO/74PAD', ( + 'Expected identifiers 10.4-2-6-25/OSF.IO/74PAD, got: ', + doi + ) + assert projects[0]['metadata']['resource_type'] == 'Other', ( + 'Expected resource_type Other, got: ', + projects[0]['metadata']['resource_type'] + ) + assert projects[0]['metadata']['resource_lang'] == 'eng', ( + 'Expected resource_lang eng, got: ', + projects[0]['metadata']['resource_lang'] + ) + assert len(projects[0]['files']) == 5 + assert '/helloworld.txt.txt' == projects[0]['files'][4][0], ( + projects[0]['files'][4][0] + ) + assert '/tf1/helloworld.txt.txt' == projects[0]['files'][1][0], ( + projects[0]['files'][1][0] + ) + assert '/tf1/tf2/file.txt' == projects[0]['files'][0][0], ( + projects[0]['files'][0][0] + ) + subjects = projects[0]['metadata']['subjects'] + assert subjects == 'Education, Literature, Geography', ( + 'Expected Education, Literature, Geography, got: ', + subjects + ) + assert len(projects[0]['wikis']) == 3 + assert projects[0]['metadata']['url'] == 'https://test.osf.io/x/', ( + 'Expected URL https://test.osf.io/x/, got: ', + projects[0]['metadata']['url'] + ) + assert projects[1]['metadata']['url'] != 'https://test.osf.io/x/', ( + 'Repeated project URL' + ) + + assert projects[0]['parent'] is None, ( + 'Expected no parent, got: ', + projects[0]['parent'] + ) + assert 'a' in projects[0]['children'] + assert 'b' in projects[0]['children'] + + assert projects[0]['metadata']['public'] + assert not projects[1]['metadata']['public'] + + assert projects[0]['metadata']['category'] == 'Methods and Measures', ( + projects[0]['metadata']['category'] + ) + assert projects[1]['metadata']['category'] == 'Uncategorized', ( + projects[1]['metadata']['category'] + ) + + assert projects[3]['parent'][0] == projects[2]['metadata']['title'], ( + projects[3]['parent'][0], + f'Expected: {projects[2]['metadata']['title']}' + ) + assert projects[3]['parent'][1] == projects[2]['metadata']['url'], ( + projects[3]['parent'][1], + f'Expected: {projects[2]['metadata']['url']}' + ) + + def test_get_paginated_projects(self): + projects, root_nodes = get_nodes( + pat='', + dryrun=True, + usetest=True, + page_size=4 + ) + assert len(projects) == 5, ( + f'Expected 5 projects in the stub data, got {len(projects)}', + projects + ) + assert len(root_nodes) == 3, ( + f'Expected 3 root nodes in the stub data, got {len(root_nodes)}' + ) + assert root_nodes[0] == 0 + assert root_nodes[1] == 1 + assert root_nodes[2] == 4 + + def test_get_single_mock_project(self): + projects, roots = get_nodes( + pat='', dryrun=True, usetest=True, + project_id='x' + ) + assert len(roots) == 1, ( + roots + ) + assert len(projects) == 3, ( + projects + ) + assert projects[0]['metadata']['id'] == 'x' + assert projects[0]['children'] == ['a', 'b'], ( + projects[0]['children'] + ) + + def test_use_dryrun_in_user_default_dir(self): + cwd = os.getcwd() + try: + # Go to user's home directory in cross-platform way + os.chdir(os.path.expanduser("~")) + projects, roots = get_nodes('', dryrun=True, usetest=True) + except Exception as e: + raise e + finally: + # Reverse state changes for reproducibility + os.chdir(cwd) + assert os.getcwd() == cwd + + def test_extract_project_id_from_strings(self): + input_expected = [ + ('https://osf.io/x/', 'x'), + ('https://api.test.osf.io/v2/nodes/x', 'x'), + ('https://api.test.osf.io/v2/nodes/x/?filter[name=lol]', 'x'), + ('x', 'x'), + ('', ''), + ('https://test.osf.io/af454s/?view_only=sdghsgfdsgj&field=name', 'af454s'), + ('https://osf.io/12ap3c/?view_only=[sdghsgfdsgj]&field=name', '12ap3c') + ] + for item in input_expected: + result = extract_project_id(item[0]) + assert result == item[1], ( + f"Expected {item[1]}, got: ", + result + ) + + @patch('osfexport.exporter.call_api') + def test_add_on_paginated_results(self, mock_get): + # Mock JSON responses + page1 = {'data': 1, 'links': {'next': 'http://api.example.com/page2'}} + page2 = {'data': 3, 'links': {'next': 'http://api.example.com/page3'}} + page3 = {'data': 5, 'links': {'next': None}} + # Configure mock to return these responses in sequence + mock_get.side_effect = [ + page1, + page2, + page3 + ] + + def add_x(json, **kwargs): + x = kwargs.get('x', 0) + return json['data'] + x + + results = paginate_json_result( + start='http://api.example.com/page1', action=add_x, x=5 + ) + assert isinstance(results, deque) + self.assertEqual(results.popleft(), 1+5) + self.assertEqual(results.popleft(), 3+5) + self.assertEqual(results.popleft(), 5+5) + + def test_get_single_component_mock_project(self): + projects, roots = get_nodes( + pat='', dryrun=True, usetest=True, + project_id='a' + ) + assert len(roots) == 1, ( + roots + ) + assert len(projects) == 1, ( + projects + ) + assert projects[0]['metadata']['id'] == 'a' + assert projects[0]['parent'][0] == 'Test1', ( + projects[0]['parent'][0], + 'Expected: Test1' + ) + assert projects[0]['parent'][1] == 'https://test.osf.io/x/', ( + projects[0]['parent'][1], + 'Expected: https://test.osf.io/x/' + ) + + +class TestFormatter(TestCase): + """Tests for the PDF formatter.""" + + def test_write_pdf_no_folder_given(self): + projects = [ + { + 'metadata': { + 'title': 'My Project Title', + 'id': 'id', + 'url': 'https://test.osf.io/x', + 'category': 'Uncategorized', + 'description': 'This is a description of the project', + 'date_created': datetime.datetime.fromisoformat( + '2025-06-12T15:54:42.105112Z' + ), + 'date_modified': datetime.datetime.fromisoformat( + '2001-01-01T01:01:01.105112Z' + ), + 'tags': 'tag1, tag2, tag3', + 'resource_type': 'na', + 'resource_lang': 'english', + 'affiliated_institutions': 'University of Manchester', + 'identifiers': 'N/A', + 'license': 'Apache 2.0', + 'subjects': 'sub1, sub2, sub3', + }, + 'contributors': [ + ('Pineapple Pizza', False, 'https://test.osf.io/userid/'), + ('Margarita', True, 'https://test.osf.io/userid/'), + ('Margarine', True, 'https://test.osf.io/userid/') + ], + 'files': [ + ('file1.txt', None, 'https://test.osf.io/userid/'), + ('file2.txt', None, None), + ], + 'funders': [], + 'wikis': { + 'Home': 'hello world', + 'Page2': 'another page' + }, + "parent": None, + 'children': ['a'] + } + ] + root_nodes = [0] + is_filename_match = False # Flag for if exported PDF has expected name + try: + pdf_one, path_one = write_pdf(projects, root_nodes[0], '') + + title_one = projects[0]['metadata']['title'].replace(' ', '-') + date_one = pdf_one.date_printed.strftime( + '%Y-%m-%d %H-%M-%S %Z' + ).replace(' ', '-') + expected_filename = f'{title_one}-{date_one}.pdf' + + is_filename_match = expected_filename in os.listdir(os.getcwd()) + assert 'Component of:' not in PdfReader(path_one).pages[0].extract_text(), ( + 'Did not expect parent URL in PDF, got: ', + PdfReader(path_one).pages[0].extract_text() + ) + except Exception as e: + if isinstance(e, AssertionError): + raise e + finally: + if os.path.exists(path_one): + os.remove(path_one) + + assert is_filename_match, ( + 'Unable to create file in current directory.' + ) + + def test_write_component_pdf_with_one_off_parent(self): + projects = [ + { + 'metadata': { + 'title': 'Component1', + 'id': 'id', + 'url': 'https://test.osf.io/x', + 'category': 'Uncategorized', + 'description': 'This is a description of the project', + 'date_created': datetime.datetime.fromisoformat( + '2025-06-12T15:54:42.105112Z' + ), + 'date_modified': datetime.datetime.fromisoformat( + '2001-01-01T01:01:01.105112Z' + ), + 'tags': 'tag1, tag2, tag3', + 'resource_type': 'na', + 'resource_lang': 'english', + 'affiliated_institutions': 'University of Manchester', + 'identifiers': 'N/A', + 'license': 'Apache 2.0', + 'subjects': 'sub1, sub2, sub3', + }, + 'contributors': [ + ('Pineapple Pizza', False, 'https://test.osf.io/userid/'), + ('Margarita', True, 'https://test.osf.io/userid/'), + ('Margarine', True, 'https://test.osf.io/userid/') + ], + 'files': [ + ('file1.txt', None, 'https://test.osf.io/userid/'), + ('file2.txt', None, None), + ], + 'funders': [], + 'wikis': { + 'Home': 'hello world', + 'Page2': 'another page' + }, + "parent": ['apple', 'https://test.osf.io/parent-id'], + 'children': ['a'] + } + ] + root_nodes = [0] + is_filename_match = False # Flag for if exported PDF has expected name + try: + pdf_one, path_one = write_pdf(projects, root_nodes[0], '') + + title_one = projects[0]['metadata']['title'].replace(' ', '-') + date_one = pdf_one.date_printed.strftime( + '%Y-%m-%d %H-%M-%S %Z' + ).replace(' ', '-') + expected_filename = f'{title_one}-{date_one}.pdf' + + is_filename_match = expected_filename in os.listdir(os.getcwd()) + + page_one = PdfReader(path_one) + text = page_one.pages[0].extract_text() + assert f'Parent: {projects[0]['parent'][0]}' in text, ( + text + ) + assert f'{projects[0]['parent'][1]}' in text, ( + text + ) + except Exception as e: + if isinstance(e, AssertionError): + raise e + finally: + if os.path.exists(path_one): + os.remove(path_one) + + assert is_filename_match, ( + 'Unable to create file in current directory.' + ) + + def test_write_unicode_pdfs_from_mock_projects(self): + # Put PDFs in a folder to keep things tidy + if os.path.exists(FOLDER_OUT): + shutil.rmtree(FOLDER_OUT) + os.mkdir(FOLDER_OUT) + + projects = [ + { + 'metadata': { + 'title': 'My Project Title', + 'id': 'id', + 'url': 'https://test.osf.io/x', + 'category': 'Uncategorized', + 'description': 'This is a description of the project ג', + 'date_created': datetime.datetime.fromisoformat( + '2025-06-12T15:54:42.105112Z' + ), + 'date_modified': datetime.datetime.fromisoformat( + '2001-01-01T01:01:01.105112Z' + ), + 'tags': 'tag1, tag2, tag3', + 'resource_type': 'na', + 'resource_lang': 'english', + # Below uses em-dash at end + 'affiliated_institutions': 'University of Manchester — Test', + 'identifiers': 'N/A', + 'license': 'Apache 2.0', + 'subjects': 'sub1, sub2, sub3', + }, + 'contributors': [ + ('Pineapple Pizza', False, 'https://test.osf.io/userid/'), + ('Margarita', True, 'https://test.osf.io/userid/'), + ('Margarine', True, 'https://test.osf.io/userid/') + ], + 'files': [ + ('file1.txt', None, 'https://test.osf.io/userid/'), + ('file2.txt', None, None), + ], + 'funders': [], + 'wikis': { + 'Home': 'hello world', + 'Page2': 'another page' + }, + "parent": None, + 'children': ['a'] + }, + { + 'metadata': { + "title": "child1", + "id": "a", + 'url': 'https://test.osf.io/a', + }, + 'contributors': [ + ( + 'Long Double-Barrelled Name and Surname', + False, 'https://test.osf.io/userid/' + ), + ( + 'name2', True, 'https://test.osf.io/userid/' + ), + ( + 'name3', True, 'https://test.osf.io/userid/' + ) + ], + 'files': [ + ('file1.txt', None, None), + ('file2.txt', None, None), + ], + 'wikis': {}, + "parent": ('My Project Title', 'https://test.osf.io/x'), + 'children': ['b'] + }, + { + 'metadata': { + "title": "Second Project in new PDF ♡", + "id": "c", + 'url': 'lol', + 'category': 'Methods and Measures' + }, + 'contributors': [ + ( + 'Long Double-Barrelled Name and Surname', + False, 'https://test.osf.io/userid/' + ), + ( + 'name2', True, 'https://test.osf.io/userid/' + ), + ( + 'name3', True, 'https://test.osf.io/userid/' + ) + ], + 'files': [ + ('file1.txt', None, None), + ('file2.txt', None, None), + ], + 'wikis': {}, + "parent": None, + 'children': [] + }, + { + 'metadata': { + "title": "child2", + "id": "b", + 'url': 'dan' + }, + 'contributors': [ + ( + 'Long Double-Barrelled Name and Surname', + False, 'https://test.osf.io/userid/' + ), + ( + 'name2', True, 'https://test.osf.io/userid/' + ), + ( + 'name3', True, 'https://test.osf.io/userid/' + ) + ], + 'files': [ + ('file1.txt', None, None), + ('file2.txt', None, None), + ], + 'wikis': {}, + "parent": ['child1', 'https://test.osf.io/a'], + 'children': [] + }, + ] + + root_nodes = [0, 2] # Indices of root nodes in projects list + + # Get URL now as it will be removed later + url = projects[0]['metadata']['url'] + url_comp = projects[1]['metadata']['url'] + + # Can we specify where to write PDFs? + pdf_one, path_one = write_pdf(projects, root_nodes[0], FOLDER_OUT) + pdf_two, path_two = write_pdf(projects, root_nodes[1], FOLDER_OUT) + files = os.listdir(FOLDER_OUT) + assert len(files) == 2 + + title_one = projects[0]['metadata']['title'].replace(' ', '-') + title_two = projects[2]['metadata']['title'].replace(' ', '-') + date_one = pdf_one.date_printed.strftime( + '%Y-%m-%d %H-%M-%S %Z' + ).replace(' ', '-') + date_two = pdf_two.date_printed.strftime( + '%Y-%m-%d %H-%M-%S %Z' + ).replace(' ', '-') + path_one_real = os.path.join( + os.getcwd(), FOLDER_OUT, + f'{title_one}-{date_one}.pdf' + ) + path_two_real = os.path.join( + os.getcwd(), FOLDER_OUT, + f'{title_two}-{date_two}.pdf' + ) + assert path_one == path_one_real, ( + path_one, + path_one_real + ) + assert path_two == path_two_real, ( + path_two, + path_two_real + ) + + import_one = PdfReader(os.path.join( + FOLDER_OUT, f'{title_one}-{date_one}.pdf' + )) + import_two = PdfReader(os.path.join( + FOLDER_OUT, f'{title_two}-{date_two}.pdf' + )) + assert len(import_one.pages) == 5, ( + 'Expected 5 pages in the first PDF, got: ', + len(import_one.pages) + ) + + content_first_page = import_one.pages[0].extract_text( + extraction_mode='layout' + ) + assert f'{projects[0]['metadata']['title']}' in content_first_page, ( + content_first_page + ) + + content_second_page = import_two.pages[0].extract_text( + extraction_mode='layout' + ) + assert 'Category: Methods and Measures' in content_second_page, ( + content_second_page + ) + + content_third_page = import_one.pages[3].extract_text( + extraction_mode='layout' + ) + assert f'{projects[0]['metadata']['title']}' in content_third_page, ( + content_third_page + ) + assert f'{url}' in content_third_page, ( + content_third_page + ) + assert f'{projects[1]['metadata']['title']}' in content_third_page + assert f'{url_comp}' in content_third_page + + assert f'{url}' in content_first_page, ( + content_third_page + ) + assert 'Category: Uncategorized' in content_first_page, ( + content_first_page + ) + timestamp = pdf_one.date_printed.strftime( + '%Y-%m-%d %H:%M:%S %Z') + assert f'Exported: {timestamp}' in content_first_page, ( + 'Actual content:', + content_first_page + ) + + # This way of string formatting compresses line lengths used + # End of headers and table rows marked by \n\n + contributors_table = ( + '2. Contributors\n\n' + 'Name' + 'Bibliographic?' + 'Profile Link\n\n' + 'Pineapple Pizza' + 'No' + 'https://test.osf.io/userid/\n\n' + 'Margarita' + 'Yes' + 'https://test.osf.io/userid/\n\n' + 'Margarine' + 'Yes' + 'https://test.osf.io/userid/\n\n' + ).replace(' ', '') + assert contributors_table in content_first_page.replace(' ', ''), ( + 'Table: ', + contributors_table, + 'Actual: ', + content_first_page.replace(' ', '') + ) + + # This way of string formatting compresses line lengths used + # End of headers and table rows marked by \n\n + files_table = ( + '3. Files in Main Project\n\n' + 'OSF Storage\n\n' + 'File Name' + 'Size (MB)' + 'Download Link\n\n' + 'file1.txt' + 'N/A' + 'https://test.osf.io/userid/\n\n' + 'file2.txt' + 'N/A' + 'N/A\n\n' + ).replace(' ', '') + assert files_table in content_first_page.replace(' ', ''), ( + 'Table: ', + files_table, + 'Actual: ', + content_first_page.replace(' ', '') + ) + + content_fourth_page = import_one.pages[4].extract_text( + extraction_mode='layout' + ) + assert f'{projects[0]['metadata']['title']}' not in content_fourth_page, ( + 'Incorrect parent title for component' + ) + assert f'Parent: {projects[3]['parent'][0]}' in content_fourth_page, ( + f'Parent: {projects[3]['parent'][0]}', + content_fourth_page + ) + assert f'{projects[3]['parent'][1]}' in content_fourth_page, ( + f'Parent: {projects[3]['parent'][1]}', + content_fourth_page + ) + + # Remove files only if all good - keep for debugging otherwise + if os.path.exists(FOLDER_OUT): + shutil.rmtree(FOLDER_OUT) + + +class TestCLI(TestCase): + @patch('osfexport.exporter.is_public', lambda x: True) + def test_prompt_pat_if_public_project_id_given(self): + pat = prompt_pat('x') + assert pat == '', ( + pat + ) + + @patch('osfexport.exporter.is_public', lambda x: False) + @patch('click.prompt', return_value='strinput') + def test_prompt_pat_if_private_project_id_given(self, mock_obj): + pat = prompt_pat('x') + assert pat == 'strinput' + + @patch('click.prompt', return_value='strinput') + def test_prompt_pat_if_exporting_all_projects(self, mock_obj): + pat = prompt_pat() + assert pat == 'strinput' + + @patch('osfexport.cli.prompt_pat') + @patch('osfexport.exporter.get_nodes') + def test_export_projects_handles_http_url_errors(self, mock_func, mock_prompt): + # Handle errors from exporting nodes + export_codes = [401, 402, 403, 404, 429, 500, -1] + for code in export_codes: + mock_prompt.return_value = '-' + if code != -1: + mock_func.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=code, + msg='HTTP Error', + hdrs={}, + fp=None + ) + else: + mock_func.side_effect = urllib.error.URLError( + reason="URL Error" + ) + runner = CliRunner() + result = runner.invoke( + cli, [ + 'projects', + '--usetest' + ], + terminal_width=60 + ) + assert "Exporting failed as an error occurred:" in result.output, ( + result.output + ) + + # Handle errors from is_public call when prompting for PAT + for code in export_codes: + if code != -1: + mock_prompt.side_effect = urllib.error.HTTPError( + url='https://test.osf.io', + code=code, + msg='HTTP Error', + hdrs={}, + fp=None + ) + else: + mock_prompt.side_effect = urllib.error.URLError( + reason="URL Error" + ) + runner = CliRunner() + result = runner.invoke( + cli, [ + 'projects', + '--usetest' + ], + terminal_width=60 + ) + assert "Exporting failed as an error occurred:" in result.output, ( + result.output + ) + + def test_pull_projects_command_on_mocks(self): + """Test generating a PDF from parsed project data. + This assumes the JSON parsing works correctly.""" + + if os.path.exists(FOLDER_OUT): + shutil.rmtree(FOLDER_OUT) + os.mkdir(FOLDER_OUT) + + runner = CliRunner() + result = runner.invoke( + cli, [ + 'projects', + '--dryrun', + '--folder', FOLDER_OUT, + '--url', '', + '--pat', '' + ], + terminal_width=60 + ) + assert not result.exception, ( + result.exc_info, + traceback.format_tb(result.exc_info[2]) + ) + + if os.path.exists(FOLDER_OUT): + shutil.rmtree(FOLDER_OUT)