diff --git a/airbyte-integrations/connectors/source-aircall/.dockerignore b/airbyte-integrations/connectors/source-aircall/.dockerignore new file mode 100644 index 000000000000..a9ca20b1887d --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_aircall +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-aircall/Dockerfile b/airbyte-integrations/connectors/source-aircall/Dockerfile new file mode 100644 index 000000000000..2b58b9659db2 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_aircall ./source_aircall + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-aircall diff --git a/airbyte-integrations/connectors/source-aircall/README.md b/airbyte-integrations/connectors/source-aircall/README.md new file mode 100644 index 000000000000..65e4417588bf --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/README.md @@ -0,0 +1,132 @@ +# Aircall Source + +This is the repository for the Aircall source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/aircall). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-aircall:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/aircall) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_aircall/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source aircall test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-aircall:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-aircall:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-aircall:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aircall:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aircall:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-aircall:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-aircall:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-aircall:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml b/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml new file mode 100644 index 000000000000..95909990f6d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml @@ -0,0 +1,32 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-aircall:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_aircall/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_data_points: true + timeout_seconds: 200 + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 200 + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 200 diff --git a/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh new file mode 100755 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-aircall/build.gradle b/airbyte-integrations/connectors/source-aircall/build.gradle new file mode 100644 index 000000000000..5a0a3be45b93 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_aircall' +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/__init__.py b/airbyte-integrations/connectors/source-aircall/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..d1a769cf85cf --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "calls": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py new file mode 100644 index 000000000000..98eca28c4928 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import docker +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + client = docker.from_env() + container = client.containers.run("airbyte/source-aircall", detach=True) + yield + container.stop() diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/catalog.json b/airbyte-integrations/connectors/source-aircall/integration_tests/catalog.json new file mode 100644 index 000000000000..cca55efc3699 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/catalog.json @@ -0,0 +1,190 @@ +{ + "streams": [ + { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "direction": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "missed_call_reason": { "type": ["string", "null"] }, + "started_at": { "type": ["null", "integer"] }, + "answered_at": { "type": ["null", "integer"] }, + "ended_at": { "type": ["null", "integer"] }, + "created_at": { "type": ["null", "integer"] }, + "duration": { "type": ["null", "integer"] }, + "voicemail": { "type": ["string", "null"] }, + "recording": { "type": ["string", "null"] }, + "asset": { "type": ["string", "null"] }, + "raw_digits": { "type": ["null", "string"] }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "contact": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, + "company_name": { "type": ["null", "string"] }, + "information": { "type": ["null", "string"] }, + "is_shared": { "type": "boolean" }, + "created_at": { "type": ["null", "integer"] }, + "updated_at": { "type": ["null", "integer"] }, + "emails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "label": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + }, + "phone_numbers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "label": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + } + } + }, + "archived": { "type": "boolean" }, + "assigned_to": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "transferred_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "transferred_to": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "cost": { "type": ["null", "string"] }, + "number": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "digits": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "open": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "is_ivr": { "type": "boolean" }, + "live_recording_activated": { "type": "boolean" }, + "priority": { "type": "null" }, + "messages": { "type": "object" } + } + }, + "comments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "content": { "type": ["null", "string"] }, + "posted_at": { "type": ["null", "integer"] }, + "posted_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] } + } + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "integer"] }, + "tagged_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] } + } + } + } + } + }, + "teams": { "type": "array", "items": { "type": ["null", "object"] } } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["started_at"], + "source_defined_primary_key": [["id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..2e1ede0290db --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json @@ -0,0 +1,198 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "direction": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "missed_call_reason": { "type": ["string", "null"] }, + "started_at": { "type": ["null", "integer"] }, + "answered_at": { "type": ["null", "integer"] }, + "ended_at": { "type": ["null", "integer"] }, + "created_at": { "type": ["null", "integer"] }, + "duration": { "type": ["null", "integer"] }, + "voicemail": { "type": ["string", "null"] }, + "recording": { "type": ["string", "null"] }, + "asset": { "type": ["string", "null"] }, + "raw_digits": { "type": ["null", "string"] }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "contact": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, + "company_name": { "type": ["null", "string"] }, + "information": { "type": ["null", "string"] }, + "is_shared": { "type": "boolean" }, + "created_at": { "type": ["null", "integer"] }, + "updated_at": { "type": ["null", "integer"] }, + "emails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "label": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + }, + "phone_numbers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "label": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } + } + } + } + }, + "archived": { "type": "boolean" }, + "assigned_to": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "transferred_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "transferred_to": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "language": { "type": ["null", "string"] } + } + }, + "cost": { "type": ["null", "string"] }, + "number": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "digits": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] }, + "open": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "is_ivr": { "type": "boolean" }, + "live_recording_activated": { "type": "boolean" }, + "priority": { "type": "null" }, + "messages": { "type": "object" } + } + }, + "comments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "content": { "type": ["null", "string"] }, + "posted_at": { "type": ["null", "integer"] }, + "posted_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] } + } + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "integer"] }, + "tagged_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "direct_link": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "available": { "type": "boolean" }, + "availability_status": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] } + } + } + } + } + }, + "teams": { + "type": "array", + "items": { "type": ["null", "object"] } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["started_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["started_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-aircall/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_config.json new file mode 100644 index 000000000000..de25981b90f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_config.json @@ -0,0 +1,8 @@ +{ + "api_id": "", + "api_token": "", + "start_date": "2022-01-01", + "time_window": { + "months":1 + } +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json new file mode 100644 index 000000000000..1a5f136aa80c --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json @@ -0,0 +1,10 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "calls" }, + "stream_state": { "started_at": 1654627306 } + }, + "data": { "calls": { "started_at": 1654627306 } } + } +] diff --git a/airbyte-integrations/connectors/source-aircall/main.py b/airbyte-integrations/connectors/source-aircall/main.py new file mode 100644 index 000000000000..2a0fb5daadfa --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aircall import SourceAircall + +if __name__ == "__main__": + source = SourceAircall() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aircall/requirements.txt b/airbyte-integrations/connectors/source-aircall/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-aircall/setup.py b/airbyte-integrations/connectors/source-aircall/setup.py new file mode 100644 index 000000000000..a4f424ab948b --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", + "basicauth~=0.4.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_aircall", + description="Source implementation for Aircall.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/__init__.py b/airbyte-integrations/connectors/source-aircall/source_aircall/__init__.py new file mode 100644 index 000000000000..74f1b750ed53 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAircall + +__all__ = ["SourceAircall"] diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/TODO.md b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/calls.json b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/calls.json new file mode 100644 index 000000000000..249cb8d66da5 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/calls.json @@ -0,0 +1,384 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "direction": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "missed_call_reason": { + "type": ["string", "null"] + }, + "started_at": { + "type": ["null", "integer"] + }, + "answered_at": { + "type": ["null", "integer"] + }, + "ended_at": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "duration": { + "type": ["null", "integer"] + }, + "voicemail": { + "type": ["string", "null"] + }, + "recording": { + "type": ["string", "null"] + }, + "asset": { + "type": ["string", "null"] + }, + "raw_digits": { + "type": ["null", "string"] + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "contact": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "company_name": { + "type": ["null", "string"] + }, + "information": { + "type": ["null", "string"] + }, + "is_shared": { + "type": "boolean" + }, + "created_at": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "integer"] + }, + "emails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "phone_numbers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + }, + "archived": { + "type": "boolean" + }, + "assigned_to": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "transferred_by": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "transferred_to": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "cost": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "digits": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "open": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "is_ivr": { + "type": "boolean" + }, + "live_recording_activated": { + "type": "boolean" + }, + "priority": { + "type": "null" + }, + "messages": { + "type": "object" + } + } + }, + "comments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "posted_at": { + "type": ["null", "integer"] + }, + "posted_by": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + } + } + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "tagged_by": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "direct_link": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "available": { + "type": "boolean" + }, + "availability_status": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + } + } + } + } + } + }, + "teams": { + "type": "array", + "items": { + "type": ["null", "object"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/company.json b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/company.json new file mode 100644 index 000000000000..4613cba1a127 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/schemas/company.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "users_count": { "type": ["null", "integer"] }, + "numbers_count": { "type": ["null", "integer"] } + } +} diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/source.py b/airbyte-integrations/connectors/source-aircall/source_aircall/source.py new file mode 100644 index 000000000000..ce232d3ed7b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/source.py @@ -0,0 +1,260 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pendulum +import requests +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from basicauth import encode + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. +""" + + +# Basic full refresh stream +class AircallStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class AircallStream(HttpStream, ABC)` which is the current class + `class Customers(AircallStream)` contains behavior to pull data for customers using v1/customers + `class Employees(AircallStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalAircallStream((AircallStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://api.aircall.io/v1/" + time_window = {"months": 1} + + def _get_end_date(self, current_date: pendulum.DateTime, end_date: pendulum.DateTime = pendulum.now()): + if current_date.add(**self.time_window).date() < end_date.date(): + end_date = current_date.add(**self.time_window) + return end_date + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + need_new_request = response.json()["meta"]["count"] != response.json()["meta"]["per_page"] + if need_new_request: + return None + return {"page": response.json()["meta"]["current_page"] + 1} + + def backoff_time(self, response: requests.Response) -> Optional[float]: + """This method is called if we run into the rate limit. + Slack puts the retry time in the `Retry-After` response header so we + we return that value. If the response is anything other than a 429 (e.g: 5XX) + fall back on default retry behavior. + Rate Limits Docs: https://api.slack.com/docs/rate-limits#web""" + + if "X-AircallApi-Reset" in response.headers: + return int(response.headers["X-AircallApi-Reset"]) - pendulum.now().int_timestamp + else: + self.logger.info("X-AircallApi-Reset header not found. Using default backoff value") + return 5 + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return self.name + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + params = {"per_page": 50} + if stream_slice: + params.update(stream_slice) + else: + params.update({"from": self.start_date.int_timestamp, "to": pendulum.now().int_timestamp}) + if next_page_token: + params.update(next_page_token) + else: + params["page"] = 1 + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + if response.json()["meta"]["current_page"] == 1: + self.logger.info(f"{response.json()['meta']['total']} {self.name} to sync in this window (Will fail if > 10000)") + yield from response.json()[self.name] + + +class Company(AircallStream): + primary_key = "id" + name = "company" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + +# Basic incremental stream +class IncrementalAircallStream(AircallStream, IncrementalMixin): + incremental_threshold = 0 + + def __init__(self, start_date: str, time_window: Mapping[str, int], incremental_threshold: int = 0, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._cursor_value = pendulum.parse(start_date).int_timestamp + self.time_window = time_window + self.incremental_threshold = incremental_threshold + + @property + @abstractmethod + def cursor_field(self) -> str: + pass + + def _update_state(self, latest_cursor): + if latest_cursor: + new_state = max(latest_cursor, self._cursor_value) if self._cursor_value else latest_cursor + if new_state != self._cursor_value: + self.logger.info(f"Advancing bookmark for {self.name} stream from {pendulum.from_timestamp(self._cursor_value).to_iso8601_string()} to {pendulum.from_timestamp(latest_cursor).to_iso8601_string()}") + self._cursor_value = new_state + + @property + def state(self) -> MutableMapping[str, Any]: + return {self.cursor_field: self._cursor_value} + + @state.setter + def state(self, value: MutableMapping[str, Any]): + self._cursor_value = value.get(self.cursor_field, self._cursor_value) + + def read_records(self, stream_slice: Mapping[str, any], *args, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_slice=stream_slice, *args, **kwargs) + latest_cursor = None + for record in records: + cursor = record[self.cursor_field] + latest_cursor = max(cursor, latest_cursor) if latest_cursor else cursor + yield record + self._update_state(latest_cursor=latest_cursor if latest_cursor else stream_slice["to"]) + +class Calls(IncrementalAircallStream): + cursor_field = "started_at" + primary_key = "id" + name = "calls" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + slices = [] + start = ( + pendulum.from_timestamp(stream_state.get(self.cursor_field)) - pendulum.duration(hours=2) if stream_state else pendulum.from_timestamp(self._cursor_value) + ) + end = pendulum.now() + slices_number = 0 + while start < end: + next = self._get_end_date(start, pendulum.now()) + slice = {"from": start.int_timestamp, "to": next.int_timestamp} + start = next + yield slice + slices_number += 1 + if slices_number >= self.incremental_threshold: + return slices + return slices + + +# Source +class SourceAircall(AbstractSource): + def _getAuth(self, config): + encoded = encode(config["api_id"], config["api_token"]).removeprefix("Basic ") + return TokenAuthenticator(token=encoded, auth_method="Basic") + + def check_connection(self, logger: AirbyteLogger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + try: + auth = self._getAuth(config) + Company(authenticator=auth) + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = auth = self._getAuth(config) + return [Calls(authenticator=auth, start_date=config["start_date"], time_window=config["time_window"],incremental_threshold=config.get("incremental_threshold", 0))] diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/spec.yaml b/airbyte-integrations/connectors/source-aircall/source_aircall/spec.yaml new file mode 100644 index 000000000000..793eeac19d5e --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/spec.yaml @@ -0,0 +1,47 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/aircall +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Aircall Spec + type: object + required: + - api_id + - api_token + - start_date + properties: + # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': + api_id: + type: string + description: The api id from Aircall (See the doc for more information on how to obtain this key.) + api_token: + type: string + airbyte_secret: true + description: The api token from Aircall (See the doc for more information on how to obtain this key.) + start_date: + type: string + title: Replication Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated + examples: + - "2018-07-15T18:00:00Z" + time_window: + type: object + description: Time window to fetch results (default 1 month) + properties: + months: + type: integer + order: 2 + years: + type: integer + order: 1 + days: + type: integer + order: 3 + minutes: + type: integer + order: 4 + seconds: + type: integer + order: 5 + incremental_threshold: + type: integer + description: number of time window slices to sync at once max (allows to checkpoint ingestion. Default 0, which means no threshold) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-aircall/unit_tests/__init__.py b/airbyte-integrations/connectors/source-aircall/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-aircall/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-aircall/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..95850f8d9dd9 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/unit_tests/test_incremental_streams.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_aircall.source import IncrementalAircallStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalAircallStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalAircallStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalAircallStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) + expected_cursor_field = ["started_at"] + assert stream.cursor_field == expected_cursor_field + + +# def test_get_updated_state(patch_incremental_base_class): +# stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) +# inputs = {"current_stream_state": None, "latest_record": None} +# # TODO: replace this with your expected updated stream state +# expected_state = {} +# assert stream.get_updated_state(**inputs) == expected_state + + +# def test_stream_slices(patch_incremental_base_class): +# stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) +# # TODO: replace this with your input parameters +# inputs = {"sync_mode": SyncMode.incremental, "cursor_field": ['created_at'], "stream_state": { "created_at": 1500 }} +# # TODO: replace this with your expected stream slices list +# expected_stream_slice = [None] +# assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalAircallStream, "cursor_field", "dummy_field") + stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalAircallStream(start_date="2022-11-29T00:00:00Z", time_window={ "days": 1 }) + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-aircall/unit_tests/test_source.py b/airbyte-integrations/connectors/source-aircall/unit_tests/test_source.py new file mode 100644 index 000000000000..b2dcf680ddda --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_aircall.source import SourceAircall + + +def test_check_connection(mocker): + source = SourceAircall() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceAircall() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 1 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-aircall/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-aircall/unit_tests/test_streams.py new file mode 100644 index 000000000000..f723a84451e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-aircall/unit_tests/test_streams.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_aircall.source import AircallStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(AircallStream, "path", "v0/example_endpoint") + mocker.patch.object(AircallStream, "primary_key", "test_primary_key") + mocker.patch.object(AircallStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = AircallStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = { "per_page": 50, "page": 1 } + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = AircallStream() + # TODO: replace this with your input parameters + inputs = {"response": { "json": lambda: { "meta": { "current_page": 2 } } } } + # TODO: replace this with your expected next page token + expected_token = {"page": 3} + assert stream.next_page_token(**inputs) == expected_token + +# def test_parse_response(patch_base_class): +# stream = AircallStream() +# # TODO: replace this with your input parameters +# inputs = {"response": MagicMock()} +# # TODO: replace this with your expected parced object +# expected_parsed_object = {} +# assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +# def test_request_headers(patch_base_class): +# stream = AircallStream() +# # TODO: replace this with your input parameters +# inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} +# # TODO: replace this with your expected request headers +# expected_headers = {} +# assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = AircallStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +# @pytest.mark.parametrize( +# ("http_status", "should_retry"), +# [ +# (HTTPStatus.OK, False), +# (HTTPStatus.BAD_REQUEST, False), +# (HTTPStatus.TOO_MANY_REQUESTS, True), +# (HTTPStatus.INTERNAL_SERVER_ERROR, True), +# ], +# ) +# def test_should_retry(patch_base_class, http_status, should_retry): +# response_mock = MagicMock() +# response_mock.status_code = http_status +# stream = AircallStream() +# assert stream.should_retry(response_mock) == should_retry + + +# def test_backoff_time(patch_base_class): +# response_mock = MagicMock() +# stream = AircallStream() +# expected_backoff_time = None +# assert stream.backoff_time(response_mock) == expected_backoff_time