From 1ba7e485150e806448a1703a634dce7529b9ecdb Mon Sep 17 00:00:00 2001 From: ttomalak-bkpr Date: Fri, 17 Jul 2026 12:46:21 +0200 Subject: [PATCH 1/3] feat: vendor MeltanoLabs/tap-github upstream source Fork of MeltanoLabs/tap-github (Apache-2.0), commit 6810bb2. Vendored wholesale as the initial source, matching sibling tap forks (tap-jira, tap-teamtailor): no upstream git history, no lockfile (pip installs straight from pyproject.toml via git URL), upstream's own CI/dependabot workflows dropped in favor of the standard lumapps/github-actions-validator workflow used by every other tap repo in the org. The round-robin repo-enumeration checkpoint fix follows in a separate PR. --- .env.template | 1 + .../workflows/github-actions-validator.yml | 17 + .gitignore | 142 + .pre-commit-config.yaml | 35 + .python-version | 1 + .secrets/.gitignore | 6 + .sonarcloud.properties | 2 + LICENSE | 201 + README.md | 184 +- config-sample.json | 8 + config.json | 4 + meltano.yml | 68 + pyproject.toml | 186 + tap_github/__init__.py | 0 tap_github/authenticator.py | 599 +++ tap_github/client.py | 567 +++ tap_github/organization_streams.py | 1075 +++++ tap_github/repository_streams.py | 3860 +++++++++++++++++ tap_github/schema_objects.py | 81 + tap_github/scraping.py | 173 + tap_github/streams.py | 161 + tap_github/tap.py | 206 + tap_github/user_streams.py | 342 ++ tap_github/utils/__init__.py | 0 tap_github/utils/filter_stdout.py | 47 + tests/__init__.py | 23 + tests/conftest.py | 3 + tests/fixtures.py | 134 + tests/test_authenticator.py | 1260 ++++++ tests/test_core.py | 71 + tests/test_discover.py | 34 + tests/test_tap.py | 518 +++ 32 files changed, 10008 insertions(+), 1 deletion(-) create mode 100644 .env.template create mode 100644 .github/workflows/github-actions-validator.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version create mode 100644 .secrets/.gitignore create mode 100644 .sonarcloud.properties create mode 100644 LICENSE create mode 100644 config-sample.json create mode 100644 config.json create mode 100644 meltano.yml create mode 100644 pyproject.toml create mode 100644 tap_github/__init__.py create mode 100644 tap_github/authenticator.py create mode 100644 tap_github/client.py create mode 100644 tap_github/organization_streams.py create mode 100644 tap_github/repository_streams.py create mode 100644 tap_github/schema_objects.py create mode 100644 tap_github/scraping.py create mode 100644 tap_github/streams.py create mode 100644 tap_github/tap.py create mode 100644 tap_github/user_streams.py create mode 100644 tap_github/utils/__init__.py create mode 100644 tap_github/utils/filter_stdout.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/fixtures.py create mode 100644 tests/test_authenticator.py create mode 100644 tests/test_core.py create mode 100644 tests/test_discover.py create mode 100644 tests/test_tap.py diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..58e4fcb --- /dev/null +++ b/.env.template @@ -0,0 +1 @@ +TAP_GITHUB_AUTH_TOKEN="****" diff --git a/.github/workflows/github-actions-validator.yml b/.github/workflows/github-actions-validator.yml new file mode 100644 index 0000000..6d34a79 --- /dev/null +++ b/.github/workflows/github-actions-validator.yml @@ -0,0 +1,17 @@ +--- + name: 'CI: github-actions-validator' + + on: + pull_request: + + concurrency: + group: ${{ github.head_ref }}-github-actions-validator + cancel-in-progress: true + + jobs: + github-actions-validator: + name: Github-actions-validator + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: lumapps/github-actions-validator@v2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a52e2c --- /dev/null +++ b/.gitignore @@ -0,0 +1,142 @@ +# Meltano hidden files +.meltano + +# Test output +.output + +# Secrets and internal config files +**/.secrets/* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# IDE +.idea/ +.vscode/ + +# Pyre type checker +.pyre/ + +#Other +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ddecc9b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +ci: + autofix_prs: true + autofix_commit_msg: 'chore: pre-commit autofix' + autoupdate_schedule: quarterly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-json + exclude: "\\.vscode/.*.json" + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 + hooks: + - id: ruff-check + args: [ --fix, --show-fixes ] + - id: ruff-format + +- repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.11.28 + hooks: + - id: uv-lock + - id: uv-sync + +- repo: https://github.com/hukkin/mdformat + rev: 1.0.0 + hooks: + - id: mdformat diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/.secrets/.gitignore b/.secrets/.gitignore new file mode 100644 index 0000000..e76051e --- /dev/null +++ b/.secrets/.gitignore @@ -0,0 +1,6 @@ +# IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, +# make sure those are never staged for commit into your git repo. You can store them here or another +# secure location. + +* +!.gitignore diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..7479be7 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,2 @@ +sonar.python.version=3.9, 3.10, 3.11, 3.12, 3.13 +sonar.cpd.exclusions=**/* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 791fa8d..7421907 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,184 @@ # tap-github -Fork of MeltanoLabs/tap-github — Singer tap for GitHub, with a repo-enumeration checkpoint fix, built with the Meltano SDK. + +`tap-github` is a Singer tap for GitHub. + +Built with the [Singer SDK](https://gitlab.com/meltano/singer-sdk). + +## Installation + +```bash +# use uv (https://docs.astral.sh/uv/) +uv tool install meltanolabs-tap-github + +# or pipx (https://pipx.pypa.io/stable/) +pipx install meltanolabs-tap-github + +# or Meltano +meltano add extractor tap-github +``` + +A list of release versions is available at https://github.com/MeltanoLabs/tap-github/releases + +## Configuration + +### Accepted Config Options + +This tap accepts the following configuration options: + +- Required: One and only one of the following modes: + 1. `repositories`: An array of strings specifying the GitHub repositories to be included. Each element of the array should be of the form `/`, e.g. `MeltanoLabs/tap-github`. + 1. `organizations`: An array of strings containing the github organizations to be included + 1. `searches`: An array of search descriptor objects with the following properties: + - `name`: A human readable name for the search query + - `query`: A github search string (generally the same as would come after `?q=` in the URL) + 1. `user_usernames`: A list of github usernames + 1. `user_ids`: A list of github user ids [int] +- Highly recommended: + - Personal access tokens (PATs) for authentication can be provided in 3 ways: + - `auth_token` - Takes a single token. + - `additional_auth_tokens` - Takes a list of tokens. Can be used together with `auth_token` or as the sole source of PATs. + - Any environment variables beginning with `GITHUB_TOKEN` will be assumed to be PATs. These tokens will be used in addition to `auth_token` (if provided), but will not be used if `additional_auth_tokens` is provided. + - GitHub App keys are another option for authentication, and can be used in combination with PATs if desired. App IDs and keys should be assembled into the format `:app_id:;;-----BEGIN RSA PRIVATE KEY-----\n_YOUR_P_KEY_\n-----END RSA PRIVATE KEY-----` (replace `:app_id:` with your actual GitHub App ID and `_YOUR_P_KEY_` with your private key content) where the key can be generated from the `Private keys` section on https://github.com/organizations/:organization_name/settings/apps/:app_name. Read more about GitHub App quotas [here](https://docs.github.com/en/enterprise-server@3.3/developers/apps/building-github-apps/rate-limits-for-github-apps#server-to-server-requests). Formatted app keys can be provided in 3 ways: + - `auth_app_keys` - List of GitHub App keys in the prescribed format. These keys are organization-agnostic and will be used as fallback for all organizations. + - `org_auth_app_keys` - Object/dictionary mapping organization names to lists of GitHub App keys. This allows you to specify different app credentials for different organizations, enabling better rate limit management across multiple organizations. Example: + ```yaml + auth_app_keys: + - "fallback_app_id;;-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + org_auth_app_keys: + my-org: + - "app_id_1;;-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + - "app_id_2;;-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + another-org: + - "app_id_3;;-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + ``` + - If `auth_app_keys` is not provided but there is an environment variable with the name `GITHUB_APP_PRIVATE_KEY`, it will be assumed to be an App key in the prescribed format (organization-agnostic). +- Optional: + - `user_agent` + - `start_date` + - `metrics_log_level` + - `stream_maps` + - `stream_maps_config` + - `stream_options`: Options which can change the behaviour of a specific stream are nested within. + - `milestones`: Valid options for the `milestones` stream are nested within. + - `state`: Determines which milestones will be extracted. One of `open` (default), `closed`, `all`. + - `project_items`: Valid options for the `project_items` stream are nested within. + - `page_size`: Number of ProjectV2 items to request per GraphQL page. Defaults to `100`, GitHub's maximum. Lower values can reduce GraphQL query complexity for projects with many custom fields at the cost of more requests. + - `rate_limit_buffer`: A buffer to avoid consuming all query points for the auth_token at hand. Defaults to 1000. + - `expiry_time_buffer`: A buffer used when determining when to refresh GitHub app tokens. Only relevant when authenticating as a GitHub app. Defaults to 10 minutes. Tokens generated by GitHub apps expire 1 hour after creation, and will be refreshed once fewer than `expiry_time_buffer` minutes remain until the anticipated expiry time. + - `backoff_max_tries`: The maximum number of backoff retry attempts for failed API requests that are retriable. Defaults to 5. + +Note that modes 1-3 are `repository` modes and 4-5 are `user` modes and will not run the same set of streams. + +A full list of supported settings and capabilities for this tap is available by running: + +```bash +tap-github --about +``` + +### Source Authentication and Authorization + +A small number of records may be pulled without an auth token. However, a Github auth token should generally be considered "required" since it gives more realistic rate limits. (See GitHub API docs for more info.) + +#### Multi-Organization Authentication + +When using `org_auth_app_keys`, the tap will automatically switch authentication contexts based on the organization being processed. This enables: + +- **Organization-specific rate limits**: Each organization can have its own set of GitHub App credentials, preventing rate limit exhaustion when processing multiple organizations. +- **Automatic token selection**: When processing repositories from a specific organization, the tap will prefer tokens configured for that organization. +- **Fallback behavior**: If no organization-specific tokens are available, the tap will fall back to: + 1. Organization-agnostic tokens (personal tokens or `auth_app_keys`) + 1. Tokens from other organizations (for accessing public data) + +## Usage + +### API Limitation - Pagination + +The GitHub API is limited for some resources such as `/events`. For some resources, users might encounter the following error: + +``` +In order to keep the API fast for everyone, pagination is limited for this resource. Check the rel=last link relation in the Link response header to see how far back you can traverse. +``` + +To avoid this, the GitHub streams will exit early. I.e. when there are no more `next page` available. If you are fecthing `/events` at the repository level, beware of letting the tap disabled for longer than a few days or you will have gaps in your data. + +You can easily run `tap-github` by itself or in a pipeline using [Meltano](www.meltano.com). + +### Notes regarding permissions + +- For the `traffic_*` streams, [you will need write access to the repository](https://docs.github.com/en/rest/metrics/traffic?apiVersion=2022-11-28). You can enable extraction for these streams by [selecting them in the catalog](https://hub.meltano.com/singer/spec/#metadata). + +### Executing the Tap Directly + +```bash +tap-github --version +tap-github --help +tap-github --config CONFIG --discover > ./catalog.json +``` + +## Contributing + +This project uses parent-child streams. Learn more about them [here.](https://gitlab.com/meltano/sdk/-/blob/main/docs/parent_streams.md) + +### Initialize your Development Environment + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh # https://docs.astral.sh/uv/getting-started/installation/ +uv sync +``` + +### Create and Run Tests + +Create tests within the `tap_github/tests` subfolder and +then run: + +```bash +uv run pytest +``` + +You can also test the `tap-github` CLI interface directly using `uv run`: + +```bash +uv run tap-github --help +``` + +### Testing with [Meltano](meltano.com) + +_**Note:** This tap will work in any Singer environment and does not require Meltano. +Examples here are for convenience and to streamline end-to-end orchestration scenarios._ + +Your project comes with a custom `meltano.yml` project file already created. Open the `meltano.yml` and follow any _"TODO"_ items listed in +the file. + +Next, install Meltano (if you haven't already) and any needed plugins: + +```bash +# Install meltano +uv tool install meltano +# Initialize meltano within this directory +cd tap-github +meltano install +``` + +Now you can test and orchestrate using Meltano: + +```bash +# Test invocation: +meltano invoke tap-github --version +# OR run a pipeline: +meltano run tap-github target-jsonl +``` + +One-liner to recreate output directory, run elt, and write out state file: + +```bash +# Update this when you want a fresh state file: +TESTJOB=testjob1 + +# Run everything in one line +mkdir -p .output && meltano elt tap-github target-jsonl --job_id $TESTJOB && meltano elt tap-github target-jsonl --job_id $TESTJOB --dump=state > .output/state.json +``` + +### Singer SDK Dev Guide + +See the [dev guide](../../docs/dev_guide.md) for more instructions on how to use the Singer SDK to +develop your own taps and targets. diff --git a/config-sample.json b/config-sample.json new file mode 100644 index 0000000..591bb8b --- /dev/null +++ b/config-sample.json @@ -0,0 +1,8 @@ +{ + "searches": [ + { + "name": "test_search", + "query": "target-athena+fork:only" + } + ] +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..8d6a94e --- /dev/null +++ b/config.json @@ -0,0 +1,4 @@ +{ + "repositories": ["indeedeng/proctor"], + "start_date": "2022-05-16" +} diff --git a/meltano.yml b/meltano.yml new file mode 100644 index 0000000..158cab1 --- /dev/null +++ b/meltano.yml @@ -0,0 +1,68 @@ +version: 1 +send_anonymous_usage_stats: false +project_id: 96584f7b-a36c-46e0-b41a-7f9074293137 +venv: + backend: uv +plugins: + extractors: + - name: tap-github + namespace: tap_github + pip_url: -e . + capabilities: + - state + - catalog + - discover + settings: + - name: user_agent + kind: string + - name: metrics_log_level + kind: string + - name: auth_token + kind: password + - name: additional_auth_tokens + kind: array + - name: auth_app_keys + kind: array + - name: org_auth_app_keys + kind: object + - name: rate_limit_buffer + kind: integer + - name: expiry_time_buffer + kind: integer + - name: backoff_max_tries + kind: integer + - name: searches + kind: array + - name: organizations + kind: array + - name: repositories + kind: array + - name: user_usernames + kind: array + - name: user_ids + kind: array + - name: stream_options.milestones.state + kind: options + options: + - label: Open + value: open + - label: Closed + value: closed + - label: All + value: all + - name: start_date + kind: date_iso8601 + value: '2010-01-01T00:00:00Z' + - name: stream_maps + kind: object + - name: stream_map_config + kind: object + select: + - '*.*' + loaders: + - name: target-jsonl + variant: andyh1203 + pip_url: target-jsonl + config: + destination_path: .output + do_timestamp_file: true diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d3db993 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,186 @@ +[project] +name = "meltanolabs-tap-github" +dynamic = [ "version" ] +description = "Singer tap for GitHub, built with the Singer SDK." +authors = [{ name = "Meltano and Meltano Community", email = "hello@meltano.com" }] +requires-python = ">=3.10" +readme = "README.md" +license = "Apache-2.0" +maintainers = [ + { name = "Meltano and Meltano Community", email = "hello@meltano.com" }, + { name = "Edgar Ramírez-Mondragón", email = "edgarrm358@gmail.com" }, +] +keywords = [ + "Meltano", + "Singer", + "Meltano SDK", + "Singer SDK", + "ELT", + "GitHub", +] +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed", +] +dependencies = [ + "backports-httpmethod>=0.2.1", + "beautifulsoup4>=4.14.2,<4.16.0", + "cryptography>=47.0,<49.1", + "nested-lookup~=0.2.25", + "PyJWT~=2.13.0", + "python-dateutil~=2.9", + "requests~=2.34.0", + "singer-sdk~=0.54.4", +] + +[project.urls] +Homepage = "https://github.com/MeltanoLabs/tap-github" +Repository = "https://github.com/MeltanoLabs/tap-github" +"Issue Tracker" = "https://github.com/MeltanoLabs/tap-github/issues" + +[project.scripts] +tap-github = "tap_github.tap:cli" + +[build-system] +requires = [ + "hatchling>=1.28,<2", + "hatch-vcs>=0.5", +] +build-backend = "hatchling.build" + +[tool.hatch.version] +fallback-version = "0.0.0" +source = "vcs" + +[tool.hatch.build.targets.sdist] +include = ["tap_github"] + +[tool.hatch.build.targets.wheel] +include = ["tap_github"] + +[tool.uv] +prerelease = "allow" +required-version = ">=0.9.17" + +[dependency-groups] +dev = [ + { include-group = "test" }, + { include-group = "type" }, +] +test = [ + "pytest~=9.0", + "requests-cache>=1.0.1", +] +type = [ + { include-group = "test" }, + "mypy>=1.19,<2.2", + "ty>=0.0.13", + "types-beautifulsoup4~=4.12", + "types-python-dateutil~=2.9", + "types-simplejson~=3.20", +] + +[tool.mypy] +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +follow_untyped_imports = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_configs = true +warn_unused_ignores = true + +[tool.pytest] +addopts = [ + "--durations=5", + "-m", + "not noconfig", +] +minversion = "9" +markers = [ + "noconfig: mark a test as not requiring a config", + "repo_list: mark a test as using a list of repos in config", + "username_list: mark a test as using a list of usernames in config", +] +testpaths = [ + "tests", +] + +[tool.ruff.lint] +ignore = [] +select = [ + "F", # Pyflakes + "E", # pycodestyle (errors) + "W", # pycodestyle (warnings) + "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "YTT", # flake8-2020 + "ANN", # flake8-annotations + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez + "FA", # flake8-future-annotations + "SIM", # flake8-simplify + "TC", # flake8-type-checking + "PERF", # Perflint + "FURB", # refurb + "RUF", # Ruff-specific rules +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN"] + +[tool.tox] +requires = [ + "tox>=4.55", + "tox-gh>=1.6.1", +] +env_list = [ + "type", + "3.14", + "3.13", + "3.12", + "3.11", + "3.10", +] + +[tool.tox.gh.python] +"3.14" = ["type", "noconfig", "3.14"] +"3.13" = ["noconfig"] +"3.12" = ["noconfig"] +"3.11" = ["noconfig"] +"3.10" = ["noconfig"] + +[tool.tox.env_run_base] +description = "Run test under {base_python}" +dependency_groups = [ + "test", +] +pass_env = [ + "LOGLEVEL", + "GITHUB_TOKEN", + "ORG_LEVEL_TOKEN", +] +commands = [["pytest", "--capture=no"]] + +[tool.tox.env.noconfig] +description = "validate that `tap-github --discover` works" +commands = [["pytest", "-m", "noconfig"]] + +[tool.tox.env.type] +description = "run type check on code base" +dependency_groups = [ + "type", +] +commands = [ + ["mypy", "tap_github"], + ["ty", "check", "tap_github"], +] diff --git a/tap_github/__init__.py b/tap_github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tap_github/authenticator.py b/tap_github/authenticator.py new file mode 100644 index 0000000..f7028f1 --- /dev/null +++ b/tap_github/authenticator.py @@ -0,0 +1,599 @@ +"""Classes to assist in authenticating to the GitHub API.""" + +from __future__ import annotations + +import logging +import time +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from os import environ +from random import choice, shuffle +from typing import TYPE_CHECKING, Any + +import jwt +import requests +import requests.exceptions +import requests.models +from singer_sdk.authenticators import APIAuthenticatorBase + +if TYPE_CHECKING: + from singer_sdk.streams import RESTStream + +logger = logging.getLogger(__name__) + + +class TokenManager: + """A class to store a token's attributes and state. + This parent class should not be used directly, use a subclass instead. + """ + + DEFAULT_RATE_LIMIT = 5000 + # The DEFAULT_RATE_LIMIT_BUFFER buffer serves two purposes: + # - keep some leeway and rotate tokens before erroring out on rate limit. + # - not consume all available calls when we rare using an org or user token. + DEFAULT_RATE_LIMIT_BUFFER = 1000 + + def __init__( + self, + token: str | None, + rate_limit_buffer: int | None = None, + logger: Any | None = None, # noqa: ANN401 + ) -> None: + """Init TokenManager info.""" + self.token = token + self.rate_limit = self.DEFAULT_RATE_LIMIT + self.rate_limit_remaining = self.DEFAULT_RATE_LIMIT + self.rate_limit_reset: datetime | None = None + self.rate_limit_used = 0 + self.rate_limit_buffer = ( + rate_limit_buffer + if rate_limit_buffer is not None + else self.DEFAULT_RATE_LIMIT_BUFFER + ) + + def update_rate_limit(self, response_headers: Any) -> None: # noqa: ANN401 + self.rate_limit = int(response_headers["X-RateLimit-Limit"]) + self.rate_limit_remaining = int(response_headers["X-RateLimit-Remaining"]) + self.rate_limit_reset = datetime.fromtimestamp( + int(response_headers["X-RateLimit-Reset"]), + tz=timezone.utc, + ) + self.rate_limit_used = int(response_headers["X-RateLimit-Used"]) + + def is_valid_token(self) -> bool: + """Try making a request with the current token. If the request succeeds return True, else False.""" # noqa: E501 + if not self.token: + return False + + try: + response = requests.get( + url="https://api.github.com/rate_limit", + headers={ + "Authorization": f"token {self.token}", + }, + ) + response.raise_for_status() + return True + except requests.exceptions.HTTPError: + msg = ( + f"A token could not be validated. " + f"{response.status_code} Client Error: " + f"{response.content!s} (Reason: {response.reason})" + ) + logger.warning(msg) + return False + + def has_calls_remaining(self) -> bool: + """Check if a token has capacity to make more calls. + + Returns: + True if the token is valid and has enough api calls remaining. + """ + if self.rate_limit_reset is None: + return True + return self.rate_limit_used <= ( + self.rate_limit - self.rate_limit_buffer + ) or self.rate_limit_reset <= datetime.now(tz=timezone.utc) + + +class PersonalTokenManager(TokenManager): + """A class to store token rate limiting information.""" + + def __init__( + self, + token: str, + rate_limit_buffer: int | None = None, + **kwargs, # noqa: ANN003 + ) -> None: + """Init PersonalTokenRateLimit info.""" + super().__init__(token, rate_limit_buffer=rate_limit_buffer, **kwargs) + + +def generate_jwt_token( + github_app_id: str, + github_private_key: str, + expiration_time: int = 600, + algorithm: str = "RS256", +) -> str: + actual_time = int(time.time()) + + payload = { + "iat": actual_time, + "exp": actual_time + expiration_time, + "iss": github_app_id, + } + token = jwt.encode(payload, github_private_key, algorithm=algorithm) + + if isinstance(token, bytes): # type: ignore[unreachable] + token = token.decode("utf-8") # type: ignore[unreachable] + + return token + + +def generate_app_access_token( + github_app_id: str, + github_private_key: str, + github_installation_id: str | None = None, +) -> tuple[str, datetime]: + produced_at = datetime.now(tz=timezone.utc) + jwt_token = generate_jwt_token(github_app_id, github_private_key) + + headers = {"Authorization": f"Bearer {jwt_token}"} + + if github_installation_id is None: + list_installations_resp = requests.get( + url="https://api.github.com/app/installations", headers=headers + ) + list_installations_resp.raise_for_status() + list_installations = list_installations_resp.json() + + if len(list_installations) == 0: + raise Exception(f"No installations found for app {github_app_id}.") + + github_installation_id = choice(list_installations)["id"] + + url = f"https://api.github.com/app/installations/{github_installation_id}/access_tokens" + resp = requests.post(url, headers=headers) + + if resp.status_code != 201: + resp.raise_for_status() + + expires_at = produced_at + timedelta(hours=1) + return resp.json()["token"], expires_at + + +class AppTokenManager(TokenManager): + """A class to store an app token's attributes and state, and handle token refreshing""" # noqa: E501 + + DEFAULT_RATE_LIMIT = 15000 + DEFAULT_EXPIRY_BUFFER_MINS = 10 + + def __init__( + self, + env_key: str, + rate_limit_buffer: int | None = None, + expiry_time_buffer: int | None = None, + **kwargs, # noqa: ANN003 + ) -> None: + rate_limit_buffer = rate_limit_buffer or self.DEFAULT_RATE_LIMIT_BUFFER + super().__init__(None, rate_limit_buffer=rate_limit_buffer, **kwargs) + + parts = env_key.split(";;") + self.github_app_id = parts[0] + self.github_private_key = (parts[1:2] or [""])[0].replace("\\n", "\n") + self.github_installation_id: str | None = parts[2] if len(parts) >= 3 else None + + if expiry_time_buffer is None: + expiry_time_buffer = self.DEFAULT_EXPIRY_BUFFER_MINS + self.expiry_time_buffer = expiry_time_buffer + + self.token_expires_at: datetime | None = None + self.claim_token() + + def claim_token(self) -> None: + """Updates the TokenManager's token and token_expires_at attributes. + + The outcome will be _either_ that self.token is updated to a newly claimed valid token and + self.token_expires_at is updated to the anticipated expiry time (erring on the side of an early estimate) + _or_ self.token and self.token_expires_at are both set to None. + """ # noqa: E501 + self.token = None + self.token_expires_at = None + + # Make sure we have the details we need + if not self.github_app_id or not self.github_private_key: + raise ValueError( + "GITHUB_APP_PRIVATE_KEY could not be parsed. The expected format is " + '":app_id:;;-----BEGIN RSA PRIVATE KEY-----\\n_YOUR_P_KEY_\\n-----END RSA PRIVATE KEY-----"' # noqa: E501 + ) + + self.token, self.token_expires_at = generate_app_access_token( + self.github_app_id, self.github_private_key, self.github_installation_id + ) + + # Check if the token isn't valid. If not, overwrite it with None + if not self.is_valid_token(): + logger.warning("An app token was generated but could not be validated.") + self.token = None + self.token_expires_at = None + + def has_calls_remaining(self) -> bool: + """Check if a token has capacity to make more calls. + + Returns: + True if the token is valid and has enough api calls remaining. + """ + if self.token_expires_at is not None: + close_to_expiry = datetime.now( + tz=timezone.utc + ) > self.token_expires_at - timedelta(minutes=self.expiry_time_buffer) + + if close_to_expiry: + self.claim_token() + if self.token is None: + logger.warning("GitHub app token refresh failed.") + return False + else: + logger.info("GitHub app token refresh succeeded.") + + return super().has_calls_remaining() + + +class GitHubTokenAuthenticator(APIAuthenticatorBase): + """Base class for offloading API auth.""" + + @staticmethod + def get_env(): # noqa: ANN205 + return dict(environ) + + def _collect_personal_tokens(self, env_dict: dict) -> set[str]: + """Collect personal tokens from config and environment variables. + + Args: + env_dict: Dictionary of environment variables. + + Returns: + Set of personal access tokens. + """ + personal_tokens: set[str] = set() + if self.auth_token: + personal_tokens.add(self.auth_token) + if self.additional_auth_tokens: + personal_tokens = personal_tokens.union(self.additional_auth_tokens) + else: + # Accept multiple tokens using environment variables GITHUB_TOKEN* + env_tokens = { + value + for key, value in env_dict.items() + if key.startswith("GITHUB_TOKEN") + } + if len(env_tokens) > 0: + logger.info( + "Found %d 'GITHUB_TOKEN' environment variables for authentication.", + len(env_tokens), + ) + personal_tokens = personal_tokens.union(env_tokens) + return personal_tokens + + def _create_personal_token_managers( + self, personal_tokens: set[str] + ) -> list[TokenManager]: + """Create validated token managers from personal tokens. + + Args: + personal_tokens: Set of personal access tokens. + + Returns: + List of validated PersonalTokenManager instances. + """ + personal_token_managers: list[TokenManager] = [] + for token in personal_tokens: + token_manager = PersonalTokenManager( + token, + rate_limit_buffer=self.rate_limit_buffer, + ) + if token_manager.is_valid_token(): + personal_token_managers.append(token_manager) + else: + logger.warning("A token was dismissed.") + return personal_token_managers + + def _collect_app_keys(self, env_dict: dict) -> dict[str | None, list[str]]: + """Collect org-agnostic app keys from config and environment variables. + + App keys can be provided via: + 1. Config as array: ["app_id;;private_key", ...] + 2. Environment variable GITHUB_APP_PRIVATE_KEY: "app_id;;private_key" + + Args: + env_dict: Dictionary of environment variables. + + Returns: + Dictionary mapping None to lists of org-agnostic app keys. + """ + app_keys: dict[str | None, list[str]] = defaultdict(list) + + if self.auth_app_keys: + for app_key in self.auth_app_keys: + app_keys[None].append(app_key) + logger.info( + "Provided %d app keys via config for authentication.", + len(self.auth_app_keys), + ) + elif "GITHUB_APP_PRIVATE_KEY" in env_dict: + app_keys[None].append(env_dict["GITHUB_APP_PRIVATE_KEY"]) + logger.info("Found 1 app key via environment variable for authentication.") + + return app_keys + + def _collect_org_auth_app_keys(self) -> dict[str | None, list[str]]: + """Collect organization-specific app keys from config. + + Returns: + Dictionary mapping organization names to lists of app keys. + """ + org_app_keys: dict[str | None, list[str]] = defaultdict(list) + + if self.org_auth_app_keys: + for org, app_keys in self.org_auth_app_keys.items(): + for app_key in app_keys: + org_app_keys[org].append(app_key) + logger.info( + "Provided %d app keys via config for authentication " + "for organization: %s", + len(app_keys), + org, + ) + + return org_app_keys + + def _create_app_token_managers( + self, + app_keys: dict[str | None, list[str]], + ) -> dict[str | None, list[TokenManager]]: + """Create validated app token managers from app keys. + + Args: + app_keys: Dictionary mapping organization names to lists of app keys. + + Returns: + Dictionary mapping organization names to lists of validated + AppTokenManager instances. + """ + token_managers: dict[str | None, list[TokenManager]] = defaultdict(list) + + for org in app_keys: + for app_key in app_keys[org]: + try: + app_token_manager = AppTokenManager( + app_key, + rate_limit_buffer=self.rate_limit_buffer, + expiry_time_buffer=self.expiry_time_buffer, + ) + if app_token_manager.is_valid_token(): + token_managers[org].append(app_token_manager) + except ValueError as e: # noqa: PERF203 + logger.warning( + f"An error was thrown while preparing an app token: {e}" + ) + + return token_managers + + def prepare_tokens(self) -> dict[str | None, list[TokenManager]]: + """Prep GitHub tokens""" + env_dict = self.get_env() + + # Collect and create personal token managers + personal_tokens = self._collect_personal_tokens(env_dict) + personal_token_managers = self._create_personal_token_managers(personal_tokens) + + # Collect and create org-agnostic app token managers + app_keys = self._collect_app_keys(env_dict) + token_managers = self._create_app_token_managers(app_keys) + + # Collect and create org-specific app token managers + org_app_keys = self._collect_org_auth_app_keys() + org_token_managers = self._create_app_token_managers(org_app_keys) + + # Merge org-specific token managers into main dict + for org, managers in org_token_managers.items(): + token_managers[org].extend(managers) + + # Log summary + logger.info( + "Tap will run with %d personal auth tokens and %d app keys.", + len(personal_token_managers), + sum(len(token_managers[org]) for org in token_managers), + ) + + # Merge personal tokens into org-agnostic tokens (None key). + if personal_token_managers: + token_managers[None].extend(personal_token_managers) + + return token_managers + + def __init__( + self, + *, + rate_limit_buffer: int | None = None, + expiry_time_buffer: int | None = None, + auth_token: str | None = None, + additional_auth_tokens: list[str] | None = None, + auth_app_keys: list[str] | None = None, + org_auth_app_keys: dict[str, list[str]] | None = None, + ) -> None: + """Init authenticator. + + Args: + stream: A stream for a RESTful endpoint. + rate_limit_buffer: A buffer to add to the rate limit. + expiry_time_buffer: A buffer used when determining when to refresh GitHub + app tokens. Only relevant when authenticating as a GitHub app. + auth_token: A personal access token. + additional_auth_tokens: A list of personal access tokens. + auth_app_keys: Organization-agnostic GitHub App keys used as fallback + for all organizations. + org_auth_app_keys: Organization-specific GitHub App keys. Dict mapping + organization names to lists of app keys. + """ + super().__init__() + self.rate_limit_buffer = rate_limit_buffer + self.expiry_time_buffer = expiry_time_buffer + self.auth_token = auth_token + self.additional_auth_tokens = additional_auth_tokens + self.auth_app_keys = auth_app_keys + self.org_auth_app_keys = org_auth_app_keys + + self.token_managers = self.prepare_tokens() + self.current_organization: str | None = None + self.active_token: TokenManager | None = None + if self.token_managers: + # Prefer org-specific tokens over org-agnostic (None key) + org_keys = [k for k in self.token_managers if k is not None] + initial_org = min(org_keys) if org_keys else None + self.logger.info( + f"Setting initial organization for authenticator: {initial_org}" + ) + self.active_token = choice(self.token_managers[initial_org]) + else: + self.logger.info("Setting initial organization for authenticator: None") + + @classmethod + def from_stream(cls, stream: RESTStream) -> GitHubTokenAuthenticator: + return cls( + rate_limit_buffer=stream.config.get("rate_limit_buffer"), + expiry_time_buffer=stream.config.get("expiry_time_buffer"), + auth_token=stream.config.get("auth_token"), + additional_auth_tokens=stream.config.get("additional_auth_tokens"), + auth_app_keys=stream.config.get("auth_app_keys"), + org_auth_app_keys=stream.config.get("org_auth_app_keys"), + ) + + def set_organization(self, org: str) -> None: + """Set the current organization and switch to an appropriate token. + + Args: + org: The organization name to switch to. + """ + # If we're already using this org, no need to switch + if self.current_organization == org: + return + + logger.info(f"Switching authentication context to organization: {org}") + self.current_organization = org + + # Get tokens for this org (check both org-specific and None keys) + available_tokens = self.token_managers.get(org, []) + if not available_tokens and None in self.token_managers: + # Fall back to org-agnostic tokens (personal tokens or env var app keys) + available_tokens = self.token_managers[None] + logger.info( + f"No org-specific tokens found for '{org}', using org-agnostic tokens" + ) + + # If still no tokens, try tokens from other orgs (for public data access) + if not available_tokens: + for other_org, tokens in self.token_managers.items(): + if other_org is not None and tokens: + available_tokens = tokens + logger.info( + f"No tokens for '{org}', using tokens from '{other_org}' " + f"for public data access" + ) + break + else: + logger.warning( + f"No authentication tokens available for organization: {org}" + ) + self.active_token = None + return + + # Select a token with remaining calls + for token_manager in available_tokens: + if token_manager.has_calls_remaining(): + self.active_token = token_manager + logger.info(f"Selected token for organization: {org}") + return + + # If no tokens have calls remaining, just pick the first one + # (it might refresh or we'll rotate later) + self.active_token = available_tokens[0] + logger.info( + f"Selected token for organization: {org} (may need rate limit refresh)" + ) + + def get_next_auth_token(self) -> None: + current_token = self.active_token.token if self.active_token else "" + + # Build a list of candidate tokens for the current organization. + # If the current org has a configured token pool, stay within that pool. + # Installation tokens are org-scoped, so falling through to another org's + # token can turn legitimate private-repo calls into misleading 404s. + candidates = [] + has_current_org_tokens = ( + self.current_organization + and self.current_organization in self.token_managers + and self.token_managers[self.current_organization] + ) + + # Priority 1: Tokens scoped to the current org + if has_current_org_tokens: + org_tokens = list(self.token_managers[self.current_organization]) + shuffle(org_tokens) + candidates.extend(org_tokens) + else: + # Priority 2: Org-agnostic tokens (stored under None key) + if None in self.token_managers: + agnostic_tokens = list(self.token_managers[None]) + shuffle(agnostic_tokens) + candidates.extend(agnostic_tokens) + + # Priority 3: Tokens from other organizations (for public data access) + # GitHub tokens can access public data from any org + for org, tokens in self.token_managers.items(): + if org != self.current_organization and org is not None: + other_org_tokens = list(tokens) + shuffle(other_org_tokens) + candidates.extend(other_org_tokens) + + # Try to find a token with remaining capacity + for token_manager in candidates: + if ( + token_manager.has_calls_remaining() + and current_token != token_manager.token + ): + self.active_token = token_manager + logger.info("Switching to fresh auth token") + return + + raise RuntimeError( + "All GitHub tokens have hit their rate limit. Stopping here." + ) + + def update_rate_limit( + self, + response_headers: requests.models.CaseInsensitiveDict, + ) -> None: + # If no token or only one token is available, return early. + # Count total tokens across all organizations + total_tokens = sum(len(tokens) for tokens in self.token_managers.values()) + if total_tokens <= 1 or self.active_token is None: + return + + self.active_token.update_rate_limit(response_headers) + + def authenticate_request( + self, + request: requests.PreparedRequest, + ) -> requests.PreparedRequest: + if self.active_token: + # Make sure that our token is still valid or update it. + if not self.active_token.has_calls_remaining(): + self.get_next_auth_token() + request.headers["Authorization"] = f"token {self.active_token.token}" + else: + logger.info( + "No auth token detected. " + "For higher rate limits, please specify `auth_token` in config." + ) + return request diff --git a/tap_github/client.py b/tap_github/client.py new file mode 100644 index 0000000..cda1aec --- /dev/null +++ b/tap_github/client.py @@ -0,0 +1,567 @@ +"""REST client handling, including GitHubStream base class.""" + +from __future__ import annotations + +import email.utils +import inspect +import random +import time +from typing import TYPE_CHECKING, Any, ClassVar, cast +from urllib.parse import parse_qs, urlparse + +from dateutil.parser import parse +from nested_lookup import nested_lookup +from singer_sdk.exceptions import FatalAPIError, RetriableAPIError +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import GraphQLStream, RESTStream + +from tap_github.authenticator import GitHubTokenAuthenticator + +if TYPE_CHECKING: + from collections.abc import Iterable + from types import FrameType + + import requests + from backoff.types import Details + from singer_sdk.helpers.types import Context + +EMPTY_REPO_ERROR_STATUS = 409 + + +class GitHubRestStream(RESTStream): + """GitHub Rest stream class.""" + + MAX_PER_PAGE = 100 # GitHub's limit is 100. + MAX_RESULTS_LIMIT: int | None = None + DEFAULT_API_BASE_URL = "https://api.github.com" + LOG_REQUEST_METRIC_URLS = True + + # GitHub is missing the "since" parameter on a few endpoints + # set this parameter to True if your stream needs to navigate data in descending order # noqa: E501 + # and try to exit early on its own. + # This only has effect on streams whose `replication_key` is `updated_at`. + use_fake_since_parameter = False + + # Set to True to use cursor-based pagination instead of page-based pagination + use_cursor_pagination = False + + _authenticator: GitHubTokenAuthenticator | None = None + + @property + def authenticator(self) -> GitHubTokenAuthenticator: + if self._authenticator is None: + self._authenticator = GitHubTokenAuthenticator.from_stream(self) + return self._authenticator + + @property + def url_base(self) -> str: + return self.config.get("api_url_base", self.DEFAULT_API_BASE_URL) + + primary_keys: ClassVar[list[str]] = ["id"] + replication_key: str | None = None + tolerated_http_errors: ClassVar[list[int]] = [] + + @property + def http_headers(self) -> dict[str, str]: + """Return the http headers needed.""" + headers = {"Accept": "application/vnd.github.v3+json"} + headers["User-Agent"] = cast("str", self.config.get("user_agent", "tap-github")) + return headers + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + """ + Override parent method to set organization-specific authentication + before fetching records. + """ + # Set organization-specific authentication before fetching records + if context is not None and "org" in context: + self.authenticator.set_organization(context["org"]) + + yield from super().get_records(context) + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """Return a token for identifying next page or None if no more pages.""" + if ( + previous_token + and self.MAX_RESULTS_LIMIT + and not self.use_cursor_pagination + and ( + cast("int", previous_token) * self.MAX_PER_PAGE + >= self.MAX_RESULTS_LIMIT + ) + ): + return None + + # Leverage header links returned by the GitHub API. + if "next" not in response.links: + return None + + resp_json = response.json() + results = ( + resp_json + if isinstance(resp_json, list) + else list(extract_jsonpath(self.records_jsonpath, input=resp_json)) + ) + + # Exit early if the response has no items. ? Maybe duplicative the "next" link check. # noqa: E501 + if not results: + return None + + # Unfortunately endpoints such as /starred, /stargazers, /events and /pulls do not support # noqa: E501 + # the "since" parameter out of the box. So we use a workaround here to exit early. # noqa: E501 + # For such streams, we sort by descending dates (most recent first), and paginate # noqa: E501 + # "back in time" until we reach records before our "fake_since" parameter. + if self.replication_key and self.use_fake_since_parameter: + request_parameters = parse_qs(str(urlparse(response.request.url).query)) + # parse_qs interprets "+" as a space, revert this to keep an aware datetime + try: + since = ( + request_parameters["fake_since"][0].replace(" ", "+") + if "fake_since" in request_parameters + else "" + ) + except IndexError: + return None + + direction = ( + request_parameters["direction"][0] + if "direction" in request_parameters + else None + ) + + # commit_timestamp is a constructed key which does not exist in the raw response # noqa: E501 + replication_date = ( + results[-1][self.replication_key] + if self.replication_key != "commit_timestamp" + else results[-1]["commit"]["committer"]["date"] + ) + # exit early if the replication_date is before our since parameter + if ( + since + and direction == "desc" + and (parse(replication_date) < parse(since)) + ): + return None + + # Handle cursor-based pagination + if self.use_cursor_pagination: + parsed_url = urlparse(response.links["next"]["url"]) + captured_after_value_list = parse_qs(parsed_url.query).get("after") + return captured_after_value_list[0] if captured_after_value_list else None + + # Use header links returned by the GitHub API for page-based pagination. + parsed_url = urlparse(response.links["next"]["url"]) + captured_page_value_list = parse_qs(parsed_url.query).get("page") + next_page_string = ( + captured_page_value_list[0] if captured_page_value_list else None + ) + if next_page_string and next_page_string.isdigit(): + return int(next_page_string) + + return (previous_token or 1) + 1 + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params: dict = {"per_page": self.MAX_PER_PAGE} + if next_page_token: + if self.use_cursor_pagination: + params["after"] = next_page_token + else: + params["page"] = next_page_token + + if self.replication_key == "updated_at": + params["sort"] = "updated" + params["direction"] = "desc" if self.use_fake_since_parameter else "asc" + + # Unfortunately the /starred, /stargazers (starred_at) and /events (created_at) endpoints do not support # noqa: E501 + # the "since" parameter out of the box. But we use a workaround in 'get_next_page_token'. # noqa: E501 + elif self.replication_key in ["starred_at", "created_at"]: + params["sort"] = "created" + params["direction"] = "desc" + + # Warning: /commits endpoint accept "since" but results are ordered by descending commit_timestamp # noqa: E501 + elif self.replication_key == "commit_timestamp": + params["direction"] = "desc" + + elif self.replication_key: + self.logger.warning( + f"The replication key '{self.replication_key}' is not fully supported by this client yet." # noqa: E501 + ) + + since = self.get_starting_timestamp(context) + since_key = "since" if not self.use_fake_since_parameter else "fake_since" + if self.replication_key and since: + params[since_key] = since.isoformat(sep="T") + # Leverage conditional requests to save API quotas + # https://github.community/t/how-does-if-modified-since-work/139627 + self.http_headers["If-modified-since"] = email.utils.format_datetime(since) + return params + + def validate_response(self, response: requests.Response) -> None: + """Validate HTTP response. + + In case an error is tolerated, continue without raising it. + + In case an error is deemed transient and can be safely retried, then this + method should raise an :class:`singer_sdk.exceptions.RetriableAPIError`. + + Args: + response: A `requests.Response`_ object. + + Raises: + FatalAPIError: If the request is not retriable. + RetriableAPIError: If the request is retriable. + + .. _requests.Response: + https://docs.python-requests.org/en/latest/api/#requests.Response + """ + full_path = urlparse(response.url).path + if response.status_code in ( + [ + *self.tolerated_http_errors, + EMPTY_REPO_ERROR_STATUS, + ] + ): + msg = ( + f"{response.status_code} Tolerated Status Code " + f"(Reason: {response.reason}) for path: {full_path}" + ) + self.logger.info(msg) + return + + github_request_id = response.headers.get("X-GitHub-Request-Id") + + if 400 <= response.status_code < 500: + msg = ( + f"{response.status_code} Client Error: " + f"{response.content!s} (Reason: {response.reason}) " + f"for path: {full_path} " + f"[GitHub-Request-Id: {github_request_id}]" + ) + # Retry on rate limiting + if ( + response.status_code == 403 + and "rate limit exceeded" in str(response.content).lower() + ): + # Update token + self.authenticator.get_next_auth_token() + # Raise an error to force a retry with the new token. + raise RetriableAPIError(msg, response) + + # Retry on secondary rate limit + if ( + response.status_code == 403 + and "secondary rate limit" in str(response.content).lower() + ): + # Wait about a minute and retry + time.sleep(60 + 30 * random.random()) + raise RetriableAPIError(msg, response) + + # The GitHub API randomly returns 401 Unauthorized errors, so we try again. + if ( + response.status_code == 401 + # if the token is invalid, we are also told about it + and "bad credentials" not in str(response.content).lower() + ): + raise RetriableAPIError(msg, response) + + # The GitHub API can return transient 404 errors, e.g. due to + # replication lag or temporary unavailability. Retry these. + if response.status_code == 404: + raise RetriableAPIError(msg, response) + + # all other errors are fatal + raise FatalAPIError(msg) + + elif 500 <= response.status_code < 600: + msg = ( + f"{response.status_code} Server Error: " + f"{response.content!s} (Reason: {response.reason}) " + f"for path: {full_path} " + f"[GitHub-Request-Id: {github_request_id}]" + ) + raise RetriableAPIError(msg, response) + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + # TODO - Split into handle_reponse and parse_response. + if response.status_code in ( + [ + *self.tolerated_http_errors, + EMPTY_REPO_ERROR_STATUS, + ] + ): + return + + # Update token rate limit info and loop through tokens if needed. + self.authenticator.update_rate_limit(response.headers) + + resp_json = response.json() + + # Handle different response structures + if isinstance(resp_json, list): + # Direct array response + results = resp_json + else: + # Use records_jsonpath to extract records + results = list(extract_jsonpath(self.records_jsonpath, input=resp_json)) + + # Fallback: if no records found via jsonpath, treat the whole response as a + # single record + if not results: + results = [resp_json] + + yield from results + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """Add `repo_id` by default to all streams.""" + if context is not None and "repo_id" in context: + row["repo_id"] = context["repo_id"] + return row + + def backoff_handler(self, details: Details) -> None: + """Handle retriable error by swapping auth token.""" + self.logger.info("Retrying request with different token") + # use python introspection to obtain the error object + # FIXME: replace this once https://github.com/litl/backoff/issues/158 + # is fixed + exc = cast( + "FrameType", + cast("FrameType", cast("FrameType", inspect.currentframe()).f_back).f_back, + ).f_locals["e"] + if ( + exc.response is not None + and exc.response.status_code == 403 + and "rate limit exceeded" in str(exc.response.content) + ): + # we hit a rate limit, rotate token + prepared_request = details["args"][0] + self.authenticator.get_next_auth_token() + prepared_request.headers.update(self.authenticator.auth_headers or {}) + + def backoff_max_tries(self) -> int: + """Return the maximum number of retry attempts.""" + return self.config["backoff_max_tries"] + + def calculate_sync_cost( + self, + request: requests.PreparedRequest, + response: requests.Response, + context: Context | None, + ) -> dict[str, int]: + """Return the cost of the last REST API call.""" + return {"rest": 1, "graphql": 0, "search": 0} + + +class GitHubDiffStream(GitHubRestStream): + """Base class for GitHub diff streams.""" + + # Known Github API errors for diff requests + tolerated_http_errors: ClassVar[list[int]] = [404, 406, 422, 500, 502, 504] + + @property + def http_headers(self) -> dict: + """Return the http headers needed for diff requests.""" + headers = super().http_headers + headers["Accept"] = "application/vnd.github.v3.diff" + return headers + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response to yield the diff text instead of an object + and prevent buffer overflow.""" + if response.status_code != 200: + contents = response.json() + self.logger.info( + "Skipping %s due to %d error: %s", + self.name.replace("_", " "), + response.status_code, + contents["message"], + ) + yield { + "success": False, + "error_message": contents["message"], + } + return + + if content_length_str := response.headers.get("Content-Length"): + content_length = int(content_length_str) + max_size = 41_943_040 # 40 MiB + if content_length > max_size: + self.logger.info( + "Skipping %s. The diff size (%.2f MiB) exceeded the maximum" + " size limit of 40 MiB.", + self.name.replace("_", " "), + content_length / 1024 / 1024, + ) + yield { + "success": False, + "error_message": "Diff exceeded the maximum size limit of 40 MiB.", + } + return + + yield {"diff": response.text, "success": True} + + +class GitHubGraphqlStream(GraphQLStream, GitHubRestStream): + """GitHub Graphql stream class.""" + + @property + def url_base(self) -> str: + return f"{self.config.get('api_url_base', self.DEFAULT_API_BASE_URL)}/graphql" + + # the jsonpath under which to fetch the list of records from the graphql response + query_jsonpath: str = "$.data.[*]" + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows. + + Args: + response: A raw `requests.Response`_ object. + + Yields: + One item for every item found in the response. + + .. _requests.Response: + https://docs.python-requests.org/en/latest/api/#requests.Response + """ + resp_json = response.json() + for record in extract_jsonpath(self.query_jsonpath, input=resp_json): + if record is not None: + yield record + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """ + Return a dict of cursors for identifying next page or None if no more pages. + + Note - pagination requires the Graphql query to have nextPageCursor_X parameters + with the assosciated hasNextPage_X, startCursor_X and endCursor_X. + + X should be an integer between 0 and 9, increasing with query depth. + + Warning - we recommend to avoid using deep (nested) pagination. + """ + + resp_json = response.json() + + # Find if results contains "hasNextPage_X" flags and if any are True. + # If so, set nextPageCursor_X to endCursor_X for X max. + + next_page_results = nested_lookup( + key="hasNextPage_", + document=resp_json, + wild=True, + with_keys=True, + ) + + has_next_page_indices: list[int] = [] + # Iterate over all the items and filter items with hasNextPage = True. + for key, value in next_page_results.items(): + # Check if key is even then add pair to new dictionary + if any(value): + pagination_index = int(str(key).split("_")[1]) + has_next_page_indices.append(pagination_index) + + # Check if any "hasNextPage" is True. Otherwise, exit early. + if not len(has_next_page_indices) > 0: + return None + + # Get deepest pagination item + max_pagination_index = max(has_next_page_indices) + + # We leverage previous_token to remember the pagination cursors + # for indices below max_pagination_index. + next_page_cursors: dict[str, str] = {} + for key, value in (previous_token or {}).items(): + # Only keep pagination info for indices below max_pagination_index. + pagination_index = int(str(key).split("_")[1]) + if pagination_index < max_pagination_index: + next_page_cursors[key] = value + + # Get the pagination cursor to update and increment it. + next_page_end_cursor_results = nested_lookup( + key=f"endCursor_{max_pagination_index}", + document=resp_json, + ) + + next_page_key = f"nextPageCursor_{max_pagination_index}" + next_page_cursor = next( + cursor for cursor in next_page_end_cursor_results if cursor is not None + ) + next_page_cursors[next_page_key] = next_page_cursor + + return next_page_cursors + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params = dict(context) if context else {} + params["per_page"] = self.MAX_PER_PAGE + if next_page_token: + params.update(next_page_token) + + since = self.get_starting_timestamp(context) + if self.replication_key and since: + params["since"] = since.isoformat(sep="T") + + return params + + def calculate_sync_cost( + self, + request: requests.PreparedRequest, + response: requests.Response, + context: Context | None, + ) -> dict[str, int]: + """Return the cost of the last graphql API call.""" + costgen = extract_jsonpath("$.data.rateLimit.cost", input=response.json()) + # calculate_sync_cost is called before the main response parsing. + # In some cases, the tap crashes here before we have been able to + # properly analyze where the error comes from, so we ignore these + # costs to allow figuring out what happened downstream, by setting + # them to 0. + cost = next(costgen, 0) + return {"rest": 0, "graphql": int(cost), "search": 0} + + def validate_response(self, response: requests.Response) -> None: + """Validate HTTP response. + + The graphql spec is a bit confusing around response codes + (https://github.com/graphql/graphql-over-http/blob/main/spec/GraphQLOverHTTP.md#response) + Github's API is a bit of a free adaptation of standards, so we + choose fail immediately on error here, so that something is logged + at the very minimum. + + Args: + response: A `requests.Response`_ object. + + Raises: + FatalAPIError: If the request is not retriable. + RetriableAPIError: If the request is retriable. + """ + super().validate_response(response) + rj = response.json() + if "errors" in rj: + msg = rj["errors"] + if any(e.get("message") == "timedout" for e in msg if isinstance(e, dict)): + raise RetriableAPIError(f"Graphql error: {msg}", response) + # If partial data is present (partial success), log errors and continue. + # This handles e.g. FORBIDDEN errors on individual nodes in a paginated + # result where other nodes are accessible. + if rj.get("data") is not None: + self.logger.warning("Graphql partial errors (ignored): %s", msg) + return + raise FatalAPIError(f"Graphql error: {msg}", response) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py new file mode 100644 index 0000000..c333213 --- /dev/null +++ b/tap_github/organization_streams.py @@ -0,0 +1,1075 @@ +"""User Stream types classes for tap-github.""" + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING, Any, ClassVar + +from singer_sdk import typing as th # JSON Schema typing helpers +from singer_sdk.exceptions import FatalAPIError + +from tap_github.client import GitHubGraphqlStream, GitHubRestStream + +if TYPE_CHECKING: + from collections.abc import Iterable + + from singer_sdk.helpers.types import Context + + +# Reusable GraphQL fragment for Actor fields +# https://docs.github.com/en/graphql/reference/interfaces#actor +ACTOR_FRAGMENT = """ + login + resource_path: resourcePath + url + type: __typename + ... on Bot { + node_id: id + id: databaseId + } + ... on User { + node_id: id + id: databaseId + } + ... on Organization { + node_id: id + id: databaseId + } + ... on Mannequin { + node_id: id + id: databaseId + } + ... on EnterpriseUserAccount { + node_id: id + } +""" + + +class OrganizationStream(GitHubRestStream): + """Defines a GitHub Organization Stream. + API Reference: https://docs.github.com/en/rest/reference/orgs#get-an-organization + """ + + name = "organizations" + path = "/orgs/{org}" + + @property + def partitions(self) -> list[dict] | None: + return [{"org": org} for org in self.config["organizations"]] + + def get_child_context(self, record: dict, context: Context | None) -> dict: + return { + "org": record["login"], + } + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + """ + Override the parent method to allow skipping API calls + if the stream is deselected and skip_parent_streams is True in config. + This allows running the tap with fewer API calls and preserving + quota when only syncing a child stream. Without this, + the API call is sent but data is discarded. + """ + if ( + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None + ): + # build a minimal mock record so that self._sync_records + # can proceed with child streams + yield { + "org": context["org"], + } + else: + yield from super().get_records(context) + + schema = th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("hooks_url", th.StringType), + th.Property("issues_url", th.StringType), + th.Property("members_url", th.StringType), + th.Property("public_members_url", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("description", th.StringType), + ).to_dict() + + +class OrganizationMembersStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members + """ + + name = "organization_members" + primary_keys: ClassVar[list[str]] = ["id"] + path = "/orgs/{org}/members" + ignore_parent_replication_key = True + parent_stream_type = OrganizationStream + state_partitioning_keys: ClassVar[list[str]] = ["org"] + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + # Rest + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() + + +class TeamsStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#list-teams + """ + + name = "teams" + primary_keys: ClassVar[list[str]] = ["id"] + path = "/orgs/{org}/teams" + ignore_parent_replication_key = True + parent_stream_type = OrganizationStream + state_partitioning_keys: ClassVar[list[str]] = ["org"] + + def get_child_context(self, record: dict, context: Context | None) -> dict: + new_context = {"team_slug": record["slug"]} + if context: + return { + **context, + **new_context, + } + return new_context + + schema = th.PropertiesList( + # Parent Keys + th.Property("org", th.StringType), + # Rest + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + th.Property( + "parent", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + ), + ), + ).to_dict() + + +class TeamMembersStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#list-team-members + """ + + name = "team_members" + primary_keys: ClassVar[list[str]] = ["id", "team_slug"] + path = "/orgs/{org}/teams/{team_slug}/members" + ignore_parent_replication_key = True + parent_stream_type = TeamsStream + state_partitioning_keys: ClassVar[list[str]] = ["team_slug", "org"] + + def get_child_context(self, record: dict, context: Context | None) -> dict: + new_context = {"username": record["login"]} + if context: + return { + **context, + **new_context, + } + return new_context + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("team_slug", th.StringType), + # Rest + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() + + +class TeamRolesStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user + """ + + name = "team_roles" + path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" + ignore_parent_replication_key = True + primary_keys: ClassVar[list[str]] = ["url"] + parent_stream_type = TeamMembersStream + state_partitioning_keys: ClassVar[list[str]] = ["username", "team_slug", "org"] + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("team_slug", th.StringType), + th.Property("username", th.StringType), + # Rest + th.Property("url", th.StringType), + th.Property("role", th.StringType), + th.Property("state", th.StringType), + ).to_dict() + + +class ProjectsStream(GitHubGraphqlStream): + """Fetches GitHub projects (new projects aka ProjectsV2) for an organization. + + API Reference: https://docs.github.com/en/graphql/reference/objects#projectv2 + """ + + name = "projects" + primary_keys: ClassVar[list[str]] = ["org", "id"] + parent_stream_type = OrganizationStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["org"] + query_jsonpath = "$.data.organization.projectsV2.nodes[*]" + + @property + def query(self) -> str: + """GraphQL query to fetch projects.""" + return f""" + query OrganizationProjects($org: String!, $nextPageCursor_0: String) {{ + organization(login: $org) {{ + projectsV2(first: 100, after: $nextPageCursor_0) {{ + nodes {{ + closed + closed_at: closedAt + created_at: createdAt + creator {{ + {ACTOR_FRAGMENT} + }} + id: fullDatabaseId + node_id: id + number + owner {{ + node_id: id + type: __typename + }} + public + readme + resource_path: resourcePath + short_description: shortDescription + template + title + updated_at: updatedAt + url + viewer_can_close: viewerCanClose + viewer_can_reopen: viewerCanReopen + viewer_can_update: viewerCanUpdate + }} + pageInfo {{ + hasNextPage_0: hasNextPage + endCursor_0: endCursor + startCursor_0: startCursor + }} + totalCount + }} + }} + rateLimit {{ + cost + }} + }} + """ + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """Return context for child streams.""" + new_context = {"project_number": record["number"]} + if context: + return {**context, **new_context} + return new_context + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """Post-process a fetched record.""" + row = super().post_process(row, context) + if context: + row["org"] = context["org"] + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + # Project fields + th.Property( + "id", th.StringType, nullable=False + ), # using fullDatabaseId from GraphQL as id, but is nullable in GraphQL + th.Property( + "node_id", th.StringType + ), # using id from GraphQL as node_id, it is required (ID!) + th.Property("number", th.IntegerType), + th.Property("title", th.StringType), + th.Property("url", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("closed", th.BooleanType), + th.Property( + "closed_at", th.DateTimeType, required=False + ), # closedAt is nullable in GraphQL + th.Property("public", th.BooleanType), + th.Property( + "readme", th.StringType, required=False + ), # readme is nullable in GraphQL + th.Property( + "short_description", th.StringType, required=False + ), # shortDescription is nullable in GraphQL + th.Property("template", th.BooleanType), + th.Property("viewer_can_close", th.BooleanType), + th.Property("viewer_can_reopen", th.BooleanType), + th.Property("viewer_can_update", th.BooleanType), + th.Property( + "owner", + th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("type", th.StringType), + ), + ), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("url", th.StringType), + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.StringType, required=False), + ), + required=False, # creator is nullable in GraphQL + ), + ).to_dict() + + +class ProjectFieldConfigurationsStream(GitHubGraphqlStream): + """Fetches all fields defined within a GitHub organization's project and outputs + a single record per project containing all its fields. + + This stream is a child of ProjectsStream. For each project, it retrieves all its + field configurations, including configurations for iteration and single-select + fields, and consolidates them into one record. + + API Reference: https://docs.github.com/en/graphql/reference/objects#projectv2fieldconfiguration + """ + + name = "project_field_configurations" + primary_keys: ClassVar[list[str]] = ["org", "project_number"] + parent_stream_type = ProjectsStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["org", "project_number"] + query_jsonpath = "$.data.organization.projectV2.fields.nodes[*]" + + @property + def query(self) -> str: + """GraphQL query to fetch a page of project fields.""" + return """ + query ProjectFieldsPage( + $org: String!, + $project_number: Int!, + $nextPageCursor_0: String + ) { + organization(login: $org) { + projectV2(number: $project_number) { + fields(first: 100, after: $nextPageCursor_0) { + nodes { + ... on ProjectV2Field { + id: databaseId + node_id: id + name + data_type: dataType + created_at: createdAt + updated_at: updatedAt + } + ... on ProjectV2IterationField { + id: databaseId + node_id: id + name + data_type: dataType + created_at: createdAt + updated_at: updatedAt + configuration { + duration + start_day: startDay + iterations { + id + title + start_date: startDate + duration + } + completed_iterations: completedIterations { + id + title + start_date: startDate + duration + } + } + } + ... on ProjectV2SingleSelectField { + id: databaseId + node_id: id + name + data_type: dataType + created_at: createdAt + updated_at: updatedAt + options { + id + name + color + description + } + } + } + pageInfo { + hasNextPage_0: hasNextPage + endCursor_0: endCursor + } + totalCount + } + } + } + rateLimit { + cost + } + } + """ + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + """ + Fetch all fields for a project, handling pagination, and yield a single record. + """ + if not context: + self.logger.warning("Received no context, skipping.") + return + + org = context.get("org") + project_number = context.get("project_number") + + if not org or project_number is None: + self.logger.warning(f"Missing org or project_number in context: {context}") + return + + all_field_configurations: list[dict] = [] + next_page_token: Any = None + + # Can't use BaseAPIPaginator - here we need to aggregate all pages of + # fields of a project into one record, while BaseAPIPaginator yields + # records incrementally as pages are fetched. + while True: + prepared_request = self.prepare_request( + context=context, next_page_token=next_page_token + ) + resp = self._request(prepared_request, context) + page_fields = list(self.parse_response(resp)) + all_field_configurations.extend(page_fields) + + current_page_info = ( + resp.json() + .get("data", {}) + .get("organization", {}) + .get("projectV2", {}) + .get("fields", {}) + .get("pageInfo", {}) + ) + if current_page_info.get("hasNextPage_0"): + next_page_token = { + "nextPageCursor_0": current_page_info.get("endCursor_0") + } + else: + break + + yield { + "org": org, + "project_number": project_number, + "all_field_configurations": all_field_configurations, + } + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """Return context for child streams.""" + child_context = dict(context or {}) # Includes org, project_number + child_context["project_field_configurations"] = record.get( + "all_field_configurations", [] + ) + + return child_context + + schema = th.PropertiesList( + th.Property("org", th.StringType), + th.Property("project_number", th.IntegerType), + th.Property( + "all_field_configurations", + th.ArrayType( + th.ObjectType( + # Schema for a single field definition + th.Property( + "id", th.StringType, nullable=False + ), # using databaseId from GraphQL as id, nullable in GraphQL + th.Property( + "node_id", th.StringType + ), # using id from GraphQL as node_id, it is required (ID!) + th.Property("name", th.StringType), + th.Property("data_type", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property( + "configuration", + th.ObjectType( + th.Property("duration", th.IntegerType), + th.Property("start_day", th.IntegerType), + th.Property( + "iterations", + th.ArrayType( + th.ObjectType( + th.Property("id", th.StringType), + th.Property("title", th.StringType), + th.Property("start_date", th.DateType), + th.Property("duration", th.IntegerType), + ) + ), + ), + th.Property( + "completed_iterations", + th.ArrayType( + th.ObjectType( + th.Property("id", th.StringType), + th.Property("title", th.StringType), + th.Property("start_date", th.DateType), + th.Property("duration", th.IntegerType), + ) + ), + ), + ), + required=False, # Only present for ProjectV2IterationField + ), + th.Property( + "options", + th.ArrayType( + th.ObjectType( + th.Property("id", th.StringType), + th.Property("name", th.StringType), + th.Property("color", th.StringType), + th.Property("description", th.StringType), + ) + ), + required=False, # Only present for ProjectV2SingleSelectField + ), + ) + ), + ), + ).to_dict() + + +class ProjectItemsStream(GitHubGraphqlStream): + """Fetches items for a project and their field values. + + This stream is a child of ProjectFieldConfigurationsStream. For each project, + it fetches all items and then for each item, it queries the values of all + known fields. + + API Reference: https://docs.github.com/en/graphql/reference/objects#projectv2item + """ + + name = "project_items" + primary_keys: ClassVar[list[str]] = ["org", "project_number", "node_id"] + parent_stream_type = ProjectFieldConfigurationsStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["org", "project_number"] + tolerated_http_errors: ClassVar[list[int]] = [414] + query_jsonpath = "$.data.organization.projectV2.items.nodes[*]" + + # Project's custom fields supports types: Text, Number, Date, SingleSelect, + # Iteration, so we fetch values from the corresponding types. + # + # Note: Other types are available in issues/pull requests so not included. + # - ProjectV2ItemFieldRepositoryValue, + # - ProjectV2ItemFieldUserValue, + # - ProjectV2ItemFieldLabelValue, + # - ProjectV2ItemFieldReviewerValue, + # - ProjectV2ItemFieldPullRequestValue, + # - ProjectV2ItemFieldMilestoneValue + _supported_project_item_field_value_types: ClassVar[tuple[str, ...]] = ( + "ProjectV2ItemFieldTextValue", + "ProjectV2ItemFieldDateValue", + "ProjectV2ItemFieldNumberValue", + "ProjectV2ItemFieldSingleSelectValue", + "ProjectV2ItemFieldIterationValue", + ) + + # These fields are automatically created by GitHub and expected to present in + # the project items. + _common_fields: ClassVar[dict[str, dict[str, str]]] = { + "Title": {"column": "title", "type": "ProjectV2ItemFieldTextValue"}, + "Status": {"column": "status", "type": "ProjectV2ItemFieldSingleSelectValue"}, + } + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + self._current_project_field_configurations: list[dict] = [] + + def request_records(self, context: Context | None) -> Iterable[dict]: + """Request records from the API, handling FORBIDDEN errors gracefully. + + TODO: should rewrite to use validate_response once + https://github.com/meltano/sdk/issues/280 is implemented. + """ + try: + yield from super().request_records(context) + except FatalAPIError as e: + # Check if the error is FORBIDDEN. This error is raised when + # the organization has security settings that block access to + # the nodes of an item of a project, e.g. allowed IP list. + error_message = str(e.args[0]) if e.args else "" + if "FORBIDDEN" in error_message: + self.logger.warning( + f"Skipping project due to FORBIDDEN error. " + f"Context: {context}. Error: {e}" + ) + return + elif "Timeout on validation of query" in error_message: + self.logger.warning( + f"Skipping project due to query validation timeout error. " + f"Context: {context}. Error: {e}" + ) + return + + raise + + def _escape_graphql_string(self, value: str) -> str: + """ + Escape special characters in a string for use in GraphQL queries. + """ + # Escape backslashes first, then quotes + return value.replace("\\", "\\\\").replace('"', '\\"') + + def _generate_gql_alias(self, field_name: str) -> str: + """ + Generate a unique GraphQL-safe alias from a field name. + """ + # Create a hash of the field name + hash_obj = hashlib.sha256(field_name.encode("utf-8")) + # Take first 8 characters of hex digest for a short but unique identifier + hash_suffix = hash_obj.hexdigest()[:8] + + # GraphQL aliases must start with a letter or underscore + # Prefix with 'field_' to ensure it's always valid + return f"field_{hash_suffix}" + + @property + def page_size(self) -> int: + """Return configured page size for ProjectV2 items.""" + stream_options = self.config.get("stream_options", {}) + project_items_options = stream_options.get("project_items", {}) + return project_items_options.get("page_size", 100) + + @property + def query(self) -> str: + """Dynamically build GraphQL query to fetch item and its field values.""" + field_value_queries = [] + for field_config in self._current_project_field_configurations: + original_field_name = field_config.get("name") + if not original_field_name: + continue + + alias = self._generate_gql_alias(original_field_name) + escaped_field_name = self._escape_graphql_string(original_field_name) + # Comprehensive inline fragments for ProjectV2ItemFieldValue. Project's + # custom fields supports types: Text, Number, Date, SingleSelect, Iteration, + # so we fetch values from the corresponding types. + field_value_query = f''' + {alias}: fieldValueByName(name: "{escaped_field_name}") {{ + __typename + ... on ProjectV2ItemFieldTextValue {{ + node_id: id + id: databaseId + text + creator {{ + {ACTOR_FRAGMENT} + }} + created_at: createdAt + updated_at: updatedAt + }} + ... on ProjectV2ItemFieldDateValue {{ + node_id: id + id: databaseId + date + creator {{ + {ACTOR_FRAGMENT} + }} + created_at: createdAt + updated_at: updatedAt + }} + ... on ProjectV2ItemFieldNumberValue {{ + node_id: id + id: databaseId + number + creator {{ + {ACTOR_FRAGMENT} + }} + created_at: createdAt + updated_at: updatedAt + }} + ... on ProjectV2ItemFieldSingleSelectValue {{ + node_id: id + id: databaseId + color + description + name + option_id: optionId + creator {{ + {ACTOR_FRAGMENT} + }} + created_at: createdAt + updated_at: updatedAt + }} + ... on ProjectV2ItemFieldIterationValue {{ + node_id: id + id: databaseId + duration + start_date: startDate + iteration_id: iterationId + title + creator {{ + {ACTOR_FRAGMENT} + }} + created_at: createdAt + updated_at: updatedAt + }} + }}''' + field_value_queries.append(field_value_query) + + all_field_values_query_part = "\n".join(field_value_queries) + + return f""" + query ProjectItemsWithFieldValues( + $org: String!, + $project_number: Int!, + $nextPageCursor_0: String + ) {{ + organization(login: $org) {{ + projectV2(number: $project_number) {{ + items(first: {self.page_size}, after: $nextPageCursor_0) {{ + nodes {{ + node_id: id + id: fullDatabaseId + created_at: createdAt + updated_at: updatedAt + is_archived: isArchived + type + creator {{ + {ACTOR_FRAGMENT} + }} + content {{ + ... on Issue {{ + type: __typename + node_id: id + }} + ... on DraftIssue {{ + type: __typename + node_id: id + }} + ... on PullRequest {{ + type: __typename + node_id: id + }} + }} + {all_field_values_query_part} + }} + pageInfo {{ + hasNextPage_0: hasNextPage + endCursor_0: endCursor + }} + totalCount + }} + }} + }} + rateLimit {{ + cost + }} + }} + """ + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + if not context: + # This should not happen if parent_stream_type is correctly set + self.logger.warning("ProjectItemFieldValuesStream received no context.") + return {} + + self._current_project_field_configurations = context.get( + "project_field_configurations", [] + ) + + params = super().get_url_params(context, next_page_token) + # org and project_number are already in params via context from parent + return params + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """Process the fetched record to extract field values and add context.""" + row = super().post_process(row, context) + if not context: + return row + + # Add context fields + row["org"] = context["org"] + row["project_number"] = context["project_number"] + + # Process dynamic field values into a list of objects + field_values_output: list[dict[str, Any]] = [] + + # Initialize dedicated fields for common project fields + for field_config in self._common_fields.values(): + row[field_config["column"]] = None + + for field_config in self._current_project_field_configurations: + original_field_name = field_config.get("name") + if not original_field_name: + continue + + alias = self._generate_gql_alias(original_field_name) + field_value_data = row.pop(alias, None) # Pop the aliased data + + if field_value_data: + value_type = field_value_data.get("__typename") + if value_type not in self._supported_project_item_field_value_types: + continue + + entry: dict[str, Any] = { + "field_name": original_field_name, + "value_type": value_type, + } + + # Copy all the values + for key in ["node_id", "id", "created_at", "updated_at"]: + if key in field_value_data: + value = field_value_data[key] + if key == "id" and value is not None: + value = str(value) + entry[key] = value + + # Copy creator if present + if "creator" in field_value_data: + entry["creator"] = field_value_data["creator"] + + # Extract the actual value based on type + if value_type == "ProjectV2ItemFieldTextValue": + text_value = field_value_data.get("text") + entry["value"] = str(text_value) if text_value is not None else None + elif value_type == "ProjectV2ItemFieldDateValue": + date_value = field_value_data.get("date") + entry["value"] = str(date_value) if date_value is not None else None + elif value_type == "ProjectV2ItemFieldNumberValue": + number_value = field_value_data.get("number") + entry["value"] = ( + str(number_value) if number_value is not None else None + ) + elif value_type == "ProjectV2ItemFieldSingleSelectValue": + name_value = field_value_data.get("name") + entry["value"] = str(name_value) if name_value is not None else None + entry["option_id"] = field_value_data.get("option_id") + entry["color"] = field_value_data.get("color") + entry["description"] = field_value_data.get("description") + elif value_type == "ProjectV2ItemFieldIterationValue": + title_value = field_value_data.get("title") + entry["value"] = ( + str(title_value) if title_value is not None else None + ) + entry["iteration_id"] = field_value_data.get("iteration_id") + entry["start_date"] = field_value_data.get("start_date") + entry["duration"] = field_value_data.get("duration") + + # Check if this is a common field that should be extracted separately + is_common_field = ( + self._common_fields.get(original_field_name) + and self._common_fields[original_field_name]["type"] == value_type + ) + + if is_common_field: + column_name = self._common_fields[original_field_name]["column"] + row[column_name] = entry + else: + field_values_output.append(entry) + + row["field_values"] = field_values_output + + return row + + @property + def schema(self) -> dict: + """Define schema with dynamic_fields as an array of name/value objects.""" + properties = th.PropertiesList( + th.Property("org", th.StringType), + th.Property("project_number", th.IntegerType), + th.Property("node_id", th.StringType), # id from GraphQL + th.Property( + "id", th.StringType, nullable=False + ), # fullDatabaseId from GraphQL, nullable + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("is_archived", th.BooleanType), + th.Property("type", th.StringType), + # Dedicated fields for common project fields + th.Property( + "title", + th.ObjectType( + th.Property("value_type", th.StringType), + th.Property("node_id", th.StringType), + th.Property( + "id", th.StringType, required=False + ), # databaseId is nullable + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property( + "value", th.StringType, required=False + ), # text value is nullable + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("url", th.StringType), + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.StringType, required=False), + ), + required=False, # creator is nullable + ), + ), + required=False, + ), + th.Property( + "status", + th.ObjectType( + th.Property("value_type", th.StringType), + th.Property("node_id", th.StringType), + th.Property( + "id", th.StringType, required=False + ), # databaseId is nullable + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property( + "value", th.StringType, required=False + ), # name value is nullable + th.Property("option_id", th.StringType, required=False), # nullable + th.Property("color", th.StringType, required=False), # nullable + th.Property( + "description", th.StringType, required=False + ), # nullable + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("url", th.StringType), + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.StringType, required=False), + ), + required=False, # creator is nullable + ), + ), + required=False, + ), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("url", th.StringType), + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.StringType, required=False), + ), + required=False, # creator can be null + ), + th.Property( + "content", + th.ObjectType( + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + ), + required=False, # content can be null for some items + ), + th.Property( + "field_values", + th.ArrayType( + th.ObjectType( + th.Property("field_name", th.StringType), + th.Property("value_type", th.StringType), + th.Property("node_id", th.StringType), + th.Property( + "id", th.StringType, required=False + ), # databaseId is nullable + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property( + "value", th.StringType, required=False + ), # All value fields are nullable in GraphQL + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("url", th.StringType), + th.Property("type", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.StringType, required=False), + ), + required=False, # creator is nullable in GraphQL + ), + # Type-specific fields + th.Property( + "option_id", th.StringType, required=False + ), # SingleSelect - nullable + th.Property( + "color", th.StringType, required=False + ), # SingleSelect - only required for SingleSelect type + th.Property( + "description", th.StringType, required=False + ), # SingleSelect - nullable + th.Property( + "iteration_id", th.StringType, required=False + ), # Iteration - required when present + th.Property( + "start_date", th.DateType, required=False + ), # Iteration - required when present + th.Property( + "duration", th.IntegerType, required=False + ), # Iteration - required when present + ) + ), + ), + ) + return properties.to_dict() diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py new file mode 100644 index 0000000..5d2cfff --- /dev/null +++ b/tap_github/repository_streams.py @@ -0,0 +1,3860 @@ +"""Repository Stream types classes for tap-github.""" + +from __future__ import annotations + +import http +from collections import defaultdict +from typing import TYPE_CHECKING, Any, ClassVar +from urllib.parse import parse_qs, urlparse + +from dateutil.parser import parse +from singer_sdk import typing as th # JSON Schema typing helpers +from singer_sdk.exceptions import FatalAPIError, RetriableAPIError +from singer_sdk.helpers.jsonpath import extract_jsonpath + +from tap_github.client import GitHubDiffStream, GitHubGraphqlStream, GitHubRestStream +from tap_github.schema_objects import ( + files_object, + label_object, + milestone_object, + reaction_type_object, + reactions_object, + user_object, +) +from tap_github.scraping import scrape_dependents, scrape_metrics + +if TYPE_CHECKING: + from collections.abc import Iterable + from datetime import datetime + + import requests + from singer_sdk import Tap + from singer_sdk.helpers.types import Context + + from tap_github.authenticator import GitHubTokenAuthenticator + + +class RepositoryStream(GitHubRestStream): + """Defines 'Repository' stream.""" + + name = "repositories" + # updated_at will be updated any time the repository object is updated, + # e.g. when the description or the primary language of the repository is updated. + replication_key = "updated_at" + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + if "search_query" in context: + # we're in search mode + params["q"] = context["search_query"] + + return params + + @property + def path(self) -> str: # type: ignore[override, return] # ty:ignore[invalid-return-type] + """Return the API endpoint path. Path options are mutually exclusive.""" + + if "searches" in self.config: + # Search API max: 1,000 total. + self.MAX_RESULTS_LIMIT = 1000 + return "/search/repositories" + if "repositories" in self.config: + # the `repo` and `org` args will be parsed from the partition's `context` + return "/repos/{org}/{repo}" + if "organizations" in self.config: + return "/orgs/{org}/repos" + + @property + def records_jsonpath(self) -> str: + if "searches" in self.config: + return "$.items[*]" + else: + return "$[*]" + + def get_repo_ids(self, repo_list: list[tuple[str, str]]) -> list[dict[str, str]]: + """Enrich the list of repos with their numeric ID from github. + + This helps maintain a stable id for context and bookmarks. + It uses the github graphql api to fetch the databaseId. + It also removes non-existant repos and corrects casing to ensure + data is correct downstream. + """ + + # use a temp handmade stream to reuse all the graphql setup of the tap + class TempStream(GitHubGraphqlStream): + name = "tempStream" + schema = th.PropertiesList( + th.Property("id", th.StringType), + th.Property("databaseId", th.IntegerType), + ).to_dict() + + def __init__( + self, + tap: Tap, + repo_list: list[tuple[str, str]], + *, + parent_authenticator: GitHubTokenAuthenticator | None = None, + ) -> None: + super().__init__(tap) + self.repo_list = repo_list + # Use parent's authenticator to maintain consistent auth state + # and rate limits + if parent_authenticator is not None: + self._authenticator = parent_authenticator + + @property + def query(self) -> str: + chunks = [] + for i, repo in enumerate(self.repo_list): + org, repo_name = repo + chunks.append( + f'repo{i}: repository(name: "{repo_name}", owner: "{org}") ' + "{ nameWithOwner databaseId }" + ) + return "query {" + " ".join(chunks) + " rateLimit { cost } }" + + def validate_response(self, response: requests.Response) -> None: + """Allow some specific errors. + Do not raise exceptions if the error is "type": "NOT_FOUND" + as we actually expect these in this stream when we send an invalid + repo name. + """ + try: + super().validate_response(response) + except FatalAPIError as e: + if "NOT_FOUND" in str(e): + return + raise + + if len(repo_list) < 1: + return [] + + repos_with_ids: list = [] + temp_stream = TempStream( + self._tap, + list(repo_list), + parent_authenticator=self.authenticator, + ) + + # replace manually provided org/repo values by the ones obtained + # from github api. This guarantees that case is correct in the output data. + # See https://github.com/MeltanoLabs/tap-github/issues/110 + # Also remove repos which do not exist to avoid crashing further down + # the line. + for record in temp_stream.request_records({}): + for item in record: + if item == "rateLimit": + continue + try: + repo_full_name = "/".join(repo_list[int(item[4:])]) + name_with_owner = record[item]["nameWithOwner"] + org, repo = name_with_owner.split("/") + except TypeError: + # one of the repos returned `None`, which means it does + # not exist, log some details, and move on to the next one + repo_full_name = "/".join(repo_list[int(item[4:])]) + self.logger.info( + f"Repository not found: {repo_full_name} \t" + "Removing it from list" + ) + continue + # check if repo has moved or been renamed + if repo_full_name.lower() != name_with_owner.lower(): + # the repo name has changed, log some details, and move on. + self.logger.info( + f"Repository name changed: {repo_full_name} \t" + f"New name: {name_with_owner}" + ) + + repos_with_ids.append( + {"org": org, "repo": repo, "repo_id": record[item]["databaseId"]} + ) + self.logger.info(f"Running the tap on {len(repos_with_ids)} repositories") + return repos_with_ids + + @property + def partitions(self) -> list[dict[str, str]] | None: + """Return a list of partitions. + + This is called before syncing records, we use it to fetch some additional + context + """ + if "searches" in self.config: + return [ + {"search_name": s["name"], "search_query": s["query"]} + for s in self.config["searches"] + ] + + if "repositories" in self.config: + split_repo_names = [s.split("/") for s in self.config["repositories"]] + + # Group repositories by organization for org-specific authentication + repos_by_org = defaultdict(list) + for org, repo in split_repo_names: + repos_by_org[org].append((org, repo)) + + augmented_repo_list = [] + # chunk requests to the graphql endpoint to avoid timeouts and other + # obscure errors that the api doesn't say much about. The actual limit + # seems closer to 1000, use half that to stay safe. + chunk_size = 500 + list_length = len(split_repo_names) + self.logger.info( + f"Filtering repository list of {list_length} repositories " + f"across {len(repos_by_org)} organizations" + ) + + # Process each organization's repos separately with org-specific auth + for org, org_repos in repos_by_org.items(): + # Set organization-specific authentication + self.authenticator.set_organization(org) + + # Process in chunks + for ndx in range(0, len(org_repos), chunk_size): + augmented_repo_list += self.get_repo_ids( + org_repos[ndx : ndx + chunk_size] + ) + + self.logger.info( + f"Running the tap on {len(augmented_repo_list)} repositories" + ) + return augmented_repo_list + + if "organizations" in self.config: + return [{"org": org} for org in self.config["organizations"]] + return None + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """Return a child context object from the record and optional provided context. + + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": record["owner"]["login"], + "repo": record["name"], + "repo_id": record["id"], + "has_discussions": record.get( + "has_discussions", False + ), # GitHub repos not updated after the feature was released in 2021 will not have this field. # noqa: E501 + "has_issues": record.get("has_issues", True), + "has_pull_requests": record.get("has_pull_requests", True), + } + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + """ + Override the parent method to allow skipping API calls + if the stream is deselected and skip_parent_streams is True in config. + This allows running the tap with fewer API calls and preserving + quota when only syncing a child stream. Without this, + the API call is sent but data is discarded. + """ + if ( + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None + ): + # build a minimal mock record so that self._sync_records + # can proceed with child streams + # the id is fetched in `get_repo_ids` above + yield { + "owner": { + "login": context["org"], + }, + "name": context["repo"], + "id": context["repo_id"], + } + else: + yield from super().get_records(context) + + schema = th.PropertiesList( + th.Property("search_name", th.StringType), + th.Property("search_query", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("name", th.StringType), + th.Property("full_name", th.StringType), + th.Property("description", th.StringType), + th.Property("html_url", th.StringType), + th.Property("owner", user_object), + th.Property( + "license", + th.ObjectType( + th.Property("key", th.StringType), + th.Property("name", th.StringType), + th.Property("url", th.StringType), + th.Property("spdx_id", th.StringType), + ), + ), + th.Property("master_branch", th.StringType), + th.Property("default_branch", th.StringType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + th.Property("pushed_at", th.DateTimeType), + th.Property("git_url", th.StringType), + th.Property("ssh_url", th.StringType), + th.Property("clone_url", th.StringType), + th.Property("homepage", th.StringType), + th.Property("private", th.BooleanType), + th.Property("archived", th.BooleanType), + th.Property("disabled", th.BooleanType), + th.Property("size", th.IntegerType), + th.Property("stargazers_count", th.IntegerType), + th.Property("fork", th.BooleanType), + th.Property( + "topics", + th.ArrayType(th.StringType), + ), + th.Property("visibility", th.StringType), + th.Property("language", th.StringType), + # These `_count` metrics appear to be duplicates but have valid data + # and are documented: https://docs.github.com/en/rest/reference/search + th.Property("forks", th.IntegerType), + th.Property("forks_count", th.IntegerType), + th.Property("watchers", th.IntegerType), + th.Property("watchers_count", th.IntegerType), + th.Property("open_issues", th.IntegerType), + th.Property("network_count", th.IntegerType), + th.Property("subscribers_count", th.IntegerType), + th.Property("open_issues_count", th.IntegerType), + th.Property("has_issues", th.BooleanType), + th.Property("has_pull_requests", th.BooleanType), + th.Property("allow_squash_merge", th.BooleanType), + th.Property("allow_merge_commit", th.BooleanType), + th.Property("allow_rebase_merge", th.BooleanType), + th.Property("allow_auto_merge", th.BooleanType), + th.Property("delete_branch_on_merge", th.BooleanType), + th.Property("organization", user_object), + ).to_dict() + + +class ReadmeStream(GitHubRestStream): + """ + A stream dedicated to fetching the object version of a README.md. + + Including its content, base64 encoded of the readme in GitHub flavored Markdown. + For html, see ReadmeHtmlStream. + """ + + name = "readme" + path = "/repos/{org}/{repo}/readme" + primary_keys: ClassVar[list[str]] = ["repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # README Keys + th.Property("type", th.StringType), + th.Property("encoding", th.StringType), + th.Property("size", th.IntegerType), + th.Property("name", th.StringType), + th.Property("path", th.StringType), + th.Property("content", th.StringType), + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + th.Property("git_url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("download_url", th.StringType), + th.Property( + "_links", + th.ObjectType( + th.Property("git", th.StringType), + th.Property("self", th.StringType), + th.Property("html", th.StringType), + ), + ), + ).to_dict() + + +class ReadmeHtmlStream(GitHubRestStream): + """ + A stream dedicated to fetching the HTML version of README.md. + + For the object details, such as path and size, see ReadmeStream. + """ + + name = "readme_html" + path = "/repos/{org}/{repo}/readme" + primary_keys: ClassVar[list[str]] = ["repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to get the raw HTML version of the readme. + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.v3.html" + return headers + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the README to yield the html response instead of an object.""" + if response.status_code in self.tolerated_http_errors: + return + + yield {"raw_html": response.text} + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Readme HTML + th.Property("raw_html", th.StringType), + ).to_dict() + + +class CommunityProfileStream(GitHubRestStream): + """Defines 'CommunityProfile' stream.""" + + name = "community_profile" + path = "/repos/{org}/{repo}/community/profile" + primary_keys: ClassVar[list[str]] = ["repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Community Profile + th.Property("health_percentage", th.IntegerType), + th.Property("description", th.StringType), + th.Property("documentation", th.StringType), + th.Property("updated_at", th.DateTimeType), + th.Property("content_reports_enabled", th.BooleanType), + th.Property( + "files", + th.ObjectType( + th.Property( + "code_of_conduct", + th.ObjectType( + th.Property("key", th.StringType), + th.Property("name", th.StringType), + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property( + "code_of_conduct_file", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + th.Property( + "contributing", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + th.Property( + "issue_template", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + th.Property( + "pull_request_template", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + th.Property( + "license", + th.ObjectType( + th.Property("key", th.StringType), + th.Property("name", th.StringType), + th.Property("spdx_id", th.StringType), + th.Property("node_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property( + "readme", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + ), + ), + ).to_dict() + + +class EventsStream(GitHubRestStream): + """ + Defines 'Events' stream. + Issue events are fetched from the repository level (as opposed to per issue) + to optimize for API quota usage. + """ + + name = "events" + path = "/repos/{org}/{repo}/events" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "created_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + # GitHub is missing the "since" parameter on this endpoint. + use_fake_since_parameter = True + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """Return a generator of row-type dictionary objects. + Each row emitted should be a dictionary of property names to their values. + """ + if context and context.get("events", None) == 0: + self.logger.debug(f"No events detected. Skipping '{self.name}' sync.") + return [] + + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + # TODO - We should think about the best approach to handle this. An alternative would be to # noqa: E501 + # do a 'dumb' tap that just keeps the same schemas as GitHub without renaming these # noqa: E501 + # objects to "target_". They are worth keeping, however, as they can be different from # noqa: E501 + # the parent stream, e.g. for fork/parent PR events. + row["target_repo"] = row.pop("repo", None) + row["target_org"] = row.pop("org", None) + return row + + schema = th.PropertiesList( + th.Property("id", th.StringType), + th.Property("type", th.StringType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("public", th.BooleanType), + th.Property("_sdc_repository", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("distinct_size", th.IntegerType), + th.Property("head", th.StringType), + th.Property("push_id", th.IntegerType), + th.Property("ref", th.StringType), + th.Property("size", th.IntegerType), + th.Property( + "target_repo", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("name", th.StringType), + ), + ), + th.Property( + "target_org", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("login", th.StringType), + ), + ), + th.Property( + "actor", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("login", th.StringType), + th.Property("display_login", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property( + "payload", + th.ObjectType( + th.Property("before", th.StringType), + th.Property("action", th.StringType), + th.Property( + "comment", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + ), + ), + th.Property( + "comments", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + ), + ), + ), + th.Property( + "issue", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("number", th.IntegerType), + th.Property("title", th.StringType), + th.Property("body", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + ), + ), + th.Property( + "pull_request", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("number", th.IntegerType), + th.Property("title", th.StringType), + th.Property("body", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + ), + ), + th.Property( + "review", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("submitted_at", th.DateTimeType), + ), + ), + th.Property("description", th.StringType), + th.Property("master_branch", th.StringType), + th.Property("pusher_type", th.StringType), + th.Property("ref", th.StringType), + th.Property("ref_type", th.StringType), + th.Property( + "commits", + th.ArrayType( + th.ObjectType( + th.Property( + "author", + th.ObjectType( + th.Property("email", th.StringType), + th.Property("name", th.StringType), + ), + ), + th.Property("distinct", th.BooleanType), + th.Property("message", th.StringType), + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + ), + ), + ), + ), + ), + ).to_dict() + + +class MilestonesStream(GitHubRestStream): + name = "milestones" + path = "/repos/{org}/{repo}/milestones" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + params["state"] = "open" + + if "milestones" in self.config.get("stream_options", {}): + params.update(self.config["stream_options"]["milestones"]) + + return params + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("labels_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property("title", th.StringType), + th.Property("description", th.StringType), + th.Property("creator", user_object), + th.Property("open_issues", th.IntegerType), + th.Property("closed_issues", th.IntegerType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), + th.Property("due_on", th.StringType), + ).to_dict() + + +class ReleasesStream(GitHubRestStream): + name = "releases" + path = "/repos/{org}/{repo}/releases" + ignore_parent_replication_key = True + primary_keys: ClassVar[list[str]] = ["id"] + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + replication_key = "created_at" + + MAX_RESULTS_LIMIT = 10000 + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("assets_url", th.StringType), + th.Property("upload_url", th.StringType), + th.Property("tarball_url", th.StringType), + th.Property("zipball_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("tag_name", th.StringType), + th.Property("target_commitish", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("draft", th.BooleanType), + th.Property("prerelease", th.BooleanType), + th.Property("created_at", th.DateTimeType), + th.Property("published_at", th.DateTimeType), + th.Property("author", user_object), + th.Property( + "assets", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), + th.Property("browser_download_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("label", th.StringType), + th.Property("state", th.StringType), + th.Property("content_type", th.StringType), + th.Property("size", th.IntegerType), + th.Property("download_count", th.IntegerType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("uploader", user_object), + ) + ), + ), + ).to_dict() + + +class LanguagesStream(GitHubRestStream): + name = "languages" + path = "/repos/{org}/{repo}/languages" + primary_keys: ClassVar[list[str]] = ["repo", "org", "language_name"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the language response and reformat to return as an iterator of [{language_name: Python, bytes: 23}].""" # noqa: E501 + if response.status_code in self.tolerated_http_errors: + return + + languages_json = response.json() + for key, value in languages_json.items(): + yield {"language_name": key, "bytes": value} + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # A list of languages parsed by GitHub is available here: + # https://github.com/github/linguist/blob/master/lib/linguist/languages.yml + th.Property("language_name", th.StringType), + th.Property("bytes", th.IntegerType), + ).to_dict() + + +class CollaboratorsStream(GitHubRestStream): + name = "collaborators" + path = "/repos/{org}/{repo}/collaborators" + primary_keys: ClassVar[list[str]] = ["id", "repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [404, 403] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + th.Property( + "permissions", + th.ObjectType( + th.Property("pull", th.BooleanType), + th.Property("triage", th.BooleanType), + th.Property("push", th.BooleanType), + th.Property("maintain", th.BooleanType), + th.Property("admin", th.BooleanType), + ), + ), + th.Property("role_name", th.StringType), + ).to_dict() + + +class AssigneesStream(GitHubRestStream): + """Defines 'Assignees' stream which returns possible assignees for issues/prs following GitHub's API convention.""" # noqa: E501 + + name = "assignees" + path = "/repos/{org}/{repo}/assignees" + primary_keys: ClassVar[list[str]] = ["id"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() + + +class IssuesStream(GitHubRestStream): + """Defines 'Issues' stream which returns Issues and PRs following GitHub's API convention.""" # noqa: E501 + + name = "issues" + path = "/repos/{org}/{repo}/issues" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + use_cursor_pagination = True + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + # Fetch all issues and PRs, regardless of state (OPEN, CLOSED, MERGED). + # To exclude PRs from the issues stream, you can use the Stream Maps in the config. # noqa: E501 + # { + # // .. + # "stream_maps": { + # "issues": { + # "__filter__": "record['type'] = 'issue'" + # } + # } + # } + params["state"] = "all" + return params + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to use beta endpoint which includes reactions as described here: + https://developer.github.com/changes/2016-05-12-reactions-api-preview/ + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.squirrel-girl-preview" + return headers + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + row["type"] = "pull_request" if "pull_request" in row else "issue" + if row["body"] is not None: + # some issue bodies include control characters such as \x00 + # that some targets (such as postgresql) choke on. This ensures + # such chars are removed from the data before we pass it on to + # the target + row["body"] = row["body"].replace("\x00", "") + if row["title"] is not None: + row["title"] = row["title"].replace("\x00", "") + + # replace +1/-1 emojis to avoid downstream column name errors. + if "reactions" in row: + row["reactions"]["plus_one"] = row["reactions"].pop("+1", None) + row["reactions"]["minus_one"] = row["reactions"].pop("-1", None) + return row + + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("number", th.IntegerType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), + th.Property("state", th.StringType), + th.Property("title", th.StringType), + th.Property("comments", th.IntegerType), + th.Property("author_association", th.StringType), + th.Property("body", th.StringType), + th.Property("type", th.StringType), + th.Property("user", user_object), + th.Property( + "labels", + th.ArrayType(label_object), + ), + th.Property("reactions", reactions_object), + th.Property("assignee", user_object), + th.Property( + "assignees", + th.ArrayType(user_object), + ), + th.Property("milestone", milestone_object), + th.Property("locked", th.BooleanType), + th.Property( + "pull_request", + th.ObjectType( + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + th.Property("diff_url", th.StringType), + th.Property("patch_url", th.StringType), + ), + ), + th.Property("state_reason", th.StringType), + th.Property("parent_issue_url", th.StringType), + th.Property( + "sub_issues_summary", + th.ObjectType( + th.Property("total", th.IntegerType), + th.Property("completed", th.IntegerType), + th.Property("percent_completed", th.IntegerType), + ), + ), + ).to_dict() + + +class IssueCommentsStream(GitHubRestStream): + """ + Defines 'IssueComments' stream. + Issue comments are fetched from the repository level (as opposed to per issue) + to optimize for API quota usage. + """ + + name = "issue_comments" + path = "/repos/{org}/{repo}/issues/comments" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + # FIXME: this allows the tap to continue on server-side timeouts but means + # we have gaps in our data + tolerated_http_errors: ClassVar[list[int]] = [502] + + # GitHub is not missing the "since" parameter on this endpoint. + # But it is too expensive on large repos and results in a lot of server errors. + use_fake_since_parameter = True + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """Return a generator of row-type dictionary objects. + + Each row emitted should be a dictionary of property names to their values. + """ + if ( + context + and context.get("has_issues", True) is False + and context.get("has_pull_requests", True) is False + ): + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + self.logger.debug( + f"Repository {org}/{repo}: Issues and pull requests not enabled, " + "skipping issue comments API call", + ) + return [] + + if context and context.get("comments", None) == 0: + self.logger.debug(f"No comments detected. Skipping '{self.name}' sync.") + return [] + + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + row["issue_number"] = int(row["issue_url"].split("/")[-1]) + if row["body"] is not None: + # some comment bodies include control characters such as \x00 + # that some targets (such as postgresql) choke on. This ensures + # such chars are removed from the data before we pass it on to + # the target + row["body"] = row["body"].replace("\x00", "") + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("issue_number", th.IntegerType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("issue_url", th.StringType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + th.Property("author_association", th.StringType), + th.Property("body", th.StringType), + th.Property("user", user_object), + ).to_dict() + + +class IssueEventsStream(GitHubRestStream): + """ + Defines 'IssueEvents' stream. + Issue events are fetched from the repository level (as opposed to per issue) + to optimize for API quota usage. + """ + + name = "issue_events" + path = "/repos/{org}/{repo}/issues/events" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "created_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + # GitHub is missing the "since" parameter on this endpoint. + use_fake_since_parameter = True + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """Return a generator of row-type dictionary objects. + + Each row emitted should be a dictionary of property names to their values. + """ + if context and context.get("events", None) == 0: + self.logger.debug(f"No events detected. Skipping '{self.name}' sync.") + return [] + + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if issue := row.get("issue"): + row["issue_number"] = int(issue.pop("number")) + row["issue_url"] = issue.pop("url") + else: + self.logger.debug( + f"No issue assosciated with event {row['id']} - {row['event']}." + ) + + return row + + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("issue_number", th.IntegerType), + th.Property("issue_url", th.StringType), + th.Property("event", th.StringType), + th.Property("commit_id", th.StringType), + th.Property("commit_url", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("actor", user_object), + th.Property( + "label", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("color", th.StringType), + ), + ), + ).to_dict() + + +class CommitsStream(GitHubRestStream): + """ + Defines the 'Commits' stream. + The stream is fetched per repository to optimize for API quota usage. + """ + + name = "commits" + path = "/repos/{org}/{repo}/commits" + primary_keys: ClassVar[list[str]] = ["node_id"] + replication_key = "commit_timestamp" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Add a timestamp top-level field to be used as state replication key. + It's not clear from github's API docs which time (author or committer) + is used to compare to the `since` argument that the endpoint supports. + """ + assert context is not None, "CommitsStream was called without context" + row = super().post_process(row, context) + row["commit_timestamp"] = row["commit"]["committer"]["date"] + return row + + def get_child_context(self, record: dict, context: Context | None) -> dict: + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "repo_id": context["repo_id"] if context else None, + "commit_id": record["sha"], + } + + schema = th.PropertiesList( + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + th.Property("html_url", th.StringType), + th.Property("commit_timestamp", th.DateTimeType), + th.Property( + "commit", + th.ObjectType( + th.Property( + "author", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.DateTimeType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.DateTimeType), + ), + ), + th.Property("message", th.StringType), + th.Property( + "tree", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + ), + ), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + ), + ), + ), + ), + th.Property("author", user_object), + th.Property("committer", user_object), + ).to_dict() + + +class CommitCommentsStream(GitHubRestStream): + name = "commit_comments" + path = "/repos/{org}/{repo}/comments" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + ignore_parent_replication_key = True + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("path", th.StringType), + th.Property("position", th.IntegerType), + th.Property("line", th.IntegerType), + th.Property("commit_id", th.StringType), + th.Property("user", user_object), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("author_association", th.StringType), + ).to_dict() + + +class CommitDiffsStream(GitHubDiffStream): + name = "commit_diffs" + path = "/repos/{org}/{repo}/commits/{commit_id}" + primary_keys: ClassVar[list[str]] = ["commit_id"] + parent_stream_type = CommitsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None: + # Get commit ID (sha) from context + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["commit_id"] = context["commit_id"] + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("commit_id", th.StringType), + # Rest + th.Property("diff", th.StringType), + th.Property("success", th.BooleanType), + th.Property("error_message", th.StringType), + ).to_dict() + + +class LabelsStream(GitHubRestStream): + """Defines 'labels' stream.""" + + name = "labels" + path = "/repos/{org}/{repo}/labels" + primary_keys: ClassVar[list[str]] = ["id"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Label Keys + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("name", th.StringType), + th.Property("description", th.StringType), + th.Property("color", th.StringType), + th.Property("default", th.BooleanType), + ).to_dict() + + +class PullRequestsStream(GitHubRestStream): + """Defines 'PullRequests' stream.""" + + name = "pull_requests" + path = "/repos/{org}/{repo}/pulls" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + # GitHub is missing the "since" parameter on this endpoint. + use_fake_since_parameter = True + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """ + Return a generator of row-type dictionary objects. + If pull requests are not enabled, skip the API call. + """ + assert context is not None, f"Context cannot be empty for '{self.name}' stream" + + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + if context.get("has_pull_requests", True) is False: + self.logger.debug( + f"Repository {org}/{repo}: Pull requests not enabled, " + "skipping API call", + ) + return [] + + return super().get_records(context) + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + # Fetch all pull requests regardless of state (OPEN, CLOSED, MERGED). + params["state"] = "all" + return params + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to use beta endpoint which includes reactions as described here: + https://developer.github.com/changes/2016-05-12-reactions-api-preview/ + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.squirrel-girl-preview" + return headers + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if row["body"] is not None: + # some pr bodies include control characters such as \x00 + # that some targets (such as postgresql) choke on. This ensures + # such chars are removed from the data before we pass it on to + # the target + row["body"] = row["body"].replace("\x00", "") + if row["title"] is not None: + row["title"] = row["title"].replace("\x00", "") + + # replace +1/-1 emojis to avoid downstream column name errors. + if "reactions" in row: + row["reactions"]["plus_one"] = row["reactions"].pop("+1", None) + row["reactions"]["minus_one"] = row["reactions"].pop("-1", None) + return row + + def get_child_context(self, record: dict, context: Context | None) -> dict: + if context: + return { + "org": context["org"], + "repo": context["repo"], + "repo_id": context["repo_id"], + "pull_number": record["number"], + "pull_id": record["id"], + } + return { + "pull_number": record["number"], + "pull_id": record["id"], + "org": record["base"]["user"]["login"], + "repo": record["base"]["repo"]["name"], + "repo_id": record["base"]["repo"]["id"], + } + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # PR keys + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("diff_url", th.StringType), + th.Property("patch_url", th.StringType), + th.Property("number", th.IntegerType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), + th.Property("merged_at", th.DateTimeType), + th.Property("state", th.StringType), + th.Property("title", th.StringType), + th.Property("locked", th.BooleanType), + th.Property("comments", th.IntegerType), + th.Property("author_association", th.StringType), + th.Property("body", th.StringType), + th.Property("merge_commit_sha", th.StringType), + th.Property("draft", th.BooleanType), + th.Property("commits_url", th.StringType), + th.Property("review_comments_url", th.StringType), + th.Property("review_comment_url", th.StringType), + th.Property("comments_url", th.StringType), + th.Property("statuses_url", th.StringType), + th.Property("user", user_object), + th.Property( + "labels", + th.ArrayType(label_object), + ), + th.Property("reactions", reactions_object), + th.Property("assignee", user_object), + th.Property( + "assignees", + th.ArrayType(user_object), + ), + th.Property( + "requested_reviewers", + th.ArrayType(user_object), + ), + th.Property("milestone", milestone_object), + th.Property("locked", th.BooleanType), + th.Property( + "pull_request", + th.ObjectType( + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + th.Property("diff_url", th.StringType), + th.Property("patch_url", th.StringType), + ), + ), + th.Property( + "head", + th.ObjectType( + th.Property("label", th.StringType), + th.Property("ref", th.StringType), + th.Property("sha", th.StringType), + th.Property("user", user_object), + th.Property( + "repo", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("full_name", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + ), + ), + th.Property( + "base", + th.ObjectType( + th.Property("label", th.StringType), + th.Property("ref", th.StringType), + th.Property("sha", th.StringType), + th.Property("user", user_object), + th.Property( + "repo", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("full_name", th.StringType), + th.Property("html_url", th.StringType), + ), + ), + ), + ), + ).to_dict() + + +class PullRequestCommitsStream(GitHubRestStream): + name = "pull_request_commits" + path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" + ignore_parent_replication_key = False + primary_keys: ClassVar[list[str]] = ["node_id"] + parent_stream_type = PullRequestsStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def get_child_context(self, record: dict, context: Context | None) -> dict: + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "repo_id": context["repo_id"] if context else None, + "pull_number": context["pull_number"] if context else None, + "commit_id": record["sha"], + } + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("pull_number", th.IntegerType), + # Rest + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + th.Property("node_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("comments_url", th.StringType), + th.Property( + "commit", + th.ObjectType( + th.Property("url", th.StringType), + th.Property( + "author", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property("message", th.StringType), + th.Property( + "tree", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + ), + ), + th.Property("comment_count", th.IntegerType), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + ), + ), + ), + ), + th.Property("author", user_object), + th.Property("committer", user_object), + th.Property( + "parents", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), th.Property("sha", th.StringType) + ) + ), + ), + th.Property("files", th.ArrayType(files_object)), + th.Property( + "stats", + th.ObjectType( + th.Property("additions", th.IntegerType), + th.Property("deletions", th.IntegerType), + th.Property("total", th.IntegerType), + ), + ), + ).to_dict() + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None and "pull_number" in context: + row["pull_number"] = context["pull_number"] + return row + + +class PullRequestDiffsStream(GitHubDiffStream): + name = "pull_request_diffs" + path = "/repos/{org}/{repo}/pulls/{pull_number}" + primary_keys: ClassVar[list[str]] = ["pull_id"] + parent_stream_type = PullRequestsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None: + # Get PR ID from context + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["pull_number"] = context["pull_number"] + row["pull_id"] = context["pull_id"] + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("pull_number", th.IntegerType), + th.Property("pull_id", th.IntegerType), + # Rest + th.Property("diff", th.StringType), + th.Property("success", th.BooleanType), + th.Property("error_message", th.StringType), + ).to_dict() + + +class PullRequestCommitDiffsStream(GitHubDiffStream): + name = "pull_request_commit_diffs" + path = "/repos/{org}/{repo}/commits/{commit_id}" + primary_keys: ClassVar[list[str]] = ["commit_id"] + parent_stream_type = PullRequestCommitsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None: + # Get commit ID (sha) from context + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["pull_number"] = context["pull_number"] + row["commit_id"] = context["commit_id"] + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("pull_number", th.IntegerType), + th.Property("commit_id", th.StringType), + # Rest + th.Property("diff", th.StringType), + th.Property("success", th.BooleanType), + th.Property("error_message", th.StringType), + ).to_dict() + + +class ReviewsStream(GitHubRestStream): + name = "reviews" + path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" + primary_keys: ClassVar[list[str]] = ["id"] + parent_stream_type = PullRequestsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent keys + th.Property("pull_id", th.IntegerType), + th.Property("pull_number", th.IntegerType), + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("user", user_object), + th.Property("body", th.StringType), + th.Property("state", th.StringType), + th.Property("html_url", th.StringType), + th.Property("pull_request_url", th.StringType), + th.Property( + "_links", + th.ObjectType( + th.Property("html", th.ObjectType(th.Property("href", th.StringType))), + th.Property( + "pull_request", th.ObjectType(th.Property("href", th.StringType)) + ), + ), + ), + th.Property("submitted_at", th.DateTimeType), + th.Property("commit_id", th.StringType), + th.Property("author_association", th.StringType), + ).to_dict() + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None: + # Get PR ID from context + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["pull_number"] = context["pull_number"] + row["pull_id"] = context["pull_id"] + return row + + +class ReviewCommentsStream(GitHubRestStream): + name = "review_comments" + path = "/repos/{org}/{repo}/pulls/comments" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + # Rest + th.Property("url", th.StringType), + th.Property("pull_request_review_id", th.IntegerType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("diff_hunk", th.StringType), + th.Property("path", th.StringType), + th.Property("position", th.IntegerType), + th.Property("original_position", th.IntegerType), + th.Property("commit_id", th.StringType), + th.Property("original_commit_id", th.StringType), + th.Property("in_reply_to_id", th.IntegerType), + th.Property("user", user_object), + th.Property("body", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("html_url", th.StringType), + th.Property("pull_request_url", th.StringType), + th.Property("author_association", th.StringType), + th.Property( + "_links", + th.ObjectType( + th.Property("self", th.ObjectType(th.Property("href", th.StringType))), + th.Property("html", th.ObjectType(th.Property("href", th.StringType))), + th.Property( + "pull_request", th.ObjectType(th.Property("href", th.StringType)) + ), + ), + ), + th.Property("start_line", th.IntegerType), + th.Property("original_start_line", th.IntegerType), + th.Property("start_side", th.StringType), + th.Property("line", th.IntegerType), + th.Property("original_line", th.IntegerType), + th.Property("side", th.StringType), + ).to_dict() + + def get_child_context(self, record: dict, context: Context | None) -> dict: + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "repo_id": context["repo_id"] if context else None, + "comment_id": record["id"] if context else None, + "comment_url": record["html_url"] if context else None, + } + + +class ReviewCommentReactionsStream(GitHubRestStream): + name = "review_comment_reactions" + path = "/repos/{org}/{repo}/pulls/comments/{comment_id}/reactions" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "created_at" + parent_stream_type = ReviewCommentsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + + if context: + row["comment_id"] = context.get("comment_id") + row["comment_url"] = context.get("comment_url") + + return row + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("comment_id", th.IntegerType), + th.Property("comment_url", th.StringType), + # Reaction properties + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("user", user_object), + th.Property("content", th.StringType), + th.Property("created_at", th.DateTimeType), + ).to_dict() + + +class ContributorsStream(GitHubRestStream): + """Defines 'Contributors' stream. Fetching User & Bot contributors.""" + + name = "contributors" + path = "/repos/{org}/{repo}/contributors" + primary_keys: ClassVar[list[str]] = ["node_id", "repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [204, 404] + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # User/Bot contributor keys + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + th.Property("contributions", th.IntegerType), + ).to_dict() + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + # TODO: update this and validate_response when + # https://github.com/meltano/sdk/pull/1754 is merged + if response.status_code != 200: + return + yield from super().parse_response(response) + + def validate_response(self, response: requests.Response) -> None: + """Allow some specific errors.""" + if response.status_code == 403: + contents = response.json() + if ( + contents["message"] + == "The history or contributor list is too large to list contributors for this repository via the API." # noqa: E501 + ): + self.logger.info( + "Skipping repo '%s'. The list of contributors is too large.", + self.name, + ) + return + super().validate_response(response) + + +class AnonymousContributorsStream(GitHubRestStream): + """Defines 'AnonymousContributors' stream.""" + + name = "anonymous_contributors" + path = "/repos/{org}/{repo}/contributors" + primary_keys: ClassVar[list[str]] = ["email", "repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [204] + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + # Fetch all contributions, including anonymous users. + params["anon"] = "true" + return params + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of anonymous contributors.""" + parsed_response = super().parse_response(response) + return filter(lambda x: x["type"] == "Anonymous", parsed_response) + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Anonymous contributor keys + th.Property("email", th.StringType), + th.Property("name", th.StringType), + th.Property("type", th.StringType), + th.Property("contributions", th.IntegerType), + ).to_dict() + + +class StargazersStream(GitHubRestStream): + """Defines 'Stargazers' stream. Warning: this stream does NOT track star deletions.""" # noqa: E501 + + name = "stargazers_rest" + path = "/repos/{org}/{repo}/stargazers" + primary_keys: ClassVar[list[str]] = ["user_id", "repo", "org"] + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + replication_key = "starred_at" + # GitHub is missing the "since" parameter on this endpoint. + use_fake_since_parameter = True + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + # TODO - remove warning with next release. + self.logger.warning( + "The stream 'stargazers_rest' is deprecated. Please use the Graphql version instead: 'stargazers'." # noqa: E501 + ) + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to use an endpoint which includes starred_at property: + https://docs.github.com/en/rest/reference/activity#custom-media-types-for-starring + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.v3.star+json" + return headers + + def validate_response(self, response: requests.Response) -> None: + """Allow 403s caused by the credentials' auth type, not by permissions. + + GitHub App/installation tokens cannot call this endpoint at all, even + with full repo access - it's restricted to personal access tokens. + """ + if response.status_code == http.HTTPStatus.FORBIDDEN: + contents = response.json() + if str(contents.get("message", "")).startswith( + ( + "Resource not accessible by integration", + "Resource not accessible by personal access token", + ) + ): + self.logger.warning( + "Skipping stargazers for '%s'. %s", + urlparse(response.url).path, + contents["message"], + ) + return + super().validate_response(response) + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + if response.status_code != 200: + return + yield from super().parse_response(response) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Add a user_id top-level field to be used as state replication key. + """ + row = super().post_process(row, context) + row["user_id"] = row["user"]["id"] + return row + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("user_id", th.IntegerType), + # Stargazer Info + th.Property("starred_at", th.DateTimeType), + th.Property("user", user_object), + ).to_dict() + + +class StargazersGraphqlStream(GitHubGraphqlStream): + """Defines 'UserContributedToStream' stream. Warning: this stream 'only' gets the first 100 projects (by stars).""" # noqa: E501 + + name = "stargazers" + query_jsonpath = "$.data.repository.stargazers.edges.[*]" + primary_keys: ClassVar[list[str]] = ["user_id", "repo_id"] + replication_key = "starred_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo_id"] + # The parent repository object changes if the number of stargazers changes. + ignore_parent_replication_key = False + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + # TODO - remove warning with next release. + self.logger.warning( + "The stream 'stargazers' might conflict with previous implementation. " + "Looking for the older version? Use 'stargazers_rest'." + ) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Add a user_id top-level field to be used as state replication key. + """ + row = super().post_process(row, context) + row["user_id"] = row["user"]["id"] + return row + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """ + Exit early if a since parameter is provided. + """ + request_parameters = parse_qs(str(urlparse(response.request.url).query)) + + # parse_qs interprets "+" as a space, revert this to keep an aware datetime + try: + since = ( + request_parameters["since"][0].replace(" ", "+") + if "since" in request_parameters + else "" + ) + except IndexError: + since = "" + + # If since parameter is present, try to exit early by looking at the last "starred_at". # noqa: E501 + # Noting that we are traversing in DESCENDING order by STARRED_AT. + if since: + results = list(extract_jsonpath(self.query_jsonpath, input=response.json())) + # If no results, return None to exit early. + if len(results) == 0: + return None + last = results[-1] + if parse(last["starred_at"]) < parse(since): + return None + return super().get_next_page_token(response, previous_token) + + @property + def query(self) -> str: + """Return dynamic GraphQL query.""" + # Graphql id is equivalent to REST node_id. To keep the tap consistent, we rename "id" to "node_id". # noqa: E501 + return """ + query repositoryStargazers($repo: String! $org: String! $nextPageCursor_0: String) { + repository(name: $repo owner: $org) { + stargazers(first: 100 orderBy: {field: STARRED_AT direction: DESC} after: $nextPageCursor_0) { + pageInfo { + hasNextPage_0: hasNextPage + startCursor_0: startCursor + endCursor_0: endCursor + } + edges { + user: node { + node_id: id + id: databaseId + login + avatar_url: avatarUrl + html_url: url + type: __typename + site_admin: isSiteAdmin + } + starred_at: starredAt + } + } + } + rateLimit { + cost + } + } + """ # noqa: E501 + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Stargazer Info + th.Property("user_id", th.IntegerType), + th.Property("starred_at", th.DateTimeType), + th.Property("user", user_object), + ).to_dict() + + +class DiscussionCategoriesStream(GitHubGraphqlStream): + """ + Defines stream fetching discussions categories from each repository. + + This stream is full drop. Categories are returned alphabetically + by GitHub, not chronologically. This means the smart pagination and + resuming if interrupted features are not supported. + + Maximum estimated cost per call: 1 point + This translates to 1 point per repository. + """ + + name = "discussion_categories" + query_jsonpath = "$.data.repository.discussionCategories.nodes.[*]" + primary_keys: ClassVar[list[str]] = [ + "node_id" + ] # Renamed id to node_id to keep tap consistent with REST streams. databaseId is not available for this object. # noqa: E501 + replication_method = "full_table" + parent_stream_type = RepositoryStream # Github allows a maximum of 25 categories per repository. # noqa: E501 + ignore_parent_replication_key = True # Repository's updated_at does not change when a discussion category is added/modified # noqa: E501 + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """ + Return a generator of row-type dictionary objects. + If discussions are not enabled, skip the API call. + """ + assert context is not None, f"Context cannot be empty for '{self.name}' stream" + + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + if not context.get("has_discussions", False): + self.logger.debug( + f"Repository {org}/{repo}: Discussions not enabled, skipping API call", + ) + return [] + + self.logger.debug( + f"Repository {org}/{repo}: Discussions enabled, making API call", + ) + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Set parent fields from context. + """ + if context: + row["org"] = context.get("org") + row["repo"] = context.get("repo") + row["repo_id"] = context.get("repo_id") + return row + + @property + def query(self) -> str: + """ + Return dynamic GraphQL query. + Note: To keep the tap consistent, we rename id to node_id. + """ + + return """ + query DiscussionCategories($repo: String!, $org: String!, $nextPageCursor_0: String) { + repository(name: $repo, owner: $org) { + discussionCategories(first: 100, after: $nextPageCursor_0) { + pageInfo { + hasNextPage_0: hasNextPage + startCursor_0: startCursor + endCursor_0: endCursor + } + nodes { + node_id: id + slug + name + description + is_answerable: isAnswerable + emoji + created_at: createdAt + updated_at: updatedAt + } + } + } + rateLimit { + cost + } + } + """ # noqa: E501 + + schema = th.PropertiesList( + # Parent Keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + # Categories Info + th.Property("node_id", th.StringType), + th.Property("slug", th.StringType), + th.Property("name", th.StringType), + th.Property("description", th.StringType), + th.Property("is_answerable", th.BooleanType), + th.Property("emoji", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + ).to_dict() + + +class DiscussionsStream(GitHubGraphqlStream): + """ + Defines stream fetching discussions from each repository. + + Note that this stream is not resumable if interrupted. + Data is extracted in descending order and we exit early when + we've seen records that go back as far as we need. + + Maximum estimated cost per call: 2 points. + """ + + name = "discussions" + query_jsonpath = "$.data.repository.discussions.nodes.[*]" + primary_keys: ClassVar[list[str]] = [ + "id" + ] # databaseId renamed to id to keep tap consistent with REST streams. + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo_id"] + ignore_parent_replication_key = True # Repository's updated_at does not change when a new discussion is added # noqa: E501 + is_sorted = False # Singer recognizes as unsorted. + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + self.cutoff: datetime | None = None + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + self.cutoff = self.get_starting_timestamp(context) + return super().get_url_params(context, next_page_token) + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """ + Exit early if oldest updated_at is older than the replication bookmark. + """ + self.logger.debug("Cutoff: %s", self.cutoff) + if self.cutoff: + results = list(extract_jsonpath(self.query_jsonpath, input=response.json())) + if results: + oldest_updated_at = parse(results[-1][self.replication_key]) + if oldest_updated_at < self.cutoff: + self.logger.info( + "Early exit: oldest=%s, cutoff=%s", + oldest_updated_at, + self.cutoff, + ) + return None # early exit + return super().get_next_page_token(response, previous_token) + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """ + Return a generator of row-type dictionary objects. + If discussions are not enabled, skip the API call + """ + assert context is not None, f"Context cannot be empty for '{self.name}' stream" + + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + if not context.get("has_discussions", False): + self.logger.debug( + f"Repository {org}/{repo}: Discussions not enabled, skipping API call", + ) + return [] + + self.logger.debug( + f"Repository {org}/{repo}: Discussions enabled, making API call", + ) + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Transform the nodes arrays to flatten the nested structure + and set parent fields. + """ + row = super().post_process(row, context) + + if context is not None: + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + + if "comments" in row: + row["comments_count"] = row["comments"].get("comments_count", 0) + row.pop("comments", None) + + if "labels" in row and "nodes" in row["labels"]: + row["labels_count"] = row["labels"].get("labels_count", 0) + row["labels"] = row["labels"]["nodes"] + + if "reactions" in row and "nodes" in row["reactions"]: + row["reactions_count"] = row["reactions"].get("reactions_count", 0) + row["reactions"] = row["reactions"]["nodes"] + + return row + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """ + Return a context dictionary for child stream(s). + """ + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "repo_id": context["repo_id"] if context else None, + "discussion_id": record["id"] if context else None, + "discussion_number": record["number"] if context else None, + "comments_count": record["comments_count"] if context else None, + } + + @property + def query(self) -> str: + """ + Return dynamic GraphQL query. + Note: To keep the tap consistent, we rename id to node_id and databaseId to id. + """ + return """ + query repositoryDiscussions($repo: String!, $org: String!, $nextPageCursor_0: String) { + repository(name: $repo, owner: $org) { + discussions(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, after: $nextPageCursor_0) { + pageInfo { + hasNextPage_0: hasNextPage + startCursor_0: startCursor + endCursor_0: endCursor + } + nodes { + node_id: id + id: databaseId + number + title + body: bodyText + url + created_at: createdAt + published_at: publishedAt + last_edited_at: lastEditedAt + updated_at: updatedAt + closed_at: closedAt + created_via_email: createdViaEmail + is_answered: isAnswered + author { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + author_association: authorAssociation + category { + node_id: id + slug + name + description + } + labels(first: 100) { + labels_count: totalCount + nodes { + node_id: id + created_at: createdAt + updated_at: updatedAt + name + description + url + resource_path: resourcePath + color + default: isDefault + } + } + locked + active_lock_reason: activeLockReason + closed + is_answered: isAnswered + answer { + id: databaseId + node_id: id + body + author { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + author_association: authorAssociation + } + answer_chosen_at: answerChosenAt + answer_chosen_by: answerChosenBy { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + upvote_count: upvoteCount + comments { + comments_count: totalCount + } + reactions(first: 100) { + reactions_count: totalCount + nodes { + reaction_type: content + reacted_at: createdAt + user { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + } + } + } + } + } + rateLimit { + cost + } + } + """ # noqa: E501 + + category_object = th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("slug", th.StringType), + th.Property("name", th.StringType), + th.Property("description", th.StringType), + ) + + answer_object = th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("author", user_object), + th.Property("author_association", th.StringType), + ) + + labels_array = th.ArrayType( + th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("name", th.StringType), + th.Property("description", th.StringType), + th.Property("url", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("color", th.StringType), + th.Property("default", th.BooleanType), + ) + ) + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Discussion Info + th.Property("node_id", th.StringType), + th.Property("id", th.IntegerType), + th.Property("number", th.IntegerType), + th.Property("title", th.StringType), + th.Property("body", th.StringType), + th.Property("url", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("published_at", th.DateTimeType), + th.Property("last_edited_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), + th.Property("created_via_email", th.BooleanType), + th.Property("author", user_object), + th.Property("author_association", th.StringType), + th.Property("category", category_object), + th.Property("labels_count", th.IntegerType), + th.Property("labels", labels_array), + th.Property("locked", th.BooleanType), + th.Property("active_lock_reason", th.StringType), + th.Property("closed", th.BooleanType), + th.Property("is_answered", th.BooleanType), + th.Property("answer", answer_object), + th.Property("answer_chosen_at", th.DateTimeType), + th.Property("answer_chosen_by", user_object), + th.Property("upvote_count", th.IntegerType), + th.Property("comments_count", th.IntegerType), + th.Property("reactions_count", th.IntegerType), + th.Property("reactions", th.ArrayType(reaction_type_object)), + ).to_dict() + + +class DiscussionCommentsStream(GitHubGraphqlStream): + """ + Defines stream fetching discussion comments from each repository. + + Edits made to comments after extraction are not reflected in the stream. + Full refresh is required to see the changes. Note that this stream is + not resumable if interrupted. + + NOTE: Special handling is needed to avoid data loss. + This stream return pages in DESC order, but records within pages + are sorted as ASC, due to the API's default ordering. + We use tail-first pagination and exit early when we've seen records + that go back as far as we need. + + Maximum estimated cost per call: 1 point. + """ + + name = "discussion_comments" + query_jsonpath = "$.data.repository.discussion.comments.nodes.[*]" + primary_keys: ClassVar[list[str]] = [ + "id" + ] # databaseId renamed to id to keep tap consistent with REST streams. + replication_key = "created_at" # API's default record ordering field. + parent_stream_type = DiscussionsStream + state_partitioning_keys: ClassVar[list[str]] = ["discussion_id"] + is_sorted = False # Set as False to avoid data loss. + # If treated as sorted, Singer will bookmark state as page-1's first record and skip older pages on incremental runs (data loss). # noqa: E501 + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + self.cutoff: datetime | None = None + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + self.cutoff = self.get_starting_timestamp(context) + return super().get_url_params(context, next_page_token) + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """ + Exit early if first (oldest) record in the page is older than the replication + bookmark. With github's default record ordering, each page contains records + in ascending order. + """ + self.logger.debug("Cutoff: %s", self.cutoff) + if self.cutoff: + results = list(extract_jsonpath(self.query_jsonpath, input=response.json())) + if results: + oldest_created_at = parse(results[0][self.replication_key]) + if oldest_created_at < self.cutoff: + self.logger.info( + "Early exit: oldest=%s, cutoff=%s", + oldest_created_at, + self.cutoff, + ) + return None # early exit + return super().get_next_page_token(response, previous_token) + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """ + Return a generator of row-type dictionary objects. + If the parent discussion has no comments, skip the API call. + """ + assert context is not None, f"Context cannot be empty for '{self.name}' stream" + + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + discussion_number = context.get("discussion_number", "unknown") + comments_count = context.get("comments_count", 0) + if not comments_count: + self.logger.debug( + f"{org}/{repo} Discussion {discussion_number}: " + f"No comments found, skipping API call", + ) + return [] + + self.logger.debug( + f"{org}/{repo} Discussion {discussion_number}: " + f"{comments_count} comments found, making API call", + ) + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Transform the nodes arrays to flatten the nested structure + and set parent fields. + """ + row = super().post_process(row, context) + + if context is not None: + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["discussion_id"] = context["discussion_id"] + row["discussion_number"] = context["discussion_number"] + + if "reactions" in row and "nodes" in row["reactions"]: + row["reactions_count"] = row["reactions"].get("reactions_count", 0) + row["reactions"] = row["reactions"]["nodes"] + + if "replies" in row: + row["replies_count"] = row["replies"].get("replies_count", 0) + row.pop("replies", None) + + return row + + @property + def query(self) -> str: + """ + Return dynamic GraphQL query. + Note: To keep the tap consistent, we rename id to node_id and databaseId to id. + The API does not support record ordering and direction, so we must use + tail-first pagination to get the newest records first. + """ + return """ + query DiscussionComments($repo: String!, $org: String!, $discussion_number: Int!, $nextPageCursor_0: String) { + repository(name: $repo, owner: $org) { + discussion(number: $discussion_number) { + comments(last: 100, before: $nextPageCursor_0) { + pageInfo { + hasNextPage_0: hasPreviousPage + startCursor_0: endCursor + endCursor_0: startCursor + } + nodes { + node_id: id + id: databaseId + author { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + author_association: authorAssociation + body + body_html: bodyHTML + body_text: bodyText + created_at: createdAt + published_at: publishedAt + last_edited_at: lastEditedAt + updated_at: updatedAt + created_via_email: createdViaEmail + deleted_at: deletedAt + includes_created_edit: includesCreatedEdit + is_answer: isAnswer + is_minimized: isMinimized + minimized_reason: minimizedReason + upvote_count: upvoteCount + html_url: url + resource_path: resourcePath + editor { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + replies { + replies_count: totalCount + } + reactions(first: 100) { + reactions_count: totalCount + nodes { + reaction_type: content + reacted_at: createdAt + user { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + } + } + } + } + } + } + rateLimit { + cost + } + } + """ # noqa: E501 + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("discussion_id", th.IntegerType), + th.Property("discussion_number", th.IntegerType), + # Discussion Comments keys + th.Property("node_id", th.StringType), + th.Property("id", th.IntegerType), + th.Property("author", user_object), + th.Property("author_association", th.StringType), + th.Property("body", th.StringType), + th.Property("body_html", th.StringType), + th.Property("body_text", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("published_at", th.DateTimeType), + th.Property("last_edited_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_via_email", th.BooleanType), + th.Property("deleted_at", th.DateTimeType), + th.Property("includes_created_edit", th.BooleanType), + th.Property("is_answer", th.BooleanType), + th.Property("is_minimized", th.BooleanType), + th.Property("minimized_reason", th.StringType), + th.Property("upvote_count", th.IntegerType), + th.Property("html_url", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("editor", user_object), + th.Property("replies_count", th.IntegerType), + th.Property("reactions_count", th.IntegerType), + th.Property("reactions", th.ArrayType(reaction_type_object)), + ).to_dict() + + +class DiscussionCommentRepliesStream(GitHubGraphqlStream): + """ + Defines stream fetching replies for each discussion comment from each repository. + + Edits made to replies after extraction are not reflected in the stream. + Full refresh is required to see the changes. Note that this stream is + not resumable if interrupted. + + NOTE: Special handling is needed to avoid data loss. + This stream return pages in DESC order, but records within pages + are sorted as ASC, due to the API's default ordering. + We use tail-first pagination and exit early when we've seen records + that go back as far as we need. + + Maximum estimated cost per call: 25 points. + + This stream batches 50 comments per discussion, and then extracts their nested + replies. This means less calls overall. + """ + + name = "discussion_comment_replies" + query_jsonpath = "$.data.repository.discussion.comments.nodes.[*]" + primary_keys: ClassVar[list[str]] = [ + "id" + ] # databaseId renamed to id to keep tap consistent with REST streams. + replication_key = "created_at" # API's default record ordering field. + parent_stream_type = DiscussionsStream # Only Discussion's timestamp is affected by replies. # noqa: E501 + state_partitioning_keys: ClassVar[list[str]] = ["discussion_id"] + is_sorted = False # Set as False to avoid data loss. + # If treated as sorted, Singer will bookmark state as page-1's first record and skip older pages on incremental runs (data loss). # noqa: E501 + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and flatten nested comments/replies structure.""" + + comments = extract_jsonpath(self.query_jsonpath, input=response.json()) + + for comment in comments: + comment_id = comment.get("comment_id") + replies = comment.get("replies", {}).get("nodes", []) + + for reply in replies: + # Add comment_id to each reply, so we can link it back + reply["comment_id"] = comment_id + yield reply + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + self.cutoff: datetime | None = None + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + self.cutoff = self.get_starting_timestamp(context) + return super().get_url_params(context, next_page_token) + + def get_next_page_token( + self, + response: requests.Response, + previous_token: Any | None, # noqa: ANN401 + ) -> Any | None: # noqa: ANN401 + """ + Exit early if first (oldest) record in the page is older than the replication + bookmark. With github's default record ordering, each page contains records + in ascending order. + """ + self.logger.debug("Cutoff: %s", self.cutoff) + if self.cutoff: + replies_jsonpath = ( + "$.data.repository.discussion.comments.nodes.[*].replies.nodes.[*]" + ) + results = list(extract_jsonpath(replies_jsonpath, input=response.json())) + if results: + oldest_created_at = parse(results[0][self.replication_key]) + if oldest_created_at < self.cutoff: + self.logger.info( + "Early exit: oldest=%s, cutoff=%s", + oldest_created_at, + self.cutoff, + ) + return None # early exit + return super().get_next_page_token(response, previous_token) + + def get_records(self, context: Context | None = None) -> Iterable[dict[str, Any]]: + """Return a generator of row-type dictionary objects. + If the parent discussion has no comments, skip the replies API call. + """ + assert context is not None, f"Context cannot be empty for '{self.name}' stream" + + comments_count = context.get("comments_count", 0) + repo = context.get("repo", "unknown") + org = context.get("org", "unknown") + discussion_number = context.get("discussion_number", "unknown") + if not comments_count: + self.logger.debug( + f"{org}/{repo} Discussion {discussion_number}/ " + f"No comments found, skipping API call for replies", + ) + return [] + + self.logger.debug( + f"{org}/{repo} Discussion {discussion_number}: " + f"{comments_count} comments found, making API call for replies", + ) + + return super().get_records(context) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Transform the nodes arrays to flatten the nested structure + and set parent fields. + """ + row = super().post_process(row, context) + + if context is not None: + row["org"] = context["org"] + row["repo"] = context["repo"] + row["repo_id"] = context["repo_id"] + row["discussion_id"] = context["discussion_id"] + row["discussion_number"] = context["discussion_number"] + + if "reactions" in row and "nodes" in row["reactions"]: + row["reactions_count"] = row["reactions"].get("reactions_count", 0) + row["reactions"] = row["reactions"]["nodes"] + return row + + @property + def query(self) -> str: + """ + Return dynamic GraphQL query. + Note: To keep the tap consistent, we rename id to node_id and databaseId to id. + The API does not support custom record ordering and direction, so we must use + tail-first pagination to get the newest records first. + """ + return """ + query DiscussionCommentReplies( + $repo: String!, + $org: String!, + $discussion_number: Int!, + $nextPageCursor_0: String, + $nextPageCursor_1: String + ) { + repository(name: $repo, owner: $org) { + discussion(number: $discussion_number) { + comments(last: 50, before: $nextPageCursor_0) { + pageInfo { + hasNextPage_0: hasPreviousPage + startCursor_0: endCursor + endCursor_0: startCursor + } + nodes { + comment_id: databaseId + replies(last: 50, before: $nextPageCursor_1) { + pageInfo { + hasNextPage_1: hasPreviousPage + startCursor_1: endCursor + endCursor_1: startCursor + } + nodes { + node_id: id + id: databaseId + author { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + author_association: authorAssociation + body + body_html: bodyHTML + body_text: bodyText + created_at: createdAt + published_at: publishedAt + last_edited_at: lastEditedAt + updated_at: updatedAt + created_via_email: createdViaEmail + deleted_at: deletedAt + includes_created_edit: includesCreatedEdit + is_answer: isAnswer + is_minimized: isMinimized + minimized_reason: minimizedReason + upvote_count: upvoteCount + html_url: url + resource_path: resourcePath + editor { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + reactions(first: 100) { + reactions_count: totalCount + nodes { + reaction_type: content + reacted_at: createdAt + user { + ... on Actor { + login + avatar_url: avatarUrl + html_url: url + type: __typename + } + ... on User { + node_id: id + id: databaseId + site_admin: isSiteAdmin + } + } + } + } + } + } + } + } + } + } + rateLimit { + limit + remaining + used + resetAt + cost + } + } + """ + + schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("discussion_id", th.IntegerType), + th.Property("discussion_number", th.IntegerType), + # Discussion Comment Replies Keys + th.Property("comment_id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("id", th.IntegerType), + th.Property("author", user_object), + th.Property("author_association", th.StringType), + th.Property("body", th.StringType), + th.Property("body_html", th.StringType), + th.Property("body_text", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("published_at", th.DateTimeType), + th.Property("last_edited_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_via_email", th.BooleanType), + th.Property("deleted_at", th.DateTimeType), + th.Property("includes_created_edit", th.BooleanType), + th.Property("is_answer", th.BooleanType), + th.Property("is_minimized", th.BooleanType), + th.Property("minimized_reason", th.StringType), + th.Property("upvote_count", th.IntegerType), + th.Property("html_url", th.StringType), + th.Property("resource_path", th.StringType), + th.Property("editor", user_object), + th.Property("reactions_count", th.IntegerType), + th.Property("reactions", th.ArrayType(reaction_type_object)), + ).to_dict() + + +class StatsContributorsStream(GitHubRestStream): + """ + Defines 'StatsContributors' stream. Fetching contributors activity. + https://docs.github.com/en/rest/reference/metrics#get-all-contributor-commit-activity + """ + + name = "stats_contributors" + path = "/repos/{org}/{repo}/stats/contributors" + primary_keys: ClassVar[list[str]] = ["user_id", "week_start", "repo", "org"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + # Note - these queries are expensive and the API might return an HTTP 202 if the response # noqa: E501 + # has not been cached recently. https://docs.github.com/en/rest/reference/metrics#a-word-about-caching + tolerated_http_errors: ClassVar[list[int]] = [202, 204] + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of flattened contributor activity.""" # noqa: E501 + replacement_keys = { + "a": "additions", + "c": "commits", + "d": "deletions", + "w": "week_start", + } + parsed_response = super().parse_response(response) + for contributor_activity in parsed_response: + weekly_data = contributor_activity["weeks"] + for week in weekly_data: + # no need to save weeks with no contributions or author. + # if a user has deleted their account, GitHub may surprisingly return author: None. # noqa: E501 + author = contributor_activity["author"] + if (sum(week[key] for key in ["a", "c", "d"]) == 0) or (author is None): + continue + week_with_author = { + replacement_keys.get(k, k): v for k, v in week.items() + } + week_with_author.update(author) + week_with_author["user_id"] = week_with_author.pop("id") + yield week_with_author + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Activity keys + th.Property("week_start", th.IntegerType), + th.Property("additions", th.IntegerType), + th.Property("deletions", th.IntegerType), + th.Property("commits", th.IntegerType), + # Contributor keys + th.Property("login", th.StringType), + th.Property("user_id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() + + +class WorkflowsStream(GitHubRestStream): + """Defines 'workflows' stream.""" + + MAX_PER_PAGE = 100 + + name = "workflows" + path = "/repos/{org}/{repo}/actions/workflows" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "$.workflows[*]" + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # PR keys + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("path", th.StringType), + th.Property("state", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("badge_url", th.StringType), + ).to_dict() + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + yield from extract_jsonpath(self.records_jsonpath, input=response.json()) + + +class WorkflowRunsStream(GitHubRestStream): + """Defines 'workflow_runs' stream.""" + + name = "workflow_runs" + path = "/repos/{org}/{repo}/actions/runs" + primary_keys: ClassVar[list[str]] = ["id"] + replication_key = "created_at" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "$.workflow_runs[*]" + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # PR keys + th.Property("id", th.IntegerType), + th.Property("name", th.StringType), + th.Property("node_id", th.StringType), + th.Property("head_branch", th.StringType), + th.Property("head_sha", th.StringType), + th.Property("run_number", th.IntegerType), + th.Property("run_attempt", th.IntegerType), + th.Property("event", th.StringType), + th.Property("status", th.StringType), + th.Property("conclusion", th.StringType), + th.Property("workflow_id", th.IntegerType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property( + "pull_requests", + th.ArrayType( + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("number", th.IntegerType), + ) + ), + ), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("jobs_url", th.StringType), + th.Property("logs_url", th.StringType), + th.Property("check_suite_url", th.StringType), + th.Property("artifacts_url", th.StringType), + th.Property("cancel_url", th.StringType), + th.Property("rerun_url", th.StringType), + th.Property("workflow_url", th.StringType), + ).to_dict() + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params = super().get_url_params(context, next_page_token) + + # GitHub Actions API uses 'created' parameter instead of 'since' + since = self.get_starting_timestamp(context) + if self.replication_key and since: + params["created"] = f"{since.isoformat(sep='T')}..*" + + return params + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + yield from extract_jsonpath(self.records_jsonpath, input=response.json()) + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """Return a child context object from the record and optional provided context. + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "run_id": record["id"], + "repo_id": context["repo_id"] if context else None, + } + + +class WorkflowRunJobsStream(GitHubRestStream): + """Defines 'workflow_run_jobs' stream.""" + + MAX_PER_PAGE = 80 + + name = "workflow_run_jobs" + path = "/repos/{org}/{repo}/actions/runs/{run_id}/jobs" + primary_keys: ClassVar[list[str]] = ["id"] + parent_stream_type = WorkflowRunsStream + ignore_parent_replication_key = False + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org", "run_id"] + records_jsonpath = "$.jobs[*]" + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # PR keys + th.Property("id", th.IntegerType), + th.Property("run_id", th.IntegerType), + th.Property("run_url", th.StringType), + th.Property("node_id", th.StringType), + th.Property("head_sha", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("status", th.StringType), + th.Property("conclusion", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("started_at", th.DateTimeType), + th.Property("completed_at", th.DateTimeType), + th.Property("name", th.StringType), + th.Property( + "steps", + th.ArrayType( + th.ObjectType( + th.Property("name", th.StringType), + th.Property("status", th.StringType), + th.Property("conclusion", th.StringType), + th.Property("number", th.IntegerType), + th.Property("started_at", th.DateTimeType), + th.Property("completed_at", th.DateTimeType), + ) + ), + ), + th.Property("check_run_url", th.StringType), + th.Property("labels", th.ArrayType(th.StringType)), + th.Property("runner_id", th.IntegerType), + th.Property("runner_name", th.StringType), + th.Property("runner_group_id", th.IntegerType), + th.Property("runner_group_name", th.StringType), + ).to_dict() + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + super().__init__(*args, **kwargs) + self._schema_emitted = False + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + yield from extract_jsonpath(self.records_jsonpath, input=response.json()) + + def get_url_params( + self, + context: Context | None, + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + params = super().get_url_params(context, next_page_token) + params["filter"] = "all" + return params + + def _write_schema_message(self) -> None: + """Write out a SCHEMA message with the stream schema.""" + if not self._schema_emitted: + super()._write_schema_message() + self._schema_emitted = True + + +class ExtraMetricsStream(GitHubRestStream): + """Defines the 'extra_metrics' stream.""" + + name = "extra_metrics" + path = "/{org}/{repo}/" + primary_keys: ClassVar[list[str]] = ["repo_id"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo_id"] + + @property + def url_base(self) -> str: + return self.config.get("api_url_base", self.DEFAULT_API_BASE_URL).replace( + "api.", "" + ) + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the repository main page to extract extra metrics.""" + yield from scrape_metrics(response, self.logger) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + row = super().post_process(row, context) + if context is not None: + row["repo"] = context["repo"] + row["org"] = context["org"] + return row + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Mock a web browser user-agent. + """ + return {"User-agent": "Mozilla/5.0"} + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Metric keys + th.Property("open_issues", th.IntegerType), + th.Property("open_prs", th.IntegerType), + th.Property("dependents", th.IntegerType), + th.Property("contributors", th.IntegerType), + th.Property("fetched_at", th.DateTimeType), + ).to_dict() + + +class DependentsStream(GitHubRestStream): + """Defines 'dependents' stream. + + This stream scrapes the website instead of using the API. + """ + + name = "dependents" + path = "/{org}/{repo}/network/dependents" + primary_keys: ClassVar[list[str]] = ["repo_id", "dependent_name_with_owner"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo_id"] + + @property + def url_base(self) -> str: + return self.config.get("api_url_base", self.DEFAULT_API_BASE_URL).replace( + "api.", "" + ) + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Get the response for the first page and scrape results, potentially iterating through pages.""" # noqa: E501 + yield from scrape_dependents(response, self.logger) + + def post_process(self, row: dict, context: Context | None = None) -> dict: + new_row = {"dependent": row} + new_row = super().post_process(new_row, context) + # we extract dependent_name_with_owner to be able to use it safely as a primary key, # noqa: E501 + # regardless of the target used. + new_row["dependent_name_with_owner"] = row["name_with_owner"] + return new_row + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Mock a web browser user-agent. + """ + return {"User-Agent": "Mozilla/5.0"} + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Dependent keys + th.Property("dependent_name_with_owner", th.StringType), + th.Property( + "dependent", + th.ObjectType( + th.Property("name_with_owner", th.StringType), + th.Property("stars", th.IntegerType), + th.Property("forks", th.IntegerType), + ), + ), + ).to_dict() + + +class DependenciesStream(GitHubGraphqlStream): + """Defines 'DependenciesStream' stream.""" + + name = "dependencies" + query_jsonpath = ( + "$.data.repository.dependencyGraphManifests.nodes.[*].dependencies.nodes.[*]" + ) + # github's api sometimes returns multiple versions of a same package as dependencies + # of a given repo. We use package_name instead of dependency_repo_id because + # the latter changes as github's resolution improves, which would lead to invalid + # duplicate values + primary_keys: ClassVar[list[str]] = [ + "repo_id", + "package_name", + "package_manager", + "requirements", + ] + parent_stream_type = RepositoryStream + state_partitioning_keys: ClassVar[list[str]] = ["repo_id"] + ignore_parent_replication_key = True + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to use the preview for Repository.dependencyGraphManifests. + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.hawkgirl-preview+json" + return headers + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + try: + yield from super().get_records(context) + except RetriableAPIError as e: + if "timedout" in str(e): + self.logger.warning( + "Skipping dependencies for %s/%s due to GitHub API timeout.", + context and context.get("org"), + context and context.get("repo"), + ) + else: + raise + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Add a dependency_repo_id top-level field to be used as primary key. + """ + row = super().post_process(row, context) + row["dependency_repo_id"] = ( + row["dependency"]["id"] if row["dependency"] else None + ) + if context is not None: + row["org"] = context["org"] + row["repo"] = context["repo"] + return row + + @property + def query(self) -> str: + """Return dynamic GraphQL query.""" + # Graphql id is equivalent to REST node_id. To keep the tap consistent, we rename "id" to "node_id". # noqa: E501 + # Due to GrapQl nested-pagination limitations, we loop through the top level dependencyGraphManifests one by one. # noqa: E501 + return """ + query repositoryDependencies($repo: String! $org: String! $nextPageCursor_0: String $nextPageCursor_1: String) { + repository(name: $repo owner: $org) { + dependencyGraphManifests (first: 1 withDependencies: true after: $nextPageCursor_0) { + totalCount + pageInfo { + hasNextPage_0: hasNextPage + startCursor_0: startCursor + endCursor_0: endCursor + } + nodes { + filename + dependenciesCount + dependencies (first: 50 after: $nextPageCursor_1) { + pageInfo { + hasNextPage_1: hasNextPage + startCursor_1: startCursor + endCursor_1: endCursor + } + nodes { + dependency: repository { + node_id: id + id: databaseId + name_with_owner: nameWithOwner + url + owner { + node_id: id + login + } + } + package_manager: packageManager + package_name: packageName + requirements + has_dependencies: hasDependencies + } + } + } + } + } + rateLimit { + cost + } + } + + """ # noqa: E501 + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Dependency Info + th.Property("dependency_repo_id", th.IntegerType), + th.Property("package_name", th.StringType), + th.Property("package_manager", th.StringType), + th.Property("requirements", th.StringType), + th.Property("has_dependencies", th.BooleanType), + th.Property( + "dependency", + th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("id", th.IntegerType), + th.Property("name_with_owner", th.StringType), + th.Property("url", th.StringType), + th.Property( + "owner", + th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("login", th.StringType), + ), + ), + ), + ), + ).to_dict() + + +class TrafficRestStream(GitHubRestStream): + """Base class for Traffic Streams""" + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + if response.status_code != 200: + return + + """Parse the response and return an iterator of result rows.""" + yield from extract_jsonpath(self.records_jsonpath, input=response.json()) + + def validate_response(self, response: requests.Response) -> None: + """Allow some specific errors. + Do not raise exceptions if the error says "Must have push access to repository" + as we actually expect these in this stream when we don't have write permissions into it. + """ # noqa: E501 + if response.status_code == 403: + contents = response.json() + if "not accessible" in contents.get("message", ""): + self.logger.info("Permissions missing to sync stream '%s'", self.name) + return + super().validate_response(response) + + +class TrafficClonesStream(TrafficRestStream): + """Defines 'traffic_clones' stream.""" + + name = "traffic_clones" + path = "/repos/{org}/{repo}/traffic/clones" + primary_keys: ClassVar[list[str]] = ["repo", "org", "timestamp"] + replication_key = "timestamp" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "$.clones[*]" + selected_by_default = False + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Clones Data + th.Property("timestamp", th.DateTimeType), + th.Property("count", th.IntegerType), + th.Property("uniques", th.IntegerType), + ).to_dict() + + +class TrafficReferralPathsStream(TrafficRestStream): + """Defines 'traffic_referral_paths' stream.""" + + name = "traffic_referral_paths" + path = "/repos/{org}/{repo}/traffic/popular/paths" + primary_keys: ClassVar[list[str]] = ["repo", "org", "path"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "[*]" + selected_by_default = False + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Referral path data + th.Property("path", th.StringType), + th.Property("title", th.StringType), + th.Property("count", th.IntegerType), + th.Property("uniques", th.IntegerType), + ).to_dict() + + +class TrafficReferrersStream(TrafficRestStream): + """Defines 'traffic_referrers' stream.""" + + name = "traffic_referrers" + path = "/repos/{org}/{repo}/traffic/popular/referrers" + primary_keys: ClassVar[list[str]] = ["repo", "org", "referrer"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "[*]" + selected_by_default = False + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Referrer data + th.Property("referrer", th.StringType), + th.Property("count", th.IntegerType), + th.Property("uniques", th.IntegerType), + ).to_dict() + + +class TrafficPageViewsStream(TrafficRestStream): + """Defines 'traffic_pageviews' stream.""" + + name = "traffic_pageviews" + path = "/repos/{org}/{repo}/traffic/views" + primary_keys: ClassVar[list[str]] = ["repo", "org", "timestamp"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + records_jsonpath = "$.views[*]" + selected_by_default = False + + schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Page view data + th.Property("timestamp", th.DateTimeType), + th.Property("count", th.IntegerType), + th.Property("uniques", th.IntegerType), + ).to_dict() + + +class BranchesStream(GitHubRestStream): + """A stream dedicated to fetching the branches of a repository.""" + + name = "branches" + path = "/repos/{org}/{repo}/branches" + primary_keys: ClassVar[list[str]] = ["repo", "org", "name"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Branch Keys + th.Property("name", th.StringType), + th.Property( + "commit", + th.ObjectType( + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property("protected", th.BooleanType), + th.Property( + "protection", + th.ObjectType( + th.Property( + "required_status_checks", + th.ObjectType( + th.Property("enforcement_level", th.StringType), + th.Property("contexts", th.ArrayType(th.StringType)), + ), + ), + ), + ), + th.Property("protection_url", th.StringType), + ).to_dict() + + +class TagsStream(GitHubRestStream): + """A stream dedicated to fetching tags in a repository.""" + + name = "tags" + path = "/repos/{org}/{repo}/tags" + primary_keys: ClassVar[list[str]] = ["repo", "org", "name"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Tag Keys + th.Property("name", th.StringType), + th.Property( + "commit", + th.ObjectType( + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property("zipball_url", th.StringType), + th.Property("tarball_url", th.StringType), + th.Property("node_id", th.StringType), + ).to_dict() + + +class DeploymentsStream(GitHubRestStream): + """A stream dedicated to fetching deployments in a repository.""" + + name = "deployments" + path = "/repos/{org}/{repo}/deployments" + primary_keys: ClassVar[list[str]] = ["node_id"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Deployment Keys + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("sha", th.StringType), + th.Property("ref", th.StringType), + th.Property("task", th.StringType), + th.Property("payload", th.StringType), + th.Property("original_environment", th.StringType), + th.Property("environment", th.StringType), + th.Property("description", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("statuses_url", th.StringType), + th.Property("repository_url", th.StringType), + th.Property("transient_environment", th.BooleanType), + th.Property("production_environment", th.BooleanType), + ).to_dict() + + def get_child_context(self, record: dict, context: Context | None) -> dict: + """Return a child context object from the record and optional provided context. + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "deployment_id": record["id"], + "repo_id": context["repo_id"] if context else None, + } + + +class DeploymentStatusesStream(GitHubRestStream): + """A stream dedicated to fetching deployment statuses in a repository.""" + + name = "deployment_statuses" + path = "/repos/{org}/{repo}/deployments/{deployment_id}/statuses" + primary_keys: ClassVar[list[str]] = ["node_id"] + parent_stream_type = DeploymentsStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org", "deployment_id"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("deployment_id", th.IntegerType), + # Deployment Status Keys + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("state", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("description", th.StringType), + th.Property("environment", th.StringType), + th.Property("target_url", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("deployment_url", th.StringType), + th.Property("repository_url", th.StringType), + th.Property("environment_url", th.StringType), + th.Property("log_url", th.StringType), + ).to_dict() + + +class CustomPropertiesStream(GitHubRestStream): + """Defines 'custom_properties' stream.""" + + name = "custom_properties" + path = "/repos/{org}/{repo}/properties/values" + primary_keys: ClassVar[list[str]] = ["repo", "org", "property_name"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Custom Property Keys + th.Property("property_name", th.StringType), + th.Property("value", th.StringType), + ).to_dict() diff --git a/tap_github/schema_objects.py b/tap_github/schema_objects.py new file mode 100644 index 0000000..363e922 --- /dev/null +++ b/tap_github/schema_objects.py @@ -0,0 +1,81 @@ +"""Reusable schema objects for tap-github. + +Below are a few common patterns in the github API +factored out as reusable objects. They help in making the +schema more readable and error-free. +""" + +from singer_sdk import typing as th # JSON Schema typing helpers + +# This user object is common throughout the API results +user_object = th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), +) + +# some objects are shared between issues and pull requests +label_object = th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("name", th.StringType), + th.Property("description", th.StringType), + th.Property("color", th.StringType), + th.Property("default", th.BooleanType), +) + +milestone_object = th.ObjectType( + th.Property("html_url", th.StringType), + th.Property("node_id", th.StringType), + th.Property("id", th.IntegerType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property("title", th.StringType), + th.Property("description", th.StringType), + th.Property("creator", user_object), + th.Property("open_issues", th.IntegerType), + th.Property("closed_issues", th.IntegerType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), + th.Property("due_on", th.DateTimeType), +) + +reactions_object = th.ObjectType( + th.Property("url", th.StringType), + th.Property("total_count", th.IntegerType), + th.Property("plus_one", th.IntegerType), + th.Property("minus_one", th.IntegerType), + th.Property("laugh", th.IntegerType), + th.Property("hooray", th.IntegerType), + th.Property("confused", th.IntegerType), + th.Property("heart", th.IntegerType), + th.Property("rocket", th.IntegerType), + th.Property("eyes", th.IntegerType), +) + +reaction_type_object = th.ObjectType( + th.Property("reaction_type", th.StringType), + th.Property("reacted_at", th.DateTimeType), + th.Property("user", user_object), +) + +files_object = th.ObjectType( + th.Property("sha", th.StringType), + th.Property("filename", th.StringType), + th.Property("status", th.StringType), + th.Property("additions", th.IntegerType), + th.Property("deletions", th.IntegerType), + th.Property("changes", th.IntegerType), + th.Property("blob_url", th.StringType), + th.Property("raw_url", th.StringType), + th.Property("contents_url", th.StringType), + th.Property("patch", th.StringType), + th.Property("previous_filename", th.StringType), +) diff --git a/tap_github/scraping.py b/tap_github/scraping.py new file mode 100644 index 0000000..16322f4 --- /dev/null +++ b/tap_github/scraping.py @@ -0,0 +1,173 @@ +"""Utility functions for scraping https://github.com + +Inspired by https://github.com/dogsheep/github-to-sqlite/pull/70 +""" + +from __future__ import annotations + +import logging +import re +import time +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse + +import requests + +if TYPE_CHECKING: + from collections.abc import Iterable + + from bs4 import NavigableString, Tag + +used_by_regex = re.compile(" {3}Used by ") +contributors_regex = re.compile(" {3}Contributors ") + + +def parse_int(s: str) -> int: + """For example, '1,808' -> 1808.""" + return int(s.strip().replace(",", "").replace("+", "")) + + +def scrape_dependents( + response: requests.Response, logger: logging.Logger | None = None +) -> Iterable[dict[str, Any]]: + from bs4 import BeautifulSoup + + logger = logger or logging.getLogger("scraping") + + soup = BeautifulSoup(response.content, "html.parser") + # Navigate through Package toggle if present + base_url = urlparse(response.url).hostname or "github.com" + options = soup.find_all("a", class_="select-menu-item") + links = [link["href"] for link in options] if len(options) > 0 else [response.url] + + logger.debug(links) + + for link in links: + yield from _scrape_dependents(f"https://{base_url}/{link}", logger) + + +def _scrape_dependents(url: str, logger: logging.Logger) -> Iterable[dict[str, Any]]: + # Optional dependency: + from bs4 import BeautifulSoup + + s = requests.Session() + + while url: + logger.debug(url) + response = s.get(url) + soup = BeautifulSoup(response.content, "html.parser") + + repo_names = [ + (a["href"] if not isinstance(a["href"], list) else a["href"][0]).lstrip("/") + for a in soup.select("a[data-hovercard-type=repository]") + ] + stars = [ + parse_int(s.next_sibling) + for s in soup.find_all("svg", {"class": "octicon octicon-star"}) + ] + forks = [ + parse_int(s.next_sibling) + for s in soup.find_all("svg", {"class": "octicon octicon-repo-forked"}) + ] + + if not len(repo_names) == len(stars) == len(forks): + raise IndexError( + "Could not find star and fork info. Maybe the GitHub page format has changed?" # noqa: E501 + ) + + repos = [ + {"name_with_owner": name, "stars": s, "forks": f} + for name, s, f in zip(repo_names, stars, forks, strict=False) + ] + + logger.debug(repos) + + yield from repos + + # next page? + try: + next_link: Tag = soup.select(".paginate-container")[0].find_all( + "a", text="Next" + )[0] + except IndexError: + break + if next_link is not None: + href = next_link["href"] + url = str(href if not isinstance(href, list) else href[0]) + time.sleep(1) + else: + url = "" # type: ignore[unreachable] + + +def parse_counter(tag: Tag | NavigableString | None) -> int: + """ + Extract a count of [issues|PR|contributors...] from an HTML tag. + For very high numbers, we only get an approximate value as github + does not provide the actual number. + """ + if not tag: + return 0 + try: + if tag == "\n": + return 0 + title = tag["title"] # type: ignore[index] # ty:ignore[invalid-argument-type] + title_string = title if isinstance(title, str) else title[0] + return parse_int(title_string) + except (KeyError, ValueError) as e: + raise IndexError( + f"Could not parse counter {tag}. Maybe the GitHub page format has changed?" + ) from e + + +def scrape_metrics( + response: requests.Response, logger: logging.Logger | None = None +) -> Iterable[dict[str, Any]]: + from bs4 import BeautifulSoup + + logger = logger or logging.getLogger("scraping") + + soup = BeautifulSoup(response.content, "html.parser") + + try: + issues = parse_counter(soup.find("span", id="issues-repo-tab-count")) + prs = parse_counter(soup.find("span", id="pull-requests-repo-tab-count")) + except IndexError as e: + # These two items should exist. We raise an error if we could not find them. + raise IndexError( + "Could not find issues or prs info. Maybe the GitHub page format has changed?" # noqa: E501 + ) from e + + dependents_node = soup.find(string=used_by_regex) + # verify that we didn't hit some random text in the page. + # sometimes the dependents section isn't shown on the page either + dependents_node_parent = getattr(dependents_node, "parent", None) + dependents: int = 0 + if dependents_node_parent is not None and "href" in dependents_node_parent: # noqa: SIM102 + if dependents_node_parent["href"].endswith("/network/dependents"): + dependents = parse_counter(getattr(dependents_node, "next_element", None)) + + # likewise, handle edge cases with contributors + contributors_node = soup.find(string=contributors_regex) + contributors_node_parent = getattr(contributors_node, "parent", None) + contributors: int = 0 + if contributors_node_parent is not None and "href" in contributors_node_parent: # noqa: SIM102 + if contributors_node_parent["href"].endswith("/graphs/contributors"): + contributors = parse_counter( + getattr(contributors_node, "next_element", None), + ) + + fetched_at = datetime.now(tz=timezone.utc) + + metrics = [ + { + "open_issues": issues, + "open_prs": prs, + "dependents": dependents, + "contributors": contributors, + "fetched_at": fetched_at, + } + ] + + logger.debug(metrics) + return metrics diff --git a/tap_github/streams.py b/tap_github/streams.py new file mode 100644 index 0000000..4d9ee1d --- /dev/null +++ b/tap_github/streams.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from enum import Enum +from typing import TYPE_CHECKING + +from tap_github.organization_streams import ( + OrganizationMembersStream, + OrganizationStream, + ProjectFieldConfigurationsStream, + ProjectItemsStream, + ProjectsStream, + TeamMembersStream, + TeamRolesStream, + TeamsStream, +) +from tap_github.repository_streams import ( + AnonymousContributorsStream, + AssigneesStream, + BranchesStream, + CollaboratorsStream, + CommitCommentsStream, + CommitDiffsStream, + CommitsStream, + CommunityProfileStream, + ContributorsStream, + CustomPropertiesStream, + DependenciesStream, + DependentsStream, + DeploymentsStream, + DeploymentStatusesStream, + DiscussionCategoriesStream, + DiscussionCommentRepliesStream, + DiscussionCommentsStream, + DiscussionsStream, + EventsStream, + ExtraMetricsStream, + IssueCommentsStream, + IssueEventsStream, + IssuesStream, + LabelsStream, + LanguagesStream, + MilestonesStream, + PullRequestCommitDiffsStream, + PullRequestCommitsStream, + PullRequestDiffsStream, + PullRequestsStream, + ReadmeHtmlStream, + ReadmeStream, + ReleasesStream, + RepositoryStream, + ReviewCommentReactionsStream, + ReviewCommentsStream, + ReviewsStream, + StargazersGraphqlStream, + StargazersStream, + StatsContributorsStream, + TagsStream, + TrafficClonesStream, + TrafficPageViewsStream, + TrafficReferralPathsStream, + TrafficReferrersStream, + WorkflowRunJobsStream, + WorkflowRunsStream, + WorkflowsStream, +) +from tap_github.user_streams import StarredStream, UserContributedToStream, UserStream + +if TYPE_CHECKING: + from singer_sdk.streams.core import Stream + + +class Streams(Enum): + """ + Represents all streams our tap supports, and which queries (by username, by organization, etc.) you can use. + """ # noqa: E501 + + valid_queries: set[str] + streams: list[type[Stream]] + + def __init__(self, valid_queries: set[str], streams: list[type[Stream]]) -> None: + self.valid_queries = valid_queries + self.streams = streams + + REPOSITORY = ( # ty:ignore[invalid-assignment] + {"repositories", "organizations", "searches"}, + [ + AnonymousContributorsStream, + AssigneesStream, + BranchesStream, + CollaboratorsStream, + CommitCommentsStream, + CommitsStream, + CommitDiffsStream, + CommunityProfileStream, + ContributorsStream, + DependenciesStream, + DependentsStream, + DeploymentsStream, + DeploymentStatusesStream, + DiscussionsStream, + DiscussionCategoriesStream, + DiscussionCommentsStream, + DiscussionCommentRepliesStream, + EventsStream, + IssueCommentsStream, + IssueEventsStream, + IssuesStream, + LabelsStream, + LanguagesStream, + MilestonesStream, + PullRequestCommitsStream, + PullRequestCommitDiffsStream, + PullRequestDiffsStream, + PullRequestsStream, + ReadmeHtmlStream, + ReadmeStream, + ReleasesStream, + ExtraMetricsStream, + RepositoryStream, + ReviewCommentReactionsStream, + ReviewCommentsStream, + ReviewsStream, + StargazersGraphqlStream, + StargazersStream, + StatsContributorsStream, + TagsStream, + TrafficClonesStream, + TrafficPageViewsStream, + TrafficReferralPathsStream, + TrafficReferrersStream, + WorkflowRunJobsStream, + WorkflowRunsStream, + WorkflowsStream, + ], + ) + USERS = ( # ty:ignore[invalid-assignment] + {"user_usernames", "user_ids"}, + [ + StarredStream, + UserContributedToStream, + UserStream, + ], + ) + ORGANIZATIONS = ( # ty:ignore[invalid-assignment] + {"organizations"}, + [ + CustomPropertiesStream, + OrganizationStream, + OrganizationMembersStream, + TeamMembersStream, + TeamRolesStream, + TeamsStream, + ProjectsStream, + ProjectFieldConfigurationsStream, + ProjectItemsStream, + ], + ) + + @classmethod + def all_valid_queries(cls) -> set[str]: + return set.union(*[stream.valid_queries for stream in Streams]) diff --git a/tap_github/tap.py b/tap_github/tap.py new file mode 100644 index 0000000..519261a --- /dev/null +++ b/tap_github/tap.py @@ -0,0 +1,206 @@ +"""GitHub tap class.""" + +from __future__ import annotations + +import logging +import os + +from singer_sdk import Stream, Tap +from singer_sdk import typing as th # JSON schema typing helpers +from singer_sdk.helpers._classproperty import classproperty + +from tap_github.streams import Streams + + +class TapGitHub(Tap): + """Singer tap for the GitHub API.""" + + name = "tap-github" + package_name = "meltanolabs-tap-github" + + @classproperty + def logger(cls) -> logging.Logger: # noqa: N805 + """Get logger. + + Returns: + Logger with local LOGLEVEL. LOGLEVEL from env takes priority. + """ + + LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() # noqa: N806 + assert LOGLEVEL in logging._levelToName.values(), ( + f"Invalid LOGLEVEL configuration: {LOGLEVEL}" + ) + logger = logging.getLogger(cls.name) + logger.setLevel(LOGLEVEL) + return logger + + config_jsonschema = th.PropertiesList( + th.Property( + "user_agent", + th.StringType, + description="User agent to use for API requests.", + ), + th.Property("metrics_log_level", th.StringType), + # Authentication options + th.Property( + "auth_token", + th.StringType, + description="GitHub token to authenticate with.", + ), + th.Property( + "additional_auth_tokens", + th.ArrayType(th.StringType), + description="List of GitHub tokens to authenticate with. Streams will loop through them when hitting rate limits.", # noqa: E501 + ), + th.Property( + "auth_app_keys", + th.ArrayType(th.StringType), + description=( + "List of GitHub App credentials to authenticate with. " + "These are organization-agnostic and will be used as " + "fallback for all organizations. Each credential should " + "be formatted as `:app_id:;;-----BEGIN RSA PRIVATE KEY-----" + "\\n_YOUR_P_KEY_\\n-----END RSA PRIVATE KEY-----`." + ), + ), + th.Property( + "org_auth_app_keys", + th.ObjectType( + additional_properties=th.ArrayType(th.StringType), + ), + description=( + "Organization-specific GitHub App credentials. " + "Maps organization names to lists of app credentials. " + "When processing repositories from a specific organization, " + "the tap will prefer tokens configured for that organization. " + "Each credential should be formatted as " + "`:app_id:;;-----BEGIN RSA PRIVATE KEY-----" + "\\n_YOUR_P_KEY_\\n-----END RSA PRIVATE KEY-----`." + ), + ), + th.Property( + "rate_limit_buffer", + th.IntegerType, + description="Add a buffer to avoid consuming all query points for the token at hand. Defaults to 1000.", # noqa: E501 + ), + th.Property( + "expiry_time_buffer", + th.IntegerType, + description=( + "When authenticating as a GitHub App, this buffer controls how many " + "minutes before expiry the GitHub app tokens will be refreshed. " + "Defaults to 10 minutes." + ), + ), + th.Property( + "backoff_max_tries", + th.IntegerType, + default=5, + description="Maximum number of retry attempts for failed API requests that are retriable. Defaults to 5.", # noqa: E501 + ), + th.Property( + "searches", + th.ArrayType( + th.ObjectType( + th.Property("name", th.StringType, required=True), + th.Property("query", th.StringType, required=True), + ) + ), + description=( + "An array of search descriptor objects with the following properties:\n" + '"name" - a human readable name for the search query.\n' + '"query" - a github search string (generally the same as would come after ?q= in the URL)"' # noqa: E501 + ), + ), + th.Property("organizations", th.ArrayType(th.StringType)), + th.Property("repositories", th.ArrayType(th.StringType)), + th.Property("user_usernames", th.ArrayType(th.StringType)), + th.Property("user_ids", th.ArrayType(th.StringType)), + th.Property( + "start_date", + th.DateTimeType, + description="Start date for incremental sync.", + ), + th.Property("stream_maps", th.ObjectType()), + th.Property("stream_map_config", th.ObjectType()), + th.Property( + "skip_parent_streams", + th.BooleanType, + description=( + "Set to true to skip API calls for the parent " + "streams (such as repositories) if it is not selected but children are" + ), + ), + th.Property( + "stream_options", + th.ObjectType( + th.Property( + "milestones", + th.ObjectType( + th.Property( + "state", + th.StringType, + description=( + "Configures which states are of interest. " + "Must be one of [open, closed, all], defaults to open." + ), + default="open", + allowed_values=["open", "closed", "all"], + ), + additional_properties=False, + ), + description="Options specific to the 'milestones' stream.", + ), + th.Property( + "project_items", + th.ObjectType( + th.Property( + "page_size", + th.IntegerType(minimum=1, maximum=100), + default=100, + description=( + "Number of ProjectV2 items to request per GraphQL " + "page. Defaults to 100, GitHub's maximum. Lower " + "values can reduce GraphQL query complexity for " + "projects with many custom fields." + ), + ), + additional_properties=False, + ), + description="Options specific to the 'project_items' stream.", + ), + additional_properties=False, + ), + description="Options which change the behaviour of a specific stream.", + ), + ).to_dict() + + def discover_streams(self) -> list[Stream]: + """Return a list of discovered streams for each query.""" + + # If the config is empty, assume we are running --help or --capabilities. + if ( + self.config + and len(Streams.all_valid_queries().intersection(self.config)) != 1 + ): + raise ValueError( + "This tap requires one and only one of the following path options: " + f"{Streams.all_valid_queries()}, provided config: {self.config}" + ) + streams = [] + for stream_type in Streams: + if (not self.config) or len( + stream_type.valid_queries.intersection(self.config) + ) > 0: + streams += [ + StreamClass(tap=self) for StreamClass in stream_type.streams + ] + + if not streams: + raise ValueError("No valid streams found.") + return streams + + +# CLI Execution: + +cli = TapGitHub.cli diff --git a/tap_github/user_streams.py b/tap_github/user_streams.py new file mode 100644 index 0000000..4db19d0 --- /dev/null +++ b/tap_github/user_streams.py @@ -0,0 +1,342 @@ +"""User Stream types classes for tap-github.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, ClassVar + +from singer_sdk import typing as th # JSON Schema typing helpers +from singer_sdk.exceptions import FatalAPIError + +from tap_github.client import GitHubGraphqlStream, GitHubRestStream +from tap_github.schema_objects import user_object + +if TYPE_CHECKING: + from collections.abc import Iterable + + from singer_sdk.helpers.types import Context + from singer_sdk.tap_base import Tap + + +class UserStream(GitHubRestStream): + """Defines 'User' stream.""" + + name = "users" + replication_key = "updated_at" + + @property + def path(self) -> str: # type: ignore[override, return] # ty:ignore[invalid-return-type] + """Return the API endpoint path.""" + if "user_usernames" in self.config: + return "/users/{username}" + elif "user_ids" in self.config: + return "/user/{id}" + + @property + def partitions(self) -> list[dict] | None: + """Return a list of partitions.""" + if "user_usernames" in self.config: + input_user_list = self.config["user_usernames"] + + augmented_user_list = [] + # chunk requests to the graphql endpoint to avoid timeouts and other + # obscure errors that the api doesn't say much about. The actual limit + # seems closer to 1000, use half that to stay safe. + chunk_size = 500 + list_length = len(input_user_list) + self.logger.info(f"Filtering user list of {list_length} users") + for ndx in range(0, list_length, chunk_size): + augmented_user_list += self.get_user_ids( + input_user_list[ndx : ndx + chunk_size] + ) + self.logger.info(f"Running the tap on {len(augmented_user_list)} users") + return augmented_user_list + + elif "user_ids" in self.config: + return [{"id": user_id} for user_id in self.config["user_ids"]] + return None + + def get_child_context(self, record: dict, context: Context | None) -> dict: + return { + "username": record["login"], + "user_id": record["id"], + } + + def get_user_ids(self, user_list: list[str]) -> list[dict[str, str]]: + """Enrich the list of userse with their numeric ID from github. + + This helps maintain a stable id for context and bookmarks. + It uses the github graphql api to fetch the databaseId. + It also removes non-existant repos and corrects casing to ensure + data is correct downstream. + """ + + # use a temp handmade stream to reuse all the graphql setup of the tap + class TempStream(GitHubGraphqlStream): + name = "tempStream" + schema = th.PropertiesList( + th.Property("id", th.StringType), + th.Property("databaseId", th.IntegerType), + ).to_dict() + + def __init__(self, tap: Tap, user_list: list[str]) -> None: + super().__init__(tap) + self.user_list = user_list + + @property + def query(self) -> str: + chunks = [] + for i, user in enumerate(self.user_list): + # we use the `repositoryOwner` query which is the only one that + # works on both users and orgs with graphql. REST is less picky + # and the /user endpoint works for all types. + chunks.append( + f'user{i}: repositoryOwner(login: "{user}") ' + "{ login avatarUrl}" + ) + return "query {" + " ".join(chunks) + " rateLimit { cost } }" + + if len(user_list) < 1: + return [] + + users_with_ids: list = [] + temp_stream = TempStream(self._tap, list(user_list)) + + database_id_pattern: re.Pattern = re.compile( + r"https://avatars.githubusercontent.com/u/(\d+)?.*" + ) + # replace manually provided org/repo values by the ones obtained + # from github api. This guarantees that case is correct in the output data. + # See https://github.com/MeltanoLabs/tap-github/issues/110 + # Also remove repos which do not exist to avoid crashing further down + # the line. + for record in temp_stream.request_records({}): + for item in record: + if item == "rateLimit": + continue + try: + username = record[item]["login"] + except TypeError: + # one of the usernames returned `None`, which means it does + # not exist, log some details, and move on to the next one + invalid_username = user_list[int(item[4:])] + self.logger.info( + f"Username not found: {invalid_username} \t" + "Removing it from list" + ) + continue + # the databaseId (in graphql language) is not available on + # repositoryOwner, so we parse the avatarUrl to get it :/ + m = database_id_pattern.match(record[item]["avatarUrl"]) + if m is not None: + db_id = m.group(1) + users_with_ids.append({"username": username, "user_id": db_id}) + else: + # If we get here, github's API is not returning what + # we expected, so it's most likely a breaking change on + # their end, and the tap's code needs updating + raise FatalAPIError("Unexpected GitHub API error: Breaking change?") + + self.logger.info(f"Running the tap on {len(users_with_ids)} users") + return users_with_ids + + def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: + """ + Override the parent method to allow skipping API calls + if the stream is deselected and skip_parent_streams is True in config. + This allows running the tap with fewer API calls and preserving + quota when only syncing a child stream. Without this, + the API call is sent but data is discarded. + """ + if ( + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None + ): + # build a minimal mock record so that self._sync_records + # can proceed with child streams + # the id is fetched in `get_user_ids` above + yield { + "login": context["username"], + "id": context["user_id"], + } + else: + yield from super().get_records(context) + + schema = th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + th.Property("name", th.StringType), + th.Property("company", th.StringType), + th.Property("blog", th.StringType), + th.Property("location", th.StringType), + th.Property("email", th.StringType), + th.Property("hireable", th.BooleanType), + th.Property("bio", th.StringType), + th.Property("twitter_username", th.StringType), + th.Property("public_repos", th.IntegerType), + th.Property("public_gists", th.IntegerType), + th.Property("followers", th.IntegerType), + th.Property("following", th.IntegerType), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + ).to_dict() + + +class StarredStream(GitHubRestStream): + """Defines 'Stars' stream. Warning: this stream does NOT track star deletions.""" + + name = "starred" + path = "/users/{username}/starred" + # "repo_id" is the starred repo's id. + primary_keys: ClassVar[list[str]] = ["repo_id", "username"] + parent_stream_type = UserStream + # TODO - change partitioning key to user_id? + state_partitioning_keys: ClassVar[list[str]] = ["username"] + replication_key = "starred_at" + ignore_parent_replication_key = True + # GitHub is missing the "since" parameter on this endpoint. + use_fake_since_parameter = True + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Overridden to use an endpoint which includes starred_at property: + https://docs.github.com/en/rest/reference/activity#custom-media-types-for-starring + """ + headers = super().http_headers + headers["Accept"] = "application/vnd.github.v3.star+json" + return headers + + def post_process(self, row: dict, context: Context | None = None) -> dict: + """ + Add a repo_id top-level field to be used as state replication key. + """ + row["repo_id"] = row["repo"]["id"] + if context is not None: + row["user_id"] = context["user_id"] + return row + + schema = th.PropertiesList( + # Parent Keys + th.Property("username", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("user_id", th.IntegerType), + # Starred Repo Info + th.Property("starred_at", th.DateTimeType), + th.Property( + "repo", + th.ObjectType( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("full_name", th.StringType), + th.Property("description", th.StringType), + th.Property("html_url", th.StringType), + th.Property("owner", user_object), + th.Property( + "license", + th.ObjectType( + th.Property("key", th.StringType), + th.Property("name", th.StringType), + th.Property("url", th.StringType), + th.Property("spdx_id", th.StringType), + ), + ), + th.Property("updated_at", th.DateTimeType), + th.Property("created_at", th.DateTimeType), + th.Property("pushed_at", th.DateTimeType), + th.Property("stargazers_count", th.IntegerType), + th.Property("fork", th.BooleanType), + th.Property( + "topics", + th.ArrayType(th.StringType), + ), + th.Property("visibility", th.StringType), + th.Property("language", th.StringType), + th.Property("forks", th.IntegerType), + th.Property("watchers", th.IntegerType), + th.Property("open_issues", th.IntegerType), + ), + ), + ).to_dict() + + +class UserContributedToStream(GitHubGraphqlStream): + """Defines 'UserContributedToStream' stream.""" + + name = "user_contributed_to" + query_jsonpath = "$.data.user.repositoriesContributedTo.nodes.[*]" + primary_keys: ClassVar[list[str]] = ["username", "name_with_owner"] + replication_key = None + parent_stream_type = UserStream + # TODO - add user_id to schema + # TODO - change partitioning key to user_id? + state_partitioning_keys: ClassVar[list[str]] = ["username"] + ignore_parent_replication_key = True + + @property + def query(self) -> str: + """Return dynamic GraphQL query.""" + # Graphql id is equivalent to REST node_id. To keep the tap consistent, + # we rename "id" to "node_id". + return """ + query userContributedTo($username: String! $nextPageCursor_0: String) { + user (login: $username) { + repositoriesContributedTo (first: 100 after: $nextPageCursor_0 includeUserRepositories: true orderBy: {field: STARGAZERS, direction: DESC}) { + pageInfo { + hasNextPage_0: hasNextPage + startCursor_0: startCursor + endCursor_0: endCursor + } + nodes { + node_id: id + database_id: databaseId + name_with_owner: nameWithOwner + open_graph_image_url: openGraphImageUrl + stargazer_count: stargazerCount + pushed_at: pushedAt + owner { + node_id: id + login + } + } + } + } + rateLimit { + cost + } + } + """ # noqa: E501 + + schema = th.PropertiesList( + th.Property("node_id", th.StringType), + th.Property("username", th.StringType), + th.Property("name_with_owner", th.StringType), + th.Property("open_graph_image_url", th.StringType), + th.Property("stargazer_count", th.IntegerType), + th.Property( + "owner", + th.ObjectType( + th.Property("node_id", th.StringType), + th.Property("login", th.StringType), + ), + ), + ).to_dict() diff --git a/tap_github/utils/__init__.py b/tap_github/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tap_github/utils/filter_stdout.py b/tap_github/utils/filter_stdout.py new file mode 100644 index 0000000..638cf21 --- /dev/null +++ b/tap_github/utils/filter_stdout.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import contextlib +import io +import re +import sys +from re import Pattern +from typing import TYPE_CHECKING, TextIO + +if TYPE_CHECKING: + from collections.abc import Generator + + +class FilterStdOutput: + """Filter out stdout/sterr given a regex pattern.""" + + def __init__(self, stream: TextIO, re_pattern: str | Pattern) -> None: + self.stream = stream + self.pattern = ( + re.compile(re_pattern) if isinstance(re_pattern, str) else re_pattern + ) + self.triggered = False + + def __getattr__(self, attr_name: str) -> object: + return getattr(self.stream, attr_name) + + def write(self, data: str) -> None: + if data == "\n" and self.triggered: + self.triggered = False + else: + if self.pattern.search(data) is None: + self.stream.write(data) + self.stream.flush() + else: + # caught bad pattern + self.triggered = True + + def flush(self) -> None: + self.stream.flush() + + +@contextlib.contextmanager +def nostdout() -> Generator[None, None, None]: + save_stdout = sys.stdout + sys.stdout = io.StringIO() + yield + sys.stdout = save_stdout diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..76c0d4f --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,23 @@ +"""Test suite for tap-github.""" + +import requests_cache + +# Setup caching for all api calls done through `requests` in order to limit +# rate limiting problems with github. +# Use the sqlite backend as it's the default option and seems to be best supported. +# To clear the cache, just delete the sqlite db file at api_calls_tests_cache.sqlite +# in the root of this repository +requests_cache.install_cache( + ".cache/api_calls_tests_cache", + backend="sqlite", + # make sure that API keys don't end up being cached + # Also ignore user-agent so that various versions of request + # can share the cache + ignored_parameters=["Authorization", "User-Agent", "If-modified-since"], + # tell requests_cache to check headers for the above parameter + match_headers=True, + # expire the cache after 24h (86400 seconds) + expire_after=24 * 60 * 60, + # make sure graphql calls get cached as well + allowable_methods=["GET", "POST"], +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..eecab73 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +pytest_plugins = "pytester" diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..343cd8a --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import datetime +import logging +import os +import sys +from typing import TYPE_CHECKING + +import pytest + +from tap_github.utils.filter_stdout import FilterStdOutput + +if TYPE_CHECKING: + from singer_sdk.helpers.types import Context + +# Filter out singer output during tests +sys.stdout = FilterStdOutput(sys.stdout, r'{"type": ') + + +@pytest.fixture +def search_config(): + return { + "metrics_log_level": "warning", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "searches": [ + { + "name": "tap_something", + "query": "tap-+language:Python", + } + ], + } + + +@pytest.fixture +def repo_list_config(request): + """ + Get a default list of repos or pass your own by decorating your test with + @pytest.mark.repo_list(['org1/repo1', 'org2/repo2']) + """ + marker = request.node.get_closest_marker("repo_list") + if marker is None: + repo_list = ["MeltanoLabs/tap-github", "mapswipe/mapswipe"] + else: + repo_list = marker.args[0] + + return { + "metrics_log_level": "warning", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "repositories": repo_list, + "rate_limit_buffer": 100, + } + + +@pytest.fixture +def username_list_config(request): + """ + Get a default list of usernames or pass your own by decorating your test with + @pytest.mark.username_list(['ericboucher', 'aaronsteers']) + """ + marker = request.node.get_closest_marker("username_list") + username_list = ["ericboucher", "aaronsteers"] if marker is None else marker.args[0] + + return { + "metrics_log_level": "warning", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "user_usernames": username_list, + "rate_limit_buffer": 100, + } + + +@pytest.fixture +def user_id_list_config(request): + """ + Get a default list of usernames or pass your own by decorating your test with + @pytest.mark.user_id_list(['ericboucher', 'aaronsteers']) + """ + marker = request.node.get_closest_marker("user_id_list") + user_id_list = [1, 2] if marker is None else marker.args[0] + + return { + "metrics_log_level": "warning", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "user_ids": user_id_list, + "rate_limit_buffer": 100, + } + + +@pytest.fixture +def organization_list_config(request): + """ + Get a default list of organizations or pass your own by decorating your test with + @pytest.mark.organization_list(['MeltanoLabs', 'oviohub']) + """ + marker = request.node.get_closest_marker("organization_list") + + organization_list = ["MeltanoLabs"] if marker is None else marker.args[0] + + return { + "metrics_log_level": "warning", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "organizations": organization_list, + "rate_limit_buffer": 100, + } + + +def alternative_sync_chidren( + self, + child_context: Context, + no_sync: bool = True, +) -> None: + """ + Override for Stream._sync_children. + Enabling us to use an ORG_LEVEL_TOKEN for the collaborators stream. + """ + for child_stream in self.child_streams: + # Use org:write access level credentials for collaborators stream + if child_stream.name == "collaborators": + ORG_LEVEL_TOKEN = os.environ.get("ORG_LEVEL_TOKEN") # noqa: N806 + # TODO - Fix collaborators tests, likely by mocking API responses directly. + # Currently we have to bypass them as they are failing frequently. + if not ORG_LEVEL_TOKEN or no_sync: + logging.warning( + 'No "ORG_LEVEL_TOKEN" found. Skipping collaborators stream sync.' + ) + continue + SAVED_GTHUB_TOKEN = os.environ.get("GITHUB_TOKEN") # noqa: N806 + os.environ["GITHUB_TOKEN"] = ORG_LEVEL_TOKEN + child_stream.sync(context=child_context) + os.environ["GITHUB_TOKEN"] = SAVED_GTHUB_TOKEN or "" + continue + + # default behavior: + if child_stream.selected or child_stream.has_selected_descendents: + child_stream.sync(context=child_context) diff --git a/tests/test_authenticator.py b/tests/test_authenticator.py new file mode 100644 index 0000000..2abcefb --- /dev/null +++ b/tests/test_authenticator.py @@ -0,0 +1,1260 @@ +import logging +import re +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +import pytest +import requests +import requests.exceptions +from singer_sdk.streams import RESTStream + +from tap_github.authenticator import ( + AppTokenManager, + GitHubTokenAuthenticator, + PersonalTokenManager, + TokenManager, +) + + +def _now(): + return datetime.now(tz=timezone.utc) + + +class TestTokenManager: + def test_default_rate_limits(self): + token_manager = TokenManager("mytoken", rate_limit_buffer=700) + + assert token_manager.rate_limit == 5000 + assert token_manager.rate_limit_remaining == 5000 + assert token_manager.rate_limit_reset is None + assert token_manager.rate_limit_used == 0 + assert token_manager.rate_limit_buffer == 700 + + token_manager_2 = TokenManager("mytoken") + assert token_manager_2.rate_limit_buffer == 1000 + + def test_update_rate_limit(self): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "4999", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "1", + } + + token_manager = TokenManager("mytoken") + token_manager.update_rate_limit(mock_response_headers) + + assert token_manager.rate_limit == 5000 + assert token_manager.rate_limit_remaining == 4999 + assert token_manager.rate_limit_reset == datetime( + 2013, + 7, + 1, + 17, + 47, + 53, + tzinfo=timezone.utc, + ) + assert token_manager.rate_limit_used == 1 + + def test_is_valid_token_successful(self): + with patch("requests.get") as mock_get: + mock_response = mock_get.return_value + mock_response.raise_for_status.return_value = None + + token_manager = TokenManager("validtoken") + + assert token_manager.is_valid_token() + mock_get.assert_called_once_with( + url="https://api.github.com/rate_limit", + headers={"Authorization": "token validtoken"}, + ) + + def test_is_valid_token_failure(self, caplog: pytest.LogCaptureFixture): + with patch("requests.get") as mock_get: + # Setup for a failed request + mock_response = mock_get.return_value + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError() + mock_response.status_code = 401 + mock_response.content = b"Unauthorized Access" + mock_response.reason = "Unauthorized" + + token_manager = TokenManager("invalidtoken") + + with caplog.at_level(logging.WARNING): + assert not token_manager.is_valid_token() + + assert "401" in caplog.text + + def test_has_calls_remaining_succeeds_if_token_never_used(self): + token_manager = TokenManager("mytoken") + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_succeeds_if_lots_remaining(self): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "4999", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "1", + } + + token_manager = TokenManager("mytoken") + token_manager.update_rate_limit(mock_response_headers) + + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_succeeds_if_reset_time_reached(self): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "1", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "4999", + } + + token_manager = TokenManager("mytoken", rate_limit_buffer=1000) + token_manager.update_rate_limit(mock_response_headers) + + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_fails_if_few_calls_remaining_and_reset_time_not_reached( # noqa: E501 + self, + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "1", + "X-RateLimit-Reset": str(int((_now() + timedelta(days=100)).timestamp())), + "X-RateLimit-Used": "4999", + } + + token_manager = TokenManager("mytoken", rate_limit_buffer=1000) + token_manager.update_rate_limit(mock_response_headers) + + assert not token_manager.has_calls_remaining() + + +class TestAppTokenManager: + def test_initialization_with_3_part_env_key(self): + with patch.object(AppTokenManager, "claim_token", return_value=None): + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + assert token_manager.github_app_id == "12345" + assert token_manager.github_private_key == "key\ncontent" + assert token_manager.github_installation_id == "67890" + + def test_initialization_with_2_part_env_key(self): + with patch.object(AppTokenManager, "claim_token", return_value=None): + token_manager = AppTokenManager("12345;;key\\ncontent") + assert token_manager.github_app_id == "12345" + assert token_manager.github_private_key == "key\ncontent" + assert token_manager.github_installation_id is None + + def test_initialization_with_malformed_env_key(self): + expected_error_expression = re.escape( + "GITHUB_APP_PRIVATE_KEY could not be parsed. The expected format is " + '":app_id:;;-----BEGIN RSA PRIVATE KEY-----\\n_YOUR_P_KEY_\\n-----END RSA PRIVATE KEY-----"' # noqa: E501 + ) + with pytest.raises(ValueError, match=expected_error_expression): + AppTokenManager("12345key\\ncontent") + + def test_generate_token_with_invalid_credentials(self): + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=False), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("some_token", MagicMock()), + ), + ): + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + assert token_manager.token is None + assert token_manager.token_expires_at is None + + def test_successful_token_generation(self): + token_time = MagicMock() + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", token_time), + ), + ): + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + token_manager.claim_token() + assert token_manager.token == "valid_token" + assert token_manager.token_expires_at == token_time + + def test_has_calls_remaining_regenerates_a_token_if_close_to_expiry( + self, + caplog: pytest.LogCaptureFixture, + ): + unexpired_time = _now() + timedelta(days=1) + expired_time = _now() - timedelta(days=1) + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "4999", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "1", + } + + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + token_manager.token_expires_at = expired_time + token_manager.update_rate_limit(mock_response_headers) + + with caplog.at_level(logging.INFO): + assert token_manager.has_calls_remaining() + + # calling has_calls_remaining() will trigger the token generation function to be called again, # noqa: E501 + # so token_expires_at should have been reset back to the mocked unexpired_time # noqa: E501 + assert token_manager.token_expires_at == unexpired_time + assert "GitHub app token refresh succeeded." in caplog.text + + def test_has_calls_remaining_logs_warning_if_token_regeneration_fails( + self, caplog: pytest.LogCaptureFixture + ): + unexpired_time = _now() + timedelta(days=1) + expired_time = _now() - timedelta(days=1) + with ( + patch.object( + AppTokenManager, "is_valid_token", return_value=True + ) as mock_is_valid, + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "4999", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "1", + } + + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + token_manager.token_expires_at = expired_time + token_manager.update_rate_limit(mock_response_headers) + + mock_is_valid.return_value = False + with caplog.at_level(logging.WARNING): + assert not token_manager.has_calls_remaining() + assert "GitHub app token refresh failed." in caplog.text + + def test_has_calls_remaining_succeeds_if_token_new_and_never_used(self): + unexpired_time = _now() + timedelta(days=1) + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_succeeds_if_time_and_requests_left(self): + unexpired_time = _now() + timedelta(days=1) + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "4999", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "1", + } + + token_manager = AppTokenManager("12345;;key\\ncontent;;67890") + token_manager.update_rate_limit(mock_response_headers) + + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_succeeds_if_time_left_and_reset_time_reached(self): + unexpired_time = _now() + timedelta(days=1) + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "1", + "X-RateLimit-Reset": "1372700873", + "X-RateLimit-Used": "4999", + } + + token_manager = AppTokenManager( + "12345;;key\\ncontent;;67890", rate_limit_buffer=1000 + ) + token_manager.update_rate_limit(mock_response_headers) + + assert token_manager.has_calls_remaining() + + def test_has_calls_remaining_fails_if_time_left_and_few_calls_remaining_and_reset_time_not_reached( # noqa: E501 + self, + ): + unexpired_time = _now() + timedelta(days=1) + with ( + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("valid_token", unexpired_time), + ), + ): + mock_response_headers = { + "X-RateLimit-Limit": "5000", + "X-RateLimit-Remaining": "1", + "X-RateLimit-Reset": str( + int((_now() + timedelta(days=100)).timestamp()) + ), + "X-RateLimit-Used": "4999", + } + + token_manager = AppTokenManager( + "12345;;key\\ncontent;;67890", rate_limit_buffer=1000 + ) + token_manager.update_rate_limit(mock_response_headers) + + assert not token_manager.has_calls_remaining() + + +@pytest.fixture +def mock_stream(): + stream = MagicMock(spec=RESTStream) + stream.tap_name = "tap_github" + stream.config = {"rate_limit_buffer": 5} + return stream + + +class TestGitHubTokenAuthenticator: + @staticmethod + def _count_total_tokens(token_managers): + """Count total tokens across all organizations.""" + return sum(len(tokens) for tokens in token_managers.values()) + + @staticmethod + def _flatten_token_managers(token_managers): + """Flatten token_managers dict to a list of all TokenManager objects.""" + return [tm for tokens in token_managers.values() for tm in tokens] + + def test_prepare_tokens_returns_empty_if_none_found(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={"GITHUB_TLJKJFDS": "gt1"}, + ), + patch.object(PersonalTokenManager, "is_valid_token", return_value=True), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=mock_stream) + token_managers = auth.prepare_tokens() + + assert len(token_managers) == 0 + + def test_config_auth_token_only(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={"OTHER_TOKEN": "blah", "NOT_THE_RIGHT_TOKEN": "meh"}, + ), + patch.object(PersonalTokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update({"auth_token": "gt5"}) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 1 + assert token_managers[None][0].token == "gt5" + + def test_config_additional_auth_tokens_only(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={"OTHER_TOKEN": "blah", "NOT_THE_RIGHT_TOKEN": "meh"}, + ), + patch.object(PersonalTokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update({"additional_auth_tokens": ["gt7", "gt8", "gt9"]}) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 3 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == ["gt7", "gt8", "gt9"] + + def test_env_personal_tokens_only(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_TOKENxyz": "gt2", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(PersonalTokenManager, "is_valid_token", return_value=True), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=mock_stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 2 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == ["gt1", "gt2"] + + def test_config_app_keys(self, mock_stream): + def generate_token_mock(app_id, private_key, installation_id): + return (f"installationtokenfor{app_id}", MagicMock()) + + with ( + patch.object(TokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + side_effect=generate_token_mock, + ), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "gt5", + "additional_auth_tokens": ["gt7", "gt8", "gt9"], + "auth_app_keys": [ + "123;;gak1;;13", + "456;;gak1;;46", + "789;;gak1;;79", + ], + } + ) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 7 + + all_tokens = self._flatten_token_managers(token_managers) + app_token_managers = { + tm for tm in all_tokens if isinstance(tm, AppTokenManager) + } + assert len(app_token_managers) == 3 + + app_tokens = {tm.token for tm in app_token_managers} + assert app_tokens == { + "installationtokenfor123", + "installationtokenfor456", + "installationtokenfor789", + } + + def test_env_app_key_only(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_APP_PRIVATE_KEY": "123;;key", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(AppTokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=mock_stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 1 + assert token_managers[None][0].token == "installationtoken12345" + + def test_all_token_types(self, mock_stream): + # Expectations: + # - the presence of additional_auth_tokens causes personal tokens in the environment to be ignored. # noqa: E501 + # - the other types all coexist + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_TOKENxyz": "gt2", + "GITHUB_APP_PRIVATE_KEY": "123;;key;;install_id", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "gt5", + "additional_auth_tokens": ["gt7", "gt8", "gt9"], + } + ) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 5 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == [ + "gt5", + "gt7", + "gt8", + "gt9", + "installationtoken12345", + ] + + def test_all_token_types_except_additional_auth_tokens(self, mock_stream): + # Expectations: + # - in the absence of additional_auth_tokens, all the other types can coexist + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_TOKENxyz": "gt2", + "GITHUB_APP_PRIVATE_KEY": "123;;key;;install_id", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "gt5", + } + ) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 4 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == [ + "gt1", + "gt2", + "gt5", + "installationtoken12345", + ] + + def test_auth_token_and_additional_auth_tokens_deduped(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_TOKENxyz": "gt2", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "gt1", + "additional_auth_tokens": ["gt1", "gt1", "gt8", "gt8", "gt9"], + } + ) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 3 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == ["gt1", "gt8", "gt9"] + + def test_auth_token_and_env_tokens_deduped(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_TOKENa": "gt2", + "GITHUB_TOKENxyz": "gt2", + "OTHER_TOKEN": "blah", + }, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + stream = mock_stream + stream.config.update({"auth_token": "gt1"}) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert self._count_total_tokens(token_managers) == 2 + all_tokens = self._flatten_token_managers(token_managers) + assert sorted({tm.token for tm in all_tokens}) == ["gt1", "gt2"] + + def test_handle_error_if_app_key_invalid( + self, + mock_stream, + caplog: pytest.LogCaptureFixture, + ): + # Confirm expected behaviour if an error is raised while setting up the app token manager: # noqa: E501 + # - don"t crash + # - print the error as a warning + # - continue with any other obtained tokens + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={"GITHUB_APP_PRIVATE_KEY": "123garbagekey"}, + ), + patch("tap_github.authenticator.AppTokenManager") as mock_app_manager, + ): + mock_app_manager.side_effect = ValueError("Invalid key format") + + auth = GitHubTokenAuthenticator.from_stream(stream=mock_stream) + auth.prepare_tokens() + + msg = "An error was thrown while preparing an app token: Invalid key format" + with caplog.at_level(logging.WARNING): + assert msg in caplog.text + + def test_exclude_generated_app_token_if_invalid(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={"GITHUB_APP_PRIVATE_KEY": "123;;key"}, + ), + patch.object(AppTokenManager, "is_valid_token", return_value=False), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=mock_stream) + token_managers = auth.prepare_tokens() + + assert len(token_managers) == 0 + + def test_prepare_tokens_returns_empty_if_all_tokens_invalid(self, mock_stream): + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={ + "GITHUB_TOKEN1": "gt1", + "GITHUB_APP_PRIVATE_KEY": "123;;key", + }, + ), + patch.object(PersonalTokenManager, "is_valid_token", return_value=False), + patch.object(AppTokenManager, "is_valid_token", return_value=False), + patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("installationtoken12345", MagicMock()), + ), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "gt5", + "additional_auth_tokens": ["gt7", "gt8", "gt9"], + } + ) + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + assert len(token_managers) == 0 + + def test_get_next_auth_token_rotates_within_org(self, mock_stream): + """Test that token rotation works correctly with org-specific token pools.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "acme-corp": ["app1;;key1", "app2;;key2"], + } + } + ) + + def mock_generate_token(app_id, private_key, installation_id): + return (f"token_for_{app_id}", MagicMock()) + + with patch( + "tap_github.authenticator.generate_app_access_token", + side_effect=mock_generate_token, + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Get the two tokens for acme-corp org + org_tokens = auth.token_managers["acme-corp"] + assert len(org_tokens) == 2 + + # Set current org and active token + auth.current_organization = "acme-corp" + auth.active_token = org_tokens[0] + + # Mock first token as exhausted, second as available + with ( + patch.object( + org_tokens[0], "has_calls_remaining", return_value=False + ), + patch.object( + org_tokens[1], "has_calls_remaining", return_value=True + ), + ): + initial_token = auth.active_token + + # Should rotate to second token + auth.get_next_auth_token() + + assert auth.active_token != initial_token + assert auth.active_token == org_tokens[1] + assert auth.current_organization == "acme-corp" + + def test_get_next_auth_token_keeps_org_specific_tokens_isolated(self, mock_stream): + """Test org-specific token rotation does not fall back to agnostic tokens.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "additional_auth_tokens": ["personal_token"], + "org_auth_app_keys": { + "acme-corp": ["app1;;key1"], + }, + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_app_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + org_token = auth.token_managers["acme-corp"][0] + agnostic_token = auth.token_managers[None][0] + + auth.current_organization = "acme-corp" + auth.active_token = org_token + + with ( + patch.object(org_token, "has_calls_remaining", return_value=False), + patch.object( + agnostic_token, "has_calls_remaining", return_value=True + ), + pytest.raises( + RuntimeError, + match="All GitHub tokens have hit their rate limit", + ), + ): + auth.get_next_auth_token() + + assert auth.active_token == org_token + assert auth.current_organization == "acme-corp" + + def test_get_next_auth_token_raises_when_all_exhausted(self, mock_stream): + """Test that get_next_auth_token raises when all tokens are exhausted.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "acme-corp": ["app1;;key1", "app2;;key2"], + } + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + org_tokens = auth.token_managers["acme-corp"] + auth.current_organization = "acme-corp" + auth.active_token = org_tokens[0] + + # Mock all tokens as exhausted + with ( + patch.object( + org_tokens[0], "has_calls_remaining", return_value=False + ), + patch.object( + org_tokens[1], "has_calls_remaining", return_value=False + ), + pytest.raises( + RuntimeError, + match="All GitHub tokens have hit their rate limit", + ), + ): + auth.get_next_auth_token() + + def test_auth_app_keys_array_format_stores_under_none_key(self, mock_stream): + """Test that array format for auth_app_keys stores tokens under None key.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "auth_app_keys": ["app1;;key1", "app2;;key2"], + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("array_format_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + # Tokens should be stored under None key (org-agnostic) + assert None in token_managers + assert len(token_managers[None]) == 2 + assert self._count_total_tokens(token_managers) == 2 + + # Should not have any org-specific keys + org_keys = [k for k in token_managers if k is not None] + assert len(org_keys) == 0 + + def test_auth_app_keys_object_format_stores_by_org(self, mock_stream): + """Test that org_auth_app_keys stores tokens by organization.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "org1": ["app1;;key1"], + "org2": ["app2;;key2", "app3;;key3"], + } + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + # Tokens should be stored under org-specific keys + assert "org1" in token_managers + assert "org2" in token_managers + assert len(token_managers["org1"]) == 1 + assert len(token_managers["org2"]) == 2 + assert self._count_total_tokens(token_managers) == 3 + + # Should not have any tokens under None key + assert None not in token_managers or len(token_managers[None]) == 0 + + def test_auth_app_keys_mixed_with_personal_tokens(self, mock_stream): + """Test that org-specific app keys and personal tokens coexist correctly.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "auth_token": "personal_token", + "org_auth_app_keys": { + "org1": ["app1;;key1"], + }, + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + token_managers = auth.prepare_tokens() + + # Should have both org-specific and org-agnostic tokens + assert "org1" in token_managers + assert None in token_managers + assert len(token_managers["org1"]) == 1 + assert len(token_managers[None]) == 1 + assert self._count_total_tokens(token_managers) == 2 + + # Verify personal token is under None key + personal_tokens = [ + tm + for tm in token_managers[None] + if isinstance(tm, PersonalTokenManager) + ] + assert len(personal_tokens) == 1 + assert personal_tokens[0].token == "personal_token" + + def test_set_organization_switches_to_org_specific_token(self, mock_stream): + """Test that set_organization switches to org-specific token.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "org1": ["app1;;key1"], + "org2": ["app2;;key2"], + } + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Initially should be on org1 (alphabetically first) + assert auth.current_organization is None # Not set yet + org1_token = auth.token_managers["org1"][0] + org2_token = auth.token_managers["org2"][0] + + # Mock has_calls_remaining to return True for all tokens + with ( + patch.object(org1_token, "has_calls_remaining", return_value=True), + patch.object(org2_token, "has_calls_remaining", return_value=True), + ): + # Switch to org1 + auth.set_organization("org1") + assert auth.current_organization == "org1" + assert auth.active_token == org1_token + + # Switch to org2 + auth.set_organization("org2") + assert auth.current_organization == "org2" + assert auth.active_token == org2_token + + # Switch back to org1 + auth.set_organization("org1") + assert auth.current_organization == "org1" + assert auth.active_token == org1_token + + def test_set_organization_falls_back_to_org_agnostic(self, mock_stream): + """Test set_organization falls back to org-agnostic tokens when no + org-specific tokens.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "additional_auth_tokens": ["personal_token"], + } + ) + + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Should have only org-agnostic token + assert None in auth.token_managers + assert len(auth.token_managers[None]) == 1 + agnostic_token = auth.token_managers[None][0] + + # Switch to an org that has no specific tokens + with patch("tap_github.authenticator.logger") as mock_logger: + auth.set_organization("some-org") + + # Should fall back to org-agnostic token + assert auth.current_organization == "some-org" + assert auth.active_token == agnostic_token + + # Verify fallback info message was logged + mock_logger.info.assert_any_call( + "No org-specific tokens found for 'some-org', " + "using org-agnostic tokens" + ) + + def test_initialization_prefers_org_specific_over_org_agnostic(self, mock_stream): + """Test that initialization prefers org-specific token over + org-agnostic when both exist.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "additional_auth_tokens": ["personal_token"], + "org_auth_app_keys": { + "acme": ["app1;;key1"], + }, + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + return_value=("org_token", MagicMock()), + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Verify both token pools exist + assert "acme" in auth.token_managers + assert None in auth.token_managers + acme_token = auth.token_managers["acme"][0] + agnostic_token = auth.token_managers[None][0] + + # Mock has_calls_remaining to return True for all tokens + with ( + patch.object(acme_token, "has_calls_remaining", return_value=True), + patch.object( + agnostic_token, "has_calls_remaining", return_value=True + ), + ): + # Set organization to acme + auth.set_organization("acme") + + # Should prefer the org-specific token over org-agnostic + assert auth.current_organization == "acme" + assert auth.active_token == acme_token + + def test_set_organization_falls_back_to_other_org_tokens(self, mock_stream): + """Test that set_organization falls back to tokens from other orgs + when no tokens exist for target org.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "acme-corp": ["app1;;key1"], + "widget-co": ["app2;;key2"], + } + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + side_effect=[ + ("acme_token", MagicMock()), + ("widget_token", MagicMock()), + ], + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Verify token pools exist for both orgs + assert "acme-corp" in auth.token_managers + assert "widget-co" in auth.token_managers + acme_token = auth.token_managers["acme-corp"][0] + widget_token = auth.token_managers["widget-co"][0] + + # Mock has_calls_remaining to return True for all tokens + with ( + patch.object(acme_token, "has_calls_remaining", return_value=True), + patch.object( + widget_token, "has_calls_remaining", return_value=True + ), + ): + # Switch to an org with no tokens (public org like matplotlib) + auth.set_organization("matplotlib") + + # Should fall back to a token from another org + assert auth.current_organization == "matplotlib" + assert auth.active_token in [acme_token, widget_token] + + def test_get_next_auth_token_falls_back_to_other_org_tokens(self, mock_stream): + """Test that get_next_auth_token falls back to tokens from other orgs.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "org_auth_app_keys": { + "acme-corp": ["app1;;key1"], + "widget-co": ["app2;;key2"], + } + } + ) + + with patch( + "tap_github.authenticator.generate_app_access_token", + side_effect=[ + ("acme_token", MagicMock()), + ("widget_token", MagicMock()), + ], + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Set current org to public org with no tokens configured + auth.current_organization = "matplotlib" + auth.active_token = None + + # Get tokens from other orgs + acme_token = auth.token_managers["acme-corp"][0] + widget_token = auth.token_managers["widget-co"][0] + + # Mock widget-co token as available + with ( + patch.object(acme_token, "has_calls_remaining", return_value=False), + patch.object( + widget_token, "has_calls_remaining", return_value=True + ), + ): + # Should fall back to token from another org + auth.get_next_auth_token() + + assert auth.active_token == widget_token + assert auth.current_organization == "matplotlib" + + def test_both_org_auth_app_keys_and_auth_app_keys_fallback(self, mock_stream): + """Test org-specific tokens preferred, with fallback to org-agnostic.""" + with ( + patch.object( + GitHubTokenAuthenticator, + "get_env", + return_value={}, + ), + patch.object(TokenManager, "is_valid_token", return_value=True), + ): + stream = mock_stream + stream.config.update( + { + "auth_app_keys": ["fallback_app;;key_fallback"], + "org_auth_app_keys": { + "acme-corp": ["acme_app;;key_acme"], + }, + } + ) + + def mock_generate_token(app_id, private_key, installation_id): + return (f"token_for_{app_id}", MagicMock()) + + with patch( + "tap_github.authenticator.generate_app_access_token", + side_effect=mock_generate_token, + ): + auth = GitHubTokenAuthenticator.from_stream(stream=stream) + + # Verify both token pools exist + assert "acme-corp" in auth.token_managers + assert None in auth.token_managers + assert len(auth.token_managers["acme-corp"]) == 1 + assert len(auth.token_managers[None]) == 1 + + acme_token = auth.token_managers["acme-corp"][0] + fallback_token = auth.token_managers[None][0] + + # Test 1: Org-specific token should be preferred + with ( + patch.object(acme_token, "has_calls_remaining", return_value=True), + patch.object( + fallback_token, "has_calls_remaining", return_value=True + ), + ): + auth.set_organization("acme-corp") + assert auth.active_token == acme_token + assert auth.current_organization == "acme-corp" + + # Test 2: Should not fall back when org-specific tokens are exhausted + with ( + patch.object(acme_token, "has_calls_remaining", return_value=False), + patch.object( + fallback_token, "has_calls_remaining", return_value=True + ), + pytest.raises( + RuntimeError, + match="All GitHub tokens have hit their rate limit", + ), + ): + auth.get_next_auth_token() + + assert auth.active_token == acme_token + assert auth.current_organization == "acme-corp" + + # Test 3: Org without specific tokens should use fallback + with ( + patch.object(acme_token, "has_calls_remaining", return_value=True), + patch.object( + fallback_token, "has_calls_remaining", return_value=True + ), + ): + auth.set_organization("other-org") + assert auth.active_token == fallback_token + assert auth.current_organization == "other-org" diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..90c5b9b --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,71 @@ +"""Tests standard tap features using the built-in SDK tests library.""" + +import logging +import os +from unittest import mock +from unittest.mock import patch + +from singer_sdk.testing import get_standard_tap_tests + +from tap_github.tap import TapGitHub +from tap_github.utils.filter_stdout import nostdout + +from .fixtures import ( # noqa: F401 + alternative_sync_chidren, + organization_list_config, + repo_list_config, + search_config, + username_list_config, +) + + +# Run standard built-in tap tests from the SDK: +def test_standard_tap_tests_for_search_mode(search_config): # noqa: F811 + """Run standard tap tests from the SDK.""" + tests = get_standard_tap_tests(TapGitHub, config=search_config) + with ( + patch( + "singer_sdk.streams.core.Stream._sync_children", alternative_sync_chidren + ), + nostdout(), + ): + for test in tests: + test() + + +def test_standard_tap_tests_for_repo_list_mode(repo_list_config): # noqa: F811 + """Run standard tap tests from the SDK.""" + tests = get_standard_tap_tests(TapGitHub, config=repo_list_config) + with ( + patch( + "singer_sdk.streams.core.Stream._sync_children", alternative_sync_chidren + ), + nostdout(), + ): + for test in tests: + test() + + +def test_standard_tap_tests_for_username_list_mode(username_list_config): # noqa: F811 + """Run standard tap tests from the SDK.""" + tests = get_standard_tap_tests(TapGitHub, config=username_list_config) + with nostdout(): + for test in tests: + test() + + +# This token needs to have read:org access for the organization listed in fixtures.py +# Default is "MeltanoLabs" +ORG_LEVEL_TOKEN = os.environ.get("ORG_LEVEL_TOKEN") + + +@mock.patch.dict(os.environ, {"GITHUB_TOKEN": ORG_LEVEL_TOKEN or ""}) +def test_standard_tap_tests_for_organization_list_mode(organization_list_config): # noqa: F811 + """Run standard tap tests from the SDK.""" + if not ORG_LEVEL_TOKEN: + logging.warning('No "ORG_LEVEL_TOKEN" found. Skipping organization tap tests.') + return + tests = get_standard_tap_tests(TapGitHub, config=organization_list_config) + with nostdout(): + for test in tests: + test() diff --git a/tests/test_discover.py b/tests/test_discover.py new file mode 100644 index 0000000..b95f388 --- /dev/null +++ b/tests/test_discover.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.noconfig +def test_catalog_changes(pytester: pytest.Pytester, tmp_path: Path) -> None: + """Fail if the catalog has changed.""" + # TODO: Discovery should be able to run any config + dummy_config = { + "repositories": [ + "meltano/meltano", + "meltano/sdk", + ], + } + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(dummy_config)) + result = pytester.run( + "tap-github", + "--discover", + f"--config={config_path.as_posix()}", + ) + assert result.ret == 0, "Tap discovery failed" + + catalog = json.loads("".join(result.outlines)) + assert "streams" in catalog + assert isinstance(catalog["streams"], list) + assert len(catalog["streams"]) > 0 diff --git a/tests/test_tap.py b/tests/test_tap.py new file mode 100644 index 0000000..c0b2520 --- /dev/null +++ b/tests/test_tap.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import json +import re +from unittest.mock import patch + +import pytest +from bs4 import BeautifulSoup +from dateutil.parser import isoparse +from requests import Response +from singer_sdk.exceptions import RetriableAPIError +from singer_sdk.helpers import _catalog as cat_helpers +from singer_sdk.singerlib import Catalog + +from tap_github.organization_streams import ProjectItemsStream +from tap_github.repository_streams import GitHubRestStream +from tap_github.scraping import parse_counter +from tap_github.tap import TapGitHub + +from .fixtures import ( # noqa: F401 + alternative_sync_chidren, + organization_list_config, + repo_list_config, + username_list_config, +) + +repo_list_2 = [ + "MeltanoLabs/tap-github", + # mistype the repo name so we can check that the tap corrects it + "MeltanoLabs/Tap-GitLab", + # mistype the org + "meltanolabs/target-athena", + # a repo that does not exist at all + # this one has no matching record below as it should be removed + # from the list by the TempStream + "brokenOrg/does_not_exist", +] +# the same list, but without typos, for validation +repo_list_2_corrected = [ + "MeltanoLabs/tap-github", + "MeltanoLabs/tap-gitlab", + "MeltanoLabs/target-athena", +] +# the github repo ids that match the repo names above +# in the same order +repo_list_2_ids = [ + 365087920, + 416891176, + 361619143, +] + + +@pytest.mark.repo_list(repo_list_2) +def test_validate_repo_list_config(repo_list_config): # noqa: F811 + """Verify that the repositories list is parsed correctly""" + repo_list_context = [ + { + "org": repo[0].split("/")[0], + "repo": repo[0].split("/")[1], + "repo_id": repo[1], + } + for repo in zip(repo_list_2_corrected, repo_list_2_ids, strict=False) + ] + tap = TapGitHub(config=repo_list_config) + partitions = tap.streams["repositories"].partitions + assert partitions == repo_list_context + + +def test_project_items_query_uses_default_page_size(organization_list_config): # noqa: F811 + """Verify that project_items uses GitHub's max page size by default.""" + tap = TapGitHub(config=organization_list_config) + stream = ProjectItemsStream(tap=tap) + + assert "items(first: 100, after: $nextPageCursor_0)" in stream.query + + +def test_project_items_query_uses_configured_page_size(organization_list_config): # noqa: F811 + """Verify that project_items page size can be configured.""" + organization_list_config["stream_options"] = { + "project_items": { + "page_size": 50, + }, + } + tap = TapGitHub(config=organization_list_config) + stream = ProjectItemsStream(tap=tap) + + assert "items(first: 50, after: $nextPageCursor_0)" in stream.query + + +def run_tap_with_config( + capsys, config_obj: dict, skip_stream: str | None, single_stream: str | None +) -> str: + """ + Run the tap with the given config and capture stdout, optionally + skipping a stream (this is meant to be the top level stream), or + running a single one. + """ + tap1 = TapGitHub(config=config_obj) + tap1.run_discovery() + catalog = Catalog.from_dict(tap1.catalog_dict) + # Reset and re-initialize with an input catalog + if skip_stream is not None: + cat_helpers.set_catalog_stream_selected( + catalog=catalog, + stream_name=skip_stream, + selected=False, + ) + elif single_stream is not None: + cat_helpers.deselect_all_streams(catalog) + cat_helpers.set_catalog_stream_selected(catalog, "repositories", selected=True) + cat_helpers.set_catalog_stream_selected( + catalog, stream_name=single_stream, selected=True + ) + + # discard previous output to stdout (potentially from other tests) + capsys.readouterr() + with patch( + "singer_sdk.streams.core.Stream._sync_children", alternative_sync_chidren + ): + tap2 = TapGitHub(config=config_obj, catalog=catalog.to_dict()) + tap2.sync_all() + captured = capsys.readouterr() + return captured.out + + +@pytest.mark.parametrize("skip_parent_streams", [False, True]) +@pytest.mark.repo_list(repo_list_2) +def test_get_a_repository_in_repo_list_mode( + capsys, + repo_list_config, # noqa: F811 + skip_parent_streams, +): + """ + Discover the catalog, and request 2 repository records. + The test is parametrized to run twice, with and without + syncing the top level `repositories` stream. + """ + repo_list_config["skip_parent_streams"] = skip_parent_streams + captured_out = run_tap_with_config( + capsys, + repo_list_config, + "repositories" if skip_parent_streams else None, + single_stream=None, + ) + # Verify we got the right number of records + # one per repo in the list only if we sync the "repositories" stream, 0 if not + assert captured_out.count('{"type":"RECORD","stream":"repositories"') == len( + repo_list_2_ids * (not skip_parent_streams) + ) + # check that the tap corrects invalid case in config input + assert '"repo": "Tap-GitLab"' not in captured_out + assert '"org": "meltanolabs"' not in captured_out + + +def test_repository_child_context_includes_pull_request_capability(repo_list_config): # noqa: F811 + """Verify repository context includes the pull request capability flag.""" + tap = TapGitHub(config=repo_list_config) + repository_stream = tap.streams["repositories"] + + context = repository_stream.get_child_context( + { + "id": 123, + "name": "issues-pi", + "owner": {"login": "shop"}, + "has_pull_requests": False, + }, + None, + ) + + assert context["has_pull_requests"] is False + + +def test_repository_child_context_defaults_pull_request_capability_to_enabled( + repo_list_config, # noqa: F811 +): + """Preserve existing behavior when GitHub does not return the capability flag.""" + tap = TapGitHub(config=repo_list_config) + repository_stream = tap.streams["repositories"] + + context = repository_stream.get_child_context( + { + "id": 123, + "name": "tap-github", + "owner": {"login": "MeltanoLabs"}, + }, + None, + ) + + assert context["has_pull_requests"] is True + + +def test_repository_child_context_includes_issue_capability(repo_list_config): # noqa: F811 + """Verify repository context includes the issue capability flag.""" + tap = TapGitHub(config=repo_list_config) + repository_stream = tap.streams["repositories"] + + context = repository_stream.get_child_context( + { + "id": 123, + "name": "no-issues", + "owner": {"login": "octo-org"}, + "has_issues": False, + }, + None, + ) + + assert context["has_issues"] is False + + +def test_repository_child_context_defaults_issue_capability_to_enabled( + repo_list_config, # noqa: F811 +): + """Preserve existing behavior when GitHub does not return the issue flag.""" + tap = TapGitHub(config=repo_list_config) + repository_stream = tap.streams["repositories"] + + context = repository_stream.get_child_context( + { + "id": 123, + "name": "tap-github", + "owner": {"login": "MeltanoLabs"}, + }, + None, + ) + + assert context["has_issues"] is True + + +def test_pull_requests_stream_skips_repos_with_pull_requests_disabled( + repo_list_config, # noqa: F811 +): + """Do not call the pull requests API when repo metadata says it is disabled.""" + tap = TapGitHub(config=repo_list_config) + pull_requests_stream = tap.streams["pull_requests"] + context = { + "org": "shop", + "repo": "issues-pi", + "repo_id": 123, + "has_pull_requests": False, + } + + with ( + patch.object(GitHubRestStream, "get_records") as get_records, + patch.object(pull_requests_stream.logger, "debug") as log_debug, + ): + records = list(pull_requests_stream.get_records(context)) + + assert records == [] + get_records.assert_not_called() + log_debug.assert_called_once_with( + "Repository shop/issues-pi: Pull requests not enabled, skipping API call", + ) + + +@pytest.mark.parametrize("has_pull_requests", [True, None, 0, "missing"]) +def test_pull_requests_stream_delegates_when_pull_request_capability_is_not_false( + repo_list_config, # noqa: F811 + has_pull_requests, +): + """Fail open when the capability flag is enabled, missing, or unknown.""" + tap = TapGitHub(config=repo_list_config) + pull_requests_stream = tap.streams["pull_requests"] + context = { + "org": "MeltanoLabs", + "repo": "tap-github", + "repo_id": 123, + } + if has_pull_requests != "missing": + context["has_pull_requests"] = has_pull_requests + + with patch.object( + GitHubRestStream, + "get_records", + return_value=iter([{"id": 456}]), + ) as get_records: + records = list(pull_requests_stream.get_records(context)) + + assert records == [{"id": 456}] + get_records.assert_called_once_with(context) + + +def test_issue_comments_stream_skips_when_issues_and_pull_requests_are_disabled( + repo_list_config, # noqa: F811 +): + """Do not call the issue comments API when repo metadata says it is disabled.""" + tap = TapGitHub(config=repo_list_config) + issue_comments_stream = tap.streams["issue_comments"] + context = { + "org": "octo-org", + "repo": "no-issues-or-pull-requests", + "repo_id": 123, + "has_issues": False, + "has_pull_requests": False, + } + + with ( + patch.object(GitHubRestStream, "get_records") as get_records, + patch.object(issue_comments_stream.logger, "debug") as log_debug, + ): + records = list(issue_comments_stream.get_records(context)) + + assert records == [] + get_records.assert_not_called() + log_debug.assert_called_once_with( + "Repository octo-org/no-issues-or-pull-requests: " + "Issues and pull requests not enabled, skipping issue comments API call", + ) + + +@pytest.mark.parametrize( + ("has_issues", "has_pull_requests"), + [ + (True, False), + (None, False), + (0, False), + ("missing", False), + (False, True), + (False, None), + (False, 0), + (False, "missing"), + ], +) +def test_issue_comments_stream_delegates_when_capabilities_are_not_false( + repo_list_config, # noqa: F811 + has_issues, + has_pull_requests, +): + """Fail open when either capability flag is enabled, missing, or unknown.""" + tap = TapGitHub(config=repo_list_config) + issue_comments_stream = tap.streams["issue_comments"] + context = { + "org": "MeltanoLabs", + "repo": "tap-github", + "repo_id": 123, + } + if has_issues != "missing": + context["has_issues"] = has_issues + if has_pull_requests != "missing": + context["has_pull_requests"] = has_pull_requests + + with patch.object( + GitHubRestStream, + "get_records", + return_value=iter([{"id": 456}]), + ) as get_records: + records = list(issue_comments_stream.get_records(context)) + + assert records == [{"id": 456}] + get_records.assert_called_once_with(context) + + +@pytest.mark.parametrize( + "stream_name", + ["issues", "issue_events", "milestones", "labels"], +) +def test_issue_adjacent_streams_delegate_when_has_issues_is_false( + repo_list_config, # noqa: F811 + stream_name, +): + """Document streams intentionally not gated by the repository Issues flag.""" + tap = TapGitHub(config=repo_list_config) + stream = tap.streams[stream_name] + context = { + "org": "octo-org", + "repo": "no-issues", + "repo_id": 123, + "has_issues": False, + } + + with patch.object( + GitHubRestStream, + "get_records", + return_value=iter([{"id": 789}]), + ) as get_records: + records = list(stream.get_records(context)) + + assert records == [{"id": 789}] + get_records.assert_called_once_with(context) + + +def test_pull_requests_stream_keeps_generic_404_retriable( + repo_list_config, # noqa: F811 +): + """Pull request gating must not broaden tolerated GitHub errors.""" + tap = TapGitHub(config=repo_list_config) + pull_requests_stream = tap.streams["pull_requests"] + response = Response() + response.status_code = 404 + response.url = "https://api.github.com/repos/shop/issues-pi/pulls" + response.reason = "Not Found" + response._content = b'{"message": "Not Found"}' + response.headers["X-GitHub-Request-Id"] = "request-id" + + with pytest.raises(RetriableAPIError, match="404 Client Error"): + pull_requests_stream.validate_response(response) + + +def test_issue_comments_stream_keeps_generic_404_retriable( + repo_list_config, # noqa: F811 +): + """Issue comments gating must not broaden tolerated GitHub errors.""" + tap = TapGitHub(config=repo_list_config) + issue_comments_stream = tap.streams["issue_comments"] + response = Response() + response.status_code = 404 + response.url = "https://api.github.com/repos/octo-org/no-issues/issues/comments" + response.reason = "Not Found" + response._content = b'{"message": "Not Found"}' + response.headers["X-GitHub-Request-Id"] = "request-id" + + with pytest.raises(RetriableAPIError, match="404 Client Error"): + issue_comments_stream.validate_response(response) + + +@pytest.mark.repo_list(["MeltanoLabs/tap-github"]) +def test_last_state_message_is_valid(capsys, repo_list_config): # noqa: F811 + """ + Validate that the last state message is not a temporary one and contains the + expected values for a stream with overridden state partitioning keys. + Run this on a single repo to avoid having to filter messages too much. + """ + repo_list_config["skip_parent_streams"] = True + captured_out = run_tap_with_config( + capsys, repo_list_config, "repositories", single_stream=None + ) + # capture the messages we're interested in + state_messages = re.findall(r'{"type":"STATE","value":.*}', captured_out) + issue_comments_records = re.findall( + r'{"type":"RECORD","stream":"issue_comments",.*}', captured_out + ) + assert state_messages is not None + last_state_msg = state_messages[-1] + + # make sure we don't have a temporary state message at the very end + assert "progress_markers" not in last_state_msg + + last_state = json.loads(last_state_msg) + last_state_updated_at = isoparse( + last_state["value"]["bookmarks"]["issue_comments"]["partitions"][0][ + "replication_key_value" + ] + ) + latest_updated_at = max( + isoparse(json.loads(record)["record"]["updated_at"]) + for record in issue_comments_records + ) + assert last_state_updated_at == latest_updated_at + + +# case is incorrect on purpose, so we can check that the tap corrects it +# and run the test twice, with and without syncing the `users` stream +@pytest.mark.parametrize("skip_parent_streams", [False, True]) +@pytest.mark.username_list(["EricBoucher", "aaRONsTeeRS"]) +def test_get_a_user_in_user_usernames_mode( + capsys, + username_list_config, # noqa: F811 + skip_parent_streams, +): + """ + Discover the catalog, and request 2 repository records + """ + username_list_config["skip_parent_streams"] = skip_parent_streams + captured_out = run_tap_with_config( + capsys, + username_list_config, + "users" if skip_parent_streams else None, + single_stream=None, + ) + # Verify we got the right number of records: + # one per user in the list if we sync the root stream, 0 otherwise + assert captured_out.count('{"type":"RECORD","stream":"users"') == len( + username_list_config["user_usernames"] * (not skip_parent_streams) + ) + # these 2 are inequalities as number will keep changing :) + assert captured_out.count('{"type":"RECORD","stream":"starred"') > 150 + assert captured_out.count('{"type":"RECORD","stream":"user_contributed_to"') > 25 + assert '{"username":"aaronsteers"' in captured_out + assert '{"username":"aaRONsTeeRS"' not in captured_out + assert '{"username":"EricBoucher"' not in captured_out + + +@pytest.mark.repo_list(["torvalds/linux"]) +def test_large_list_of_contributors(capsys, repo_list_config): # noqa: F811 + """ + Check that the github error message for very large lists of contributors + is handled properly (does not return any records). + """ + captured_out = run_tap_with_config( + capsys, repo_list_config, skip_stream=None, single_stream="contributors" + ) + assert captured_out.count('{"type":"RECORD","stream":"contributors"') == 0 + + +def test_web_tag_parse_counter(): + """ + Check that the parser runs ok on various forms of counters. + Used in extra_metrics stream. + """ + # regular int + tag = BeautifulSoup( + '57', + "html.parser", + ).span + assert parse_counter(tag) == 57 + + # 2k + tag = BeautifulSoup( + '2k', + "html.parser", + ).span + assert parse_counter(tag) == 2028 + + # 5k+. The real number is not available in the page, use this approx value + tag = BeautifulSoup( + '5k+', + "html.parser", + ).span + assert parse_counter(tag) == 5_000 From 2c6ffcd8a29670f8244ad4a91e537f558a48deb0 Mon Sep 17 00:00:00 2001 From: ttomalak-bkpr Date: Fri, 17 Jul 2026 13:15:57 +0200 Subject: [PATCH 2/3] chore: drop github-actions-validator workflow lumapps/github-actions-validator is a private repo and tap-github isn't on its access allowlist yet, so this check can never pass as-is (Unable to resolve action, not found). Several sibling tap repos (tap-qoben-comeen, tap-netsuitesuiteql, tap-lucca, tap-travel-perk) ship with no CI workflow at all, so this just matches that existing pattern instead of shipping a permanently-broken check. --- .github/workflows/github-actions-validator.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .github/workflows/github-actions-validator.yml diff --git a/.github/workflows/github-actions-validator.yml b/.github/workflows/github-actions-validator.yml deleted file mode 100644 index 6d34a79..0000000 --- a/.github/workflows/github-actions-validator.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - name: 'CI: github-actions-validator' - - on: - pull_request: - - concurrency: - group: ${{ github.head_ref }}-github-actions-validator - cancel-in-progress: true - - jobs: - github-actions-validator: - name: Github-actions-validator - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: lumapps/github-actions-validator@v2 From fdfd0864c89a8cb2f44977cd7e0b2855b0873bb8 Mon Sep 17 00:00:00 2001 From: ttomalak-bkpr Date: Fri, 17 Jul 2026 12:54:04 +0200 Subject: [PATCH 3/3] feat: round-robin repo-enumeration checkpoint + secondary rate-limit throttle The upstream RepositoryStream re-walks its full repo list in the same endpoint-imposed order every run (by updated_at, which the "since" param can't actually filter on either the org-listing or search endpoints), so it perpetually re-verifies the same front-of-queue repos and never advances into the rest of the catalog before hitting GitHub's secondary rate limit. Most of an org's repos never get PR/commit data synced at all, despite running on schedule for months. - RepositoryStream now round-robins through organizations/searches mode partitions: resumes right after the last repo whose children fully synced last run (tracked via a `round_robin_checkpoint` state key, deliberately not nested under `progress_markers`, which singer_sdk wipes on every partition finalize), wrapping at the end. Traversal order is sorted by immutable repo `id`, not the API's volatile `updated_at` order, so reshuffling from ordinary repo activity doesn't cause redundant re-syncs of already-current repos before genuinely new ones are reached. - Logs a one-line round-robin summary at the end of each partition's enumeration (repos processed, elapsed time, full lap or not, next resume point) - fires even on a mid-run crash via a `finally` block, since a crash from child-stream syncing tears down the paused generator (GeneratorExit) rather than skipping straight to process exit. - Adds an adaptive, tap-wide throttle for the secondary (frequency) rate limit: escalates a shared inter-request delay on each hit (capped), applied to every subsequent request from any stream, and decays it back down after a run of clean requests. Previously a secondary-limit hit only paced the one retried request while everything else kept firing at the same rate that triggered it. Verified against live GitHub data (fresh + simulated mid-cycle-crash resume) and via fake-clock/logic simulations for the throttle and reshuffling-robustness properties - see PR description for details. --- tap_github/client.py | 80 ++++++++++++++++++++++++++++++++ tap_github/repository_streams.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/tap_github/client.py b/tap_github/client.py index cda1aec..c63332b 100644 --- a/tap_github/client.py +++ b/tap_github/client.py @@ -45,6 +45,19 @@ class GitHubRestStream(RESTStream): # Set to True to use cursor-based pagination instead of page-based pagination use_cursor_pagination = False + # Shared by every GitHubRestStream instance/subclass in this process: once + # any stream trips the secondary (frequency-based) rate limit, we throttle + # all subsequent requests - from any stream - instead of resuming at the + # same pace that triggered it. Decays back down after a run of clean + # requests so a brief burst doesn't slow down an otherwise-healthy sync + # for the rest of the run. Reset each time the tap process starts. + _secondary_rate_limit_delay: ClassVar[float] = 0.0 + _consecutive_clean_requests: ClassVar[int] = 0 + _last_request_at: ClassVar[float | None] = None + _MIN_THROTTLE_DELAY: ClassVar[float] = 1.0 # seconds + _MAX_THROTTLE_DELAY: ClassVar[float] = 30.0 # seconds + _DECAY_AFTER_CLEAN_REQUESTS: ClassVar[int] = 25 + _authenticator: GitHubTokenAuthenticator | None = None @property @@ -79,6 +92,72 @@ def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: yield from super().get_records(context) + def _throttle_before_request(self) -> None: + """Sleep if we're currently backing off from a secondary rate limit.""" + delay = GitHubRestStream._secondary_rate_limit_delay + last_request_at = GitHubRestStream._last_request_at + GitHubRestStream._last_request_at = time.monotonic() + if delay <= 0 or last_request_at is None: + return + remaining = delay - (time.monotonic() - last_request_at) + if remaining > 0: + time.sleep(remaining) + + def _note_clean_request(self) -> None: + """Gradually undo the throttle once requests stop tripping the limit.""" + if GitHubRestStream._secondary_rate_limit_delay <= 0: + return + GitHubRestStream._consecutive_clean_requests += 1 + if ( + GitHubRestStream._consecutive_clean_requests + >= self._DECAY_AFTER_CLEAN_REQUESTS + ): + GitHubRestStream._consecutive_clean_requests = 0 + halved = GitHubRestStream._secondary_rate_limit_delay / 2 + GitHubRestStream._secondary_rate_limit_delay = ( + halved if halved >= self._MIN_THROTTLE_DELAY else 0.0 + ) + + def _register_secondary_rate_limit_hit(self) -> None: + """Ratchet up the shared inter-request delay after a secondary rate limit. + + A single sleep-and-retry (see `validate_response`) only paces the one + request that got throttled; every other stream keeps firing at the + same rate that tripped the limit in the first place. This applies a + floor to the gap between *all* subsequent requests, tap-wide, until + enough clean requests go by to relax it again. + """ + GitHubRestStream._consecutive_clean_requests = 0 + current = GitHubRestStream._secondary_rate_limit_delay + new_delay = min( + self._MAX_THROTTLE_DELAY, + max(self._MIN_THROTTLE_DELAY, current * 2), + ) + GitHubRestStream._secondary_rate_limit_delay = new_delay + self.logger.warning( + "Secondary (frequency) rate limit hit - throttling all requests " + f"in this run to >= {new_delay:.1f}s apart until things settle down." + ) + + def _request( + self, + prepared_request: requests.PreparedRequest, + context: Context | None, + ) -> requests.Response: + """Pace requests around the parent implementation. + + Args: + prepared_request: The request to send. + context: Stream partition or context dictionary. + + Returns: + The HTTP response. + """ + self._throttle_before_request() + response = super()._request(prepared_request, context) + self._note_clean_request() + return response + def get_next_page_token( self, response: requests.Response, @@ -261,6 +340,7 @@ def validate_response(self, response: requests.Response) -> None: response.status_code == 403 and "secondary rate limit" in str(response.content).lower() ): + self._register_secondary_rate_limit_hit() # Wait about a minute and retry time.sleep(60 + 30 * random.random()) raise RetriableAPIError(msg, response) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 5d2cfff..0d2c6c5 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -3,6 +3,7 @@ from __future__ import annotations import http +import time from collections import defaultdict from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import parse_qs, urlparse @@ -272,9 +273,84 @@ def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: "name": context["repo"], "id": context["repo_id"], } + elif context is not None and ( + "organizations" in self.config or "searches" in self.config + ): + # `organizations`/`searches` mode sorts by `updated_at`, in a stable, + # endpoint-imposed order that the "since" param can't actually filter + # (neither endpoint supports it). Every run therefore restarts at the + # same front-of-list repos, so once heavy child streams (commits, + # reviews) exhaust the secondary rate limit, repos further down the + # list are never reached. We round-robin instead: resume right after + # the last repo whose children fully synced last run, wrapping + # around at the end. + yield from self._get_records_round_robin(context) else: yield from super().get_records(context) + def _get_records_round_robin(self, context: Context) -> Iterable[dict[str, Any]]: + """Yield this partition's repos starting after the last completed one. + + Traversal order is our own ascending sort by `id` (immutable), not the + API's `updated_at` order. `updated_at` shifts every run as repos see + real activity - the very thing this stream feeds downstream - so + resuming a position in an `updated_at`-sorted list keeps re-deriving a + different list to resume in. It never skips a repo (every repo the + API returns this run is still yielded exactly once), but each reshuffle + can push already-synced repos back in front of the resume point, + forcing a re-sync of their children before genuinely new repos are + reached. Sorting by `id` removes that churn entirely: new repos always + sort to the end (ids only increase), so the round-robin position is + undisturbed by anyone else's activity. + """ + partition_label = context.get("org") or context.get("search_name") or self.name + + all_records = list(super().get_records(context)) + if not all_records: + self.logger.info(f"[{self.name}:{partition_label}] round-robin summary: 0 repos returned by the API.") + return + + all_records.sort(key=lambda record: record["id"]) + + checkpoint = self.get_context_state(context).setdefault( + "round_robin_checkpoint", {} + ) + last_repo_id = checkpoint.get("last_enumerated_repo_id") + + start_index = 0 + if last_repo_id is not None: + repo_ids = [record["id"] for record in all_records] + if last_repo_id in repo_ids: + start_index = (repo_ids.index(last_repo_id) + 1) % len(all_records) + + ordered_records = all_records[start_index:] + all_records[:start_index] + started_at = time.monotonic() + processed = 0 + + # `finally` runs even on a mid-run crash: when an exception from + # child-stream syncing unwinds the outer sync loop, this generator is + # torn down (GeneratorExit thrown at the `yield` below) before the + # process exits, so the summary still gets logged either way. + try: + for record in ordered_records: + yield record + # Reached only once this record's child streams have fully synced + # (singer_sdk syncs children before pulling the next parent record). + checkpoint["last_enumerated_repo_id"] = record["id"] + checkpoint["last_enumerated_repo"] = record["full_name"] # for readability in state.json + processed += 1 + self._write_state_message() + finally: + full_lap = processed >= len(ordered_records) + self.logger.info( + f"[{self.name}:{partition_label}] round-robin summary: " + f"processed {processed}/{len(ordered_records)} repos in " + f"{time.monotonic() - started_at:.1f}s " + f"({'completed a full lap' if full_lap else 'stopped early - crash or run end'}); " + f"next run resumes after {checkpoint.get('last_enumerated_repo', 'n/a')} " + f"(id={checkpoint.get('last_enumerated_repo_id', 'n/a')})." + ) + schema = th.PropertiesList( th.Property("search_name", th.StringType), th.Property("search_query", th.StringType),