Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .aws-architecture
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
linux/arm64
10 changes: 7 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
name: CI
on: push
permissions: read-all
on:
pull_request:
paths-ignore:
- '.github/**'
jobs:
test:
uses: mitlibraries/.github/.github/workflows/python-shared-test.yml@main
uses: mitlibraries/.github/.github/workflows/python-uv-shared-test.yml@main
lint:
uses: mitlibraries/.github/.github/workflows/python-shared-lint.yml@main
uses: mitlibraries/.github/.github/workflows/python-uv-shared-lint.yml@main
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ cython_debug/
#.idea/

.DS_Store
.arch_tag
output/
.vscode/
.idea/
25 changes: 11 additions & 14 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,29 +1,26 @@
default_language_version:
python: python3.11 # set for project python version
python: python3.13
default_stages:
- pre-push
repos:
- repo: local
hooks:
- id: black-apply
name: black-apply
entry: pipenv run black
- id: ruff-format
name: ruff-format
entry: uv run ruff format --diff
language: system
pass_filenames: true
types: ["python"]
- id: mypy
name: mypy
entry: pipenv run mypy
entry: uv run mypy
language: system
pass_filenames: true
types: ["python"]
exclude: "tests/"
- id: ruff-apply
name: ruff-apply
entry: pipenv run ruff check --fix
exclude: "(tests/|output/|migrations/)"
- id: ruff-check
name: ruff-check
entry: uv run ruff check
language: system
pass_filenames: true
types: ["python"]
- id: pip-audit
name: pip-audit
entry: pipenv run pip-audit
language: system
pass_filenames: false
2 changes: 1 addition & 1 deletion .python-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.12
3.13
25 changes: 18 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,23 @@
FROM python:3.12-slim AS build
FROM python:3.13-slim

RUN apt-get update && \
apt-get install -y --no-install-recommends git ca-certificates && \
rm -rf /var/lib/apt/lists/*

COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv

ENV UV_SYSTEM_PYTHON=1

WORKDIR /app
COPY . .

RUN pip install --no-cache-dir --upgrade pip pipenv
# Copy project metadata
COPY pyproject.toml uv.lock* ./

RUN apt-get update && apt-get upgrade -y && apt-get install -y git
# Copy source
COPY tim ./tim

COPY Pipfile* /
RUN pipenv install
# Install package into system python, includes entry point script
RUN uv pip install --system .

ENTRYPOINT ["pipenv", "run", "tim"]
ENTRYPOINT ["tim"]
CMD []
197 changes: 128 additions & 69 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,97 +1,156 @@
### This is the Terraform-generated header for timdex-index-manager-dev ###
### This is the Terraform-generated header for timdex-index-manager-dev. If ###
### this is a Lambda repo, uncomment the FUNCTION line below ###
### and review the other commented lines in the document. ###
.PHONY: test
ECR_NAME_DEV:=timdex-index-manager-dev
ECR_URL_DEV:=222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-index-manager-dev
### End of Terraform-generated header ###
ECR_NAME_DEV := timdex-index-manager-dev
ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-index-manager-dev
CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64")
### End of Terraform-generated header ###

SHELL=/bin/bash
DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)

help: # preview Makefile commands
help: # Preview Makefile commands
@awk 'BEGIN { FS = ":.*#"; print "Usage: make <target>\n\nTargets:" } \
/^[-_[:alpha:]]+:.?*#/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)

#######################
# Dependency commands
#######################
# ensure OS binaries aren't called if naming conflict with Make recipes
.PHONY: help install venv update test coveralls lint lint-fix security tim

install: # Install Python dependencies
pipenv install --dev
pipenv run pre-commit install
##############################################
# Python Environment and Dependency commands
##############################################

update: install # Update Python dependencies
pipenv clean
pipenv update git+https://github.com/MITLibraries/timdex-dataset-api.git
pipenv update --dev
install: .venv .git/hooks/pre-commit .git/hooks/pre-push # Install Python dependencies and create virtual environment if not exists
uv sync --dev

.venv: # Creates virtual environment if not found
@echo "Creating virtual environment at .venv..."
uv venv .venv

.git/hooks/pre-commit: # Sets up pre-commit commit hooks if not setup
@echo "Installing pre-commit commit hooks..."
uv run pre-commit install --hook-type pre-commit

.git/hooks/pre-push: # Sets up pre-commit push hooks if not setup
@echo "Installing pre-commit push hooks..."
uv run pre-commit install --hook-type pre-push

venv: .venv # Create the Python virtual environment

update: # Update Python dependencies
uv lock --upgrade
uv sync --dev

######################
# Unit test commands
######################

test: # Run tests and print a coverage report
pipenv run coverage run --source=tim -m pytest -vv
pipenv run coverage report -m
uv run coverage run --source=tim -m pytest -vv
uv run coverage report -m

coveralls: test # Write coverage data to an LCOV report
pipenv run coverage lcov -o ./coverage/lcov.info
uv run coverage lcov -o ./coverage/lcov.info

####################################
# Code quality and safety commands
# Code linting and formatting
####################################

lint: black mypy ruff safety # Run linters

black: # Run 'black' linter and print a preview of suggested changes
pipenv run black --check --diff .

mypy: # Run 'mypy' linter
pipenv run mypy .
lint: # Run linting, alerts only, no code changes
uv run ruff format --diff
uv run mypy .
uv run ruff check .

ruff: # Run 'ruff' linter and print a preview of errors
pipenv run ruff check .
lint-fix: # Run linting, auto fix behaviors where supported
uv run ruff format .
uv run ruff check --fix .

safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-date
pipenv run pip-audit
pipenv verify
security: # Run security / vulnerability checks
uv run pip-audit

lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'

black-apply: # Apply changes with 'black'
pipenv run black .

ruff-apply: # Resolve 'fixable errors' with 'ruff'
pipenv run ruff check --fix .

## Terraform-generated commands for container build and deployment in dev
dist-dev: # build docker container (intended for developer-based manual build)
docker build --platform linux/amd64 \
-t $(ECR_URL_DEV):latest \
-t $(ECR_URL_DEV):`git describe --always` \
-t $(ECR_NAME_DEV):latest .

publish-dev: dist-dev # build, tag and push (intended for developer-based manual publish)
docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_DEV)
docker push $(ECR_URL_DEV):latest
docker push $(ECR_URL_DEV):`git describe --always`

## Terraform-generated commands for container build and deployment in stage \
This requires that ECR_NAME_STAGE and ECR_URL_STAGE environment variables \
are set locally by the developer and that the developer has \
authenticated to the correct AWS Account. The values for the environment \
variables can be found in the stage_build.yml caller workflow. \
While Stage should generally only be used in an emergency for most repos, \
it is necessary for any testing requiring access to the Data Warehouse \
because Cloud Connector is not enabled on Dev1.
dist-stage:
docker build --platform linux/amd64 \
-t $(ECR_URL_STAGE):latest \
-t $(ECR_URL_STAGE):`git describe --always` \
-t $(ECR_NAME_STAGE):latest .
##############################
# CLI convenience commands
##############################

publish-stage:
docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_STAGE)
docker push $(ECR_URL_STAGE):latest
docker push $(ECR_URL_STAGE):`git describe --always`
tim: # CLI without any arguments, utilizing uv script entrypoint
uv run tim

#############
# Terraform
#############

### Terraform-generated Developer Deploy Commands for Dev environment ###
check-arch:
@ARCH_FILE=".aws-architecture"; \
if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \
echo "Invalid CPU_ARCH: $(CPU_ARCH)"; exit 1; \
fi; \
if [[ -f $$ARCH_FILE ]]; then \
echo "latest-$(shell echo $(CPU_ARCH) | cut -d'/' -f2)" > .arch_tag; \
else \
echo "latest" > .arch_tag; \
fi

dist-dev: check-arch # Build docker container (intended for developer-based manual build)
@ARCH_TAG=$$(cat .arch_tag); \
docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \
docker buildx use $(ECR_NAME_DEV); \
docker buildx build --platform $(CPU_ARCH) \
--load \
--tag $(ECR_URL_DEV):$$ARCH_TAG \
--tag $(ECR_URL_DEV):make-$$ARCH_TAG \
--tag $(ECR_URL_DEV):make-$(shell git describe --always) \
--tag $(ECR_NAME_DEV):$$ARCH_TAG \
.

publish-dev: dist-dev # Build, tag and push (intended for developer-based manual publish)
@ARCH_TAG=$$(cat .arch_tag); \
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \
docker push $(ECR_URL_DEV):$$ARCH_TAG; \
docker push $(ECR_URL_DEV):make-$$ARCH_TAG; \
docker push $(ECR_URL_DEV):make-$(shell git describe --always); \
echo "Cleaning up dangling Docker images..."; \
docker image prune -f --filter "dangling=true"

### Terraform-generated Developer Deploy Commands for Stage environment ###
## This requires that ECR_NAME_STAGE and ECR_URL_STAGE environment variables
## are set locally by the developer and that the developer has
## authenticated to the correct AWS Account. The values for the environment
## variables can be found in the stage_build.yml caller workflow.
## While Stage should generally only be used in an emergency for most repos,
## it is necessary for any testing requiring access to the Data Warehouse
## because Cloud Connector is not enabled on Dev1.
dist-stage: check-arch
@ARCH_TAG=$$(cat .arch_tag); \
docker buildx inspect $(ECR_NAME_STAGE) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_STAGE) --use; \
docker buildx use $(ECR_NAME_STAGE); \
docker buildx build --platform $(CPU_ARCH) \
--load \
--tag $(ECR_URL_STAGE):$$ARCH_TAG \
--tag $(ECR_URL_STAGE):make-$$ARCH_TAG \
--tag $(ECR_URL_STAGE):make-$(shell git describe --always) \
--tag $(ECR_NAME_STAGE):$$ARCH_TAG \
.

publish-stage: dist-stage
@ARCH_TAG=$$(cat .arch_tag); \
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_STAGE); \
docker push $(ECR_URL_STAGE):$$ARCH_TAG; \
docker push $(ECR_URL_STAGE):make-$$ARCH_TAG; \
docker push $(ECR_URL_STAGE):make-$(shell git describe --always); \
echo "Cleaning up dangling Docker images..."; \
docker image prune -f --filter "dangling=true"

docker-clean: # Clean up Docker detritus
@ARCH_TAG=$$(cat .arch_tag); \
echo "Cleaning up Docker leftovers (containers, images, builders)"; \
docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \
docker rmi -f $(ECR_URL_DEV):make-$$ARCH_TAG; \
docker rmi -f $(ECR_URL_DEV):make-$(shell git describe --always) || true; \
docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \
docker buildx rm $(ECR_NAME_DEV) || true
@rm -rf .arch_tag

##############################
# Local Opensearch commands
Expand All @@ -104,4 +163,4 @@ local-opensearch-stop: # Stop local instance of Opensearch
docker compose --env-file .env stop

local-opensearch-teardown: # Teardown local instance of Opensearch (includes data volume)
docker compose --env-file .env down -v
docker compose --env-file .env down -v
33 changes: 0 additions & 33 deletions Pipfile

This file was deleted.

Loading