diff --git a/Dockerfile b/Dockerfile index 2f07154..238564e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bookworm +FROM python:3.10-slim-alpine WORKDIR / RUN apt-get update RUN apt-get upgrade -y diff --git a/bin/transform_resources.py b/bin/transform_resources.py index 5358421..25f37f8 100755 --- a/bin/transform_resources.py +++ b/bin/transform_resources.py @@ -2,7 +2,7 @@ import sys import click from pathlib import Path -from multiprocessing import Pool, cpu_count +from multiprocessing import get_context, cpu_count from tqdm import tqdm from digital_land import __version__ as dl_version @@ -22,6 +22,10 @@ logger = logging.getLogger(__name__) +def _init_worker(log_level): + logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s: %(message)s') + + def process_single_resource(args): """Process a single resource through the digital-land pipeline using Python library directly. @@ -218,7 +222,12 @@ def process_resources( failed = 0 errors = [] - with Pool(processes=max_workers) as pool: + log_level = logging.getLogger().getEffectiveLevel() + with get_context("spawn").Pool( + processes=max_workers, + initializer=_init_worker, + initargs=(log_level,), + ) as pool: if use_progress_bar: # Interactive mode with progress bar results = list(tqdm( diff --git a/requirements.txt b/requirements.txt index 972b5e6..87fcd42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ --e git+https://github.com/digital-land/pipeline.git@main#egg=digital-land +-e git+https://github.com/digital-land/pipeline.git@fix/remove-dask#egg=digital-land -e . tqdm cloudpathlib[s3] \ No newline at end of file