diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 08c97ab5f3..9c6bf37dea 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -81,8 +81,7 @@ jobs: # in .github/workflows/python-ci.yml to catch import-time regressions early. CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project" CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py" - # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support - CIBW_SKIP: "cp3*t-*" + CIBW_FREE_THREADED_SUPPORT: "true" - name: Add source distribution diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 86c9ef6885..c132300832 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -52,7 +52,7 @@ jobs: max-parallel: 15 fail-fast: true matrix: - python: ['3.10', '3.11', '3.12', '3.13', '3.14'] + python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t'] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/svn-build-artifacts.yml b/.github/workflows/svn-build-artifacts.yml index a6e2261c65..b25283a7a4 100644 --- a/.github/workflows/svn-build-artifacts.yml +++ b/.github/workflows/svn-build-artifacts.yml @@ -73,8 +73,7 @@ jobs: CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.15" CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project" CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py" - # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support - CIBW_SKIP: "cp3*t-*" + CIBW_FREE_THREADED_SUPPORT: "true" - name: Add source distribution if: matrix.os == 'ubuntu-latest' diff --git a/Makefile b/Makefile index d262de45a9..be7f077501 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,19 @@ else PYTHON_ARG = endif +# Extras that lack free-threaded (PEP 703) wheels +FREE_THREADED_INCOMPATIBLE_EXTRAS = bodo ray sql-postgres hive-kerberos datafusion + +# Detect free-threaded Python (e.g. PYTHON=3.14t) and exclude incompatible extras +ifneq ($(findstring t,$(PYTHON)),) + EXTRAS_ARG = --all-extras $(foreach extra,$(FREE_THREADED_INCOMPATIBLE_EXTRAS),--no-extra $(extra)) + # Ignore test files that import excluded extras at module level + FREE_THREADED_PYTEST_IGNORES = --ignore=tests/table/test_datafusion.py --ignore=tests/table/test_upsert.py -k "not kerberos" +else + EXTRAS_ARG = --all-extras + FREE_THREADED_PYTEST_IGNORES = +endif + ifeq ($(COVERAGE),1) TEST_RUNNER = uv run $(PYTHON_ARG) python -m coverage run --parallel-mode --source=pyiceberg -m else @@ -74,11 +87,11 @@ install-uv: ## Ensure uv is installed fi install: install-uv ## Install uv, dependencies, and pre-commit hooks - uv sync $(PYTHON_ARG) --all-extras + uv sync $(PYTHON_ARG) $(EXTRAS_ARG) @# Reinstall pyiceberg if Cython extensions (.so) are missing after `make clean` (see #2869) @if ! find pyiceberg -name "*.so" 2>/dev/null | grep -q .; then \ echo "Cython extensions not found, reinstalling pyiceberg..."; \ - uv sync $(PYTHON_ARG) --all-extras --reinstall-package pyiceberg; \ + uv sync $(PYTHON_ARG) $(EXTRAS_ARG) --reinstall-package pyiceberg; \ fi @# Install pre-commit hooks (skipped outside git repo, e.g. release tarballs) @if [ -d .git ]; then \ @@ -104,7 +117,7 @@ lint: ## Run code linters via prek (pre-commit hooks) ##@ Testing test: ## Run all unit tests (excluding integration) - $(TEST_RUNNER) pytest tests/ -m "(unmarked or parametrize) and not integration" $(PYTEST_ARGS) + $(TEST_RUNNER) pytest tests/ -m "(unmarked or parametrize) and not integration" $(FREE_THREADED_PYTEST_IGNORES) $(PYTEST_ARGS) test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests @@ -115,7 +128,7 @@ test-integration-setup: install ## Start Docker services for integration tests uv run $(PYTHON_ARG) python dev/provision.py test-integration-exec: ## Run integration tests (excluding provision) - $(TEST_RUNNER) pytest tests/ -m integration $(PYTEST_ARGS) + $(TEST_RUNNER) pytest tests/ -m integration $(FREE_THREADED_PYTEST_IGNORES) $(PYTEST_ARGS) test-integration-cleanup: ## Clean up integration test environment @if [ "${KEEP_COMPOSE}" != "1" ]; then \ diff --git a/pyiceberg/avro/decoder_fast.pyx b/pyiceberg/avro/decoder_fast.pyx index 52caec3308..f984c567f3 100644 --- a/pyiceberg/avro/decoder_fast.pyx +++ b/pyiceberg/avro/decoder_fast.pyx @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# cython: freethreading_compatible=True import cython from cython.cimports.cpython import array from pyiceberg.avro import STRUCT_DOUBLE, STRUCT_FLOAT diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index b59f43fbcd..01043668f7 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -28,6 +28,7 @@ More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses """ +import threading from typing import Any, ClassVar @@ -41,11 +42,14 @@ def _convert_to_hashable_type(element: Any) -> Any: class Singleton: _instances: ClassVar[dict] = {} # type: ignore + _lock: ClassVar[threading.Lock] = threading.Lock() def __new__(cls, *args, **kwargs): # type: ignore key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: - cls._instances[key] = super().__new__(cls) + with cls._lock: + if key not in cls._instances: + cls._instances[key] = super().__new__(cls) return cls._instances[key] def __deepcopy__(self, memo: dict[int, Any]) -> Any: diff --git a/pyproject.toml b/pyproject.toml index ac1177db44..0a2f3c2496 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,7 +148,7 @@ default-groups = [ ] [build-system] -requires = ["setuptools>=80", "wheel", "Cython>=3.0.0"] +requires = ["setuptools>=80", "wheel", "Cython>=3.1.0"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] diff --git a/setup.py b/setup.py index 34eee94bbd..156a9b6f7c 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ # under the License. import os +import sysconfig from setuptools import Extension, find_packages, setup from setuptools.command.sdist import sdist as _sdist @@ -62,10 +63,16 @@ def make_release_tree(self, base_dir: str, files: list[str]) -> None: ) ] + compiler_directives = {"language_level": "3"} + + # Enable free-threading support when building on free-threaded Python (PEP 703) + if sysconfig.get_config_var("Py_GIL_DISABLED"): + compiler_directives["freethreading_compatible"] = True # type: ignore[assignment] + ext_modules = cythonize( extensions, include_path=[package_path], - compiler_directives={"language_level": "3"}, + compiler_directives=compiler_directives, annotate=True, ) except Exception: diff --git a/uv.lock b/uv.lock index 5a3c46dc44..c0ee0d417b 100644 --- a/uv.lock +++ b/uv.lock @@ -1377,7 +1377,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -2222,10 +2222,10 @@ wheels = [ [[package]] name = "impi-rt" -version = "2021.17.0" +version = "2021.18.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/55/376f8c095326b10109d89e08eb46d05600c494d813df94300d65e94e5ab6/impi_rt-2021.17.0-py2.py3-none-win_amd64.whl", hash = "sha256:34993304c1034474f1b52b94e484807029d571e246c1c842bab2dce44252c20a", size = 17439414, upload-time = "2025-10-22T17:57:07.971Z" }, + { url = "https://files.pythonhosted.org/packages/39/06/6527fb776213f3fd1432b84f35421e3c848c1b4dc40ec9f38aafeabce7fd/impi_rt-2021.18.0-py2.py3-none-win_amd64.whl", hash = "sha256:344eb2c1ec364e6f060ca97481e51d913699e75934f52f0ceacfa87122047bb4", size = 21846717, upload-time = "2026-04-24T14:14:26.77Z" }, ] [[package]] @@ -3792,11 +3792,11 @@ wheels = [ [[package]] name = "openmpi" -version = "5.0.9" +version = "5.0.10" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/ed/51c9a4c0b56a0a41e28d5282d0d1621910054266a54f36bbf0a249536c07/openmpi-5.0.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:18d38be62afa692854b0f3c575a589c191651fe30b7478c5f1333f3f2de0d3d7", size = 3145187, upload-time = "2025-11-14T16:11:59.939Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ff/b16d51f4e282c4629816f60bda56ebfed0d483a7e168f3b50576c8ab6714/openmpi-5.0.9-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:a0d8c35c6ced31253c8e27c2dfb6900ba6cb89da65d7e2f56ac2d8658271b507", size = 3571687, upload-time = "2025-11-14T16:12:02.242Z" }, + { url = "https://files.pythonhosted.org/packages/65/48/47b429693ecfb64bb97caead4ef274a09e1c960242a4239929c942c6ad46/openmpi-5.0.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6e9cd0cb0c29abd5157449343c93d31906f4aaaabf9e84fff5da67d305d1d55b", size = 3150518, upload-time = "2026-02-25T10:48:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/87/16/5f6260bd68e8bd1ed4939ebb3e3fda89e874426ee3d62db9e2e56bc7242f/openmpi-5.0.10-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:f4f524543cf443072b79b1600a71fa0019e20c2ff3291968092f4f0552a17829", size = 3527026, upload-time = "2026-02-25T10:49:01.444Z" }, ] [[package]]