dependencies.yaml

# Dependency list for https://github.com/rapidsai/dependency-file-generator
files:
  all:
    output: conda
    matrix:
      cuda: ["11.8", "12.2"]
      arch: [x86_64]
    includes:
      - build_base
      - build_all
      - build_cpp
      - build_python_common
      - build_python_cudf
      - cuda
      - cuda_version
      - develop
      - docs
      - libarrow_build
      - notebooks
      - py_version
      - rapids_build_skbuild
      - rapids_build_setuptools
      - run_common
      - run_cudf
      - run_dask_cudf
      - run_custreamz
      - test_cpp
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
      - depends_on_cupy
  test_static_build:
    output: none
    includes:
      - build_base
  test_cpp:
    output: none
    includes:
      - cuda_version
      - libarrow_run
      - test_cpp
  test_python:
    output: none
    includes:
      - cuda_version
      - py_version
      - pyarrow_run
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
  test_java:
    output: none
    includes:
      - build_base
      - build_all
      - cuda
      - cuda_version
      - libarrow_run
      - test_java
  test_notebooks:
    output: none
    includes:
      - cuda_version
      - notebooks
      - py_version
  checks:
    output: none
    includes:
      - develop
      - py_version
  docs:
    output: none
    includes:
      - cuda
      - cuda_version
      - docs
      - libarrow_run
      - py_version
  py_rapids_build_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: build-system
    includes:
      - rapids_build_skbuild
  py_build_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: tool.rapids-build-backend
      key: requires
    includes:
      - build_base
      - build_python_common
      - build_python_cudf
  py_run_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project
    includes:
      - run_common
      - run_cudf
      - pyarrow_run
      - depends_on_cupy
  py_test_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_cudf
  py_test_pandas_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project.optional-dependencies
      key: pandas-tests
    includes:
      - test_python_pandas_cudf
  py_test_cudf_pandas:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project.optional-dependencies
      key: cudf-pandas-tests
    includes:
      - test_python_cudf_pandas
  py_rapids_build_cudf_polars:
    output: pyproject
    pyproject_dir: python/cudf_polars
    extras:
      table: build-system
    includes:
      - rapids_build_setuptools
  py_run_cudf_polars:
    output: pyproject
    pyproject_dir: python/cudf_polars
    extras:
      table: project
    includes:
      - run_cudf_polars
      - depends_on_cudf
  py_test_cudf_polars:
    output: pyproject
    pyproject_dir: python/cudf_polars
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
  py_build_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: build-system
    includes:
      - rapids_build_setuptools
  py_run_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project
    includes:
      - run_common
      - run_dask_cudf
      - depends_on_cudf
      - depends_on_cupy
  py_test_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_dask_cudf
  py_rapids_build_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: build-system
    includes:
      - rapids_build_skbuild
  py_build_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: tool.rapids-build-backend
      key: requires
    includes:
      - build_base
      - build_python_common
  py_run_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project
    includes:
      - depends_on_cudf
  py_test_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
  py_build_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: build-system
    includes:
      - rapids_build_setuptools
  py_run_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project
    includes:
      - run_custreamz
      - depends_on_cudf
      - depends_on_cudf_kafka
  py_test_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
channels:
  - rapidsai
  - rapidsai-nightly
  - dask/label/dev
  - conda-forge
  - nvidia
dependencies:
  build_base:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - &cmake_ver cmake>=3.26.4,!=3.30.0
          - &ninja ninja
  build_all:
    common:
      - output_types: conda
        packages:
          - c-compiler
          - cxx-compiler
          - dlpack>=0.8,<1.0
          - zlib>=1.2.13
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              - gcc_linux-64=11.*
              - sysroot_linux-64==2.17
          - matrix:
              arch: aarch64
            packages:
              - gcc_linux-aarch64=11.*
              - sysroot_linux-aarch64==2.17
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.*"
            packages:
              - cuda-nvcc
          - matrix:
              arch: x86_64
              cuda: "11.8"
            packages:
              - nvcc_linux-64=11.8
          - matrix:
              arch: aarch64
              cuda: "11.8"
            packages:
              - nvcc_linux-aarch64=11.8
  build_cpp:
    common:
      - output_types: conda
        packages:
          - fmt>=10.1.1,<11
          - librmm==24.8.*,>=0.0.0a0
          - libkvikio==24.8.*,>=0.0.0a0
          - librdkafka>=1.9.0,<1.10.0a0
          # Align nvcomp version with rapids-cmake
          - nvcomp==3.0.6
          - spdlog>=1.12.0,<1.13
  rapids_build_skbuild:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - &rapids_build_backend rapids-build-backend>=0.3.0,<0.4.0.dev0
      - output_types: conda
        packages:
          - scikit-build-core>=0.7.0
      - output_types: [requirements, pyproject]
        packages:
          - scikit-build-core[pyproject]>=0.7.0
  rapids_build_setuptools:
    common:
      - output_types: [requirements, pyproject]
        packages:
          - *rapids_build_backend
          - setuptools
          - wheel
  build_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cython>=3.0.3
          # Hard pin the patch version used during the build. This must be kept
          # in sync with the version pinned in get_arrow.cmake.
          - pyarrow==16.1.0.*
      - output_types: pyproject
        packages:
          # Hard pin the patch version used during the build.
          # Sync with conda build constraint & wheel run constraint.
          - numpy==1.23.*
  build_python_cudf:
    common:
      - output_types: conda
        packages:
          - &rmm_conda rmm==24.8.*,>=0.0.0a0
          - pip
          - pip:
              - git+https://github.com/python-streamz/streamz.git@master
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm-cu{11,12}.
          - --extra-index-url=https://pypi.nvidia.com
          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
          - git+https://github.com/python-streamz/streamz.git@master
    specific:
      - output_types: [requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages: &build_python_packages_cu12
              - rmm-cu12==24.8.*,>=0.0.0a0
          - matrix: {cuda: "11.*"}
            packages: &build_python_packages_cu11
              - rmm-cu11==24.8.*,>=0.0.0a0
          - {matrix: null, packages: [*rmm_conda] }
  libarrow_build:
    common:
      - output_types: conda
        packages:
          # Hard pin the Arrow patch version used during the build. This must
          # be kept in sync with the version pinned in get_arrow.cmake.
          - libarrow-acero==16.1.0.*
          - libarrow-dataset==16.1.0.*
          - libarrow==16.1.0.*
          - libparquet==16.1.0.*
  libarrow_run:
    common:
      - output_types: conda
        packages:
          # Allow runtime version to float up to patch version
          - libarrow-acero>=16.1.0,<16.2.0a0
          - libarrow-dataset>=16.1.0,<16.2.0a0
          - libarrow>=16.1.0,<16.2.0a0
          - libparquet>=16.1.0,<16.2.0a0
  pyarrow_run:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          # Allow runtime version to float up to patch version
          - pyarrow>=16.1.0,<16.2.0a0
  cuda_version:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "11.2"
            packages:
              - cuda-version=11.2
          - matrix:
              cuda: "11.4"
            packages:
              - cuda-version=11.4
          - matrix:
              cuda: "11.5"
            packages:
              - cuda-version=11.5
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-version=11.8
          - matrix:
              cuda: "12.0"
            packages:
              - cuda-version=12.0
          - matrix:
              cuda: "12.2"
            packages:
              - cuda-version=12.2
  cuda:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.*"
            packages:
              - cuda-cudart-dev
              - cuda-nvrtc-dev
              - cuda-nvtx-dev
              - libcurand-dev
          - matrix:
              cuda: "11.8"
            packages:
              - cudatoolkit
              - cuda-nvtx=11.8
              - libcurand-dev=10.3.0.86
              - libcurand=10.3.0.86
          - matrix:
              cuda: "11.5"
            packages:
              - cudatoolkit
              - cuda-nvtx=11.5
                # Can't hard pin the version since 11.x is missing many
                # packages for specific versions
              - libcurand-dev>=10.2.6.48,<=10.2.7.107
              - libcurand>=10.2.6.48,<=10.2.7.107
          - matrix:
              cuda: "11.4"
            packages:
              - cudatoolkit
              - &cudanvtx114 cuda-nvtx=11.4
              - &libcurand_dev114 libcurand-dev>=10.2.5.43,<=10.2.5.120
              - &libcurand114 libcurand>=10.2.5.43,<=10.2.5.120
          - matrix:
              cuda: "11.2"
            packages:
              - cudatoolkit
                # The NVIDIA channel doesn't publish pkgs older than 11.4 for
                # these libs, so 11.2 uses 11.4 packages (the oldest
                # available).
              - *cudanvtx114
              - *libcurand_dev114
              - *libcurand114
      - output_types: conda
        matrices:
          - matrix:
              arch: aarch64
            packages:
          - matrix:
              cuda: "12.*"
              arch: x86_64
            packages:
              - libcufile-dev
          - matrix:
              cuda: "11.8"
              arch: x86_64
            packages:
              - libcufile=1.4.0.31
              - libcufile-dev=1.4.0.31
          - matrix:
              cuda: "11.5"
              arch: x86_64
            packages:
              - libcufile>=1.1.0.37,<=1.1.1.25
              - libcufile-dev>=1.1.0.37,<=1.1.1.25
          - matrix:
              cuda: "11.4"
              arch: x86_64
            packages:
              - &libcufile_114 libcufile>=1.0.0.82,<=1.0.2.10
              - &libcufile_dev114 libcufile-dev>=1.0.0.82,<=1.0.2.10
          - matrix:
              cuda: "11.2"
              arch: x86_64
            packages:
              # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs,
              # so 11.2 uses 11.4 packages (the oldest available).
              - *libcufile_114
              - *libcufile_dev114
  develop:
    common:
      - output_types: [conda, requirements]
        packages:
          - pre-commit
          # pre-commit requires identify minimum version 1.0, but clang-format requires textproto support and that was
          # added in 2.5.20, so we need to call out the minimum version needed for our plugins
          - identify>=2.5.20
      - output_types: conda
        packages:
          - clang==16.0.6
          - clang-tools=16.0.6
          - &doxygen doxygen=1.9.1 # pre-commit hook needs a specific version.
  docs:
    common:
      - output_types: [conda]
        packages:
          - breathe>=4.35.0
          - dask-cuda==24.8.*,>=0.0.0a0
          - *doxygen
          - make
          - myst-nb
          - nbsphinx
          - numpydoc
          - pandoc
          # https://github.com/pydata/pydata-sphinx-theme/issues/1539
          - pydata-sphinx-theme!=0.14.2
          - scipy
          - sphinx
          - sphinx-autobuild
          - sphinx-copybutton
          - sphinx-markdown-tables
          - sphinx-remove-toctrees
          - sphinxcontrib-websupport
  notebooks:
    common:
      - output_types: [conda, requirements]
        packages:
          - ipython
          - notebook
          - scipy
  py_version:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              py: "3.9"
            packages:
              - python=3.9
          - matrix:
              py: "3.10"
            packages:
              - python=3.10
          - matrix:
              py: "3.11"
            packages:
              - python=3.11
          - matrix:
            packages:
              - python>=3.9,<3.12
  run_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - fsspec>=0.6.0
          - numpy>=1.23,<2.0a0
          - pandas>=2.0,<2.2.3dev0
  run_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cachetools
          - &numba numba>=0.57
          - nvtx>=0.2.1
          - packaging
          - rich
          - typing_extensions>=4.0.0
      - output_types: conda
        packages:
          - *rmm_conda
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm, cubinlinker, ptxcompiler.
          - --extra-index-url=https://pypi.nvidia.com
          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
    specific:
      - output_types: [conda, requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - cuda-python>=12.0,<13.0a0
          - matrix: {cuda: "11.*"}
            packages: &run_cudf_packages_all_cu11
              - cuda-python>=11.7.1,<12.0a0
          - {matrix: null, packages: *run_cudf_packages_all_cu11}
      - output_types: conda
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - pynvjitlink>=0.0.0a0
          - matrix: {cuda: "11.*"}
            packages:
              - cubinlinker
              - ptxcompiler
      - output_types: [requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - rmm-cu12==24.8.*,>=0.0.0a0
              - pynvjitlink-cu12>=0.0.0a0
          - matrix: {cuda: "11.*"}
            packages:
              - rmm-cu11==24.8.*,>=0.0.0a0
              - cubinlinker-cu11
              - ptxcompiler-cu11
          - {matrix: null, packages: [cubinlinker, ptxcompiler, *rmm_conda]}
  run_cudf_polars:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - polars>=1.0
  run_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - rapids-dask-dependency==24.8.*,>=0.0.0a0
  run_custreamz:
    common:
      - output_types: conda
        packages:
          - python-confluent-kafka>=1.9.0,<1.10.0a0
      - output_types: [conda, requirements, pyproject]
        packages:
          - streamz
      - output_types: [requirements, pyproject]
        packages:
          - confluent-kafka>=1.9.0,<1.10.0a0
  test_cpp:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.*"
            packages:
              - cuda-sanitizer-api
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-sanitizer-api=11.8.86
          - matrix:  # Fallback for CUDA 11 or no matrix
            packages:
  test_java:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
          - maven
          - openjdk=8.*
  test_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - pytest<8
          - pytest-cov
          - pytest-xdist
  test_python_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cramjam
          - fastavro>=0.22.9
          - hypothesis
          - pytest-benchmark
          - pytest-cases>=3.8.2
          - scipy
      - output_types: conda
        packages:
          - aiobotocore>=2.2.0
          - boto3>=1.21.21
          - botocore>=1.24.21
          - msgpack-python
          - moto>=4.0.8
          - s3fs>=2022.3.0
      - output_types: pyproject
        packages:
          - msgpack
          - &tokenizers tokenizers==0.15.2
          - &transformers transformers==4.39.3
          - tzdata
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              # Currently, CUDA + aarch64 builds of pytorch do not exist on conda-forge.
              - pytorch>=2.1.0
              # We only install these on x86_64 to avoid pulling pytorch as a
              # dependency of transformers.
              - *tokenizers
              - *transformers
          - matrix:
            packages:
  test_python_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - dask-cuda==24.8.*,>=0.0.0a0
          - *numba
  depends_on_cudf:
    common:
      - output_types: conda
        packages:
          - &cudf_conda cudf==24.8.*,>=0.0.0a0
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm, cubinlinker, ptxcompiler.
          - --extra-index-url=https://pypi.nvidia.com
          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
    specific:
      - output_types: [requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - cudf-cu12==24.8.*,>=0.0.0a0
          - matrix: {cuda: "11.*"}
            packages:
              - cudf-cu11==24.8.*,>=0.0.0a0
          - {matrix: null, packages: [*cudf_conda]}
  depends_on_cudf_kafka:
    common:
      - output_types: conda
        packages:
          - &cudf_kafka_conda cudf_kafka==24.8.*,>=0.0.0a0
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm, cubinlinker, ptxcompiler.
          - --extra-index-url=https://pypi.nvidia.com
          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
    specific:
      - output_types: [requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - cudf_kafka-cu12==24.8.*,>=0.0.0a0
          - matrix: {cuda: "11.*"}
            packages:
              - cudf_kafka-cu11==24.8.*,>=0.0.0a0
          - {matrix: null, packages: [*cudf_kafka_conda]}
  depends_on_cupy:
    common:
      - output_types: conda
        packages:
          - cupy>=12.0.0
    specific:
      - output_types: [requirements, pyproject]
        matrices:
          - matrix: {cuda: "12.*"}
            packages:
              - cupy-cuda12x>=12.0.0
          - matrix: {cuda: "11.*"}
            packages: &cupy_packages_cu11
              - cupy-cuda11x>=12.0.0
          - {matrix: null, packages: *cupy_packages_cu11}
  test_python_pandas_cudf:
    common:
      - output_types: [requirements, pyproject]
        packages:
          # dependencies to run pandas tests
          # https://github.com/pandas-dev/pandas/blob/main/environment.yml
          # pandas[...] includes all of the required dependencies.
          # Intentionally excluding `postgresql` because of
          # installation issues with `psycopg2`.
          - pandas[test, pyarrow, performance, computation, fss, excel, parquet, feather, hdf5, spss, html, xml, plot, output-formatting, clipboard, compression]
          - pytest-reportlog
  test_python_cudf_pandas:
    common:
      - output_types: [requirements, pyproject]
        packages:
          - ipython
          - openpyxl