diff --git a/sdks/python/apache_beam/examples/dataframe/flight_delays.py b/sdks/python/apache_beam/examples/dataframe/flight_delays.py index c2c467b39eae..4ba4b77fa9dd 100644 --- a/sdks/python/apache_beam/examples/dataframe/flight_delays.py +++ b/sdks/python/apache_beam/examples/dataframe/flight_delays.py @@ -44,7 +44,7 @@ def get_mean_delay_at_top_airports(airline_df): top_airports = total.nlargest(10, keep='all').dropna() at_top_airports = airline_df['arrival_airport'].isin( top_airports.index.values) - return airline_df[at_top_airports].mean() + return airline_df[at_top_airports].mean(numeric_only=True) def input_date(date): diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 4310daf6fea8..529c3ee38bd9 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -154,7 +154,7 @@ def cythonize(*args, **kwargs): # Exclude 1.5.0 and 1.5.1 because of # https://github.com/pandas-dev/pandas/issues/45725 dataframe_dependency = [ - 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"', + 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.1;python_version>="3.8"', ] def find_by_ext(root_dir, ext): @@ -323,13 +323,13 @@ def get_portability_package_data(): # Pinning docutils as a workaround for Sphinx issue: # https://github.com/sphinx-doc/sphinx/issues/9727 'docutils==0.17.1', - 'pandas<2.0.0', + 'pandas<2.1.0', ], 'test': [ 'freezegun>=0.3.12', 'joblib>=1.0.1', 'mock>=1.0.1,<6.0.0', - 'pandas<2.0.0', + 'pandas<2.1.0', 'parameterized>=0.7.1,<0.10.0', 'pyhamcrest>=1.9,!=1.10.0,<3.0.0', 'pyyaml>=3.12,<7.0.0', diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index b1ed5f88c7c9..f01caa3607d0 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -89,6 +89,10 @@ toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}" test.dependsOn "testPy38pandas-15" preCommitPyCoverage.dependsOn "testPy38pandas-15" +toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}" +test.dependsOn "testPy38pandas-20" +preCommitPyCoverage.dependsOn "testPy38pandas-20" + // Create a test task for each minor version of pytorch toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}" test.dependsOn "testPy38pytorch-19" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index e4cf09cacba4..ab8f080a55bc 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -270,12 +270,18 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs} + [testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}] deps = + # Pandas 2 minimum for pyarrow is 7 3: pyarrow>=3,<4 + 3: pandas<2 4: pyarrow>=4,<5 + 4: pandas<2 5: pyarrow>=5,<6 + 5: pandas<2 6: pyarrow>=6,<7 + 6: pandas<2 7: pyarrow>=7,<8 8: pyarrow>=8,<9 9: pyarrow>=9,<10 @@ -300,11 +306,13 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py{38,39,310,311}-pandas-{14,15}] +[testenv:py{38,39,310,311}-pandas-{14,15,20}] deps = 14: pandas>=1.4.3,<1.5.0 # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725 15: pandas>=1.5.2,<1.6.0 + 20: pandas>=2.0.0,<2.1.0 + 20: pyarrow>=7 commands = # Log pandas and numpy version for debugging /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"