From c0763d2066e4b668e5a78ac9d9745856530e5a75 Mon Sep 17 00:00:00 2001
From: Christopher Neffshade <caneff@google.com>
Date: Sun, 24 Sep 2023 22:24:13 +0000
Subject: [PATCH 1/3] Add support for pandas 2.0

---
 sdks/python/setup.py                          | 6 +++---
 sdks/python/test-suites/tox/py38/build.gradle | 4 ++++
 sdks/python/tox.ini                           | 3 ++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 917e4eaba940..87a3045375aa 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -148,7 +148,7 @@ def cythonize(*args, **kwargs):
 # Exclude 1.5.0 and 1.5.1 because of
 # https://github.com/pandas-dev/pandas/issues/45725
 dataframe_dependency = [
-    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"',
+    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.1;python_version>="3.8"',
 ]
 
 
@@ -285,13 +285,13 @@ def get_portability_package_data():
               # Pinning docutils as a workaround for Sphinx issue:
               # https://github.com/sphinx-doc/sphinx/issues/9727
               'docutils==0.17.1',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
           ],
           'test': [
               'freezegun>=0.3.12',
               'joblib>=1.0.1',
               'mock>=1.0.1,<6.0.0',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
               'parameterized>=0.7.1,<0.10.0',
               'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
               'pyyaml>=3.12,<7.0.0',
diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
index 208a1d9d39ca..ab9ce6dddc26 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -91,6 +91,10 @@ toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
 test.dependsOn "testPy38pandas-15"
 preCommitPyCoverage.dependsOn "testPy38pandas-15"
 
+toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}"
+test.dependsOn "testPy38pandas-20"
+preCommitPyCoverage.dependsOn "testPy38pandas-20"
+
 // Create a test task for each minor version of pytorch
 toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
 test.dependsOn "testPy38pytorch-19"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 1caf25caf080..9aeaded00d13 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -316,11 +316,12 @@ commands =
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
 
 
-[testenv:py{38,39,310,311}-pandas-{14,15}]
+[testenv:py{38,39,310,311}-pandas-{14,15,20}]
 deps =
   14: pandas>=1.4.3,<1.5.0
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
+  20: pandas>=2.0.0,<2.1.0
 commands =
   # Log pandas and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"

From 95f076e010bb5013487bad33f4e721a7ff2d2c65 Mon Sep 17 00:00:00 2001
From: Christopher Neffshade <caneff@google.com>
Date: Tue, 26 Sep 2023 18:33:39 +0000
Subject: [PATCH 2/3] Fix pyarrow tests for pandas 2 compat

---
 sdks/python/tox.ini | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 9aeaded00d13..a1bbc8001f51 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -177,7 +177,7 @@ deps =
   holdup==1.8.0
 extras =
   gcp
-allowlist_externals = 
+allowlist_externals =
   bash
   echo
   sleep
@@ -210,7 +210,7 @@ deps =
 extras =
   azure
 passenv = REQUESTS_CA_BUNDLE
-allowlist_externals = 
+allowlist_externals =
   wget
   az
   bash
@@ -286,12 +286,22 @@ extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
 
+
 [testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
 deps =
-  3: pyarrow>=3,<4
-  4: pyarrow>=4,<5
-  5: pyarrow>=5,<6
-  6: pyarrow>=6,<7
+  # Pandas 2 minimum for pyarrow is 7
+  3:
+    pyarrow>=3,<4
+    pandas<2
+  4:
+    pyarrow>=4,<5
+    pandas<2
+  5:
+    pyarrow>=5,<6
+    pandas<2
+  6:
+    pyarrow>=6,<7
+    pandas<2
   7: pyarrow>=7,<8
   8: pyarrow>=8,<9
   9: pyarrow>=9,<10

From 66796bd2fde94029c6df294691eba3076abc779f Mon Sep 17 00:00:00 2001
From: Christopher Neffshade <caneff@google.com>
Date: Tue, 14 Nov 2023 15:37:33 +0000
Subject: [PATCH 3/3] Fix tox.ini maybe

---
 sdks/python/tox.ini | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 79a3bfd3ae81..ab8f080a55bc 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -274,18 +274,14 @@ commands =
 [testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
 deps =
   # Pandas 2 minimum for pyarrow is 7
-  3:
-    pyarrow>=3,<4
-    pandas<2
-  4:
-    pyarrow>=4,<5
-    pandas<2
-  5:
-    pyarrow>=5,<6
-    pandas<2
-  6:
-    pyarrow>=6,<7
-    pandas<2
+  3: pyarrow>=3,<4
+  3: pandas<2
+  4: pyarrow>=4,<5
+  4: pandas<2
+  5: pyarrow>=5,<6
+  5: pandas<2
+  6: pyarrow>=6,<7
+  6: pandas<2
   7: pyarrow>=7,<8
   8: pyarrow>=8,<9
   9: pyarrow>=9,<10
@@ -316,6 +312,7 @@ deps =
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
   20: pandas>=2.0.0,<2.1.0
+  20: pyarrow>=7
 commands =
   # Log pandas and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"