fix: allow protobuf 5.x (#2042)

gapic/templates/noxfile.py.j2

@@ -43,11 +43,23 @@ nox.sessions = [
 ]
 
 @nox.session(python=ALL_PYTHON)
-def unit(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def unit(session, protobuf_implementation):
     """Run the unit test suite."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
-    session.install('-e', '.')
+    session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt")
+
+    # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped.
+    # The 'cpp' implementation requires Protobuf<4.
+    if protobuf_implementation == "cpp":
+        session.install("protobuf<4")
 
     session.run(
         'py.test',
@@ -57,13 +69,23 @@ def unit(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 @nox.session(python=ALL_PYTHON[-1])
-def prerelease_deps(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def prerelease_deps(session, protobuf_implementation):
     """Run the unit test suite against pre-release versions of dependencies."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     # Install test environment dependencies
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
 
@@ -131,7 +153,10 @@ def prerelease_deps(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 

gapic/templates/setup.py.j2

@@ -36,7 +36,7 @@ dependencies = [
     "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0",
     "proto-plus >= 1.22.3, <2.0.0dev",
     {# Explicitly exclude protobuf versions mentioned in https://cloud.google.com/support/bulletins#GCP-2022-019 #}
-    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+    "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     {% for package_tuple, package_info in pypi_packages.items() %}
     {# Quick check to make sure the package is different from this setup.py #}
     {% if api.naming.warehouse_package_name != package_info.package_name %}

gapic/templates/testing/constraints-3.7.txt.j2

@@ -8,7 +8,7 @@
 google-api-core==1.34.1
 google-auth==2.14.1
 proto-plus==1.22.3
-protobuf==3.19.5
+protobuf==3.20.2
 {% for package_tuple, package_info in pypi_packages.items() %}
 {# Quick check to make sure the package is different from this setup.py #}
 {% if api.naming.warehouse_package_name != package_info.package_name %}

requirements.in

@@ -3,7 +3,7 @@ google-api-core
 googleapis-common-protos
 jinja2
 MarkupSafe
-protobuf<5 # See https://github.com/googleapis/gapic-generator-python/issues/1996
+protobuf
 pypandoc
 PyYAML
 grpc-google-iam-v1

requirements.txt

@@ -201,7 +201,7 @@ iniconfig==2.0.0 \
     --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
     --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
     # via pytest
-Jinja2==3.1.4 \
+jinja2==3.1.4 \
     --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
     --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
     # via -r requirements.in

setup.py

@@ -35,7 +35,7 @@
     # https://jinja.palletsprojects.com/en/3.0.x/templates/#jinja-filters.map
     # https://jinja.palletsprojects.com/en/2.11.x/changelog/#version-2-11-0
     "jinja2 >= 2.11",
-    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+    "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     "pypandoc >= 1.4",
     "PyYAML >= 5.1.1",
     "grpc-google-iam-v1 >= 0.12.4, < 1.0.0dev",

tests/integration/goldens/asset/noxfile.py

@@ -54,11 +54,23 @@
 ]
 
 @nox.session(python=ALL_PYTHON)
-def unit(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def unit(session, protobuf_implementation):
     """Run the unit test suite."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
-    session.install('-e', '.')
+    session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt")
+
+    # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped.
+    # The 'cpp' implementation requires Protobuf<4.
+    if protobuf_implementation == "cpp":
+        session.install("protobuf<4")
 
     session.run(
         'py.test',
@@ -68,13 +80,23 @@ def unit(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 @nox.session(python=ALL_PYTHON[-1])
-def prerelease_deps(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def prerelease_deps(session, protobuf_implementation):
     """Run the unit test suite against pre-release versions of dependencies."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     # Install test environment dependencies
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
 
@@ -142,7 +164,10 @@ def prerelease_deps(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 

tests/integration/goldens/asset/setup.py

@@ -44,7 +44,7 @@
     # See https://github.com/googleapis/google-cloud-python/issues/12364
     "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0",
     "proto-plus >= 1.22.3, <2.0.0dev",
-    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+    "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     "google-cloud-access-context-manager >= 0.1.2, <1.0.0dev",
     "google-cloud-os-config >= 1.0.0, <2.0.0dev",
     "grpc-google-iam-v1 >= 0.12.4, <1.0.0dev",

tests/integration/goldens/asset/testing/constraints-3.7.txt

@@ -7,7 +7,7 @@
 google-api-core==1.34.1
 google-auth==2.14.1
 proto-plus==1.22.3
-protobuf==3.19.5
+protobuf==3.20.2
 google-cloud-access-context-manager==0.1.2
 google-cloud-os-config==1.0.0
 grpc-google-iam-v1==0.12.4

tests/integration/goldens/credentials/noxfile.py

@@ -54,11 +54,23 @@
 ]
 
 @nox.session(python=ALL_PYTHON)
-def unit(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def unit(session, protobuf_implementation):
     """Run the unit test suite."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
-    session.install('-e', '.')
+    session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt")
+
+    # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped.
+    # The 'cpp' implementation requires Protobuf<4.
+    if protobuf_implementation == "cpp":
+        session.install("protobuf<4")
 
     session.run(
         'py.test',
@@ -68,13 +80,23 @@ def unit(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 @nox.session(python=ALL_PYTHON[-1])
-def prerelease_deps(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def prerelease_deps(session, protobuf_implementation):
     """Run the unit test suite against pre-release versions of dependencies."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     # Install test environment dependencies
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
 
@@ -142,7 +164,10 @@ def prerelease_deps(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 

tests/integration/goldens/credentials/setup.py

@@ -44,7 +44,7 @@
     # See https://github.com/googleapis/google-cloud-python/issues/12364
     "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0",
     "proto-plus >= 1.22.3, <2.0.0dev",
-    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+    "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
 ]
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-iam-credentials"
 

tests/integration/goldens/credentials/testing/constraints-3.7.txt

@@ -7,4 +7,4 @@
 google-api-core==1.34.1
 google-auth==2.14.1
 proto-plus==1.22.3
-protobuf==3.19.5
+protobuf==3.20.2

tests/integration/goldens/eventarc/noxfile.py

@@ -54,11 +54,23 @@
 ]
 
 @nox.session(python=ALL_PYTHON)
-def unit(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def unit(session, protobuf_implementation):
     """Run the unit test suite."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
-    session.install('-e', '.')
+    session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt")
+
+    # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped.
+    # The 'cpp' implementation requires Protobuf<4.
+    if protobuf_implementation == "cpp":
+        session.install("protobuf<4")
 
     session.run(
         'py.test',
@@ -68,13 +80,23 @@ def unit(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 @nox.session(python=ALL_PYTHON[-1])
-def prerelease_deps(session):
+@nox.parametrize(
+    "protobuf_implementation",
+    [ "python", "upb", "cpp" ],
+)
+def prerelease_deps(session, protobuf_implementation):
     """Run the unit test suite against pre-release versions of dependencies."""
 
+    if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"):
+        session.skip("cpp implementation is not supported in python 3.11+")
+
     # Install test environment dependencies
     session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"')
 
@@ -142,7 +164,10 @@ def prerelease_deps(session):
         '--cov-config=.coveragerc',
         '--cov-report=term',
         '--cov-report=html',
-        os.path.join('tests', 'unit', ''.join(session.posargs))
+        os.path.join('tests', 'unit', ''.join(session.posargs)),
+        env={
+            "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+        },
     )
 
 

tests/integration/goldens/eventarc/setup.py

@@ -44,7 +44,7 @@
     # See https://github.com/googleapis/google-cloud-python/issues/12364
     "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0",
     "proto-plus >= 1.22.3, <2.0.0dev",
-    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+    "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     "grpc-google-iam-v1 >= 0.12.4, <1.0.0dev",
 ]
 url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-eventarc"

tests/integration/goldens/eventarc/testing/constraints-3.7.txt

@@ -7,5 +7,5 @@
 google-api-core==1.34.1
 google-auth==2.14.1
 proto-plus==1.22.3
-protobuf==3.19.5
+protobuf==3.20.2
 grpc-google-iam-v1==0.12.4