diff --git a/Jenkinsfile b/Jenkinsfile index dc70acc6c5..498e01589c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,7 +34,7 @@ slack = new Slack() DEFAULT_CASSANDRA = ['3.11', '4.0', '4.1', '5.0'] DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30', 'dse-6.9.0'] DEFAULT_HCD = ['hcd-1.0.0'] -DEFAULT_RUNTIME = ['3.9.23', '3.10.18', '3.11.13', '3.12.11', '3.13.5'] +DEFAULT_RUNTIME = ['3.10.18', '3.11.13', '3.12.11', '3.13.5'] DEFAULT_CYTHON = ["True", "False"] matrices = [ "FULL": [ @@ -119,7 +119,7 @@ def getBuildContext() { def buildAndTest(context) { initializeEnvironment() - installDriverAndCompileExtensions() + installDriver() try { executeTests() @@ -165,15 +165,18 @@ def getMatrixBuilds(buildContext) { def initializeEnvironment() { sh label: 'Initialize the environment', script: '''#!/bin/bash -lex - pyenv global ${PYTHON_VERSION} - sudo apt-get install socat - pip install --upgrade pip - pip install -U setuptools - # install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023 + # One of the integration tests relies on socat so let's install that here + sudo apt-get install -y socat moreutils + + pyenv shell ${PYTHON_VERSION} + python -m venv jenkins-venv + . ./jenkins-venv/bin/activate + pip install --upgrade pip setuptools wheel + + # Install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023 # this works around the python-3.10+ compatibility problem as described in DSP-23524 - pip install wheel - pip install "Cython<3.0" "pyyaml<6.0" --no-build-isolation + pip install "pyyaml<6.0" --no-build-isolation pip install ${HOME}/ccm ''' @@ -182,35 +185,28 @@ def initializeEnvironment() { if (env.PYTHON_VERSION =~ /3\.12\.\d+/) { echo "Cannot install DSE dependencies for Python 3.12.x; installing Apache CassandraⓇ requirements only. See PYTHON-1368 for more detail." sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate pip install -r test-requirements.txt ''' } else { sh label: 'Install DataStax Enterprise requirements', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate pip install -r test-datastax-requirements.txt ''' } } else { sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate pip install -r test-requirements.txt ''' sh label: 'Uninstall the geomet dependency since it is not required for Cassandra', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate pip uninstall -y geomet ''' } - sh label: 'Install unit test modules', script: '''#!/bin/bash -lex - pip install --no-deps nose-ignore-docstring nose-exclude - pip install service_identity - ''' - - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install cython modules', script: '''#!/bin/bash -lex - pip install cython numpy - ''' - } - sh label: 'Download Apache CassandraⓇ or DataStax Enterprise', script: '''#!/bin/bash -lex . ${CCM_ENVIRONMENT_SHELL} ${CASSANDRA_VERSION} ''' @@ -244,6 +240,8 @@ ENVIRONMENT_EOF } sh label: 'Display Python and environment information', script: '''#!/bin/bash -le + . ./jenkins-venv/bin/activate + # Load CCM environment variables set -o allexport . ${HOME}/environment.txt @@ -251,27 +249,46 @@ ENVIRONMENT_EOF python --version pip --version - pip freeze printenv | sort ''' } -def installDriverAndCompileExtensions() { - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace - ''' - } else { - sh label: 'Install the driver and compile with C extensions without Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace --no-cython - ''' - } +def installDriver() { + sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex + # Update libev includes and libs to point to the right spot for this install + pyenv shell ${PYTHON_VERSION} + python -m venv libev-venv + . ./libev-venv/bin/activate + pip install toml + python fix-jenkinsfile-libev.py ./pyproject.toml "/usr/include" "/usr/lib/x86_64-linux-gnu" | sponge ./pyproject.toml + deactivate + + ls /usr/include/ev.h + ls /usr/lib/x86_64-linux-gnu/libev* + + # Now that we've made relevant mods to our local pyproject.toml we're ready to build the driver + . ./jenkins-venv/bin/activate + + # Load CCM environment variables + set -o allexport + . ${HOME}/environment.txt + set +o allexport + + cat ./pyproject.toml + pip install --verbose --editable . + + # After install display a list of packages in the venv for auditing + pip list + ''' } + def executeStandardTests() { try { sh label: 'Execute unit tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variables set -o allexport . ${HOME}/environment.txt @@ -289,6 +306,8 @@ def executeStandardTests() { try { sh label: 'Execute Simulacron integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variables set -o allexport . ${HOME}/environment.txt @@ -314,6 +333,8 @@ def executeStandardTests() { try { sh label: 'Execute CQL engine integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variables set -o allexport . ${HOME}/environment.txt @@ -330,6 +351,8 @@ def executeStandardTests() { try { sh label: 'Execute Apache CassandraⓇ integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variables set -o allexport . ${HOME}/environment.txt @@ -351,6 +374,8 @@ def executeStandardTests() { else { try { sh label: 'Execute DataStax Enterprise integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variable set -o allexport . ${HOME}/environment.txt @@ -369,6 +394,8 @@ def executeStandardTests() { try { sh label: 'Execute DataStax Astra integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variable set -o allexport . ${HOME}/environment.txt @@ -386,6 +413,8 @@ def executeStandardTests() { if (env.PROFILE == 'FULL') { try { sh label: 'Execute long running integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variable set -o allexport . ${HOME}/environment.txt @@ -404,6 +433,8 @@ def executeStandardTests() { def executeDseSmokeTests() { sh label: 'Execute profile DataStax Enterprise smoke test integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variable set -o allexport . ${HOME}/environment.txt @@ -418,6 +449,8 @@ def executeDseSmokeTests() { def executeEventLoopTests() { sh label: 'Execute profile event loop manager integration tests', script: '''#!/bin/bash -lex + . ./jenkins-venv/bin/activate + # Load CCM environment variable set -o allexport . ${HOME}/environment.txt diff --git a/fix-jenkinsfile-libev.py b/fix-jenkinsfile-libev.py new file mode 100644 index 0000000000..80bd4cb1c8 --- /dev/null +++ b/fix-jenkinsfile-libev.py @@ -0,0 +1,9 @@ +import sys +import toml + +pyproject = toml.load(sys.argv[1]) +base = pyproject["tool"]["cassandra-driver"] +base["libev-includes"] = [sys.argv[2]] +base["libev-libs"] = [sys.argv[3]] + +print(toml.dumps(pyproject)) diff --git a/pyproject.toml b/pyproject.toml index 170d96dff5..0a3fb577d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,6 @@ include = ['cassandra', 'cassandra.io', 'cassandra.cqlengine', 'cassandra.graph' build-murmur3-extension = true build-libev-extension = true build-cython-extensions = true -libev-includes = [] -libev-libs = [] +libev-includes = ["/usr/include/libev", "/usr/local/include", "/opt/local/include", "/usr/include"] +libev-libs = ["/usr/local/lib", "/opt/local/lib", "/usr/lib64"] build-concurrency = 0 \ No newline at end of file diff --git a/setup.py b/setup.py index a3f067c840..07ea384420 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ is_pypy = "PyPy" in sys.version is_supported_platform = sys.platform != "cli" and not sys.platform.startswith("java") is_supported_arch = sys.byteorder != "big" +is_supported = is_supported_platform and is_supported_arch # ========================== A few upfront checks ========================== platform_unsupported_msg = \ @@ -66,62 +67,58 @@ pyproject_data = toml.load(f) driver_project_data = pyproject_data["tool"]["cassandra-driver"] -murmur3_ext = Extension('cassandra.cmurmur3', sources=['cassandra/cmurmur3.c']) - -DEFAULT_LIBEV_INCLUDES = ['/usr/include/libev', '/usr/local/include', '/opt/local/include', '/usr/include'] -DEFAULT_LIBEV_LIBS = ['/usr/local/lib', '/opt/local/lib', '/usr/lib64'] -libev_includes = driver_project_data["libev-includes"] or DEFAULT_LIBEV_INCLUDES -libev_libs = driver_project_data["libev-libs"] or DEFAULT_LIBEV_LIBS -if is_macos: - libev_includes.extend(['/opt/homebrew/include', os.path.expanduser('~/homebrew/include')]) - libev_libs.extend(['/opt/homebrew/lib']) -libev_ext = Extension('cassandra.io.libevwrapper', - sources=['cassandra/io/libevwrapper.c'], - include_dirs=libev_includes, - libraries=['ev'], - library_dirs=libev_libs) - -try_murmur3 = driver_project_data["build-murmur3-extension"] and is_supported_platform and is_supported_arch -try_libev = driver_project_data["build-libev-extension"] and is_supported_platform and is_supported_arch -try_cython = driver_project_data["build-cython-extensions"] and is_supported_platform and is_supported_arch and not is_pypy +def key_or_false(k): + return driver_project_data[k] if k in driver_project_data else False + +try_murmur3 = key_or_false("build-murmur3-extension") and is_supported +try_libev = key_or_false("build-libev-extension") and is_supported +try_cython = key_or_false("build-cython-extensions") and is_supported and not is_pypy build_concurrency = driver_project_data["build-concurrency"] -def build_extension_list(): - - rv = [] - - if try_murmur3: - sys.stderr.write("Appending murmur extension %s\n" % murmur3_ext) - rv.append(murmur3_ext) - - if try_libev: - sys.stderr.write("Appending libev extension %s\n" % libev_ext) - rv.append(libev_ext) - - if try_cython: - sys.stderr.write("Trying Cython builds in order to append Cython extensions\n") - try: - from Cython.Build import cythonize - cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'metadata', - 'pool', 'protocol', 'query', 'util'] - compile_args = [] if is_windows else ['-Wno-unused-function'] - rv.extend(cythonize( - [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], - extra_compile_args=compile_args) - for m in cython_candidates], - nthreads=build_concurrency, - exclude_failures=True)) - - rv.extend(cythonize(Extension("*", ["cassandra/*.pyx"], extra_compile_args=compile_args), - nthreads=build_concurrency)) - except Exception as exc: - sys.stderr.write("Failed to cythonize one or more modules. These will not be compiled as extensions (optional).\n") - sys.stderr.write("Cython error: %s\n" % exc) - - return rv +exts = [] +if try_murmur3: + murmur3_ext = Extension('cassandra.cmurmur3', sources=['cassandra/cmurmur3.c']) + sys.stderr.write("Appending murmur extension %s\n" % murmur3_ext) + exts.append(murmur3_ext) + +if try_libev: + libev_includes = driver_project_data["libev-includes"] + libev_libs = driver_project_data["libev-libs"] + if is_macos: + libev_includes.extend(['/opt/homebrew/include', os.path.expanduser('~/homebrew/include')]) + libev_libs.extend(['/opt/homebrew/lib']) + libev_ext = Extension('cassandra.io.libevwrapper', + sources=['cassandra/io/libevwrapper.c'], + include_dirs=libev_includes, + libraries=['ev'], + library_dirs=libev_libs) + sys.stderr.write("Appending libev extension %s\n" % libev_ext) + exts.append(libev_ext) + +if try_cython: + sys.stderr.write("Trying Cython builds in order to append Cython extensions\n") + try: + from Cython.Build import cythonize + cython_candidates = ['cluster', 'concurrent', 'connection', 'cqltypes', 'metadata', + 'pool', 'protocol', 'query', 'util'] + compile_args = [] if is_windows else ['-Wno-unused-function'] + exts.extend(cythonize( + [Extension('cassandra.%s' % m, ['cassandra/%s.py' % m], + extra_compile_args=compile_args) + for m in cython_candidates], + nthreads=build_concurrency, + exclude_failures=True)) + + exts.extend(cythonize( + Extension("*", ["cassandra/*.pyx"], + extra_compile_args=compile_args), + nthreads=build_concurrency)) + except Exception as exc: + sys.stderr.write("Failed to cythonize one or more modules. These will not be compiled as extensions (optional).\n") + sys.stderr.write("Cython error: %s\n" % exc) # ========================== And finally setup() itself ========================== setup( - ext_modules = build_extension_list() + ext_modules = exts ) \ No newline at end of file