diff --git a/.github/actions/build-manylinux-aarch64/entrypoint.sh b/.github/actions/build-manylinux-aarch64/entrypoint.sh index 99b70dac8..102d3b014 100755 --- a/.github/actions/build-manylinux-aarch64/entrypoint.sh +++ b/.github/actions/build-manylinux-aarch64/entrypoint.sh @@ -4,12 +4,39 @@ set -e # setup cd /github/workspace yum -y update -yum -y install python3 python3-devel gcc-gfortran +yum -y install gcc gcc-c++ gcc-gfortran make wget openssl-devel bzip2-devel libffi-devel xz-devel zlib-devel + +# python +cd /usr/src +wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz +tar xzf openssl-1.1.1w.tar.gz +cd openssl-1.1.1w +./config --prefix=/opt/openssl --openssldir=/opt/openssl no-shared +make -j2 +make install + +cd /usr/src +wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz +tar xzf Python-3.11.9.tgz +cd Python-3.11.9 +export LD_RUN_PATH=/opt/python311/lib +./configure \ + --prefix=/opt/python311 \ + --with-openssl=/opt/openssl \ + --enable-optimizations \ + --enable-shared \ + --without-static-libpython \ + LDFLAGS="-Wl,-rpath=/opt/python311/lib" +make -j2 +make altinstall +export PATH="/opt/python311/bin:$PATH" +alias python=python3.11 # env +cd /github/workspace export PYTHONPATH=$(pwd)/test/python -export CODON_PYTHON=$(python3 test/python/find-python-library.py) -python3 -m pip install -Iv pip==21.3.1 numpy==1.17.5 +export CODON_PYTHON=$(python test/python/find-python-library.py) +python -m pip install -Iv pip==21.3.1 numpy==2.0.2 # deps if [ ! -d ./llvm ]; then @@ -30,10 +57,10 @@ cmake --install build --prefix=codon-deploy # build cython export PATH=$PATH:$(pwd)/llvm/bin -python3 -m pip install cython wheel astunparse -(cd codon-deploy/python && python3 setup.py sdist) -CODON_DIR=$(pwd)/codon-deploy python3 -m pip install -v codon-deploy/python/dist/*.gz -python3 test/python/cython_jit.py +python -m pip install cython wheel astunparse +(cd codon-deploy/python && python setup.py sdist) +CODON_DIR=$(pwd)/codon-deploy python -m pip install -v codon-deploy/python/dist/*.gz +python test/python/cython_jit.py # test export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH diff --git a/.github/actions/build-manylinux-x86_64/entrypoint.sh b/.github/actions/build-manylinux-x86_64/entrypoint.sh index 99b70dac8..102d3b014 100755 --- a/.github/actions/build-manylinux-x86_64/entrypoint.sh +++ b/.github/actions/build-manylinux-x86_64/entrypoint.sh @@ -4,12 +4,39 @@ set -e # setup cd /github/workspace yum -y update -yum -y install python3 python3-devel gcc-gfortran +yum -y install gcc gcc-c++ gcc-gfortran make wget openssl-devel bzip2-devel libffi-devel xz-devel zlib-devel + +# python +cd /usr/src +wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz +tar xzf openssl-1.1.1w.tar.gz +cd openssl-1.1.1w +./config --prefix=/opt/openssl --openssldir=/opt/openssl no-shared +make -j2 +make install + +cd /usr/src +wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tgz +tar xzf Python-3.11.9.tgz +cd Python-3.11.9 +export LD_RUN_PATH=/opt/python311/lib +./configure \ + --prefix=/opt/python311 \ + --with-openssl=/opt/openssl \ + --enable-optimizations \ + --enable-shared \ + --without-static-libpython \ + LDFLAGS="-Wl,-rpath=/opt/python311/lib" +make -j2 +make altinstall +export PATH="/opt/python311/bin:$PATH" +alias python=python3.11 # env +cd /github/workspace export PYTHONPATH=$(pwd)/test/python -export CODON_PYTHON=$(python3 test/python/find-python-library.py) -python3 -m pip install -Iv pip==21.3.1 numpy==1.17.5 +export CODON_PYTHON=$(python test/python/find-python-library.py) +python -m pip install -Iv pip==21.3.1 numpy==2.0.2 # deps if [ ! -d ./llvm ]; then @@ -30,10 +57,10 @@ cmake --install build --prefix=codon-deploy # build cython export PATH=$PATH:$(pwd)/llvm/bin -python3 -m pip install cython wheel astunparse -(cd codon-deploy/python && python3 setup.py sdist) -CODON_DIR=$(pwd)/codon-deploy python3 -m pip install -v codon-deploy/python/dist/*.gz -python3 test/python/cython_jit.py +python -m pip install cython wheel astunparse +(cd codon-deploy/python && python setup.py sdist) +CODON_DIR=$(pwd)/codon-deploy python -m pip install -v codon-deploy/python/dist/*.gz +python test/python/cython_jit.py # test export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48b80e9bf..c78c94d54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,7 @@ jobs: - name: macOS Setup if: startsWith(matrix.os, 'macos') run: | + brew install gcc echo "LIBEXT=dylib" >> $GITHUB_ENV echo "OS_NAME=osx" >> $GITHUB_ENV echo "CODON_SYSTEM_LIBRARIES=$(brew --prefix gcc)/lib/gcc/current" >> $GITHUB_ENV @@ -132,7 +133,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel python -m pip install cython wheel astunparse - python -m pip install --force-reinstall -v "numpy==1.26.4" + python -m pip install --force-reinstall -v "numpy==2.0.2" which python which pip echo "CODON_PYTHON=$(python test/python/find-python-library.py)" >> $GITHUB_ENV diff --git a/CMakeLists.txt b/CMakeLists.txt index 5014ea04a..c1fcdbf66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -482,7 +482,7 @@ target_link_libraries(codon PUBLIC fmt codonc codon_jupyter Threads::Threads) include(FetchContent) FetchContent_Declare( googletest - URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip ) # For Windows: Prevent overriding the parent project's compiler/linker settings set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) diff --git a/README.md b/README.md index 8e893e38f..e4427a329 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ print(total) ``` Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic -reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md). +reduction to avoid race conditions. Learn more in the [multithreading docs](https://docs.exaloop.io/codon/advanced/parallel). Codon also supports writing and executing GPU kernels. Here's an example that computes the [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set): diff --git a/cmake/deps.cmake b/cmake/deps.cmake index a5cb67e98..11dfcaa07 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -1,8 +1,8 @@ -set(CPM_DOWNLOAD_VERSION 0.32.3) +set(CPM_DOWNLOAD_VERSION 0.40.8) set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION})) message(STATUS "Downloading CPM.cmake...") - file(DOWNLOAD https://github.com/TheLartians/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION}) + file(DOWNLOAD https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake ${CPM_DOWNLOAD_LOCATION}) endif() include(${CPM_DOWNLOAD_LOCATION}) @@ -77,9 +77,9 @@ endif() CPMAddPackage( NAME bdwgc - GITHUB_REPOSITORY "ivmai/bdwgc" + GITHUB_REPOSITORY "exaloop/bdwgc" VERSION 8.0.5 - GIT_TAG d0ba209660ea8c663e06d9a68332ba5f42da54ba + GIT_TAG e16c67244aff26802203060422545d38305e0160 EXCLUDE_FROM_ALL YES OPTIONS "CMAKE_POSITION_INDEPENDENT_CODE ON" "BUILD_SHARED_LIBS OFF" @@ -169,7 +169,7 @@ if(NOT APPLE) CPMAddPackage( NAME openblas GITHUB_REPOSITORY "OpenMathLib/OpenBLAS" - GIT_TAG v0.3.28 + GIT_TAG v0.3.29 EXCLUDE_FROM_ALL YES OPTIONS "DYNAMIC_ARCH ON" "BUILD_TESTING OFF" diff --git a/codon/app/main.cpp b/codon/app/main.cpp index 54419810e..1bf285eb8 100644 --- a/codon/app/main.cpp +++ b/codon/app/main.cpp @@ -11,6 +11,7 @@ #include #include +#include "codon/cir/util/format.h" #include "codon/compiler/compiler.h" #include "codon/compiler/error.h" #include "codon/compiler/jit.h" @@ -87,7 +88,7 @@ void initLogFlags(const llvm::cl::opt &log) { codon::getLogger().parse(std::string(d)); } -enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect }; +enum BuildKind { LLVM, Bitcode, Object, Executable, Library, PyExtension, Detect, CIR }; enum OptMode { Debug, Release }; enum Numerics { C, Python }; } // namespace @@ -333,6 +334,7 @@ int buildMode(const std::vector &args, const std::string &argv0) { clEnumValN(Executable, "exe", "Generate executable"), clEnumValN(Library, "lib", "Generate shared library"), clEnumValN(PyExtension, "pyext", "Generate Python extension module"), + clEnumValN(CIR, "cir", "Generate Codon Intermediate Representation"), clEnumValN(Detect, "detect", "Detect output type based on output file extension")), llvm::cl::init(Detect)); @@ -372,6 +374,9 @@ int buildMode(const std::vector &args, const std::string &argv0) { case BuildKind::Detect: extension = ""; break; + case BuildKind::CIR: + extension = ".cir"; + break; default: seqassertn(0, "unknown build kind"); } @@ -401,6 +406,11 @@ int buildMode(const std::vector &args, const std::string &argv0) { compiler->getLLVMVisitor()->writeToPythonExtension(*compiler->getCache()->pyModule, filename); break; + case BuildKind::CIR: { + std::ofstream out(filename); + codon::ir::util::format(out, compiler->getModule()); + break; + } case BuildKind::Detect: compiler->getLLVMVisitor()->compile(filename, argv0, libsVec, lflags); break; diff --git a/docs/README.md b/docs/README.md index b8475d0b1..3776b135f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -124,7 +124,7 @@ print(total) ``` Note that Codon automatically turns the `total += 1` statement in the loop body into an atomic -reduction to avoid race conditions. Learn more in the [multitheading docs](advanced/parallel.md). +reduction to avoid race conditions. Learn more in the [multithreading docs](advanced/parallel.md). Codon also supports writing and executing GPU kernels. Here's an example that computes the [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set): diff --git a/stdlib/internal/python.codon b/stdlib/internal/python.codon index 9d8ecefd5..9f5cf4740 100644 --- a/stdlib/internal/python.codon +++ b/stdlib/internal/python.codon @@ -24,6 +24,7 @@ PyFloat_AsDouble = Function[[cobj], float](cobj()) PyFloat_FromDouble = Function[[float], cobj](cobj()) PyBool_FromLong = Function[[int], cobj](cobj()) PyBytes_AsString = Function[[cobj], cobj](cobj()) +PyBytes_Size = Function[[cobj], int](cobj()) PyList_New = Function[[int], cobj](cobj()) PyList_Size = Function[[cobj], int](cobj()) PyList_GetItem = Function[[cobj, int], cobj](cobj()) @@ -130,6 +131,7 @@ PyLong_Type = cobj() PyFloat_Type = cobj() PyBool_Type = cobj() PyUnicode_Type = cobj() +PyBytes_Type = cobj() PyComplex_Type = cobj() PyList_Type = cobj() PyDict_Type = cobj() @@ -213,6 +215,7 @@ def init_handles_dlopen(py_handle: cobj): global PyFloat_FromDouble global PyBool_FromLong global PyBytes_AsString + global PyBytes_Size global PyList_New global PyList_Size global PyList_GetItem @@ -303,6 +306,7 @@ def init_handles_dlopen(py_handle: cobj): global PyFloat_Type global PyBool_Type global PyUnicode_Type + global PyBytes_Type global PyComplex_Type global PyList_Type global PyDict_Type @@ -347,6 +351,7 @@ def init_handles_dlopen(py_handle: cobj): PyFloat_FromDouble = dlsym(py_handle, "PyFloat_FromDouble") PyBool_FromLong = dlsym(py_handle, "PyBool_FromLong") PyBytes_AsString = dlsym(py_handle, "PyBytes_AsString") + PyBytes_Size = dlsym(py_handle, "PyBytes_Size") PyList_New = dlsym(py_handle, "PyList_New") PyList_Size = dlsym(py_handle, "PyList_Size") PyList_GetItem = dlsym(py_handle, "PyList_GetItem") @@ -437,6 +442,7 @@ def init_handles_dlopen(py_handle: cobj): PyFloat_Type = dlsym(py_handle, "PyFloat_Type") PyBool_Type = dlsym(py_handle, "PyBool_Type") PyUnicode_Type = dlsym(py_handle, "PyUnicode_Type") + PyBytes_Type = dlsym(py_handle, "PyBytes_Type") PyComplex_Type = dlsym(py_handle, "PyComplex_Type") PyList_Type = dlsym(py_handle, "PyList_Type") PyDict_Type = dlsym(py_handle, "PyDict_Type") @@ -482,6 +488,7 @@ def init_handles_static(): from C import PyFloat_FromDouble(float) -> cobj as _PyFloat_FromDouble from C import PyBool_FromLong(int) -> cobj as _PyBool_FromLong from C import PyBytes_AsString(cobj) -> cobj as _PyBytes_AsString + from C import PyBytes_Size(cobj) -> int as _PyBytes_Size from C import PyList_New(int) -> cobj as _PyList_New from C import PyList_Size(cobj) -> int as _PyList_Size from C import PyList_GetItem(cobj, int) -> cobj as _PyList_GetItem @@ -572,6 +579,7 @@ def init_handles_static(): from C import PyFloat_Type: cobj as _PyFloat_Type from C import PyBool_Type: cobj as _PyBool_Type from C import PyUnicode_Type: cobj as _PyUnicode_Type + from C import PyBytes_Type: cobj as _PyBytes_Type from C import PyComplex_Type: cobj as _PyComplex_Type from C import PyList_Type: cobj as _PyList_Type from C import PyDict_Type: cobj as _PyDict_Type @@ -616,6 +624,7 @@ def init_handles_static(): global PyFloat_FromDouble global PyBool_FromLong global PyBytes_AsString + global PyBytes_Size global PyList_New global PyList_Size global PyList_GetItem @@ -706,6 +715,7 @@ def init_handles_static(): global PyFloat_Type global PyBool_Type global PyUnicode_Type + global PyBytes_Type global PyComplex_Type global PyList_Type global PyDict_Type @@ -750,6 +760,7 @@ def init_handles_static(): PyFloat_FromDouble = _PyFloat_FromDouble PyBool_FromLong = _PyBool_FromLong PyBytes_AsString = _PyBytes_AsString + PyBytes_Size = _PyBytes_Size PyList_New = _PyList_New PyList_Size = _PyList_Size PyList_GetItem = _PyList_GetItem @@ -840,6 +851,7 @@ def init_handles_static(): PyFloat_Type = __ptr__(_PyFloat_Type).as_byte() PyBool_Type = __ptr__(_PyBool_Type).as_byte() PyUnicode_Type = __ptr__(_PyUnicode_Type).as_byte() + PyBytes_Type = __ptr__(_PyBytes_Type).as_byte() PyComplex_Type = __ptr__(_PyComplex_Type).as_byte() PyList_Type = __ptr__(_PyList_Type).as_byte() PyDict_Type = __ptr__(_PyDict_Type).as_byte() @@ -1174,7 +1186,7 @@ class pyobj: return pyobj.to_str(self.p, errors, empty) def to_str(p: cobj, errors: str, empty: str = "") -> str: - obj = PyUnicode_AsEncodedString(p, "utf-8".c_str(), errors.c_str()) + obj = PyUnicode_AsEncodedString(p, "utf-8".ptr, errors.c_str() if errors else "".ptr) if obj == cobj(): return empty bts = PyBytes_AsString(obj) @@ -1292,8 +1304,11 @@ class _PyObject_Struct: def _conversion_error(name: Static[str]): raise PyError("conversion error: Python object did not have type '" + name + "'") +def _get_type(o: cobj): + return Ptr[_PyObject_Struct](o)[0].pytype + def _ensure_type(o: cobj, t: cobj, name: Static[str]): - if Ptr[_PyObject_Struct](o)[0].pytype != t: + if _get_type(o) != t: _conversion_error(name) @@ -1350,7 +1365,14 @@ class str: return pyobj.exc_wrap(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len)) def __from_py__(s: cobj) -> str: - return pyobj.exc_wrap(pyobj.to_str(s, "strict")) + if _get_type(s) == PyBytes_Type: + n = PyBytes_Size(s) + p0 = PyBytes_AsString(s) + p1 = cobj(n) + str.memcpy(p1, p0, n) + return str(p1, n) + else: + return pyobj.exc_wrap(pyobj.to_str(s, "strict")) @extend class complex: diff --git a/stdlib/numpy/pybridge.codon b/stdlib/numpy/pybridge.codon index 6fb202069..7e353903f 100644 --- a/stdlib/numpy/pybridge.codon +++ b/stdlib/numpy/pybridge.codon @@ -120,17 +120,19 @@ class PyArrayDescr: kind: u8 type: u8 byteorder: u8 - flags: u8 + former_flags: u8 type_num: i32 - elsize: i32 - alignment: i32 + flags: u64 + elsize: int + alignment: int + metadata: cobj + hash: int + reserved0: cobj + reserved1: cobj subarray: cobj fields: cobj names: cobj - f: cobj - metadata: cobj c_metadata: cobj - hash: int @tuple class PyArrayObject: @@ -149,21 +151,18 @@ NPY_ARRAY_F_CONTIGUOUS: Static[int] = 2 PyArray_Type = cobj() PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](cobj()) +PyArray_NewFromDescr = Function[[cobj, cobj, i32, cobj, cobj, cobj, i32, cobj], cobj](cobj()) +PyArray_DescrNewFromType = Function[[i32], cobj](cobj()) def _pyobj_type(p: cobj): return Ptr[PyObject](p)[0].typptr -def _set_datetime_descr(a: Ptr[PyArrayObject], T: type): - p = Ptr[PyArray_DatetimeDTypeMetaData](a[0].descr[0].c_metadata) - meta = PyArray_DatetimeMetaData(i32(T._code()), i32(T.num)) - p[0] = PyArray_DatetimeDTypeMetaData(p[0].base, meta) - def _setup_numpy_bridge(): import python from internal.python import PyImport_ImportModule, PyObject_GetAttrString, PyCapsule_Type, PyCapsule_GetPointer - global PyArray_Type, PyArray_New + global PyArray_Type, PyArray_New, PyArray_NewFromDescr, PyArray_DescrNewFromType - module = PyImport_ImportModule("numpy.core._multiarray_umath".ptr) + module = PyImport_ImportModule("numpy._core._multiarray_umath".ptr) if not module: raise RuntimeError("Failed to import 'numpy.core._multiarray_umath'") @@ -177,6 +176,8 @@ def _setup_numpy_bridge(): # https://github.com/numpy/numpy/blob/main/numpy/_core/code_generators/numpy_api.py PyArray_Type = api[2] PyArray_New = Function[[cobj, i32, cobj, i32, cobj, cobj, i32, i32, cobj], cobj](api[93]) + PyArray_NewFromDescr = Function[[cobj, cobj, i32, cobj, cobj, cobj, i32, cobj], cobj](api[94]) + PyArray_DescrNewFromType = Function[[i32], cobj](api[96]) _setup_numpy_bridge() @@ -185,8 +186,19 @@ class ndarray: def __to_py__(self): dims = self.shape code = _type_code(self.dtype) - arr = PyArray_New(PyArray_Type, i32(self.ndim), __ptr__(dims).as_byte(), - i32(code), cobj(), cobj(), i32(0), i32(0), cobj()) + + if isinstance(self.dtype, datetime64) or isinstance(self.dtype, timedelta64): + descr = Ptr[PyArrayDescr](PyArray_DescrNewFromType(i32(code))) + p = Ptr[PyArray_DatetimeDTypeMetaData](descr[0].c_metadata) + meta = PyArray_DatetimeMetaData(i32(self.dtype._code()), i32(self.dtype.num)) + p[0] = PyArray_DatetimeDTypeMetaData(p[0].base, meta) + arr = PyArray_NewFromDescr(PyArray_Type, descr.as_byte(), i32(self.ndim), + __ptr__(dims).as_byte(), cobj(), cobj(), i32(0), + cobj()) + else: + arr = PyArray_New(PyArray_Type, i32(self.ndim), __ptr__(dims).as_byte(), + i32(code), cobj(), cobj(), i32(0), i32(0), cobj()) + arr_ptr = Ptr[PyArrayObject](arr) data = arr_ptr[0].data @@ -210,9 +222,6 @@ class ndarray: p[k] = e k += 1 - if isinstance(self.dtype, datetime64) or isinstance(self.dtype, timedelta64): - _set_datetime_descr(arr_ptr, self.dtype) - return arr def _from_py(a: cobj, copy: bool): diff --git a/test/python/pyext.py b/test/python/pyext.py index b9f19599c..5939cad81 100644 --- a/test/python/pyext.py +++ b/test/python/pyext.py @@ -88,6 +88,7 @@ def test_codon_extensions(m): assert m.f4(a=2.2) == (2.2, 2.22) assert m.f4(b=3.3) == (1.11, 3.3) assert m.f4('foo') == ('foo', 'foo') + assert m.f4(b'foo') == ('foo', 'foo') assert m.f4({1}) == {1} assert m.f5() is None assert equal(m.f6(1.9, 't'), 1.9, 1.9, 't')