diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/.gitignore b/sdk/python/feast/infra/feature_servers/multicloud/offline/.gitignore new file mode 100644 index 00000000000..b83f21c6338 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/.gitignore @@ -0,0 +1 @@ +arrow-thirdparty-offline-versions \ No newline at end of file diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.arrow b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.arrow deleted file mode 100644 index f5719d672b6..00000000000 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.arrow +++ /dev/null @@ -1,114 +0,0 @@ -FROM yum-builder:latest - -ENV ARROW_HOME=${APP_ROOT}/arrow-dist -ENV LD_LIBRARY_PATH=${ARROW_HOME}/lib:$LD_LIBRARY_PATH -ENV CMAKE_PREFIX_PATH=${ARROW_HOME}:$CMAKE_PREFIX_PATH -ENV THIRD_PARTY_PATH=/tmp/arrow/cpp/arrow-thirdparty - -COPY --chown=default cpp/cmake_modules ${APP_ROOT}/src/arrow/cpp/cmake_modules -COPY --chown=default python ${APP_ROOT}/src/arrow/python - -# configured for Arrow version 17.0.0 -RUN mkdir ${ARROW_HOME} ${APP_ROOT}/src/arrow-build -RUN export \ - # Environment variables for an offline Arrow build - ARROW_ABSL_URL="${THIRD_PARTY_PATH}/absl-20211102.0.tar.gz" \ - ARROW_AWS_C_AUTH_URL="${THIRD_PARTY_PATH}/aws-c-auth-v0.6.22.tar.gz" \ - ARROW_AWS_C_CAL_URL="${THIRD_PARTY_PATH}/aws-c-cal-v0.5.20.tar.gz" \ - ARROW_AWS_C_COMMON_URL="${THIRD_PARTY_PATH}/aws-c-common-v0.8.9.tar.gz" \ - ARROW_AWS_C_COMPRESSION_URL="${THIRD_PARTY_PATH}/aws-c-compression-v0.2.16.tar.gz" \ - ARROW_AWS_C_EVENT_STREAM_URL="${THIRD_PARTY_PATH}/aws-c-event-stream-v0.2.18.tar.gz" \ - ARROW_AWS_C_HTTP_URL="${THIRD_PARTY_PATH}/aws-c-http-v0.7.3.tar.gz" \ - ARROW_AWS_C_IO_URL="${THIRD_PARTY_PATH}/aws-c-io-v0.13.14.tar.gz" \ - ARROW_AWS_C_MQTT_URL="${THIRD_PARTY_PATH}/aws-c-mqtt-v0.8.4.tar.gz" \ - ARROW_AWS_C_S3_URL="${THIRD_PARTY_PATH}/aws-c-s3-v0.2.3.tar.gz" \ - ARROW_AWS_C_SDKUTILS_URL="${THIRD_PARTY_PATH}/aws-c-sdkutils-v0.1.6.tar.gz" \ - ARROW_AWS_CHECKSUMS_URL="${THIRD_PARTY_PATH}/aws-checksums-v0.1.13.tar.gz" \ - ARROW_AWS_CRT_CPP_URL="${THIRD_PARTY_PATH}/aws-crt-cpp-v0.18.16.tar.gz" \ - ARROW_AWS_LC_URL="${THIRD_PARTY_PATH}/aws-lc-v1.3.0.tar.gz" \ - ARROW_AWSSDK_URL="${THIRD_PARTY_PATH}/aws-sdk-cpp-1.10.55.tar.gz" \ - ARROW_BOOST_URL="${THIRD_PARTY_PATH}/boost-1.81.0.tar.gz" \ - ARROW_BROTLI_URL="${THIRD_PARTY_PATH}/brotli-v1.0.9.tar.gz" \ - ARROW_BZIP2_URL="${THIRD_PARTY_PATH}/bzip2-1.0.8.tar.gz" \ - ARROW_CARES_URL="${THIRD_PARTY_PATH}/cares-1.17.2.tar.gz" \ - ARROW_CRC32C_URL="${THIRD_PARTY_PATH}/crc32c-1.1.2.tar.gz" \ - ARROW_GBENCHMARK_URL="${THIRD_PARTY_PATH}/gbenchmark-v1.8.3.tar.gz" \ - ARROW_GFLAGS_URL="${THIRD_PARTY_PATH}/gflags-v2.2.2.tar.gz" \ - ARROW_GLOG_URL="${THIRD_PARTY_PATH}/glog-v0.5.0.tar.gz" \ - ARROW_GOOGLE_CLOUD_CPP_URL="${THIRD_PARTY_PATH}/google-cloud-cpp-v2.22.0.tar.gz" \ - ARROW_GRPC_URL="${THIRD_PARTY_PATH}/grpc-v1.46.3.tar.gz" \ - ARROW_GTEST_URL="${THIRD_PARTY_PATH}/gtest-1.11.0.tar.gz" \ - ARROW_JEMALLOC_URL="${THIRD_PARTY_PATH}/jemalloc-5.3.0.tar.bz2" \ - ARROW_LZ4_URL="${THIRD_PARTY_PATH}/lz4-v1.9.4.tar.gz" \ - ARROW_MIMALLOC_URL="${THIRD_PARTY_PATH}/mimalloc-v2.0.6.tar.gz" \ - ARROW_NLOHMANN_JSON_URL="${THIRD_PARTY_PATH}/nlohmann-json-v3.10.5.tar.gz" \ - ARROW_OPENTELEMETRY_URL="${THIRD_PARTY_PATH}/opentelemetry-cpp-v1.13.0.tar.gz" \ - ARROW_OPENTELEMETRY_PROTO_URL="${THIRD_PARTY_PATH}/opentelemetry-proto-v0.17.0.tar.gz" \ - ARROW_ORC_URL="${THIRD_PARTY_PATH}/orc-2.0.1.tar.gz" \ - ARROW_PROTOBUF_URL="${THIRD_PARTY_PATH}/protobuf-v21.3.tar.gz" \ - ARROW_RAPIDJSON_URL="${THIRD_PARTY_PATH}/rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" \ - ARROW_RE2_URL="${THIRD_PARTY_PATH}/re2-2022-06-01.tar.gz" \ - ARROW_S2N_TLS_URL="${THIRD_PARTY_PATH}/s2n-v1.3.35.tar.gz" \ - ARROW_SUBSTRAIT_URL="${THIRD_PARTY_PATH}/substrait-0.44.0.tar.gz" \ - ARROW_SNAPPY_URL="${THIRD_PARTY_PATH}/snappy-1.1.10.tar.gz" \ - ARROW_THRIFT_URL="${THIRD_PARTY_PATH}/thrift-0.16.0.tar.gz" \ - ARROW_UCX_URL="${THIRD_PARTY_PATH}/ucx-1.12.1.tar.gz" \ - ARROW_UTF8PROC_URL="${THIRD_PARTY_PATH}/utf8proc-v2.7.0.tar.gz" \ - ARROW_XSIMD_URL="${THIRD_PARTY_PATH}/xsimd-13.0.0.tar.gz" \ - ARROW_ZLIB_URL="${THIRD_PARTY_PATH}/zlib-1.3.1.tar.gz" \ - ARROW_ZSTD_URL="${THIRD_PARTY_PATH}/zstd-1.5.6.tar.gz" \ - && \ - cmake \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_COMPUTE=ON \ - -DARROW_ACERO=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_FLIGHT=ON \ - -DARROW_FLIGHT_SQL=ON \ - -DARROW_GANDIVA=ON \ - -DARROW_GCS=ON \ - -DARROW_HDFS=ON \ - -DARROW_JEMALLOC=ON \ - -DARROW_JSON=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_MIMALLOC=ON \ - -DARROW_PARQUET=ON \ - -DARROW_S3=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_SUBSTRAIT=ON \ - -DARROW_WITH_RE2=ON \ - -DARROW_WITH_UTF8PROC=ON \ - -DARROW_TENSORFLOW=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_BUILD_SHARED=ON \ - -S /tmp/arrow/cpp \ - -B ${APP_ROOT}/src/arrow-build && \ - \ - cmake --build ${APP_ROOT}/src/arrow-build --target install && \ - \ - source /tmp/hermeto.env && \ - pip install -r ${APP_ROOT}/src/arrow/python/requirements-wheel-build.txt && \ - \ - cd ${APP_ROOT}/src/arrow/python && \ - PYARROW_PARALLEL=4 python setup.py build_ext --bundle-arrow-cpp bdist_wheel && \ - pip install dist/pyarrow-*.whl && \ - \ - cd ${APP_ROOT}/src && \ - rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${APP_ROOT}/arrow-dist - -RUN python -c "import pyarrow; print(pyarrow.__version__)" -RUN python -c "import pyarrow.lib as _lib; print(_lib.__name__)" -RUN python -c "import pyarrow.parquet as parquet; print(parquet.__name__)" -RUN python -c "import pyarrow.dataset as dataset; print(dataset.__name__)" -RUN python -c "import pyarrow.flight as flight; print(flight.__name__)" -RUN python -c "import pyarrow.substrait as substrait; print(substrait.__name__)" - -# a higher numpy was required for the pyarrow wheel build, but the pyarrow module itself can run w/ a lesser version, "numpy>=1.16.6". -# feast requires "numpy<2", so here we install numpy 1.x, as well as some other packages which will be needed for the feast build. -RUN source /tmp/hermeto.env && \ - pip install "numpy>=1.16.6,<2" diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.ibis b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.ibis deleted file mode 100644 index 74fcf30e5ef..00000000000 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.ibis +++ /dev/null @@ -1,8 +0,0 @@ -FROM arrow-builder:latest - -COPY --chown=default ibis ${APP_ROOT}/src/ibis -RUN cd ${APP_ROOT}/src/ibis && \ - source /tmp/hermeto.env && \ - pip install .[duckdb] && \ - cd ${APP_ROOT}/src && \ - rm -rf ${APP_ROOT}/src/ibis diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum index cffb3bcbb6c..75a082b8210 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum @@ -1,10 +1,17 @@ FROM registry.access.redhat.com/ubi9/python-311:1 ARG RELEASE +ENV IBIS_VERSION="9.5.0" +ENV MILVUS_LITE_VERSION="2.4.12" + USER 0 RUN yum install -y ninja-build llvm-devel cmake llvm-toolset ncurses-devel rust cargo RUN if [[ -z "$RELEASE" ]] ; then npm install -g yarn yalc && rm -rf .npm ; fi USER 1001 # milvus dep -RUN pip install milvus-lite==2.4.12 +RUN pip install milvus-lite==${MILVUS_LITE_VERSION} + +# ibis-framework must build from a clone of its git repo... versioning requirement +## RuntimeError: Unable to detect version control system. Checked: Git. Not installed: Mercurial, Darcs, Subversion, Bazaar, Fossil, Pijul. +RUN git clone -b ${IBIS_VERSION} --single-branch https://github.com/ibis-project/ibis ${APP_ROOT}/src/ibis diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist index 2aa77e5e3d0..a96c365b164 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist @@ -1,4 +1,123 @@ -FROM ibis-builder:latest +FROM yum-builder:dev + +ENV APACHE_ARROW_VERSION="17.0.0" + +ENV ARROW_HOME=${APP_ROOT}/arrow-dist +ENV LD_LIBRARY_PATH=${ARROW_HOME}/lib:$LD_LIBRARY_PATH +ENV CMAKE_PREFIX_PATH=${ARROW_HOME}:$CMAKE_PREFIX_PATH +ENV THIRD_PARTY_PATH=/tmp/hermeto-generic-output/deps/generic + +# configured for Arrow version 17.0.0 +RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \ + tar xfz ${THIRD_PARTY_PATH}/apache-arrow-${APACHE_ARROW_VERSION}.tar.gz -C ${APP_ROOT}/src/arrow --strip-components=1 && \ + export \ + # Environment variables for an offline Arrow build + ARROW_ABSL_URL="${THIRD_PARTY_PATH}/absl-20211102.0.tar.gz" \ + ARROW_AWS_C_AUTH_URL="${THIRD_PARTY_PATH}/aws-c-auth-v0.6.22.tar.gz" \ + ARROW_AWS_C_CAL_URL="${THIRD_PARTY_PATH}/aws-c-cal-v0.5.20.tar.gz" \ + ARROW_AWS_C_COMMON_URL="${THIRD_PARTY_PATH}/aws-c-common-v0.8.9.tar.gz" \ + ARROW_AWS_C_COMPRESSION_URL="${THIRD_PARTY_PATH}/aws-c-compression-v0.2.16.tar.gz" \ + ARROW_AWS_C_EVENT_STREAM_URL="${THIRD_PARTY_PATH}/aws-c-event-stream-v0.2.18.tar.gz" \ + ARROW_AWS_C_HTTP_URL="${THIRD_PARTY_PATH}/aws-c-http-v0.7.3.tar.gz" \ + ARROW_AWS_C_IO_URL="${THIRD_PARTY_PATH}/aws-c-io-v0.13.14.tar.gz" \ + ARROW_AWS_C_MQTT_URL="${THIRD_PARTY_PATH}/aws-c-mqtt-v0.8.4.tar.gz" \ + ARROW_AWS_C_S3_URL="${THIRD_PARTY_PATH}/aws-c-s3-v0.2.3.tar.gz" \ + ARROW_AWS_C_SDKUTILS_URL="${THIRD_PARTY_PATH}/aws-c-sdkutils-v0.1.6.tar.gz" \ + ARROW_AWS_CHECKSUMS_URL="${THIRD_PARTY_PATH}/aws-checksums-v0.1.13.tar.gz" \ + ARROW_AWS_CRT_CPP_URL="${THIRD_PARTY_PATH}/aws-crt-cpp-v0.18.16.tar.gz" \ + ARROW_AWS_LC_URL="${THIRD_PARTY_PATH}/aws-lc-v1.3.0.tar.gz" \ + ARROW_AWSSDK_URL="${THIRD_PARTY_PATH}/aws-sdk-cpp-1.10.55.tar.gz" \ + ARROW_BOOST_URL="${THIRD_PARTY_PATH}/boost-1.81.0.tar.gz" \ + ARROW_BROTLI_URL="${THIRD_PARTY_PATH}/brotli-v1.0.9.tar.gz" \ + ARROW_BZIP2_URL="${THIRD_PARTY_PATH}/bzip2-1.0.8.tar.gz" \ + ARROW_CARES_URL="${THIRD_PARTY_PATH}/cares-1.17.2.tar.gz" \ + ARROW_CRC32C_URL="${THIRD_PARTY_PATH}/crc32c-1.1.2.tar.gz" \ + ARROW_GBENCHMARK_URL="${THIRD_PARTY_PATH}/gbenchmark-v1.8.3.tar.gz" \ + ARROW_GFLAGS_URL="${THIRD_PARTY_PATH}/gflags-v2.2.2.tar.gz" \ + ARROW_GLOG_URL="${THIRD_PARTY_PATH}/glog-v0.5.0.tar.gz" \ + ARROW_GOOGLE_CLOUD_CPP_URL="${THIRD_PARTY_PATH}/google-cloud-cpp-v2.22.0.tar.gz" \ + ARROW_GRPC_URL="${THIRD_PARTY_PATH}/grpc-v1.46.3.tar.gz" \ + ARROW_GTEST_URL="${THIRD_PARTY_PATH}/gtest-1.11.0.tar.gz" \ + ARROW_JEMALLOC_URL="${THIRD_PARTY_PATH}/jemalloc-5.3.0.tar.bz2" \ + ARROW_LZ4_URL="${THIRD_PARTY_PATH}/lz4-v1.9.4.tar.gz" \ + ARROW_MIMALLOC_URL="${THIRD_PARTY_PATH}/mimalloc-v2.0.6.tar.gz" \ + ARROW_NLOHMANN_JSON_URL="${THIRD_PARTY_PATH}/nlohmann-json-v3.10.5.tar.gz" \ + ARROW_OPENTELEMETRY_URL="${THIRD_PARTY_PATH}/opentelemetry-cpp-v1.13.0.tar.gz" \ + ARROW_OPENTELEMETRY_PROTO_URL="${THIRD_PARTY_PATH}/opentelemetry-proto-v0.17.0.tar.gz" \ + ARROW_ORC_URL="${THIRD_PARTY_PATH}/orc-2.0.1.tar.gz" \ + ARROW_PROTOBUF_URL="${THIRD_PARTY_PATH}/protobuf-v21.3.tar.gz" \ + ARROW_RAPIDJSON_URL="${THIRD_PARTY_PATH}/rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" \ + ARROW_RE2_URL="${THIRD_PARTY_PATH}/re2-2022-06-01.tar.gz" \ + ARROW_S2N_TLS_URL="${THIRD_PARTY_PATH}/s2n-v1.3.35.tar.gz" \ + ARROW_SUBSTRAIT_URL="${THIRD_PARTY_PATH}/substrait-0.44.0.tar.gz" \ + ARROW_SNAPPY_URL="${THIRD_PARTY_PATH}/snappy-1.1.10.tar.gz" \ + ARROW_THRIFT_URL="${THIRD_PARTY_PATH}/thrift-0.16.0.tar.gz" \ + ARROW_UCX_URL="${THIRD_PARTY_PATH}/ucx-1.12.1.tar.gz" \ + ARROW_UTF8PROC_URL="${THIRD_PARTY_PATH}/utf8proc-v2.7.0.tar.gz" \ + ARROW_XSIMD_URL="${THIRD_PARTY_PATH}/xsimd-13.0.0.tar.gz" \ + ARROW_ZLIB_URL="${THIRD_PARTY_PATH}/zlib-1.3.1.tar.gz" \ + ARROW_ZSTD_URL="${THIRD_PARTY_PATH}/zstd-1.5.6.tar.gz" \ + && \ + cmake \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_COMPUTE=ON \ + -DARROW_ACERO=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_FLIGHT=ON \ + -DARROW_FLIGHT_SQL=ON \ + -DARROW_GANDIVA=ON \ + -DARROW_GCS=ON \ + -DARROW_HDFS=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_JSON=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_MIMALLOC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_S3=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_SUBSTRAIT=ON \ + -DARROW_WITH_RE2=ON \ + -DARROW_WITH_UTF8PROC=ON \ + -DARROW_TENSORFLOW=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_BUILD_SHARED=ON \ + -S ${APP_ROOT}/src/arrow/cpp \ + -B ${APP_ROOT}/src/arrow-build && \ + \ + cmake --build ${APP_ROOT}/src/arrow-build --target install && \ + \ + source /tmp/hermeto.env && \ + pip install -r ${APP_ROOT}/src/arrow/python/requirements-wheel-build.txt && \ + \ + cd ${APP_ROOT}/src/arrow/python && \ + PYARROW_PARALLEL=4 python setup.py build_ext --bundle-arrow-cpp bdist_wheel && \ + pip install dist/pyarrow-*.whl && \ + \ + cd ${APP_ROOT}/src && \ + rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${ARROW_HOME} + +RUN python -c "import pyarrow; print(pyarrow.__version__)" +RUN python -c "import pyarrow.lib as _lib; print(_lib.__name__)" +RUN python -c "import pyarrow.parquet as parquet; print(parquet.__name__)" +RUN python -c "import pyarrow.dataset as dataset; print(dataset.__name__)" +RUN python -c "import pyarrow.flight as flight; print(flight.__name__)" +RUN python -c "import pyarrow.substrait as substrait; print(substrait.__name__)" + +# a higher numpy was required for the pyarrow wheel build, but the pyarrow module itself can run w/ a lesser version, "numpy>=1.16.6". +# feast requires "numpy<2", so here we install numpy 1.x, as well as some other packages which will be needed for the feast build. +RUN source /tmp/hermeto.env && \ + pip install "numpy>=1.16.6,<2" + +RUN cd ${APP_ROOT}/src/ibis && \ + source /tmp/hermeto.env && \ + pip install .[duckdb] && \ + cd ${APP_ROOT}/src && \ + rm -rf ${APP_ROOT}/src/ibis # This section only necessary when building from local feast source ... e.g. ".[minimal]" ######################## diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release index 5ad0e9ba739..dd5b07b58d7 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release @@ -1,17 +1,16 @@ -FROM yum-builder:latest +FROM yum-builder:release + +ENV APACHE_ARROW_VERSION="17.0.0" -ENV OFFLINE_BUILD_DIR=offline_build ENV ARROW_HOME=${APP_ROOT}/arrow-dist ENV LD_LIBRARY_PATH=${ARROW_HOME}/lib:$LD_LIBRARY_PATH ENV CMAKE_PREFIX_PATH=${ARROW_HOME}:$CMAKE_PREFIX_PATH -ENV THIRD_PARTY_PATH=/tmp/arrow/cpp/arrow-thirdparty - -COPY --chown=default ${OFFLINE_BUILD_DIR}/arrow/cpp/cmake_modules ${APP_ROOT}/src/arrow/cpp/cmake_modules -COPY --chown=default ${OFFLINE_BUILD_DIR}/arrow/python ${APP_ROOT}/src/arrow/python +ENV THIRD_PARTY_PATH=/tmp/hermeto-generic-output/deps/generic # configured for Arrow version 17.0.0 -RUN mkdir ${ARROW_HOME} ${APP_ROOT}/src/arrow-build -RUN export \ +RUN mkdir ${APP_ROOT}/src/arrow ${ARROW_HOME} ${APP_ROOT}/src/arrow-build && \ + tar xfz ${THIRD_PARTY_PATH}/apache-arrow-${APACHE_ARROW_VERSION}.tar.gz -C ${APP_ROOT}/src/arrow --strip-components=1 && \ + export \ # Environment variables for an offline Arrow build ARROW_ABSL_URL="${THIRD_PARTY_PATH}/absl-20211102.0.tar.gz" \ ARROW_AWS_C_AUTH_URL="${THIRD_PARTY_PATH}/aws-c-auth-v0.6.22.tar.gz" \ @@ -87,7 +86,7 @@ RUN export \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DARROW_BUILD_SHARED=ON \ - -S /tmp/arrow/cpp \ + -S ${APP_ROOT}/src/arrow/cpp \ -B ${APP_ROOT}/src/arrow-build && \ \ cmake --build ${APP_ROOT}/src/arrow-build --target install && \ @@ -100,7 +99,7 @@ RUN export \ pip install dist/pyarrow-*.whl && \ \ cd ${APP_ROOT}/src && \ - rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${APP_ROOT}/arrow-dist + rm -rf ${APP_ROOT}/src/arrow-build ${APP_ROOT}/src/arrow ${ARROW_HOME} RUN python -c "import pyarrow; print(pyarrow.__version__)" RUN python -c "import pyarrow.lib as _lib; print(_lib.__name__)" @@ -114,14 +113,13 @@ RUN python -c "import pyarrow.substrait as substrait; print(substrait.__name__)" RUN source /tmp/hermeto.env && \ pip install "numpy>=1.16.6,<2" -COPY --chown=default ${OFFLINE_BUILD_DIR}/ibis ${APP_ROOT}/src/ibis RUN cd ${APP_ROOT}/src/ibis && \ source /tmp/hermeto.env && \ pip install .[duckdb] && \ cd ${APP_ROOT}/src && \ rm -rf ${APP_ROOT}/src/ibis -COPY sdk/python/feast/infra/feature_servers/multicloud/requirements.txt requirements.txt +COPY requirements.txt requirements.txt RUN source /tmp/hermeto.env && \ pip install -r requirements.txt diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/artifacts.lock.yaml b/sdk/python/feast/infra/feature_servers/multicloud/offline/artifacts.lock.yaml new file mode 100644 index 00000000000..978cbbf31dd --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/artifacts.lock.yaml @@ -0,0 +1,142 @@ +--- +metadata: + version: "1.0" +artifacts: + - download_url: "https://github.com/apache/arrow/archive/apache-arrow-17.0.0.tar.gz" + checksum: "sha256:8379554d89f19f2c8db63620721cabade62541f47a4e706dfb0a401f05a713ef" + filename: "apache-arrow-17.0.0.tar.gz" + - download_url: "https://github.com/abseil/abseil-cpp/archive/20211102.0.tar.gz" + checksum: "sha256:dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4" + filename: "absl-20211102.0.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-auth/archive/v0.6.22.tar.gz" + checksum: "sha256:691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c" + filename: "aws-c-auth-v0.6.22.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-cal/archive/v0.5.20.tar.gz" + checksum: "sha256:acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0" + filename: "aws-c-cal-v0.5.20.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-common/archive/v0.8.9.tar.gz" + checksum: "sha256:2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a" + filename: "aws-c-common-v0.8.9.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-compression/archive/v0.2.16.tar.gz" + checksum: "sha256:044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4" + filename: "aws-c-compression-v0.2.16.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-event-stream/archive/v0.2.18.tar.gz" + checksum: "sha256:310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f" + filename: "aws-c-event-stream-v0.2.18.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-http/archive/v0.7.3.tar.gz" + checksum: "sha256:07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae" + filename: "aws-c-http-v0.7.3.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-io/archive/v0.13.14.tar.gz" + checksum: "sha256:12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71" + filename: "aws-c-io-v0.13.14.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-mqtt/archive/v0.8.4.tar.gz" + checksum: "sha256:232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f" + filename: "aws-c-mqtt-v0.8.4.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-s3/archive/v0.2.3.tar.gz" + checksum: "sha256:a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe" + filename: "aws-c-s3-v0.2.3.tar.gz" + - download_url: "https://github.com/awslabs/aws-c-sdkutils/archive/v0.1.6.tar.gz" + checksum: "sha256:8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91" + filename: "aws-c-sdkutils-v0.1.6.tar.gz" + - download_url: "https://github.com/awslabs/aws-checksums/archive/v0.1.13.tar.gz" + checksum: "sha256:0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d" + filename: "aws-checksums-v0.1.13.tar.gz" + - download_url: "https://github.com/awslabs/aws-crt-cpp/archive/v0.18.16.tar.gz" + checksum: "sha256:9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483" + filename: "aws-crt-cpp-v0.18.16.tar.gz" + - download_url: "https://github.com/awslabs/aws-lc/archive/v1.3.0.tar.gz" + checksum: "sha256:ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37" + filename: "aws-lc-v1.3.0.tar.gz" + - download_url: "https://github.com/aws/aws-sdk-cpp/archive/1.10.55.tar.gz" + checksum: "sha256:2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de" + filename: "aws-sdk-cpp-1.10.55.tar.gz" + - download_url: "https://apache.jfrog.io/artifactory/arrow/thirdparty/7.0.0/boost_1_81_0.tar.gz" + checksum: "sha256:9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574" + filename: "boost-1.81.0.tar.gz" + - download_url: "https://github.com/google/brotli/archive/v1.0.9.tar.gz" + checksum: "sha256:f9e8d81d0405ba66d181529af42a3354f838c939095ff99930da6aa9cdf6fe46" + filename: "brotli-v1.0.9.tar.gz" + - download_url: "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz" + checksum: "sha256:ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" + filename: "bzip2-1.0.8.tar.gz" + - download_url: "https://github.com/c-ares/c-ares/releases/download/cares-1_17_2/c-ares-1.17.2.tar.gz" + checksum: "sha256:4803c844ce20ce510ef0eb83f8ea41fa24ecaae9d280c468c582d2bb25b3913d" + filename: "cares-1.17.2.tar.gz" + - download_url: "https://github.com/google/crc32c/archive/refs/tags/1.1.2.tar.gz" + checksum: "sha256:ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56" + filename: "crc32c-1.1.2.tar.gz" + - download_url: "https://github.com/google/benchmark/archive/v1.8.3.tar.gz" + checksum: "sha256:6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce" + filename: "gbenchmark-v1.8.3.tar.gz" + - download_url: "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz" + checksum: "sha256:34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf" + filename: "gflags-v2.2.2.tar.gz" + - download_url: "https://github.com/google/glog/archive/v0.5.0.tar.gz" + checksum: "sha256:eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5" + filename: "glog-v0.5.0.tar.gz" + - download_url: "https://github.com/googleapis/google-cloud-cpp/archive/v2.22.0.tar.gz" + checksum: "sha256:0c68782e57959c82e0c81def805c01460a042c1aae0c2feee905acaa2a2dc9bf" + filename: "google-cloud-cpp-v2.22.0.tar.gz" + - download_url: "https://github.com/grpc/grpc/archive/v1.46.3.tar.gz" + checksum: "sha256:d6cbf22cb5007af71b61c6be316a79397469c58c82a942552a62e708bce60964" + filename: "grpc-v1.46.3.tar.gz" + - download_url: "https://github.com/google/googletest/archive/release-1.11.0.tar.gz" + checksum: "sha256:b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5" + filename: "gtest-1.11.0.tar.gz" + - download_url: "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" + checksum: "sha256:2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + filename: "jemalloc-5.3.0.tar.bz2" + - download_url: "https://github.com/lz4/lz4/archive/v1.9.4.tar.gz" + checksum: "sha256:0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b" + filename: "lz4-v1.9.4.tar.gz" + - download_url: "https://github.com/microsoft/mimalloc/archive/v2.0.6.tar.gz" + checksum: "sha256:9f05c94cc2b017ed13698834ac2a3567b6339a8bde27640df5a1581d49d05ce5" + filename: "mimalloc-v2.0.6.tar.gz" + - download_url: "https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.tar.gz" + checksum: "sha256:5daca6ca216495edf89d167f808d1d03c4a4d929cef7da5e10f135ae1540c7e4" + filename: "nlohmann-json-v3.10.5.tar.gz" + - download_url: "https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.13.0.tar.gz" + checksum: "sha256:7735cc56507149686e6019e06f588317099d4522480be5f38a2a09ec69af1706" + filename: "opentelemetry-cpp-v1.13.0.tar.gz" + - download_url: "https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v0.17.0.tar.gz" + checksum: "sha256:f269fbcb30e17b03caa1decd231ce826e59d7651c0f71c3b28eb5140b4bb5412" + filename: "opentelemetry-proto-v0.17.0.tar.gz" + - download_url: "https://archive.apache.org/dist/orc/orc-2.0.1/orc-2.0.1.tar.gz" + checksum: "sha256:1ffac0228aa83f04a1b1cf2788a3af5953e82587ae3a77c41900e99f2557132d" + filename: "orc-2.0.1.tar.gz" + - download_url: "https://github.com/google/protobuf/releases/download/v21.3/protobuf-all-21.3.tar.gz" + checksum: "sha256:2f723218f6cb709ae4cdc4fb5ed56a5951fc5d466f0128ce4c946b8c78c8c49f" + filename: "protobuf-v21.3.tar.gz" + - download_url: "https://github.com/miloyip/rapidjson/archive/232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" + checksum: "sha256:b9290a9a6d444c8e049bd589ab804e0ccf2b05dc5984a19ed5ae75d090064806" + filename: "rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" + - download_url: "https://github.com/google/re2/archive/2022-06-01.tar.gz" + checksum: "sha256:f89c61410a072e5cbcf8c27e3a778da7d6fd2f2b5b1445cd4f4508bee946ab0f" + filename: "re2-2022-06-01.tar.gz" + - download_url: "https://github.com/aws/s2n-tls/archive/v1.3.35.tar.gz" + checksum: "sha256:9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d" + filename: "s2n-v1.3.35.tar.gz" + - download_url: "https://github.com/google/snappy/archive/1.1.10.tar.gz" + checksum: "sha256:49d831bffcc5f3d01482340fe5af59852ca2fe76c3e05df0e67203ebbe0f1d90" + filename: "snappy-1.1.10.tar.gz" + - download_url: "https://archive.apache.org/dist/thrift/0.16.0/thrift-0.16.0.tar.gz" + checksum: "sha256:f460b5c1ca30d8918ff95ea3eb6291b3951cf518553566088f3f2be8981f6209" + filename: "thrift-0.16.0.tar.gz" + - download_url: "https://github.com/openucx/ucx/archive/v1.12.1.tar.gz" + checksum: "sha256:9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71" + filename: "ucx-1.12.1.tar.gz" + - download_url: "https://github.com/JuliaStrings/utf8proc/archive/v2.7.0.tar.gz" + checksum: "sha256:4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1" + filename: "utf8proc-v2.7.0.tar.gz" + - download_url: "https://github.com/xtensor-stack/xsimd/archive/13.0.0.tar.gz" + checksum: "sha256:8bdbbad0c3e7afa38d88d0d484d70a1671a1d8aefff03f4223ab2eb6a41110a3" + filename: "xsimd-13.0.0.tar.gz" + - download_url: "https://zlib.net/fossils/zlib-1.3.1.tar.gz" + checksum: "sha256:9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + filename: "zlib-1.3.1.tar.gz" + - download_url: "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz" + checksum: "sha256:8c29e06cf42aacc1eafc4077ae2ec6c6fcb96a626157e0593d5e82a34fd403c1" + filename: "zstd-1.5.6.tar.gz" + - download_url: "https://github.com/substrait-io/substrait/archive/v0.44.0.tar.gz" + checksum: "sha256:f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2" + filename: "substrait-0.44.0.tar.gz" diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/hermeto-generic-artifacts.sh b/sdk/python/feast/infra/feature_servers/multicloud/offline/hermeto-generic-artifacts.sh new file mode 100755 index 00000000000..bebe446ffb7 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/hermeto-generic-artifacts.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# modified from existing arrow script - +# https://github.com/apache/arrow/blob/apache-arrow-17.0.0/cpp/thirdparty/download_dependencies.sh + +set -eu + +APACHE_ARROW_VERSION="17.0.0" +APACHE_ARCHIVE_SHA256_CHECKSUM="8379554d89f19f2c8db63620721cabade62541f47a4e706dfb0a401f05a713ef" + +SUBSTRAIT_VERSION="0.44.0" +SUBSTRAIT_ARCHIVE_SHA256_CHECKSUM="f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2" + +PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) +DESTDIR=${PROJECT_ROOT_DIR}/sdk/python/feast/infra/feature_servers/multicloud/offline +artifacts_file=${DESTDIR}/artifacts.lock.yaml + +set_dependency() { + local url=$1 + local hash=$2 + local file=$3 + + echo " - download_url: \"${url}\"" >> ${artifacts_file} + echo " checksum: \"sha256:${hash}\"" >> ${artifacts_file} + echo " filename: \"${file}\"" >> ${artifacts_file} +} + +main() { + ARROW_VERSION_TAG=apache-arrow-${APACHE_ARROW_VERSION} + ARROW_ARCHIVE=${ARROW_VERSION_TAG}.tar.gz + + versions_file=${DESTDIR}/arrow-thirdparty-offline-versions + wget https://raw.githubusercontent.com/apache/arrow/refs/tags/${ARROW_VERSION_TAG}/cpp/thirdparty/versions.txt -O ${versions_file} + + # Load `DEPENDENCIES` variable. + hash_suffix="_BUILD_SHA256_CHECKSUM" + while IFS= read -r line; do + if [[ "$line" == *"${hash_suffix}"* ]]; then + export "$line" + fi + done < ${versions_file} + + echo '--- +metadata: + version: "1.0" +artifacts:' > ${artifacts_file} + + set_dependency "https://github.com/apache/arrow/archive/${ARROW_ARCHIVE}" "${APACHE_ARCHIVE_SHA256_CHECKSUM}" "${ARROW_ARCHIVE}" + + source ${versions_file} + for ((i = 0; i < ${#DEPENDENCIES[@]}; i++)); do + local dep_packed=${DEPENDENCIES[$i]} + + # Unpack each entry of the form "$home_var $tar_out $dep_url" + IFS=" " read -r dep_url_var dep_tar_name dep_url <<< "${dep_packed}" + + dep_hash_var=${dep_url_var%"_URL"}"${hash_suffix}" + + set_dependency "${dep_url}" "${!dep_hash_var}" "${dep_tar_name}" + done + + # set substrait manually until using an arrow version w/ this fix - https://github.com/apache/arrow/pull/46191 + substrait_tar_name="substrait-${SUBSTRAIT_VERSION}.tar.gz" + set_dependency "https://github.com/substrait-io/substrait/archive/v${SUBSTRAIT_VERSION}.tar.gz" "${SUBSTRAIT_ARCHIVE_SHA256_CHECKSUM}" ${substrait_tar_name} + + cat ${artifacts_file} +} + +main diff --git a/infra/scripts/offline-binary-build.sh b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-build.sh similarity index 96% rename from infra/scripts/offline-binary-build.sh rename to sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-build.sh index f2d51d9cd07..34c24202bce 100755 --- a/infra/scripts/offline-binary-build.sh +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-build.sh @@ -1,4 +1,4 @@ -# to run -> source ./infra/scripts/offline-binary-build.sh +# to run -> source ./sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-build.sh # on the build host... requires docker/podman # Get Feast project repository root directory diff --git a/infra/scripts/offline-binary-release-build.sh b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-release-build.sh similarity index 92% rename from infra/scripts/offline-binary-release-build.sh rename to sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-release-build.sh index 6c8e1c0084e..5ff58ad4a53 100755 --- a/infra/scripts/offline-binary-release-build.sh +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-release-build.sh @@ -1,4 +1,4 @@ -# to run -> source ./infra/scripts/offline-binary-release-build.sh +# to run -> source ./sdk/python/feast/infra/feature_servers/multicloud/offline/offline-binary-release-build.sh # on the build host... requires docker/podman # Get Feast project repository root directory diff --git a/infra/scripts/offline-build.sh b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-build.sh similarity index 63% rename from infra/scripts/offline-build.sh rename to sdk/python/feast/infra/feature_servers/multicloud/offline/offline-build.sh index 5b4bd3535ef..5950287f9ae 100755 --- a/infra/scripts/offline-build.sh +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-build.sh @@ -1,10 +1,6 @@ -# to run -> source ./infra/scripts/offline-build.sh +# to run -> source ./sdk/python/feast/infra/feature_servers/multicloud/offline/offline-build.sh # on the build host... requires docker/podman, git, wget -APACHE_ARROW_VERSION="17.0.0" -SUBSTRAIT_VERSION="0.44.0" -IBIS_VERSION="9.5.0" - # Get Feast project repository root directory PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) OFFLINE_BUILD_DIR=${PROJECT_ROOT_DIR}/offline_build @@ -15,15 +11,16 @@ mkdir offline_build # yum builder docker build \ - --tag yum-builder \ + --tag yum-builder:dev \ -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \ sdk/python/feast/infra/feature_servers/multicloud/offline -git clone --branch apache-arrow-${APACHE_ARROW_VERSION} https://github.com/apache/arrow ${OFFLINE_BUILD_DIR}/arrow -${OFFLINE_BUILD_DIR}/arrow/cpp/thirdparty/download_dependencies.sh ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty -wget https://github.com/substrait-io/substrait/archive/v${SUBSTRAIT_VERSION}.tar.gz -O ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty/substrait-${SUBSTRAIT_VERSION}.tar.gz - alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0' +hermeto fetch-deps \ + --source sdk/python/feast/infra/feature_servers/multicloud/offline \ + --output ${OFFLINE_BUILD_DIR}/hermeto-generic-output \ + generic + # not needed for downstream build from release ############################### hermeto fetch-deps \ @@ -60,27 +57,6 @@ hermeto fetch-deps \ hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output hermeto inject-files --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output -# arrow OFFLINE builder -rm -f ${OFFLINE_BUILD_DIR}/arrow/.dockerignore -docker build \ - --network none \ - --volume ${OFFLINE_BUILD_DIR}/arrow:/tmp/arrow:Z \ - --volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \ - --volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \ - --tag arrow-builder \ - -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.arrow \ - ${OFFLINE_BUILD_DIR}/arrow - -# ibis OFFLINE builder - ibis-framework must build from its own git repo... versioning requirement -git clone -b ${IBIS_VERSION} https://github.com/ibis-project/ibis ${OFFLINE_BUILD_DIR}/ibis -docker build \ - --network none \ - --volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \ - --volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \ - --tag ibis-builder \ - -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.ibis \ - ${OFFLINE_BUILD_DIR} - # feast OFFLINE builder docker build \ --network none \ @@ -90,6 +66,7 @@ docker build \ --volume ${OFFLINE_BUILD_DIR}/hermeto-yarn.env:/tmp/hermeto-yarn.env:Z \ --volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \ --volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \ + --volume ${OFFLINE_BUILD_DIR}/hermeto-generic-output:/tmp/hermeto-generic-output:Z \ --tag feature-server:sdist-build \ -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist \ ${PROJECT_ROOT_DIR} diff --git a/infra/scripts/offline-release-build.sh b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-release-build.sh similarity index 65% rename from infra/scripts/offline-release-build.sh rename to sdk/python/feast/infra/feature_servers/multicloud/offline/offline-release-build.sh index cde3589eb44..bfea7ed41c6 100755 --- a/infra/scripts/offline-release-build.sh +++ b/sdk/python/feast/infra/feature_servers/multicloud/offline/offline-release-build.sh @@ -1,10 +1,6 @@ -# to run -> source ./infra/scripts/offline-release-build.sh +# to run -> source ./sdk/python/feast/infra/feature_servers/multicloud/offline/offline-release-build.sh # on the build host... requires docker/podman, git, wget -APACHE_ARROW_VERSION="17.0.0" -SUBSTRAIT_VERSION="0.44.0" -IBIS_VERSION="9.5.0" - # Get Feast project repository root directory PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) OFFLINE_BUILD_DIR=${PROJECT_ROOT_DIR}/offline_build @@ -16,13 +12,15 @@ mkdir offline_build # yum builder docker build \ --build-arg RELEASE=true \ - --tag yum-builder \ + --tag yum-builder:release \ -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \ sdk/python/feast/infra/feature_servers/multicloud/offline -git clone --branch apache-arrow-${APACHE_ARROW_VERSION} https://github.com/apache/arrow ${OFFLINE_BUILD_DIR}/arrow -${OFFLINE_BUILD_DIR}/arrow/cpp/thirdparty/download_dependencies.sh ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty -wget https://github.com/substrait-io/substrait/archive/v${SUBSTRAIT_VERSION}.tar.gz -O ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty/substrait-${SUBSTRAIT_VERSION}.tar.gz +alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0' +hermeto fetch-deps \ + --source sdk/python/feast/infra/feature_servers/multicloud/offline \ + --output ${OFFLINE_BUILD_DIR}/hermeto-generic-output \ + generic alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0' hermeto fetch-deps \ @@ -42,17 +40,14 @@ hermeto fetch-deps \ hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output hermeto inject-files --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output -# ibis OFFLINE builder - ibis-framework must build from its own git repo... versioning requirement -git clone -b ${IBIS_VERSION} https://github.com/ibis-project/ibis ${OFFLINE_BUILD_DIR}/ibis - # feast OFFLINE builder docker build \ --network none \ - --volume ${OFFLINE_BUILD_DIR}/arrow:/tmp/arrow:Z \ --volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \ --volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \ + --volume ${OFFLINE_BUILD_DIR}/hermeto-generic-output:/tmp/hermeto-generic-output:Z \ --tag feature-server:sdist-release-build \ -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release \ - ${PROJECT_ROOT_DIR} + sdk/python/feast/infra/feature_servers/multicloud docker run --rm -ti feature-server:sdist-release-build feast version diff --git a/sdk/python/feast/infra/feature_servers/multicloud/offline/pip_install_binary_exceptions.txt b/sdk/python/feast/infra/feature_servers/multicloud/offline/pip_install_binary_exceptions.txt deleted file mode 100644 index 0faedffdbaf..00000000000 --- a/sdk/python/feast/infra/feature_servers/multicloud/offline/pip_install_binary_exceptions.txt +++ /dev/null @@ -1,2 +0,0 @@ -# EXCEPTION for milvus -milvus-lite==2.4.12