From e4fcbb6ff98e4d98ea47ab260ae607b80118021a Mon Sep 17 00:00:00 2001 From: Daniel Wagner Date: Wed, 28 Jun 2023 12:38:19 +0200 Subject: [PATCH] build: Use containers with matrix build Use a matrix build approach and a base container which already contains all the libraries installed. Signed-off-by: Daniel Wagner --- .github/cross/clang.txt | 5 - ...buntu-armhf.txt => ubuntu-cross-armhf.txt} | 0 ...u-ppc64le.txt => ubuntu-cross-ppc64le.txt} | 0 ...buntu-s390x.txt => ubuntu-cross-s390x.txt} | 0 .github/workflows/build.yml | 362 +++--------------- scripts/build.sh | 177 +++++++++ 6 files changed, 229 insertions(+), 315 deletions(-) delete mode 100644 .github/cross/clang.txt rename .github/cross/{ubuntu-armhf.txt => ubuntu-cross-armhf.txt} (100%) rename .github/cross/{ubuntu-ppc64le.txt => ubuntu-cross-ppc64le.txt} (100%) rename .github/cross/{ubuntu-s390x.txt => ubuntu-cross-s390x.txt} (100%) create mode 100755 scripts/build.sh diff --git a/.github/cross/clang.txt b/.github/cross/clang.txt deleted file mode 100644 index 1484a3e7..00000000 --- a/.github/cross/clang.txt +++ /dev/null @@ -1,5 +0,0 @@ -[binaries] -c = 'clang' -cpp = 'clang++' -strip = 'strip' -pkgconfig = 'pkg-config' diff --git a/.github/cross/ubuntu-armhf.txt b/.github/cross/ubuntu-cross-armhf.txt similarity index 100% rename from .github/cross/ubuntu-armhf.txt rename to .github/cross/ubuntu-cross-armhf.txt diff --git a/.github/cross/ubuntu-ppc64le.txt b/.github/cross/ubuntu-cross-ppc64le.txt similarity index 100% rename from .github/cross/ubuntu-ppc64le.txt rename to .github/cross/ubuntu-cross-ppc64le.txt diff --git a/.github/cross/ubuntu-s390x.txt b/.github/cross/ubuntu-cross-s390x.txt similarity index 100% rename from .github/cross/ubuntu-s390x.txt rename to .github/cross/ubuntu-cross-s390x.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6137be5..e3e2fd4e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,357 +10,99 @@ on: workflow_dispatch: jobs: - gcc-debug: - name: gcc debug + default: runs-on: ubuntu-latest + strategy: + matrix: + compiler: [gcc, clang] + buildtype: [debug, release] + container: + image: ghcr.io/igaw/linux-nvme/debian:0.30 steps: - - name: install dependencies - run: sudo apt-get install gcc pkg-config libjson-c-dev libhugetlbfs-dev - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.x' - - uses: bsfishy/meson-build@v1.0.3 - name: build - with: - action: build - setup-options: > - --werror - --buildtype=debug - --force-fallback-for=libnvme - -Dlibnvme:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: log files - path: | - build/meson-logs/*.txt - - gcc-release: - name: gcc release - runs-on: ubuntu-latest - steps: - - name: install dependencies - run: sudo apt-get install gcc pkg-config libjson-c-dev libhugetlbfs-dev - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - uses: bsfishy/meson-build@v1.0.3 - name: build - with: - action: build - setup-options: > - --werror - --buildtype=release - --force-fallback-for=libnvme - -Dlibnvme:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: log files - path: | - build/meson-logs/*.txt - - clang-release: - name: clang release - runs-on: ubuntu-latest - steps: - - name: install dependencies - run: sudo apt-get install clang pkg-config libjson-c-dev libhugetlbfs-dev - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - uses: bsfishy/meson-build@v1.0.3 - name: build - with: - action: build - setup-options: > - --werror - --buildtype=release - --cross-file=.github/cross/clang.txt - --force-fallback-for=libnvme - -Dlibnvme:werror=false - -Dopenssl:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: log files - path: | - build/meson-logs/*.txt - - fallback-shared-libraries: - name: fallback shared libraries - runs-on: ubuntu-latest - steps: - - name: install dependencies - run: sudo apt-get install libpam-dev libcap-ng-dev - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - uses: BSFishy/meson-build@v1.0.3 - name: build - with: - action: build - setup-options: > - --werror - --buildtype=release - --wrap-mode=forcefallback - --default-library=shared - -Dlibnvme:werror=false - -Dopenssl:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: log files - path: | - build/meson-logs/*.txt - - fallback-static-libraries: - name: fallback static libraries - runs-on: ubuntu-latest - steps: - - name: install dependencies - run: sudo apt-get install gcc meson ninja-build libpam-dev libcap-ng-dev - - uses: actions/checkout@v3 - name: build run: | - LDFLAGS=-static meson setup \ - --werror \ - --buildtype=release \ - --wrap-mode=forcefallback \ - --default-library=static \ - -Dlibnvme:openssl=disabled \ - -Dlibnvme:python=disabled \ - -Dlibnvme:werror=false \ - .build - meson compile -C .build - - name: test - run: | - meson test -C .build - ldd .build/nvme 2>&1 | grep 'not a dynamic executable' + scripts/build.sh -b ${{ matrix.buildtype }} -c ${{ matrix.compiler }} - uses: actions/upload-artifact@v3 + name: upload logs if: failure() with: - name: log files + name: logs files path: | - build/meson-logs/*.txt + .build-ci/meson-logs/*.txt - cross-armhf: - name: cross armhf + cross: runs-on: ubuntu-latest + strategy: + matrix: + include: + - arch: armhf + port: armhf + compiler: gcc-arm-linux-gnueabihf + packages: + - arch: s390x + port: s390x + compiler: gcc-s390x-linux-gnu + packages: libgcc-s1:s390x + - arch: ppc64le + port: ppc64el + compiler: gcc-powerpc64le-linux-gnu + packges: steps: + - uses: actions/checkout@v3 - name: set up arm architecture run: | export release=$(lsb_release -c -s) - sudo dpkg --add-architecture armhf + sudo dpkg --add-architecture ${{ matrix.port }} sudo sed -i -e 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list - sudo dd of=/etc/apt/sources.list.d/armhf.list < - --werror - --buildtype=release - --cross-file=.github/cross/ubuntu-armhf.txt - --force-fallback-for=libnvme - -Dlibnvme:python=disabled - -Dopenssl:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: log files - path: | - build/meson-logs/*.txt - - cross-ppc64le: - name: cross ppc64le - runs-on: ubuntu-latest - steps: - - name: set up ppc64le architecture + sudo apt install -y meson pkg-config qemu-user-static ${{ matrix.compiler}} libjson-c-dev:${{ matrix.port }} ${{ matrix.packages }} + - name: build run: | - export release=$(lsb_release -c -s) - sudo dpkg --add-architecture ppc64el - sudo sed -i -e 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list - sudo dd of=/etc/apt/sources.list.d/ppc64el.list < - --werror - --buildtype=release - --cross-file=.github/cross/ubuntu-ppc64le.txt - --force-fallback-for=libnvme - -Dlibnvme:werror=false - -Dlibnvme:python=disabled - -Dopenssl:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test - with: - action: test - meson-version: 0.61.2 + scripts/build.sh -b release -c gcc -t ${{ matrix.arch }} cross - uses: actions/upload-artifact@v3 + name: upload logs if: failure() with: name: log files path: | - build/meson-logs/*.txt + .build-ci/meson-logs/*.txt - cross-s390x: - name: cross s390x + fallback-shared-libraries: + name: fallback shared libraries runs-on: ubuntu-latest + container: + image: ghcr.io/igaw/linux-nvme/debian:0.30 + if: github.ref == 'refs/heads/master' steps: - - name: set up s390x architecture - run: | - export release=$(lsb_release -c -s) - sudo dpkg --add-architecture s390x - sudo sed -i -e 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list - sudo dd of=/etc/apt/sources.list.d/s390x.list < - --werror - --buildtype=release - --cross-file=.github/cross/ubuntu-s390x.txt - --force-fallback-for=libnvme - -Dlibnvme:werror=false - -Dlibnvme:python=disabled - -Dopenssl:werror=false - meson-version: 0.61.2 - - uses: bsfishy/meson-build@v1.0.3 - name: test + - uses: actions/setup-python@v4 with: - action: test - meson-version: 0.61.2 + python-version: '3.x' + - name: build + run: | + scripts/build.sh -b release -c gcc fallback - uses: actions/upload-artifact@v3 if: failure() with: name: log files path: | - build/meson-logs/*.txt + .build-ci/meson-logs/*.txt build-muon: - name: muon + name: muon minimal static runs-on: ubuntu-latest + container: + image: ghcr.io/igaw/linux-nvme/debian:0.30 steps: - - name: install dependencies - run: sudo apt install gcc make pkg-config libcurl4-openssl-dev libarchive-dev libpkgconf-dev - uses: actions/checkout@v3 - - name: build samurai - run: | - cd .. - - export CC=gcc - export ROOT_PATH=$(pwd)/root - - git clone --depth 1 https://github.com/michaelforney/samurai.git - - cd samurai - make - make PREFIX=${ROOT_PATH} install - - name: build muon - run: | - cd .. - - export CC=gcc - export ROOT_PATH=$(pwd)/root - export PATH=${ROOT_PATH}/bin:${PATH} - - - git clone https://git.sr.ht/~lattis/muon - git -C muon reset --hard 0.2.0 - - cd muon - ./bootstrap.sh stage1 - stage1/muon setup \ - -Dprefix=${ROOT_PATH} \ - -Ddocs=disabled \ - -Dsamurai=disabled \ - .build - samu -C .build - .build/muon -C .build install - name: build run: | - export ROOT_PATH=$(pwd)/../root - export PATH=${ROOT_PATH}/bin:${PATH} - - muon setup \ - -Dprefix=${ROOT_PATH} \ - -Dwrap_mode=forcefallback \ - -Dlibnvme:json-c=disabled \ - -Dlibnvme:python=disabled \ - -Dlibnvme:openssl=disabled \ - -Djson-c=disabled \ - .build - samu -C .build - muon -C .build install - - name: test - run: | - export ROOT_PATH=$(pwd)/../root - export PATH=${ROOT_PATH}/bin:${PATH} - - muon -C .build test + scripts/build.sh -m muon diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..94c62b86 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,177 @@ +#!/bin/bash + +usage() { + echo "Usage: build.sh [-b [release|debug]] " + echo " [-c [gcc|clang]]" + echo " [-m [meson|muon]" + echo " [config]" + echo "" + echo "CI build script." + echo "" + echo " -b [release]|debug build type" + echo " -c [gcc]|clang compiler to use" + echo " -m [meson]|muon use meson or muon" + echo " -t [arm]|ppc64le|s390x cross compile target" + echo "" + echo "configs with meson:" + echo " [default] default settings" + echo " fallback download all dependencies" + echo " and build them as shared libaries" + echo " cross use cross toolchain to build" + echo "" + echo "configs with muon:" + echo " [default] minimal static build" +} + +BUILDTOOL=meson +MESON=meson +BUILDTYPE=release +CROSS_TARGET=arm +CC=${CC:-"gcc"} + +while getopts "b:c:m:t:" o; do + case "${o}" in + b) + BUILDTYPE="${OPTARG}" + ;; + c) + CC="${OPTARG}" + ;; + m) + BUILDTOOL="${OPTARG}" + ;; + t) + CROSS_TARGET="${OPTARG}" + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +CONFIG=${1:-"default"} + +cd "$(git rev-parse --show-toplevel)" || exit 1 + +BUILDDIR="$(pwd)/.build-ci" + +config_meson_default() { + CC="${CC}" "${MESON}" setup \ + --werror \ + --buildtype="${BUILDTYPE}" \ + --force-fallback-for=libnvme \ + -Dlibnvme:werror=false \ + "${BUILDDIR}" +} + +config_meson_fallback() { + CC="${CC}" "${MESON}" setup \ + --werror \ + --buildtype="${BUILDTYPE}" \ + --wrap-mode=forcefallback \ + --default-library=both \ + -Dlibnvme:werror=false \ + -Dopenssl:werror=false \ + "${BUILDDIR}" +} + +config_meson_cross() { + CC="${CC}" "${MESON}" setup \ + --werror \ + --buildtype="${BUILDTYPE}" \ + --cross-file=.github/cross/ubuntu-cross-${CROSS_TARGET}.txt \ + --force-fallback-for=libnvme \ + -Dlibnvme:werror=false \ + -Dlibnvme:python=disabled \ + -Dlibnvme:openssl=disabled \ + "${BUILDDIR}" +} + +build_meson() { + "${MESON}" compile \ + -C "${BUILDDIR}" +} + +test_meson() { + "${MESON}" test \ + -C "${BUILDDIR}" +} + +tools_build_samurai() { + mkdir -p "${BUILDDIR}"/build-tools + git clone --depth 1 https://github.com/michaelforney/samurai.git \ + "${BUILDDIR}/build-tools/samurai" + pushd "${BUILDDIR}/build-tools/samurai" || exit 1 + + CC="${CC}" make + SAMU="${BUILDDIR}/build-tools/samurai/samu" + + popd || exit 1 +} + +tools_build_muon() { + mkdir -p "${BUILDDIR}"/build-tools + git clone --depth 1 https://git.sr.ht/~lattis/muon \ + "${BUILDDIR}/build-tools/muon" + pushd "${BUILDDIR}/build-tools/muon" || exit 1 + + CC="${CC}" ninja="${SAMU}" ./bootstrap.sh stage1 + + CC="${CC}" ninja="${SAMU}" stage1/muon setup \ + -Dprefix="${BUILDDIR}/build-tools" \ + -Dlibcurl=enabled \ + -Dlibarchive=enabled \ + -Dlibpkgconf=enabled \ + -Ddocs=disabled \ + -Dsamurai=disabled \ + "${BUILDDIR}/build-tools/.build-muon" + "${SAMU}" -C "${BUILDDIR}/build-tools/.build-muon" + MUON="${BUILDDIR}/build-tools/.build-muon/muon" + + # "${MUON}" -C "${BUILDDIR}/build-tools/.build-muon" test + + popd || exit 1 +} + +config_muon_default() { + CC="${CC}" CFLAGS="${CFLAGS} -static" \ + ninja="${SAMU}" "${MUON}" setup \ + -Dwrap_mode=forcefallback \ + -Dlibnvme:json-c=disabled \ + -Dlibnvme:python=disabled \ + -Dlibnvme:openssl=disabled \ + -Dlibnvme:keyutils=disabled \ + -Djson-c=disabled \ + "${BUILDDIR}" +} + +build_muon() { + "${SAMU}" -C "${BUILDDIR}" +} + +test_muon() { + ninja="${SAMU}" "${MUON}" -C "${BUILDDIR}" test + ldd "${BUILDDIR}/nvme" 2>&1 | grep 'not a dynamic executable' || exit 1 +} + +rm -rf "${BUILDDIR}" + +if [[ "${BUILDTOOL}" == "muon" ]]; then + if ! which samu ; then + tools_build_samurai + else + SAMU="$(which samu)" + fi + + if ! which muon ; then + tools_build_muon + else + MUON="$(which muon)" + fi +fi + +config_"${BUILDTOOL}"_"${CONFIG}" +build_"${BUILDTOOL}" +test_"${BUILDTOOL}" -- 2.49.0