From 6fca4b03f6ea08e12e34fc90c404e26db9f87b06 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 20 Oct 2022 02:35:07 +0000 Subject: [PATCH] Add win build for CI --- .github/workflows/CI.yml | 68 +++++++++++++++++++++++++++++++++++ patch/zlib-msvc/zconf.h.patch | 11 ++++++ scripts/build_win_exiv2.bat | 28 +++++++++++++++ scripts/build_win_openssl.bat | 19 ++++++++++ scripts/build_win_pkgconf.bat | 15 ++++++++ scripts/build_win_zlib.bat | 32 +++++++++++++++++ scripts/download_resource.bat | 23 ++++++++++++ scripts/download_resource.py | 42 ++++++++++++++++++++++ scripts/get_cache_key.py | 36 ++++++++++++------- 9 files changed, 261 insertions(+), 13 deletions(-) create mode 100644 patch/zlib-msvc/zconf.h.patch create mode 100644 scripts/build_win_exiv2.bat create mode 100644 scripts/build_win_openssl.bat create mode 100644 scripts/build_win_pkgconf.bat create mode 100644 scripts/build_win_zlib.bat create mode 100644 scripts/download_resource.bat create mode 100644 scripts/download_resource.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bf83fa8..c8beb66 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -161,3 +161,71 @@ jobs: export "LIBRARY_PATH=$LIBRARY_PATH:`pwd`/clib/lib" export "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/clib/lib" cargo test --features all --verbose -- --show-output || exit 1 + build-win: + runs-on: windows-latest + defaults: + run: + shell: cmd + - name: Check Out + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: true + - name: Get Cache Key + id: cache_key + run: | + cd scripts + python get_cache_key.py --prefix=win zlib pkgconf exiv2 openssl || exit 1 + - name: Cache + id: cache + uses: actions/cache@v2 + with: + path: clib/ + key: win-${{ steps.cache_key.outputs.cache_key }} + - name: Install Ninja + run: | + python -m pip install --upgrade ninja + - name: Install Meson + if: steps.cache.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade meson + - name: Setup CL + if: steps.cache.outputs.cache-hit != 'true' + uses: ilammy/msvc-dev-cmd@v1 + - name: Set up NASM + if: steps.cache.outputs.cache-hit != 'true' + uses: ilammy/setup-nasm@v1.3.0 + - name: Setup MSYS2 + if: steps.cache.outputs.cache-hit != 'true' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + base-devel + - name: Build zlib + if: steps.cache.outputs.cache-hit != 'true' + run: | + COPY /Y scripts\build_win_zlib.bat || exit 1 + CALL build_win_zlib.bat || exit 1 + - name: Build pkgconf + if: steps.cache.outputs.cache-hit != 'true' + run: | + COPY /Y scripts\build_win_pkgconf.bat || exit 1 + CALL build_win_pkgconf.bat || exit 1 + - name: Build exiv2 + if: steps.cache.outputs.cache-hit != 'true' + run: | + COPY /Y build_win_exiv2.bat || exit 1 + CALL build_win_exiv2.bat || exit 1 + - name: Build openssl + if: steps.cache.outputs.cache-hit != 'true' + run: | + COPY /Y scripts\build_win_openssl.bat || exit 1 + CALL build_win_openssl.bat || exit 1 + - name: Build + run: | + SET PKG_CONFIG_PATH=%CD%\clib\lib\pkgconfig + SET CMAKE_PREFIX_PATH=%CD%\clib + SET PATH=%PATH%;%CD%\clib\bin + cargo build --features exif -vv || exit 1 diff --git a/patch/zlib-msvc/zconf.h.patch b/patch/zlib-msvc/zconf.h.patch new file mode 100644 index 0000000..f4ad002 --- /dev/null +++ b/patch/zlib-msvc/zconf.h.patch @@ -0,0 +1,11 @@ +--- zconf.h.orig 2022-04-26 09:20:44 +0000 ++++ zconf.h 2022-04-26 09:23:39 +0000 +@@ -474,7 +474,7 @@ + #endif + #ifndef Z_SOLO + # if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +-# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ ++//# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ + # ifdef VMS + # include /* for off_t */ + # endif diff --git a/scripts/build_win_exiv2.bat b/scripts/build_win_exiv2.bat new file mode 100644 index 0000000..5e38668 --- /dev/null +++ b/scripts/build_win_exiv2.bat @@ -0,0 +1,28 @@ +@ECHO OFF +SETLOCAL +SET TOP=%CD% +SET PREFIX=%CD%\clib +SET PKG_CONFIG_DIR=%PREFIX%\lib\pkgconfig +SET PATCH_DIR=%CD%\exif\patchs +IF NOT EXIST cbuild ( + MD cbuild || EXIT /B 1 +) +CD cbuild || EXIT /B 1 +git clone --depth 1 'https://github.com/Exiv2/exiv2' || EXIT /B %ERRORLEVEL% +CD exiv2 || EXIT /B %ERRORLEVEL% +git apply %PATCH_DIR%\basicio.cpp.patch || EXIT /B %ERRORLEVEL% +IF NOT EXIST build ( + MD build || EXIT /B 1 +) +CD build || EXIT /B 1 +cmake ^ + -G Ninja ^ + -DCMAKE_PREFIX_PATH=%PREFIX% ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%PREFIX% ^ + -DINSTALL_PKGCONFIG_DIR=%PKG_CONFIG_DIR% ^ + ../ || EXIT /B %ERRORLEVEL% +cmake --build . --config Release && cmake --build . --config Release --target INSTALL ^ + || cmake --build . --config Release && cmake --build . --config Release --target INSTALL ^ + || EXIT /B %ERRORLEVEL% +ENDLOCAL diff --git a/scripts/build_win_openssl.bat b/scripts/build_win_openssl.bat new file mode 100644 index 0000000..1f54de2 --- /dev/null +++ b/scripts/build_win_openssl.bat @@ -0,0 +1,19 @@ +@ECHO OFF +SETLOCAL +SET TOP=%CD% +SET SCRIPTS_DIR=%CD%\scripts +SET DOWNLOAD_RESOURCE=%SCRIPTS_DIR%\download_resource.bat +SET PREFIX=%CD%\clib +SET PKG_CONFIG_DIR=%PREFIX%\lib\pkgconfig +SET OPENSSLDIR=%PREFIX%\ssl +IF NOT EXIST cbuild ( + MD cbuild || EXIT /B 1 +) +CD cbuild || EXIT /B 1 +CALL %DOWNLOAD_RESOURCE% -o "openssl-3.0.5.tar.gz" "https://www.openssl.org/source/openssl-3.0.5.tar.gz" || EXIT /B %ERRORLEVEL% +tar -xzvf "openssl-3.0.5.tar.gz" || EXIT /B %ERRORLEVEL% +CD openssl-3.0.5 || EXIT /B 1 +perl Configure shared zlib-dynamic --prefix=%PREFIX% --openssldir=%OPENSSLDIR% || EXIT /B %ERRORLEVEL% +nmake -j8 || EXIT /B %ERRORLEVEL% +nmake install || EXIT /B %ERRORLEVEL% +ENDLOCAL diff --git a/scripts/build_win_pkgconf.bat b/scripts/build_win_pkgconf.bat new file mode 100644 index 0000000..92e0c02 --- /dev/null +++ b/scripts/build_win_pkgconf.bat @@ -0,0 +1,15 @@ +@ECHO OFF +SETLOCAL +SET PREFIX=%CD%\clib +IF NOT EXIST cbuild ( + MD cbuild || EXIT /B 1 +) +CD cbuild || EXIT /B 1 +git clone --depth 1 "https://github.com/pkgconf/pkgconf" || EXIT /B %ERRORLEVEL% +CD pkgconf || EXIT /B %ERRORLEVEL% +meson setup build --prefix %PREFIX% -Dtests=false || EXIT /B %ERRORLEVEL% +meson compile -C build || EXIT /B %ERRORLEVEL% +meson install -C build || EXIT /B %ERRORLEVEL% +CD %PREFIX%\bin || EXIT /B 1 +COPY /Y pkgconf.exe pkg-config.exe || EXIT /B %ERRORLEVEL% +ENDLOCAL diff --git a/scripts/build_win_zlib.bat b/scripts/build_win_zlib.bat new file mode 100644 index 0000000..d03c5bf --- /dev/null +++ b/scripts/build_win_zlib.bat @@ -0,0 +1,32 @@ +@ECHO OFF +SETLOCAL +SET TOP=%CD% +SET SCRIPTS_DIR=%CD%\scripts +SET DOWNLOAD_RESOURCE=%SCRIPTS_DIR%\download_resource.bat +SET PREFIX=%CD%\clib +SET PKG_CONFIG_DIR=%PREFIX%\lib\pkgconfig +SET PATCH_DIR=%CD%\patch\zlib-msvc +IF NOT EXIST cbuild ( + MD cbuild || EXIT /B 1 +) +CD cbuild || EXIT /B 1 +CALL %DOWNLOAD_RESOURCE% -o "zlib-1.2.13.tar.gz" "https://www.zlib.net/zlib-1.2.13.tar.gz" || EXIT /B %ERRORLEVEL% +tar -xzvf "zlib-1.2.13.tar.gz" || EXIT /B %ERRORLEVEL% +CD zlib-1.2.13 || EXIT /B 1 +IF NOT EXIST build ( + MD build || EXIT /B 1 +) +CD build || EXIT /B 1 +cmake ^ + -G Ninja ^ + -DCMAKE_PREFIX_PATH=%PREFIX% ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%PREFIX% ^ + -DINSTALL_PKGCONFIG_DIR=%PKG_CONFIG_DIR% ^ + ../ || EXIT /B %ERRORLEVEL% +cmake --build . --config Release && cmake --build . --config Release --target INSTALL ^ + || cmake --build . --config Release && cmake --build . --config Release --target INSTALL ^ + || EXIT /B %ERRORLEVEL% +CD %PREFIX%\include || EXIT /B 1 +patch -p1 zconf.h %PATCH_DIR%\zconf.h.patch || EXIT /B %ERRORLEVEL% +ENDLOCAL diff --git a/scripts/download_resource.bat b/scripts/download_resource.bat new file mode 100644 index 0000000..950c6a2 --- /dev/null +++ b/scripts/download_resource.bat @@ -0,0 +1,23 @@ +@ECHO OFF +SETLOCAL +SET DOWNLOAD_RESOURCE_PY=download_resource.py +IF NOT EXIST %DOWNLOAD_RESOURCE_PY% ( + SET DOWNLOAD_RESOURCE_PY=%CD%\download_resource.py + IF NOT EXIST %DOWNLOAD_RESOURCE_PY% ( + IF DEFINED SCRIPTS_DIR ( + SET DOWNLOAD_RESOURCE_PY=%SCRIPTS_DIR%\download_resource.py + ) + ) +) +IF NOT EXIST %DOWNLOAD_RESOURCE_PY% ( + EXIT /B 1 +) +python %DOWNLOAD_RESOURCE_PY% %* +IF ERRORLEVEL 2 ( + python -m pip install --upgrade requests + python %DOWNLOAD_RESOURCE_PY% %* +) +IF %ERRORLEVEL% NEQ 0 ( + EXIT /B %ERRORLEVEL% +) +ENDLOCAL diff --git a/scripts/download_resource.py b/scripts/download_resource.py new file mode 100644 index 0000000..0e771fa --- /dev/null +++ b/scripts/download_resource.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser, Namespace +from sys import exit as _exit +from traceback import print_exc +from urllib.parse import urlparse +from os.path import join +try: + from requests import Session +except ImportError: + _exit(2) +except Exception: + print_exc() + _exit(1) + + +def download_file(arg: Namespace): + s = Session() + fn = urlparse(arg.URL).path.split('/')[-1] + output = fn + if arg.output: + output = arg.output + elif arg.dest: + output = join(arg.dest, fn) + print(f'Downloading {arg.URL} to {output}') + r = s.get(arg.URL, stream=True) + if r.status_code >= 400: + raise Exception(f'Failed to download {arg.URL}') + with open(output, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + + +try: + p = ArgumentParser(description='Download resources from github.') + p.add_argument('URL', help='The URL of the resource.') + p.add_argument('-o', '--output', help='The full path of output file.') + p.add_argument('-d', '--dest', help='The destination directory.') + arg = p.parse_intermixed_args() + download_file(arg) +except Exception: + print_exc() + _exit(1) diff --git a/scripts/get_cache_key.py b/scripts/get_cache_key.py index 9c0d427..3226592 100644 --- a/scripts/get_cache_key.py +++ b/scripts/get_cache_key.py @@ -6,7 +6,7 @@ from time import gmtime, time, strftime from typing import List -ALL_FEATURES = ['exiv2', 'ffmpeg', 'libzip', 'x264'] +ALL_FEATURES = ['exiv2', 'ffmpeg', 'libzip', 'openssl', 'pkgconf', 'x264', 'zlib'] def sha256(data) -> str: if isinstance(data, str): @@ -18,22 +18,32 @@ def sha256(data) -> str: return s.hexdigest() -def hash_file(feature) -> str: - fn = f"build_{feature}.sh" - if not exists(fn): - return '' - with open(fn, 'rb') as f: - c = f.read(256) - s = _sha256() - while len(c) > 0: - s.update(c) - c = f.read(256) - return s.hexdigest() +def hash_file(feature, prefix) -> str: + if prefix is None: + fns = [f"build_{feature}.sh"] + else: + fns = [] + fns.append(f"build_{prefix}_{feature}.sh") + fns.append(f"build_{prefix}_{feature}.bat") + fns.append(f"download_{prefix}_{feature}.sh") + fns.append(f"download_{prefix}_{feature}.bat") + s = None + for fn in fns: + if exists(fn): + if s is None: + s = _sha256() + with open(fn, 'rb') as f: + c = f.read(256) + while len(c) > 0: + s.update(c) + c = f.read(256) + return s.hexdigest() if s is not None else '' try: p = ArgumentParser(description='Get the cache key which used in action/cache') p.add_argument("features", help="The feature's name", action='append', nargs='+', choices=['all'] + ALL_FEATURES) + p.add_argument('--prefix', help='The prefix of the cache key') args = p.parse_intermixed_args(sys.argv[1:]) features: List[str] = args.features[0] if 'all' in features: @@ -42,7 +52,7 @@ try: now = time() for i in features: dt = strftime('%Y-%m', gmtime(now)) - h = hash_file(i) + h = hash_file(i, args.prefix) d += f"{i}={dt}:{h}\n" print(d) print(f"::set-output name=cache_key::{sha256(d)}")