From d52ef79363aebccfbbaf66a7063b93d004186e68 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 8 Feb 2024 07:43:24 +0100 Subject: [PATCH 1/5] start --- extensions/commands/bin/README.md | 0 extensions/commands/bin/cmd_lipo.py | 43 +++++++++++++++++++++++++++++ tests/test_lipo_command.py | 43 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 extensions/commands/bin/README.md create mode 100644 extensions/commands/bin/cmd_lipo.py create mode 100644 tests/test_lipo_command.py diff --git a/extensions/commands/bin/README.md b/extensions/commands/bin/README.md new file mode 100644 index 00000000..e69de29b diff --git a/extensions/commands/bin/cmd_lipo.py b/extensions/commands/bin/cmd_lipo.py new file mode 100644 index 00000000..7960eab2 --- /dev/null +++ b/extensions/commands/bin/cmd_lipo.py @@ -0,0 +1,43 @@ +import os +import pathlib +from conan.api.conan_api import ConanAPI +from conan.cli.command import conan_command, conan_subcommand +from conan.errors import ConanException + + +@conan_command(group="Binary manipulation") +def lipo(conan_api: ConanAPI, parser, *args): + """ + Wrapper over lipo to manage universal binaries for Apple OS's. + """ + + +@conan_subcommand() +def lipo_create(conan_api: ConanAPI, parser, subparser, *args): + """ + Create lipo binaries from the results of a Conan full_deploy. It expects a folder structure as: + //// + """ + subparser.add_argument("input_path", help="Root path for the Conan deployment. Needs ") + subparser.add_argument("-a", "--architecture", help="Each architecture that will be added to the resulting " + "universal binary. If not used, all found architectures will be added.", + action="append") + subparser.add_argument("--name-filter", help="", + action="append") + subparser.add_argument("--build-type-filter", help="", + action="append") + args = parser.parse_args(*args) + + input_path = pathlib(args.input_path) + + if not input_path.exists() or not input_path.is_dir(): + raise ConanException(f"The input path is not valid.") + + +@conan_subcommand() +def lipo_info(conan_api: ConanAPI, parser, subparser, *args): + """ + Get information for lipo files + """ + + args = parser.parse_args(*args) diff --git a/tests/test_lipo_command.py b/tests/test_lipo_command.py new file mode 100644 index 00000000..775a9353 --- /dev/null +++ b/tests/test_lipo_command.py @@ -0,0 +1,43 @@ +import sys +import tempfile +import textwrap +import os + +from tools import load, save, run + +import pytest + +from tools import load, save, run + + +@pytest.fixture(autouse=True) +def conan_test(): + old_env = dict(os.environ) + env_vars = {"CONAN_HOME": tempfile.mkdtemp(suffix='conans')} + os.environ.update(env_vars) + current = tempfile.mkdtemp(suffix="conans") + cwd = os.getcwd() + os.chdir(current) + try: + yield + finally: + os.chdir(cwd) + os.environ.clear() + os.environ.update(old_env) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="Universal binaries tests only for macOS") +def test_lipo_create(): + repo = os.path.join(os.path.dirname(__file__), "..") + run(f"conan config install {repo}") + run("conan --help") + run("conan new cmake_lib -d name=require -d version=1.0") + run("conan create . -tf="" -s arch=armv8") + run("conan create . -tf="" -s arch=x86_64") + run("conan new cmake_lib -d name=mylibrary -d version=1.0 -d requires=require/1.0 --force") + run("conan create . -tf="" -s arch=armv8") + run("conan create . -tf="" -s arch=x86_64") + run("conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=armv8") + run("conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=x86_64") + run("conan bin:lipo full_deploy -a x86_64 -a armv8") + From 1553728f87f03d541604032ab6d20b58f1a13781 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 8 Feb 2024 10:02:43 +0100 Subject: [PATCH 2/5] wip --- extensions/commands/bin/cmd_lipo.py | 104 +++++++++++++++++++++++++--- tests/test_lipo_command.py | 10 ++- 2 files changed, 98 insertions(+), 16 deletions(-) diff --git a/extensions/commands/bin/cmd_lipo.py b/extensions/commands/bin/cmd_lipo.py index 7960eab2..c3c86a23 100644 --- a/extensions/commands/bin/cmd_lipo.py +++ b/extensions/commands/bin/cmd_lipo.py @@ -1,6 +1,9 @@ import os import pathlib +import shutil +import subprocess from conan.api.conan_api import ConanAPI +from conan.api.output import ConanOutput from conan.cli.command import conan_command, conan_subcommand from conan.errors import ConanException @@ -18,20 +21,100 @@ def lipo_create(conan_api: ConanAPI, parser, subparser, *args): Create lipo binaries from the results of a Conan full_deploy. It expects a folder structure as: //// """ - subparser.add_argument("input_path", help="Root path for the Conan deployment. Needs ") - subparser.add_argument("-a", "--architecture", help="Each architecture that will be added to the resulting " - "universal binary. If not used, all found architectures will be added.", - action="append") - subparser.add_argument("--name-filter", help="", - action="append") - subparser.add_argument("--build-type-filter", help="", - action="append") + subparser.add_argument( + "input_path", help="Root path for the Conan deployment.") + subparser.add_argument("--output-folder", help="Optional root path for the output." + "If not specified, output will be generated in a 'universal' folder inside input_path.", + default=None) + subparser.add_argument("-a", "--architecture", nargs='+', help="Each architecture that will be added to the resulting " + "universal binary. If not used, all found architectures will be added.", + default=[]) args = parser.parse_args(*args) - input_path = pathlib(args.input_path) + output = ConanOutput() + + input_path = pathlib.Path(args.input_path) + output_path = pathlib.Path(args.output_folder or ".") if not input_path.exists() or not input_path.is_dir(): - raise ConanException(f"The input path is not valid.") + raise ConanException( + f"The input path '{args.input_path}' is not valid or does not exist.") + + # These are for optimization only, to avoid unnecessarily reading files. + _binary_exts = ['.a', '.dylib'] + _regular_exts = [ + '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png' + ] + + def is_macho_binary(file_path): + """ + Determines if file_path is a Mach-O binary or fat binary + """ + ext = os.path.splitext(file_path)[1] + if ext in _binary_exts: + return True + if ext in _regular_exts: + return False + with open(file_path, "rb") as f: + header = f.read(4) + if header == b'\xcf\xfa\xed\xfe': + # cffaedfe is Mach-O binary + return True + elif header == b'\xca\xfe\xba\xbe': + # cafebabe is Mach-O fat binary + return True + elif header == b'!\n': + # ar archive + return True + return False + + # Function to process each build_type directory + def process_build_type(build_type_path, output_build_type_path, valid_architectures): + if valid_architectures: + architectures = [d for d in build_type_path.iterdir() if d.is_dir() and d.name in valid_architectures] + else: + architectures = [d for d in build_type_path.iterdir() if d.is_dir()] + + all_archs = valid_architectures or [d.name for d in architectures] + + output.info(f"Creating universal binaries for architectures: {', '.join(all_archs)}") + + if len(architectures) < 2: + raise ConanException(f"Less than two architectures found in folder {build_type_path}") + + combined_arch_name = ".".join(sorted([d.name for d in architectures])) + + # Identify all files in the first architecture to check if they are Mach-O binaries + first_arch_files = list(architectures[0].glob( + "**/*")) # Recursively find all files + for file in first_arch_files: + if file.is_file(): + relative_path = file.relative_to(build_type_path) + relative_path_without_arch = pathlib.Path(*list(relative_path.parts)[1:]) + output_relative_path = combined_arch_name / pathlib.Path(*list(relative_path.parts)[1:]) + ouput_path = output_build_type_path / output_relative_path + if is_macho_binary(str(file)): + # This file is a Mach-O binary, attempt to create a lipo binary with files from other architectures + arch_files = [str(architecture / relative_path_without_arch) for architecture in architectures] + ouput_path.parent.mkdir(parents=True, exist_ok=True) + lipo_args = ["lipo", "-create"] + arch_files + ["-output", str(ouput_path)] + output.info(f"Creating universal binary {ouput_path} for: {', '.join(arch_files)}") + subprocess.run(lipo_args) + else: + # Not a Mach-O binary, simply copy the file to the destination tree + ouput_path.parent.mkdir(parents=True, exist_ok=True) + output.info(f"Copying: {file} -> {ouput_path}") + shutil.copy(file, ouput_path) + + # Traverse the input_path + for lib_name in input_path.iterdir(): + if lib_name.is_dir(): + for version in lib_name.iterdir(): + if version.is_dir(): + for build_type in version.iterdir(): + if build_type.is_dir(): + output_build_type_path = output_path / lib_name.name / version.name / build_type.name + process_build_type(build_type, output_build_type_path, args.architecture) @conan_subcommand() @@ -39,5 +122,6 @@ def lipo_info(conan_api: ConanAPI, parser, subparser, *args): """ Get information for lipo files """ + # TODO: implement args = parser.parse_args(*args) diff --git a/tests/test_lipo_command.py b/tests/test_lipo_command.py index 775a9353..2415ade7 100644 --- a/tests/test_lipo_command.py +++ b/tests/test_lipo_command.py @@ -1,13 +1,10 @@ import sys import tempfile -import textwrap import os -from tools import load, save, run - import pytest -from tools import load, save, run +from tools import run @pytest.fixture(autouse=True) @@ -39,5 +36,6 @@ def test_lipo_create(): run("conan create . -tf="" -s arch=x86_64") run("conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=armv8") run("conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=x86_64") - run("conan bin:lipo full_deploy -a x86_64 -a armv8") - + run("conan bin:lipo full_deploy --output-folder=universal") + out = run("lipo universal/mylibrary/1.0/Release/armv8.x86_64/lib/libmylibrary.a -info") + assert 'x86_64 arm64' in out From bd282b2b84ae22b2f7239f38c8e31d9fef08c3f1 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 8 Feb 2024 10:10:05 +0100 Subject: [PATCH 3/5] minor changes --- extensions/commands/bin/cmd_lipo.py | 61 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/extensions/commands/bin/cmd_lipo.py b/extensions/commands/bin/cmd_lipo.py index c3c86a23..5c47ea49 100644 --- a/extensions/commands/bin/cmd_lipo.py +++ b/extensions/commands/bin/cmd_lipo.py @@ -8,6 +8,35 @@ from conan.errors import ConanException +# These are for optimization only, to avoid unnecessarily reading files. +_binary_exts = ['.a', '.dylib'] +_regular_exts = [ + '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png' +] + +def is_macho_binary(file_path): + """ + Determines if file_path is a Mach-O binary or fat binary + """ + ext = os.path.splitext(file_path)[1] + if ext in _binary_exts: + return True + if ext in _regular_exts: + return False + with open(file_path, "rb") as f: + header = f.read(4) + if header == b'\xcf\xfa\xed\xfe': + # cffaedfe is Mach-O binary + return True + elif header == b'\xca\xfe\xba\xbe': + # cafebabe is Mach-O fat binary + return True + elif header == b'!\n': + # ar archive + return True + return False + + @conan_command(group="Binary manipulation") def lipo(conan_api: ConanAPI, parser, *args): """ @@ -40,35 +69,6 @@ def lipo_create(conan_api: ConanAPI, parser, subparser, *args): raise ConanException( f"The input path '{args.input_path}' is not valid or does not exist.") - # These are for optimization only, to avoid unnecessarily reading files. - _binary_exts = ['.a', '.dylib'] - _regular_exts = [ - '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png' - ] - - def is_macho_binary(file_path): - """ - Determines if file_path is a Mach-O binary or fat binary - """ - ext = os.path.splitext(file_path)[1] - if ext in _binary_exts: - return True - if ext in _regular_exts: - return False - with open(file_path, "rb") as f: - header = f.read(4) - if header == b'\xcf\xfa\xed\xfe': - # cffaedfe is Mach-O binary - return True - elif header == b'\xca\xfe\xba\xbe': - # cafebabe is Mach-O fat binary - return True - elif header == b'!\n': - # ar archive - return True - return False - - # Function to process each build_type directory def process_build_type(build_type_path, output_build_type_path, valid_architectures): if valid_architectures: architectures = [d for d in build_type_path.iterdir() if d.is_dir() and d.name in valid_architectures] @@ -85,8 +85,7 @@ def process_build_type(build_type_path, output_build_type_path, valid_architectu combined_arch_name = ".".join(sorted([d.name for d in architectures])) # Identify all files in the first architecture to check if they are Mach-O binaries - first_arch_files = list(architectures[0].glob( - "**/*")) # Recursively find all files + first_arch_files = list(architectures[0].glob("**/*")) # Recursively find all files for file in first_arch_files: if file.is_file(): relative_path = file.relative_to(build_type_path) From ec06c5c60f2fff8c95207685d43daefba8364eda Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 8 Feb 2024 10:12:15 +0100 Subject: [PATCH 4/5] wip --- extensions/commands/bin/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extensions/commands/bin/README.md b/extensions/commands/bin/README.md index e69de29b..e1c4823d 100644 --- a/extensions/commands/bin/README.md +++ b/extensions/commands/bin/README.md @@ -0,0 +1,9 @@ +## Binary manipulation commands + +### Creating universal binaries + +```bash +conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=armv8 +conan install --requires=mylibrary/1.0 --deployer=full_deploy -s arch=x86_64 +conan bin:lipo create full_deploy/host --output-folder=universal +``` From 517eb7efc2568d4fff5307b5fa8e15580f8fedcc Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 8 Feb 2024 10:58:12 +0100 Subject: [PATCH 5/5] wip --- extensions/commands/bin/cmd_lipo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/commands/bin/cmd_lipo.py b/extensions/commands/bin/cmd_lipo.py index 5c47ea49..d8a9849b 100644 --- a/extensions/commands/bin/cmd_lipo.py +++ b/extensions/commands/bin/cmd_lipo.py @@ -50,8 +50,7 @@ def lipo_create(conan_api: ConanAPI, parser, subparser, *args): Create lipo binaries from the results of a Conan full_deploy. It expects a folder structure as: //// """ - subparser.add_argument( - "input_path", help="Root path for the Conan deployment.") + subparser.add_argument("input_path", help="Root path for the Conan deployment.") subparser.add_argument("--output-folder", help="Optional root path for the output." "If not specified, output will be generated in a 'universal' folder inside input_path.", default=None)