From ff08270b6b2cf7c76cf38150a5c629a54ac3d25c Mon Sep 17 00:00:00 2001 From: David Gonzalez Martin Date: Wed, 19 Feb 2025 19:21:09 -0600 Subject: [PATCH] Add CI coverage --- .github/workflows/ci.yml | 22 ++++--- build.zig | 122 ++++++++++++++++++++++++++++++++++----- src/LLVM.zig | 92 ++++++++++++++++++----------- src/lib.zig | 21 ++++++- src/llvm.cpp | 26 ++++++++- src/main.zig | 1 + 6 files changed, 224 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b9eb18..84b99e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: BIRTH_GITHUB_TARGETS: ${{ steps.generate-config.outputs.BIRTH_GITHUB_TARGETS }} BIRTH_BUILD_TYPES: ${{ steps.generate-config.outputs.BIRTH_BUILD_TYPES }} BIRTH_CMAKE_BUILD_TYPES: ${{ steps.generate-config.outputs.BIRTH_CMAKE_BUILD_TYPES }} + BIRTH_ZIG_BUILD_TYPES: ${{ steps.generate-config.outputs.BIRTH_ZIG_BUILD_TYPES }} BIRTH_COMPILERS: ${{ steps.generate-config.outputs.BIRTH_COMPILERS }} BIRTH_LINUX_IMAGE: ${{ steps.generate-config.outputs.BIRTH_LINUX_IMAGE }} BIRTH_MACOS_IMAGE: ${{ steps.generate-config.outputs.BIRTH_MACOS_IMAGE }} @@ -32,7 +33,7 @@ jobs: uses: actions/checkout@v4 - name: Generate config id: generate-config - uses: birth-software/github-config@v4 + uses: birth-software/github-config@v6 - name: Create tag if: github.ref == 'refs/heads/main' shell: bash @@ -59,7 +60,9 @@ jobs: fail-fast: false matrix: os: [ x86_64-linux-znver4 ] - BIRTH_BUILD_TYPE: ${{ fromJSON(needs.generate-config.outputs.BIRTH_BUILD_TYPES) }} + BIRTH_ZIG_BUILD_TYPE: ${{ fromJSON(needs.generate-config.outputs.BIRTH_ZIG_BUILD_TYPES) }} + ENABLE_LLVM: [ true, false ] + SYSTEM_LLVM: [ true, false ] runs-on: ${{ matrix.os }} env: BIRTH_LINUX_IMAGE: ${{ needs.generate-config.outputs.BIRTH_LINUX_IMAGE }} @@ -68,10 +71,11 @@ jobs: RELEASE_TAG_NAME: ${{ needs.generate-config.outputs.RELEASE_TAG_NAME }} steps: - uses: actions/checkout@v4 - - name: Build - env: - CC: clang - BB_BUILD_TYPE: ${{matrix.BIRTH_BUILD_TYPE}} - run: ./build.sh - - name: Run - run: ./cache/bb + - uses: mlugg/setup-zig@v1 + with: + version: master + - name: Build and test + run: | + zig build test --summary all -Denable_llvm=${{matrix.ENABLE_LLVM}} -Doptimize=${{matrix.BIRTH_ZIG_BUILD_TYPE}} -Dsystem_llvm=${{matrix.SYSTEM_LLVM}} + zig build install -Denable_llvm=${{matrix.ENABLE_LLVM}} -Doptimize=${{matrix.BIRTH_ZIG_BUILD_TYPE}} -Dsystem_llvm=${{matrix.SYSTEM_LLVM}} + ldd zig-out/bin/bloat-buster diff --git a/build.zig b/build.zig index 8daab98..adca59b 100644 --- a/build.zig +++ b/build.zig @@ -45,10 +45,26 @@ fn executable_find_in_path(allocator: std.mem.Allocator, file_name: []const u8, return file_find_in_path(allocator, file_name, path_env, extension); } +const CmakeBuildType = enum { + Debug, + RelWithDebInfo, + MinSizeRel, + Release, + + fn from_zig_build_type(o: std.builtin.OptimizeMode) CmakeBuildType { + return switch (o) { + .Debug => .Debug, + .ReleaseSafe => .RelWithDebInfo, + .ReleaseSmall => .MinSizeRel, + .ReleaseFast => .Release, + }; + } +}; + const LLVM = struct { module: *std.Build.Module, - fn setup(b: *std.Build, path: []const u8, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) !LLVM { + fn setup(b: *std.Build, path: []const u8) !LLVM { if (enable_llvm) { var llvm_libs = std.ArrayList([]const u8).init(b.allocator); var flags = std.ArrayList([]const u8).init(b.allocator); @@ -57,7 +73,17 @@ const LLVM = struct { const f = std.fs.cwd().openFile(full_path, .{}) catch return error.llvm_not_found; f.close(); break :blk full_path; - } else executable_find_in_path(b.allocator, "llvm-config", path) orelse return error.llvm_not_found; + } else if (system_llvm) executable_find_in_path(b.allocator, "llvm-config", path) orelse return error.llvm_not_found else blk: { + const home_env = switch (@import("builtin").os.tag) { + .windows => "USERPROFILE", + else => "HOME", + }; + const home_path = env.get(home_env) orelse unreachable; + const full_path = try std.mem.concat(b.allocator, u8, &.{ home_path, "/Downloads/llvm-", @tagName(target.result.cpu.arch), "-", @tagName(target.result.os.tag), "-", @tagName(CmakeBuildType.from_zig_build_type(optimize)), "/bin/llvm-config" }); + const f = std.fs.cwd().openFile(full_path, .{}) catch return error.llvm_not_found; + f.close(); + break :blk full_path; + }; const llvm_components_result = try run_process_and_capture_stdout(b, &.{ llvm_config_path, "--components" }); var it = std.mem.splitScalar(u8, llvm_components_result, ' '); var args = std.ArrayList([]const u8).init(b.allocator); @@ -96,15 +122,76 @@ const LLVM = struct { llvm.addLibraryPath(.{ .cwd_relative = llvm_lib_dir }); + const a = std.fs.cwd().openDir("/usr/lib/x86_64-linux-gnu/", .{}); + if (a) |_| { + var dir = a catch unreachable; + dir.close(); + llvm.addLibraryPath(.{ .cwd_relative = "/usr/lib/x86_64-linux-gnu/" }); + } else |err| { + err catch {}; + } + llvm.addCSourceFiles(.{ .files = &.{"src/llvm.cpp"}, .flags = flags.items, }); - llvm.addIncludePath(.{ .cwd_relative = "/usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/14.2.1/../../../../include/c++/14.2.1" }); - llvm.addIncludePath(.{ .cwd_relative = "/usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/14.2.1/../../../../include/c++/14.2.1/x86_64-pc-linux-gnu" }); - llvm.addObjectFile(.{ .cwd_relative = "/usr/lib/libstdc++.so.6" }); - const needed_libraries: []const []const u8 = &.{ "unwind", "z" }; + var dir = try std.fs.cwd().openDir("/usr/include/c++", .{ + .iterate = true, + }); + var iterator = dir.iterate(); + const gcc_version = while (try iterator.next()) |entry| { + if (entry.kind == .directory) { + break entry.name; + } + } else return error.include_cpp_dir_not_found; + dir.close(); + const general_cpp_include_dir = try std.mem.concat(b.allocator, u8, &.{ "/usr/include/c++/", gcc_version }); + llvm.addIncludePath(.{ .cwd_relative = general_cpp_include_dir }); + + { + const arch_cpp_include_dir = try std.mem.concat(b.allocator, u8, &.{ general_cpp_include_dir, "/x86_64-pc-linux-gnu" }); + const d2 = std.fs.cwd().openDir(arch_cpp_include_dir, .{}); + if (d2) |_| { + var d = d2 catch unreachable; + d.close(); + llvm.addIncludePath(.{ .cwd_relative = arch_cpp_include_dir }); + } else |err| err catch {}; + } + + { + const arch_cpp_include_dir = try std.mem.concat(b.allocator, u8, &.{ "/usr/include/x86_64-linux-gnu/c++/", gcc_version }); + const d2 = std.fs.cwd().openDir(arch_cpp_include_dir, .{}); + if (d2) |_| { + var d = d2 catch unreachable; + d.close(); + llvm.addIncludePath(.{ .cwd_relative = arch_cpp_include_dir }); + } else |err| err catch {}; + } + + var found_libcpp = false; + + if (std.fs.cwd().openFile("/usr/lib/libstdc++.so.6", .{})) |file| { + file.close(); + found_libcpp = true; + llvm.addObjectFile(.{ .cwd_relative = "/usr/lib/libstdc++.so.6" }); + } else |err| { + err catch {}; + } + + if (std.fs.cwd().openFile("/usr/lib/x86_64-linux-gnu/libstdc++.so.6", .{})) |file| { + file.close(); + found_libcpp = true; + llvm.addObjectFile(.{ .cwd_relative = "/usr/lib/x86_64-linux-gnu/libstdc++.so.6" }); + } else |err| { + err catch {}; + } + + if (!found_libcpp) { + return error.libcpp_not_found; + } + + const needed_libraries: []const []const u8 = &.{ "unwind", "z", "zstd" }; const lld_libs: []const []const u8 = &.{ "lldCommon", "lldCOFF", "lldELF", "lldMachO", "lldMinGW", "lldWasm" }; @@ -128,9 +215,9 @@ const LLVM = struct { } } - fn link(llvm: LLVM, target: *std.Build.Step.Compile) void { - if (target.root_module != llvm.module) { - target.root_module.addImport("llvm", llvm.module); + fn link(llvm: LLVM, compile: *std.Build.Step.Compile) void { + if (compile.root_module != llvm.module) { + compile.root_module.addImport("llvm", llvm.module); } else { // TODO: should we allow this case? unreachable; @@ -149,12 +236,17 @@ fn debug_binary(b: *std.Build, exe: *std.Build.Step.Compile) *std.Build.Step.Run } var enable_llvm: bool = undefined; +var system_llvm: bool = undefined; +var target: std.Build.ResolvedTarget = undefined; +var optimize: std.builtin.OptimizeMode = undefined; +var env: std.process.EnvMap = undefined; pub fn build(b: *std.Build) !void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); + env = try std.process.getEnvMap(b.allocator); + target = b.standardTargetOptions(.{}); + optimize = b.standardOptimizeOption(.{}); enable_llvm = b.option(bool, "enable_llvm", "Enable LLVM") orelse false; - const env = try std.process.getEnvMap(b.allocator); + system_llvm = b.option(bool, "system_llvm", "Link against system LLVM libraries") orelse true; const path = env.get("PATH") orelse unreachable; const configuration = b.addOptions(); @@ -167,7 +259,7 @@ pub fn build(b: *std.Build) !void { }); exe_mod.addOptions("configuration", configuration); - const llvm = try LLVM.setup(b, path, target, optimize); + const llvm = try LLVM.setup(b, path); const exe = b.addExecutable(.{ .name = "bloat-buster", @@ -206,4 +298,8 @@ pub fn build(b: *std.Build) !void { const test_step = b.step("test", "Run unit tests"); test_step.dependOn(&run_exe_unit_tests.step); + + const debug_test_cmd = debug_binary(b, exe_unit_tests); + const debug_test_step = b.step("debug_test", "Debug the tests"); + debug_test_step.dependOn(&debug_test_cmd.step); } diff --git a/src/LLVM.zig b/src/LLVM.zig index 24d9c21..4dec75e 100644 --- a/src/LLVM.zig +++ b/src/LLVM.zig @@ -1,4 +1,5 @@ const lib = @import("lib.zig"); +const builtin = @import("builtin"); const Arena = lib.Arena; const assert = lib.assert; const api = @import("llvm_api.zig"); @@ -763,6 +764,9 @@ test "experiment" { return error.SkipZigTest; } + lib.GlobalState.initialize(); + initialize_all(); + const thread = &global.threads[0]; thread.initialize(); const module = thread.context.create_module("first_module"); @@ -782,15 +786,17 @@ test "experiment" { builder.create_ret(return_value.to_value()); const function_verify = function.verify(); if (!function_verify.success) { - unreachable; + return error.function_failed_to_verify; } const module_verify = module.verify(); if (!module_verify.success) { - unreachable; + return error.module_failed_to_verify; } - const module_string = module.to_string(); - lib.print_string(module_string); + if (!builtin.is_test) { + const module_string = module.to_string(); + lib.print_string(module_string); + } var error_message: String = undefined; const target_machine = Target.Machine.create(.{ @@ -803,12 +809,12 @@ test "experiment" { .code_model = .none, .jit = false, }, &error_message) orelse { - unreachable; + return error.target_machine_creation_failed; }; module.set_target(target_machine); module.run_optimization_pipeline(target_machine, OptimizationPipelineOptions.default(.{ .optimization_level = .O3, .debug_info = 1 })); - const object_path = ".zig-cache/foo.o"; + const object_path = "foo.o"; const result = module.run_code_generation_pipeline(target_machine, CodeGenerationPipelineOptions{ .output_file_path = String.from_slice(object_path), .output_dwarf_file_path = .{}, @@ -818,51 +824,67 @@ test "experiment" { .verify_module = @intFromBool(lib.optimization_mode == .Debug or lib.optimization_mode == .ReleaseSafe), }, }); - if (result != .success) { - unreachable; + switch (result) { + .success => {}, + .failed_to_create_file => return error.failed_to_create_file, + .failed_to_add_emit_passes => return error.failed_to_add_emit_passes, } var arg_builder = LldArgvBuilder{}; arg_builder.add("ld.lld"); arg_builder.add("--error-limit=0"); arg_builder.add("-o"); - arg_builder.add(".zig-cache/foo"); + arg_builder.add("foo"); const objects: []const [*:0]const u8 = &.{object_path}; for (objects) |object| { arg_builder.add(object); } - arg_builder.add("-L/usr/lib"); + const library_paths = [_][:0]const u8{ "/usr/lib", "/usr/lib/x86_64-linux-gnu" }; - const link_libcpp = false; - if (link_libcpp) { - arg_builder.add("-lstdc++"); - } + inline for (library_paths) |library_path| { + const scrt1_path = library_path ++ "/" ++ "Scrt1.o"; + const file = lib.os.File.open(scrt1_path, .{ .read = 1 }, .{}); + if (file.is_valid()) { + file.close(); - const link_libc = true; + arg_builder.add("-L" ++ library_path); - const dynamic_linker = true; - if (dynamic_linker) { - arg_builder.add("-dynamic-linker"); - arg_builder.add("/usr/lib/ld-linux-x86-64.so.2"); - } + const link_libcpp = false; + if (link_libcpp) { + arg_builder.add("-lstdc++"); + } - if (link_libc) { - arg_builder.add("/usr/lib/Scrt1.o"); - arg_builder.add("-lc"); - } + const link_libc = true; - const lld_args = arg_builder.flush(); - const lld_result = api.lld_elf_link(lld_args.ptr, lld_args.len, true, false); - const success = lld_result.success and lld_result.stderr.length == 0; - if (!success) { - if (lld_result.stdout.length != 0) { - lib.print_string(lld_result.stdout.to_slice() orelse unreachable); + const dynamic_linker = true; + if (dynamic_linker) { + arg_builder.add("-dynamic-linker"); + arg_builder.add("/usr/lib/ld-linux-x86-64.so.2"); + } + + if (link_libc) { + arg_builder.add(scrt1_path); + arg_builder.add("-lc"); + } + + const lld_args = arg_builder.flush(); + const lld_result = api.lld_elf_link(lld_args.ptr, lld_args.len, true, false); + const success = lld_result.success and lld_result.stderr.length == 0; + if (!success) { + if (lld_result.stdout.length != 0) { + lib.print_string_stderr(lld_result.stdout.to_slice() orelse unreachable); + } + + if (lld_result.stderr.length != 0) { + lib.print_string_stderr(lld_result.stderr.to_slice() orelse unreachable); + } + return error.linking_failed; + } + break; } - - if (lld_result.stderr.length != 0) { - lib.print_string(lld_result.stderr.to_slice() orelse unreachable); - } - lib.libc.exit(1); + } else { + lib.print_string_stderr("Failed to find directory for Scrt1.o\n"); + lib.os.abort(); } } diff --git a/src/lib.zig b/src/lib.zig index 24c9e1c..5af86c3 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -157,7 +157,7 @@ pub const os = struct { .windows => @compileError("TODO"), else => { const o = posix.O{ - .ACCMODE = if (flags.read | flags.write != 0) .RDWR else if (flags.read != 0) .RDONLY else if (flags.write != 0) .WRONLY else unreachable, + .ACCMODE = if (flags.read & flags.write != 0) .RDWR else if (flags.read != 0) .RDONLY else if (flags.write != 0) .WRONLY else unreachable, .TRUNC = flags.truncate, .CREAT = flags.create, .DIRECTORY = flags.directory, @@ -495,6 +495,17 @@ pub const os = struct { }, }; } + + pub fn get_stderr() File { + return switch (builtin.os.tag) { + .windows => @compileError("TODO"), + else => { + return File{ + .fd = 2, + }; + }, + }; + } }; pub const libc = struct { @@ -2480,6 +2491,12 @@ pub fn print(format_string: [*:0]const u8, ...) callconv(.C) void { @cVaEnd(&args); } -pub fn print_string(str: []const u8) void { +pub const print_string = print_string_stdout; + +pub fn print_string_stdout(str: []const u8) void { os.get_stdout().write(str); } + +pub fn print_string_stderr(str: []const u8) void { + os.get_stderr().write(str); +} diff --git a/src/llvm.cpp b/src/llvm.cpp index db8b8a6..333f415 100644 --- a/src/llvm.cpp +++ b/src/llvm.cpp @@ -8,6 +8,8 @@ typedef uint64_t u64; #define EXPORT extern "C" #define fn static +#include "llvm/Config/llvm-config.h" + #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" @@ -17,6 +19,7 @@ typedef uint64_t u64; #include "llvm/Passes/PassBuilder.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Frontend/Driver/CodeGenOptions.h" @@ -158,7 +161,16 @@ EXPORT BBLLVMString llvm_host_cpu_name() EXPORT BBLLVMString llvm_host_cpu_features() { SubtargetFeatures Features; - for (const auto &[Feature, IsEnabled] : sys::getHostCPUFeatures()) +#if LLVM_VERSION_MAJOR >= 19 + auto host_cpu_features = sys::getHostCPUFeatures(); +#else + StringMap host_cpu_features; + if (!sys::getHostCPUFeatures(host_cpu_features)) { + return {}; + } +#endif + + for (const auto &[Feature, IsEnabled] : host_cpu_features) { Features.AddFeature(Feature, IsEnabled); } @@ -510,7 +522,9 @@ EXPORT TargetMachine* llvm_create_target_machine(const BBLLVMTargetMachineCreate target_options.XCOFFTracebackTable = create.target_options.xcoff_traceback_table; target_options.UniqueSectionNames = create.target_options.unique_section_names; target_options.UniqueBasicBlockSectionNames = create.target_options.unique_basic_block_section_names; +#if LLVM_VERSION_MAJOR >= 19 target_options.SeparateNamedSections = create.target_options.separate_named_sections; +#endif target_options.TrapUnreachable = create.target_options.trap_unreachable; target_options.NoTrapAfterNoreturn = create.target_options.no_trap_after_noreturn; target_options.TLSSize = create.target_options.tls_size; @@ -522,7 +536,9 @@ EXPORT TargetMachine* llvm_create_target_machine(const BBLLVMTargetMachineCreate target_options.EnableMachineFunctionSplitter = create.target_options.enable_machine_function_splitter; target_options.SupportsDefaultOutlining = create.target_options.supports_default_outlining; target_options.EmitAddrsig = create.target_options.emit_address_significance_table; +#if LLVM_VERSION_MAJOR >= 19 target_options.BBAddrMap = create.target_options.bb_address_map; +#endif auto bb_sections = (BBLLVMBasicBlockSection) create.target_options.bb_sections; switch (bb_sections) @@ -649,15 +665,19 @@ EXPORT TargetMachine* llvm_create_target_machine(const BBLLVMTargetMachineCreate target_options.MCOptions.MCNoTypeCheck = create.target_options.mc.no_type_check; target_options.MCOptions.MCSaveTempLabels = create.target_options.mc.save_temp_labels; target_options.MCOptions.MCIncrementalLinkerCompatible = create.target_options.mc.incremental_linker_compatible; +#if LLVM_VERSION_MAJOR >= 19 target_options.MCOptions.FDPIC = create.target_options.mc.fdpic; +#endif target_options.MCOptions.ShowMCEncoding = create.target_options.mc.show_mc_encoding; target_options.MCOptions.ShowMCInst = create.target_options.mc.show_mc_inst; target_options.MCOptions.AsmVerbose = create.target_options.mc.asm_verbose; target_options.MCOptions.PreserveAsmComments = create.target_options.mc.preserve_asm_comments; target_options.MCOptions.Dwarf64 = create.target_options.mc.dwarf64; +#if LLVM_VERSION_MAJOR >= 19 target_options.MCOptions.Crel = create.target_options.mc.crel; target_options.MCOptions.X86RelaxRelocations = create.target_options.mc.x86_relax_relocations; target_options.MCOptions.X86Sse2Avx = create.target_options.mc.x86_sse2_avx; +#endif auto emit_dwarf_unwind = (BBLLVMEmitDwarfUnwindType) create.target_options.mc.emit_dwarf_unwind; switch (emit_dwarf_unwind) @@ -675,6 +695,7 @@ EXPORT TargetMachine* llvm_create_target_machine(const BBLLVMTargetMachineCreate case BBLLVMDwarfDirectory::normal: target_options.MCOptions.MCUseDwarfDirectory = MCTargetOptions::DwarfDirectory::DefaultDwarfDirectory; break; } +#if LLVM_VERSION_MAJOR >= 19 auto debug_compression_type = (BBLLVMDebugCompressionType) create.target_options.mc.debug_compression_type; switch (debug_compression_type) { @@ -682,6 +703,7 @@ EXPORT TargetMachine* llvm_create_target_machine(const BBLLVMTargetMachineCreate case BBLLVMDebugCompressionType::zlib: target_options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib; break; case BBLLVMDebugCompressionType::zstd: target_options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd; break; } +#endif target_options.MCOptions.EmitCompactUnwindNonCanonical = create.target_options.mc.emit_compact_unwind_non_canonical; target_options.MCOptions.PPCUseFullRegisterNames = create.target_options.mc.ppc_use_full_register_names; @@ -853,10 +875,12 @@ EXPORT BBLLVMCodeGenerationPipelineResult llvm_module_run_code_generation_pipeli // does not work with the codegen pipeline. // FIXME: make the new PM work with the codegen pipeline. legacy::PassManager CodeGenPasses; +#if LLVM_VERSION_MAJOR >= 19 if (options.optimize_when_possible) { CodeGenPasses.add(createTargetTransformInfoWrapperPass(target_machine.getTargetIRAnalysis())); } +#endif raw_pwrite_stream* dwarf_object_file = 0; if (options.output_dwarf_file_path.length) diff --git a/src/main.zig b/src/main.zig index cf1ec89..75122cb 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,4 +1,5 @@ const lib = @import("lib.zig"); +const configuration = @import("configuration"); const llvm = @import("LLVM.zig"); const parser = @import("parser.zig"); const Arena = lib.Arena;