From 5eb77c3daf6568c36aaa3a94081f26cacbcb9820 Mon Sep 17 00:00:00 2001 From: David Gonzalez Martin Date: Mon, 22 Apr 2024 11:47:03 -0600 Subject: [PATCH] Enable CI --- .github/workflows/ci.yml | 54 +++---------------- bootstrap/Compilation.zig | 12 +++-- bootstrap/linker/lld.zig | 57 ++++++++++++++------ build.zig | 103 +++++++++++++++++++++---------------- build/fetcher.zig | 2 +- build/test_runner.zig | 3 +- ci/github_ubuntu_runner.sh | 10 ++++ src/main.nat | 69 ++++++------------------- 8 files changed, 143 insertions(+), 167 deletions(-) create mode 100644 ci/github_ubuntu_runner.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a6f9ed..9105a78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,57 +7,17 @@ on: - main schedule: - cron: "0 0 * * *" -concurrency: - # Cancels pending runs when a PR gets updated. - group: ${{ github.head_ref || github.run_id }}-${{ github.actor }} - cancel-in-progress: true jobs: - self_hosted_linux: - #runs-on: [ self-hosted, Linux, x64 ] + build_and_test: runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v4 - - name: Test - run: | - echo "TODO" - #zig build test -Dself_hosted_ci=true -Dstatic=true -Dllvm_path=../../../../../dev/llvm/llvm-static-release-zen4-17.0.6/out/x86_64-linux-musl-native - #zig build test -Dself_hosted_ci=true -Dstatic=false - # macos_m1: - # runs-on: macos-14 - # timeout-minutes: 15 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Set up Zig - # uses: davidgm94/setup-zig@foo - # with: - # version: master - # - name: Build - # run: zig build -Dthird_party_ci=true -Dtarget=aarch64-macos-none -Dcpu=apple_m1 - # linux_x86_64_v3: - # runs-on: ubuntu-latest - # timeout-minutes: 15 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Set up Zig - # uses: davidgm94/setup-zig@foo - # with: - # version: master - # - name: Build - # run: zig build -Dthird_party_ci=true -Dtarget=x86_64-linux-musl -Dcpu=x86_64_v3 - # windows_x86_64_v3: - # runs-on: windows-latest - # timeout-minutes: 15 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Set up Zig - # uses: davidgm94/setup-zig@foo - # with: - # version: master - # - name: Build - # run: zig build -Dthird_party_ci=true -Dtarget=x86_64-windows-gnu -Dcpu=x86_64_v3 + - name: Set up Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: master + - name: Build and test + run: sh ci/github_ubuntu_runner.sh diff --git a/bootstrap/Compilation.zig b/bootstrap/Compilation.zig index 706306f..864fe8e 100644 --- a/bootstrap/Compilation.zig +++ b/bootstrap/Compilation.zig @@ -657,7 +657,13 @@ pub fn compileCSourceFile(context: *const Context, arguments: []const []const u8 "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include", }, .linux => switch (@import("builtin").abi) { - .gnu => &.{ + .gnu => if (@import("configuration").ci) &.{ + "/usr/include/c++/11", + "/usr/include/x86_64-linux-gnu/c++/11", + "/usr/lib/clang/17/include", + "/usr/include", + "/usr/include/x86_64-linux-gnu", + } else &.{ "/usr/include/c++/13.2.1", "/usr/include/c++/13.2.1/x86_64-pc-linux-gnu", "/usr/lib/clang/17/include", @@ -13925,8 +13931,8 @@ pub const Builder = struct { .@"comptime" = .{ .enum_value = enum_fields[0], }, - }, - }, emit, null); + }, + }, emit, null); _ = comptime_payload; // autofix const identifier = unit.getExpectedTokenBytes(payload_node.token, .identifier); const hash = try unit.processIdentifier(context, identifier); diff --git a/bootstrap/linker/lld.zig b/bootstrap/linker/lld.zig index 1f55fa2..f437bc8 100644 --- a/bootstrap/linker/lld.zig +++ b/bootstrap/linker/lld.zig @@ -30,6 +30,7 @@ pub fn link(context: *const Compilation.Context, options: linker.Options) !void try argv.append(context.my_allocator, object.path); } + const ci = @import("configuration").ci; switch (@import("builtin").os.tag) { .macos => { try argv.append(context.my_allocator, "-dynamic"); @@ -54,24 +55,46 @@ pub fn link(context: *const Compilation.Context, options: linker.Options) !void } }, .linux => { - if (options.link_libcpp) { - assert(options.link_libc); - try argv.append(context.my_allocator, "/usr/lib/libstdc++.so"); - } + if (ci) { + if (options.link_libcpp) { + assert(options.link_libc); + try argv.append(context.my_allocator, "/lib/x86_64-linux-gnu/libstdc++.so.6"); + } - if (options.link_libc) { - try argv.append(context.my_allocator, "/usr/lib/crt1.o"); - try argv.append(context.my_allocator, "/usr/lib/crti.o"); - try argv.append_slice(context.my_allocator, &.{ "-L", "/usr/lib" }); - try argv.append_slice(context.my_allocator, &.{ "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2" }); - try argv.append(context.my_allocator, "--as-needed"); - try argv.append(context.my_allocator, "-lm"); - try argv.append(context.my_allocator, "-lpthread"); - try argv.append(context.my_allocator, "-lc"); - try argv.append(context.my_allocator, "-ldl"); - try argv.append(context.my_allocator, "-lrt"); - try argv.append(context.my_allocator, "-lutil"); - try argv.append(context.my_allocator, "/usr/lib/crtn.o"); + if (options.link_libc) { + try argv.append(context.my_allocator, "/lib/x86_64-linux-gnu/crt1.o"); + try argv.append(context.my_allocator, "/lib/x86_64-linux-gnu/crti.o"); + try argv.append_slice(context.my_allocator, &.{ "-L", "/lib/x86_64-linux-gnu" }); + try argv.append_slice(context.my_allocator, &.{ "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2" }); + try argv.append(context.my_allocator, "--as-needed"); + try argv.append(context.my_allocator, "-lm"); + try argv.append(context.my_allocator, "-lpthread"); + try argv.append(context.my_allocator, "-lc"); + try argv.append(context.my_allocator, "-ldl"); + try argv.append(context.my_allocator, "-lrt"); + try argv.append(context.my_allocator, "-lutil"); + try argv.append(context.my_allocator, "/lib/x86_64-linux-gnu/crtn.o"); + } + } else { + if (options.link_libcpp) { + assert(options.link_libc); + try argv.append(context.my_allocator, "/usr/lib/libstdc++.so"); + } + + if (options.link_libc) { + try argv.append(context.my_allocator, "/usr/lib/crt1.o"); + try argv.append(context.my_allocator, "/usr/lib/crti.o"); + try argv.append_slice(context.my_allocator, &.{ "-L", "/usr/lib" }); + try argv.append_slice(context.my_allocator, &.{ "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2" }); + try argv.append(context.my_allocator, "--as-needed"); + try argv.append(context.my_allocator, "-lm"); + try argv.append(context.my_allocator, "-lpthread"); + try argv.append(context.my_allocator, "-lc"); + try argv.append(context.my_allocator, "-ldl"); + try argv.append(context.my_allocator, "-lrt"); + try argv.append(context.my_allocator, "-lutil"); + try argv.append(context.my_allocator, "/usr/lib/crtn.o"); + } } }, .windows => {}, diff --git a/build.zig b/build.zig index f911d8f..bd68abc 100644 --- a/build.zig +++ b/build.zig @@ -34,6 +34,7 @@ pub fn build(b: *std.Build) !void { }; const compiler_options = b.addOptions(); compiler_options.addOption(bool, "print_stack_trace", print_stack_trace); + compiler_options.addOption(bool, "ci", is_ci); const fetcher = b.addExecutable(.{ .name = "llvm_fetcher", @@ -119,7 +120,6 @@ pub fn build(b: *std.Build) !void { compiler.want_lto = false; compiler.linkLibC(); - //if (target.result.os.tag == .windows) compiler.linkage = .dynamic; const zstd = if (target.result.os.tag == .windows) "zstd.lib" else "libzstd.a"; @@ -372,20 +372,24 @@ pub fn build(b: *std.Build) !void { assert(!self_hosted_ci); if (third_party_ci or (!target.query.isNativeOs() or !target.query.isNativeCpu())) { var llvm_directory = try std.ArrayListUnmanaged(u8).initCapacity(b.allocator, 128); - llvm_directory.appendSliceAssumeCapacity(prefix ++ "/"); - llvm_directory.appendSliceAssumeCapacity("llvm-"); - llvm_directory.appendSliceAssumeCapacity(llvm_version); - llvm_directory.appendSliceAssumeCapacity("-"); - llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.cpu.arch)); - llvm_directory.appendSliceAssumeCapacity("-"); - llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.os.tag)); - llvm_directory.appendSliceAssumeCapacity("-"); - llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.abi)); - llvm_directory.appendSliceAssumeCapacity("-"); const cpu = if (std.mem.eql(u8, target.result.cpu.model.name, @tagName(target.result.cpu.arch))) "baseline" else target.result.cpu.model.name; - llvm_directory.appendSliceAssumeCapacity(cpu); + if (!is_ci) { + llvm_directory.appendSliceAssumeCapacity(prefix ++ "/"); + llvm_directory.appendSliceAssumeCapacity("llvm-"); + llvm_directory.appendSliceAssumeCapacity(llvm_version); + llvm_directory.appendSliceAssumeCapacity("-"); + llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.cpu.arch)); + llvm_directory.appendSliceAssumeCapacity("-"); + llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.os.tag)); + llvm_directory.appendSliceAssumeCapacity("-"); + llvm_directory.appendSliceAssumeCapacity(@tagName(target.result.abi)); + llvm_directory.appendSliceAssumeCapacity("-"); + llvm_directory.appendSliceAssumeCapacity(cpu); + } else { + llvm_directory.appendSliceAssumeCapacity(prefix ++ "/x86_64-linux-gnu-x86_64_v3-release-static"); + } - const url = try std.mem.concat(b.allocator, u8, &.{ "https://github.com/birth-software/fetch-llvm/releases/download/v", llvm_version, "/llvm-", llvm_version, "-", @tagName(target.result.cpu.arch), "-", @tagName(target.result.os.tag), "-", @tagName(target.result.abi), "-", cpu, ".tar.xz" }); + const url = if (is_ci) "https://github.com/birth-software/fetch-llvm/releases/download/v17.0.6/x86_64-linux-gnu-x86_64_v3-release-static.tar.gz" else try std.mem.concat(b.allocator, u8, &.{ "https://github.com/birth-software/fetch-llvm/releases/download/v", llvm_version, "/llvm-", llvm_version, "-", @tagName(target.result.cpu.arch), "-", @tagName(target.result.os.tag), "-", @tagName(target.result.abi), "-", cpu, ".tar.xz" }); var dir = std.fs.cwd().openDir(llvm_directory.items, .{}) catch { const run = b.addRunArtifact(fetcher); @@ -424,42 +428,55 @@ pub fn build(b: *std.Build) !void { compiler.linkSystemLibrary("lldMachO"); compiler.linkSystemLibrary("lldWasm"); compiler.linkSystemLibrary("unwind"); - compiler.linkSystemLibrary("zlib"); + compiler.linkSystemLibrary(if (is_ci) "z" else "zlib"); compiler.linkSystemLibrary("zstd"); switch (target.result.os.tag) { .linux => { - const result = try std.ChildProcess.run(.{ - .allocator = b.allocator, - .argv = &.{ "c++", "--version" }, - }); - const success = switch (result.term) { - .Exited => |exit_code| exit_code == 0, - else => false, - }; - - if (!success) { - unreachable; - } - - var tokenizer = std.mem.tokenize(u8, result.stdout, " "); - const cxx_version = while (tokenizer.next()) |chunk| { - if (std.ascii.isDigit(chunk[0])) { - if (std.SemanticVersion.parse(chunk)) |_| { - break chunk; - } else |err| err catch {}; - } + if (third_party_ci) { + compiler.addObjectFile(.{ .cwd_relative = "/lib/x86_64-linux-gnu/libstdc++.so.6" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/include" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/include/x86_64-linux-gnu" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/include/c++/11" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/include/x86_64-linux-gnu/c++/11" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/lib/llvm-17/include" }); + compiler.addLibraryPath(.{ .cwd_relative = "/lib/x86_64-linux-gnu" }); + compiler.addLibraryPath(.{ .cwd_relative = "/usr/lib/llvm-17/lib" }); } else { - unreachable; - }; + const result = try std.ChildProcess.run(.{ + .allocator = b.allocator, + .argv = &.{ + "cc", "--version", + }, + .max_output_bytes = 0xffffffffffffff, + }); + const success = switch (result.term) { + .Exited => |exit_code| exit_code == 0, + else => false, + }; - const cxx_include_base = try std.mem.concat(b.allocator, u8, &.{"/usr/include/c++/", cxx_version} ); + if (!success) { + unreachable; + } - compiler.addObjectFile(.{ .cwd_relative = "/usr/lib/libstdc++.so" }); - compiler.addIncludePath(.{ .cwd_relative = "/usr/include" }); - compiler.addIncludePath(.{ .cwd_relative = cxx_include_base}); - compiler.addIncludePath(.{ .cwd_relative = try std.mem.concat(b.allocator, u8, &.{cxx_include_base, "/x86_64-pc-linux-gnu"}) }); - compiler.addLibraryPath(.{ .cwd_relative = "/usr/lib" }); + var tokenizer = std.mem.tokenize(u8, result.stdout, " "); + const cxx_version = while (tokenizer.next()) |chunk| { + if (std.ascii.isDigit(chunk[0])) { + if (std.SemanticVersion.parse(chunk)) |_| { + break chunk; + } else |err| err catch {}; + } + } else { + unreachable; + }; + + const cxx_include_base = try std.mem.concat(b.allocator, u8, &.{ "/usr/include/c++/", cxx_version }); + compiler.addObjectFile(.{ .cwd_relative = "/usr/lib/libstdc++.so" }); + compiler.addIncludePath(.{ .cwd_relative = "/usr/include" }); + compiler.addIncludePath(.{ .cwd_relative = cxx_include_base }); + compiler.addIncludePath(.{ .cwd_relative = try std.mem.concat(b.allocator, u8, &.{ cxx_include_base, "/x86_64-pc-linux-gnu" }) }); + compiler.addLibraryPath(.{ .cwd_relative = "/usr/lib" }); + } }, .macos => { compiler.linkLibCpp(); @@ -522,12 +539,10 @@ pub fn build(b: *std.Build) !void { .windows => blk: { const result = b.addSystemCommand(&.{"remedybg"}); result.addArg("-g"); - // result.addArg(compiler_exe_path); break :blk result; }, .macos => blk: { - // not tested const result = b.addSystemCommand(&.{"lldb"}); result.addArg("--"); break :blk result; diff --git a/build/fetcher.zig b/build/fetcher.zig index f820bf5..74487bd 100644 --- a/build/fetcher.zig +++ b/build/fetcher.zig @@ -65,7 +65,7 @@ pub fn main() !void { .server_header_buffer = &buffer, }); defer request.deinit(); - try request.send(.{}); + try request.send(); try request.wait(); if (request.response.status != .ok) { diff --git a/build/test_runner.zig b/build/test_runner.zig index 689d600..ead7ff9 100644 --- a/build/test_runner.zig +++ b/build/test_runner.zig @@ -312,7 +312,8 @@ fn runCmakeTests(allocator: Allocator, args: struct { "..", // "--debug-trycompile", // "--debug-output", - // "-G", "Unix Makefiles", + "-G", + "Ninja", // "-DCMAKE_VERBOSE_MAKEFILE=On", try std.mem.concat(allocator, u8, &.{ "-DCMAKE_C_COMPILER=", compiler_realpath, ";cc" }), try std.mem.concat(allocator, u8, &.{ "-DCMAKE_CXX_COMPILER=", compiler_realpath, ";c++" }), diff --git a/ci/github_ubuntu_runner.sh b/ci/github_ubuntu_runner.sh new file mode 100644 index 0000000..3cc9f05 --- /dev/null +++ b/ci/github_ubuntu_runner.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -ex + +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 17 all + +sudo apt install liblld-17-dev libclang-17-dev liblld-17 ninja-build cmake -y + +zig build test -Dthird_party_ci=true diff --git a/src/main.nat b/src/main.nat index d317907..9cf3f0c 100644 --- a/src/main.nat +++ b/src/main.nat @@ -58,6 +58,13 @@ const FixedKeyword = enum{ "continue", }; +const FileStartToken = enum{ + "comptime", + "test", + "const", + "var", +}; + const lex = fn (arena: &Arena, bytes: []const u8) *!void { if (bytes.length >= 0xffffffff) { unreachable; @@ -66,60 +73,14 @@ const lex = fn (arena: &Arena, bytes: []const u8) *!void { const length: u32 = #cast(bytes.length); var i: u32 = 0; - const line_offset_arena = try Arena.init(bytes.length + #size(Arena)); - var line_offsets = try line_offset_arena.new_array($u32, bytes.length); - line_offsets[0] = 0; - var line_count: u32 = 1; - - var index: u32 = 0; - while (index < length) { - const byte = bytes[index]; - line_count += #cast(byte == '\n'); - line_offsets[line_count] = index; - - index += 1; - } - - index = 0; - - while (index < length) { - const start_index = index; - const start_character = bytes[index]; - var token_id = Token.Id.invalid; - - switch (start_character) { - 'a'...'z', 'A'...'Z', '_' => { - while (true) { - const ch = bytes[index]; - if ((ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z') or ch == '_' or (ch >= '0' and ch <= '9')) { - index += 1; - continue; - } - - break; - } - - if (start_character == 'u' or start_character == 's' and bytes[start_index + 1] >= '0' and bytes[start_index + 1] <= '9') { - var index_integer = start_index + 1; - while (bytes[index_integer] >= '0' and bytes[index_integer] <= '9') { - index_integer += 1; - } - - if (index_integer == index) { - token_id = switch (start_character) { - 'u' => .keyword_unsigned_integer, - 's' => .keyword_signed_integer, - else => unreachable, - }; - } - } - - const string_length = index - start_index; - const string = bytes[start_index..][0..string_length]; - }, - else => index += 1, - } - } + //var index: u32 = 0; + //while (index + 64 < length) { + // var i = index; + // const top = index + 64; + // while (i < top) { + // i += 1; + // } + //} } const get_argument = fn (real_argument: []const u8, wanted_argument: []const u8, command_arguments: []const [&:0]const u8, i_ptr: &usize) ?[]const u8 {