Merge pull request #2 from birth-software/self-hosted

ci: try to add self-hosted

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/ci.yml"
+      - "**.zig"
+  push:
+    paths:
+      - ".github/workflows/ci.yml"
+      - "**.zig"
+    branches:
+      - main
+  schedule:
+    - cron: "0 0 * * *"
+concurrency:
+  # Cancels pending runs when a PR gets updated.
+  group: ${{ github.head_ref || github.run_id }}-${{ github.actor }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [
+          ubuntu-latest,
+          macos-latest,
+          windows-latest,
+        ]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Zig
+        uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: master
+      - name: Zig environment variables
+        run: zig env
+      - name: Build test executables
+        run: zig build all_tests -Dci --verbose
+  build_and_test:
+    runs-on: [self-hosted, Linux, X64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Zig
+        uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: master
+      - name: Zig environment variables
+        run: zig env
+      - name: Build test executables
+        run: zig build all_tests -Dci -Dci_native --verbose
+      - name: Test with QEMU
+        run: zig build test_all -Dci -Dci_native --verbose

.github/workflows/lightning.yml

@@ -1,40 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/ci.yml"
-      - "**.zig"
-  push:
-    paths:
-      - ".github/workflows/ci.yml"
-      - "**.zig"
-    branches:
-      - main
-  schedule:
-    - cron: "0 0 * * *"
-concurrency:
-  # Cancels pending runs when a PR gets updated.
-  group: ${{ github.head_ref || github.run_id }}-${{ github.actor }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        os: [
-        ]
-    runs-on: [
-          ubuntu-latest,
-          macos-latest,
-          windows-latest,
-          self-hosted,
-    ]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: goto-bus-stop/setup-zig@v2
-        with:
-          version: master
-
-      - name: Build test executables
-        run: zig build all_tests -Dci --verbose

build.zig

@@ -37,6 +37,7 @@ const source_root_dir = "src";
 const user_program_dir_path = "src/user/programs";
 
 var ci = false;
+var ci_native = false;
 var debug_user = false;
 var debug_loader = false;
 var modules = Modules{};
@@ -49,6 +50,7 @@ var options = Options{};
 pub fn build(b_arg: *Build) !void {
     b = b_arg;
     ci = b.option(bool, "ci", "CI mode") orelse false;
+    ci_native = b.option(bool, "ci_native", "CI mode in self-hosted runner") orelse false;
     debug_user = b.option(bool, "debug_user", "Debug user program") orelse false;
     debug_loader = b.option(bool, "debug_loader", "Debug loader program") orelse false;
     const default_cfg_override = b.option([]const u8, "default", "Default configuration JSON file") orelse "config/default.json";
@@ -157,6 +159,7 @@ pub fn build(b_arg: *Build) !void {
         root_project_path: []const u8,
         modules: []const ModuleID,
         c: ?C = null,
+        run_native: bool = true,
 
         const C = struct {
             include_paths: []const []const u8,
@@ -190,6 +193,7 @@ pub fn build(b_arg: *Build) !void {
                 .link_libc = true,
                 .link_libcpp = false,
             },
+            .run_native = false,
         },
     };
 
@@ -222,12 +226,25 @@ pub fn build(b_arg: *Build) !void {
             }
         }
 
-        const run_test_step = b.addRunArtifact(test_exe);
         //run_test_step.condition = .always;
-        build_steps.test_all.dependOn(&run_test_step.step);
+        const should_run = !ci_native or (ci_native and native_test.run_native);
-        build_steps.test_host.dependOn(&run_test_step.step);
+        if (should_run) {
+            const run_test_step = b.addRunArtifact(test_exe);
+            build_steps.test_all.dependOn(&run_test_step.step);
+            build_steps.test_host.dependOn(&run_test_step.step);
+        }
     }
 
+    const ovmf_downloader = try addCompileStep(.{
+        .name = "ovmf_downloader",
+        .root_project_path = "src/host/ovmf_downloader",
+        .optimize_mode = .Debug,
+        .modules = &.{ .lib, .host },
+        .kind = .exe,
+    });
+    const ovmf_downloader_run_step = b.addRunArtifact(ovmf_downloader);
+    const ovmf_path = ovmf_downloader_run_step.addOutputFileArg("OVMF.fd");
+
     {
         var user_module_list = std.ArrayList(common.Module).init(b.allocator);
         var user_program_dir = try std.fs.cwd().openIterableDir(user_program_dir_path, .{ .access_sub_paths = true });
@@ -461,6 +478,7 @@ pub fn build(b_arg: *Build) !void {
                                         .is_debug = false,
                                         .is_test = is_test,
                                     },
+                                    .ovmf_path = ovmf_path,
                                 });
                                 const runner_debug = try newRunnerRunArtifact(.{
                                     .configuration = configuration,
@@ -473,6 +491,7 @@ pub fn build(b_arg: *Build) !void {
                                         .is_debug = true,
                                         .is_test = is_test,
                                     },
+                                    .ovmf_path = ovmf_path,
                                 });
 
                                 if (is_test) {
@@ -572,9 +591,12 @@ fn newRunnerRunArtifact(arguments: struct {
     cpu_driver: *CompileStep,
     user_init: *CompileStep,
     qemu_options: QEMUOptions,
+    ovmf_path: FileSource,
 }) !*RunStep {
     const runner = b.addRunArtifact(arguments.runner);
+
     var argument_parser = common.ArgumentParser.Runner{};
+
     while (argument_parser.next()) |argument_type| switch (argument_type) {
         .configuration => inline for (common.fields(Configuration)) |field| runner.addArg(@tagName(@field(arguments.configuration, field.name))),
         .image_configuration_path => runner.addArg(common.ImageConfig.default_path),
@@ -586,6 +608,7 @@ fn newRunnerRunArtifact(arguments: struct {
         .ci => runner.addArg(if (ci) "true" else "false"),
         .debug_user => runner.addArg(if (debug_user) "true" else "false"),
         .debug_loader => runner.addArg(if (debug_loader) "true" else "false"),
+        .ovmf_path => runner.addFileSourceArg(arguments.ovmf_path),
     };
 
     return runner;

src/bootloader/rise/uefi/main.zig

@@ -97,7 +97,6 @@ const Filesystem = extern struct {
 
     pub fn getFileSize(filesystem: *Filesystem, file_path: []const u8) !u32 {
         const file = try filesystem.openFile(file_path);
-        log.debug("File size", .{});
         var file_info_buffer: [@sizeOf(uefi.FileInfo) + 0x100]u8 align(@alignOf(uefi.FileInfo)) = undefined;
         var file_info_size = file_info_buffer.len;
         try uefi.Try(file.handle.getInfo(&uefi.FileInfo.guid, &file_info_size, &file_info_buffer));
@@ -112,25 +111,20 @@ const Filesystem = extern struct {
             return Error.boot_services_exited;
         }
 
-        log.debug("opening file: {s}", .{file_path});
         var file: *FileProtocol = undefined;
         var path_buffer: [256:0]u16 = undefined;
         const length = try lib.unicode.utf8ToUtf16Le(&path_buffer, file_path);
         path_buffer[length] = 0;
         const path = path_buffer[0..length :0];
         const uefi_path = if (path[0] == '/') path[1..] else path;
-        log.debug("uefi path: {any}", .{uefi_path});
 
         try uefi.Try(filesystem.root.open(&file, uefi_path, FileProtocol.efi_file_mode_read, 0));
-        log.debug("Opened", .{});
 
         const result = FileDescriptor{
             .handle = file,
             .path_size = @as(u32, @intCast(path.len * @sizeOf(u16))),
         };
 
-        log.debug("opened file: {s}", .{file_path});
-
         return result;
     }
 
@@ -251,9 +245,7 @@ const Initialization = struct {
                 },
             },
             .framebuffer = blk: {
-                log.debug("Locating GOP", .{});
                 const gop = try Protocol.locate(uefi.GraphicsOutputProtocol, boot_services);
-                log.debug("Located GOP", .{});
 
                 const pixel_format_info: struct {
                     red_color_mask: bootloader.Framebuffer.ColorMask,
@@ -303,7 +295,6 @@ const Initialization = struct {
             },
         };
 
-        log.debug("Memory map size: {}", .{init.memory_map.size});
         _ = boot_services.getMemoryMap(&init.memory_map.size, @as([*]MemoryDescriptor, @ptrCast(&init.memory_map.buffer)), &init.memory_map.key, &init.memory_map.descriptor_size, &init.memory_map.descriptor_version);
         init.memory_map.entry_count = @as(u32, @intCast(@divExact(init.memory_map.size, init.memory_map.descriptor_size)));
         assert(init.memory_map.entry_count > 0);
@@ -321,8 +312,6 @@ const Initialization = struct {
             const expected_memory_map_descriptor_size = init.memory_map.descriptor_size;
             const expected_memory_map_descriptor_version = init.memory_map.descriptor_version;
 
-            log.debug("Getting memory map before exiting boot services...", .{});
-
             blk: while (init.memory_map.size < MemoryMap.buffer_len) : (init.memory_map.size += init.memory_map.descriptor_size) {
                 uefi.Try(init.boot_services.getMemoryMap(&init.memory_map.size, @as([*]MemoryDescriptor, @ptrCast(&init.memory_map.buffer)), &init.memory_map.key, &init.memory_map.descriptor_size, &init.memory_map.descriptor_version)) catch continue;
                 init.exited_boot_services = true;

src/common.zig

@@ -481,6 +481,7 @@ pub const ArgumentParser = struct {
             debug_user,
             debug_loader,
             init,
+            ovmf_path,
         };
 
         pub const Result = struct {
@@ -494,6 +495,7 @@ pub const ArgumentParser = struct {
             debug_user: bool,
             debug_loader: bool,
             init: []const u8,
+            ovmf_path: []const u8,
         };
     };
 };

src/host.zig

@@ -15,6 +15,9 @@ pub const cwd = fs.cwd;
 pub const Dir = fs.Dir;
 pub const basename = fs.path.basename;
 pub const dirname = fs.path.dirname;
+pub const realpathAlloc = fs.realpathAlloc;
+
+pub const os = std.os;
 
 const io = std.io;
 pub const getStdOut = std.io.getStdOut;

src/host/ovmf_downloader/main.zig

@@ -0,0 +1,57 @@
+const host = @import("host");
+
+const Error = error{
+    wrong_arguments,
+};
+
+pub fn main() !void {
+    const allocator = @import("std").heap.page_allocator;
+    const arguments = try host.allocateArguments(allocator);
+    if (arguments.len != 2) {
+        return Error.wrong_arguments;
+    }
+
+    const ovmf_path = arguments[1];
+
+    const file_descriptor = blk: {
+        if (host.fs.openFileAbsolute(ovmf_path, .{})) |file_descriptor| {
+            break :blk file_descriptor;
+        } else |_| {
+            const url = "https://retrage.github.io/edk2-nightly/bin/RELEASEX64_OVMF.fd";
+            const uri = try host.Uri.parse(url);
+
+            var http_client = host.http.Client{ .allocator = allocator };
+            defer http_client.deinit();
+
+            var request_headers = host.http.Headers{ .allocator = allocator };
+            defer request_headers.deinit();
+
+            var request = try http_client.request(.GET, uri, request_headers, .{});
+            defer request.deinit();
+
+            try request.start();
+            try request.wait();
+
+            if (request.response.status != .ok) {
+                return error.ResponseNotOk;
+            }
+
+            const content_length = request.response.content_length orelse {
+                return error.OutOfMemory;
+            };
+
+            const buffer = try allocator.alloc(u8, content_length);
+            const read_byte_count = try request.readAll(buffer);
+            if (read_byte_count != buffer.len) {
+                return error.OutOfMemory;
+            }
+
+            const ovmf_file_descriptor = try host.fs.createFileAbsolute(ovmf_path, .{});
+            try ovmf_file_descriptor.writeAll(buffer);
+
+            break :blk ovmf_file_descriptor;
+        }
+    };
+
+    file_descriptor.close();
+}

src/host/runner/main.zig

@@ -21,6 +21,7 @@ const Error = error{
     not_implemented,
     architecture_not_supported,
     execution_environment_not_supported,
+    ovmf_path_not_found,
 };
 
 pub fn main() anyerror!void {
@@ -44,6 +45,7 @@ pub fn main() anyerror!void {
         var argument_debug_loader: ?bool = null;
         var argument_init_path: ?[]const u8 = null;
         var argument_index: usize = 0;
+        var argument_ovmf_path: ?[]const u8 = null;
 
         while (argument_parser.next()) |argument_type| switch (argument_type) {
             .disk_image_path => {
@@ -95,6 +97,10 @@ pub fn main() anyerror!void {
                 argument_init_path = arguments[argument_index];
                 argument_index += 1;
             },
+            .ovmf_path => {
+                argument_ovmf_path = arguments[argument_index];
+                argument_index += 1;
+            },
         };
 
         if (argument_index != arguments.len) return Error.wrong_argument_count;
@@ -110,6 +116,7 @@ pub fn main() anyerror!void {
             .debug_user = argument_debug_user orelse return Error.debug_user_not_found,
             .debug_loader = argument_debug_loader orelse return Error.debug_loader_not_found,
             .init = argument_init_path orelse return Error.init_not_found,
+            .ovmf_path = argument_ovmf_path orelse return Error.ovmf_path_not_found,
         };
     };
 
@@ -130,40 +137,7 @@ pub fn main() anyerror!void {
             }
 
             switch (arguments_result.configuration.boot_protocol) {
-                .uefi => {
+                .uefi => try argument_list.appendSlice(&.{ "-bios", arguments_result.ovmf_path }),
-                    const ovmf_path = "tools/OVMF.fd";
-                    if (host.cwd().openFile(ovmf_path, .{})) |file_descriptor| {
-                        file_descriptor.close();
-                    } else |_| {
-                        const url = "https://retrage.github.io/edk2-nightly/bin/RELEASEX64_OVMF.fd";
-                        const uri = try host.Uri.parse(url);
-
-                        var http_client = host.http.Client{ .allocator = wrapped_allocator.zigUnwrap() };
-                        defer http_client.deinit();
-
-                        var request_headers = host.http.Headers{ .allocator = wrapped_allocator.zigUnwrap() };
-                        defer request_headers.deinit();
-
-                        var request = try http_client.request(.GET, uri, request_headers, .{});
-                        defer request.deinit();
-
-                        try request.start();
-                        try request.wait();
-
-                        if (request.response.status != .ok) return error.ResponseNotOk;
-                        const content_length = request.response.content_length orelse return error.OutOfMemory;
-
-                        const buffer = try wrapped_allocator.zigUnwrap().alloc(u8, content_length);
-                        const read_byte_count = try request.readAll(buffer);
-                        if (read_byte_count != buffer.len) {
-                            return error.OutOfMemory;
-                        }
-
-                        try host.cwd().writeFile(ovmf_path, buffer);
-                    }
-
-                    try argument_list.appendSlice(&.{ "-bios", ovmf_path });
-                },
                 else => {},
             }