From 25114a053d24847509ea4bf6bb806200376ee5de Mon Sep 17 00:00:00 2001 From: David Gonzalez Martin Date: Thu, 28 Mar 2024 10:20:51 -0600 Subject: [PATCH] Rewrite test runner to be easier to work with --- bootstrap/Compilation.zig | 6 +- bootstrap/main.zig | 2 +- build.zig | 2 + build/test_runner.zig | 871 ++++++++++++++++---------------------- 4 files changed, 381 insertions(+), 500 deletions(-) diff --git a/bootstrap/Compilation.zig b/bootstrap/Compilation.zig index 65eba9f..a3dbcf3 100644 --- a/bootstrap/Compilation.zig +++ b/bootstrap/Compilation.zig @@ -2907,7 +2907,11 @@ pub fn buildExecutable(context: *const Context, arguments: []const []const u8, o break :blk path; } else unreachable; - const executable_name = if (maybe_executable_name) |name| name else std.fs.path.basename(main_package_path[0 .. main_package_path.len - "/main.nat".len]); + const executable_name = if (maybe_executable_name) |name| name else b: { + const slash_index = data_structures.last_byte(main_package_path, '/') orelse unreachable; + const p = std.fs.path.basename(main_package_path[0 .. slash_index]); + break :b p; + }; const executable_path = maybe_executable_path orelse blk: { assert(executable_name.len > 0); diff --git a/bootstrap/main.zig b/bootstrap/main.zig index 2221750..b00bb49 100644 --- a/bootstrap/main.zig +++ b/bootstrap/main.zig @@ -56,7 +56,7 @@ pub fn main() !void { try args.append(context.my_allocator, arg); } const arguments = args.slice(); - const debug_args = true; + const debug_args = false; if (debug_args) { assert(arguments.len > 0); const home_dir = std.posix.getenv("HOME") orelse unreachable; diff --git a/build.zig b/build.zig index 2d73559..7aca7ae 100644 --- a/build.zig +++ b/build.zig @@ -498,6 +498,8 @@ pub fn build(b: *std.Build) !void { b.default_step.dependOn(&test_runner.step); const test_command = b.addRunArtifact(test_runner); + const install_runner = b.addInstallArtifact(test_runner, .{}); + b.getInstallStep().dependOn(&install_runner.step); test_command.step.dependOn(&compiler.step); test_command.step.dependOn(b.getInstallStep()); diff --git a/build/test_runner.zig b/build/test_runner.zig index 698e05d..b0a0c1e 100644 --- a/build/test_runner.zig +++ b/build/test_runner.zig @@ -7,532 +7,407 @@ const TestError = error{ signaled, stopped, unknown, + internal, fail, }; -fn collectDirectoryDirEntries(allocator: Allocator, path: []const u8) ![]const []const u8 { - var dir = try std.fs.cwd().openDir(path, .{ - .iterate = true, - }); - var dir_iterator = dir.iterate(); - var dir_entries = std.ArrayListUnmanaged([]const u8){}; +const TestGroup = struct { + state: States = .{}, + tests: std.ArrayListUnmanaged(TestRecord) = .{}, + path: []const u8, + path_policy: PathPolicy, + category: Category, + compilation_kind: CompilationKind, + collect_directory_entries: ?[]const []const u8, - while (try dir_iterator.next()) |entry| { - switch (entry.kind) { - .directory => try dir_entries.append(allocator, try allocator.dupe(u8, entry.name)), - else => return error.junk_in_test_directory, - } - } - - dir.close(); - - return dir_entries.items; -} - -fn runStandalone(allocator: Allocator, args: struct { - directory_path: []const u8, - group_name: []const u8, - is_test: bool, -}) !void { - const test_names = try collectDirectoryDirEntries(allocator, args.directory_path); - - const total_compilation_count = test_names.len; - var ran_compilation_count: usize = 0; - var failed_compilation_count: usize = 0; - - var ran_test_count: usize = 0; - var failed_test_count: usize = 0; - const total_test_count = test_names.len; - - std.debug.print("\n[{s} START]\n\n", .{args.group_name}); - - for (test_names) |test_name| { - std.debug.print("{s}... ", .{test_name}); - const source_file_path = try std.mem.concat(allocator, u8, &.{ args.directory_path, "/", test_name, "/main.nat" }); - const compile_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{ "zig-out/bin/nat", if (args.is_test) "test" else "exe", "-main_source_file", source_file_path }, - .max_output_bytes = std.math.maxInt(u64), - }); - ran_compilation_count += 1; - - const compilation_result: TestError!bool = switch (compile_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const compilation_success = compilation_result catch b: { - failed_compilation_count += 1; - break :b false; - }; - - std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (compile_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); - } - if (compile_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); - } - - if (compilation_success) { - const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name }); - const test_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{test_path}, - .max_output_bytes = std.math.maxInt(u64), - }); - ran_test_count += 1; - const test_result: TestError!bool = switch (test_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const test_success = test_result catch b: { - failed_test_count += 1; - break :b false; - }; - std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (test_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); - } - if (test_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); - } - } else { - std.debug.print("\n", .{}); - } - } - - std.debug.print("\n{s} COMPILATIONS: {}. FAILED: {}\n", .{ args.group_name, total_compilation_count, failed_compilation_count }); - std.debug.print("{s} TESTS: {}. RAN: {}. FAILED: {}\n", .{ args.group_name, total_test_count, ran_test_count, failed_test_count }); - - if (failed_compilation_count > 0 or failed_test_count > 0) { - return error.fail; - } -} - -fn runStandaloneTests(allocator: Allocator) !void { - const standalone_test_dir_path = "test/standalone"; - const standalone_test_names = try collectDirectoryDirEntries(allocator, standalone_test_dir_path); - - const total_compilation_count = standalone_test_names.len; - var ran_compilation_count: usize = 0; - var failed_compilation_count: usize = 0; - - var ran_test_count: usize = 0; - var failed_test_count: usize = 0; - const total_test_count = standalone_test_names.len; - - for (standalone_test_names) |standalone_test_name| { - std.debug.print("{s}... ", .{standalone_test_name}); - const source_file_path = try std.mem.concat(allocator, u8, &.{ standalone_test_dir_path, "/", standalone_test_name, "/main.nat" }); - const compile_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{ "zig-out/bin/nat", "exe", "-main_source_file", source_file_path }, - .max_output_bytes = std.math.maxInt(u64), - }); - ran_compilation_count += 1; - - const compilation_result: TestError!bool = switch (compile_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const compilation_success = compilation_result catch b: { - failed_compilation_count += 1; - break :b false; - }; - - std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (compile_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); - } - if (compile_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); - } - - if (compilation_success) { - const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", standalone_test_name }); - const test_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{test_path}, - .max_output_bytes = std.math.maxInt(u64), - }); - ran_test_count += 1; - const test_result: TestError!bool = switch (test_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const test_success = test_result catch b: { - failed_test_count += 1; - break :b false; - }; - std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (test_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); - } - if (test_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); - } - } else { - std.debug.print("\n", .{}); - } - } - - std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count }); - std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count }); - - if (failed_compilation_count > 0 or failed_test_count > 0) { - return error.fail; - } -} - -fn runBuildTests(allocator: Allocator) !void { - std.debug.print("\n[BUILD TESTS]\n\n", .{}); - const previous_cwd = try std.fs.cwd().realpathAlloc(allocator, "."); - const test_dir_path = "test/build"; - const test_names = try collectDirectoryDirEntries(allocator, test_dir_path); - const test_dir_realpath = try std.fs.cwd().realpathAlloc(allocator, test_dir_path); - const compiler_realpath = try std.fs.cwd().realpathAlloc(allocator, "zig-out/bin/nat"); - try std.posix.chdir(test_dir_realpath); - - const total_compilation_count = test_names.len; - var ran_compilation_count: usize = 0; - var failed_compilation_count: usize = 0; - - var ran_test_count: usize = 0; - var failed_test_count: usize = 0; - const total_test_count = test_names.len; - - for (test_names) |test_name| { - std.debug.print("{s}... ", .{test_name}); - try std.posix.chdir(test_name); - - const compile_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{ compiler_realpath, "build" }, - .max_output_bytes = std.math.maxInt(u64), - }); - - ran_compilation_count += 1; - - const compilation_result: TestError!bool = switch (compile_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const compilation_success = compilation_result catch b: { - failed_compilation_count += 1; - break :b false; - }; - - std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (compile_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); - } - if (compile_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); - } - - if (compilation_success) { - const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name }); - const test_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{test_path}, - .max_output_bytes = std.math.maxInt(u64), - }); - ran_test_count += 1; - const test_result: TestError!bool = switch (test_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const test_success = test_result catch b: { - failed_test_count += 1; - break :b false; - }; - std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (test_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); - } - if (test_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); - } - } else { - std.debug.print("\n", .{}); - } - - try std.posix.chdir(test_dir_realpath); - } - - std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count }); - std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count }); - - try std.posix.chdir(previous_cwd); - - if (failed_compilation_count > 0 or failed_test_count > 0) { - return error.fail; - } -} - -fn runStdTests(allocator: Allocator) !void { - var errors = false; - std.debug.print("std... ", .{}); - - const result = try std.ChildProcess.run(.{ - .allocator = allocator, - .argv = &.{ "zig-out/bin/nat", "test", "-main_source_file", "lib/std/std.nat", "-name", "std" }, - .max_output_bytes = std.math.maxInt(u64), - }); - const compilation_result: TestError!bool = switch (result.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, + const PathPolicy = enum { + join_path, + take_entry, }; - const compilation_success = compilation_result catch b: { - errors = true; - break :b false; + const CompilationKind = enum { + exe, + @"test", + build, + cmake, + }; +}; + +const TestRecord = struct { + name: []const u8, + configuration: ?TestResult = null, + compilation: ?TestResult = null, + execution: ?TestResult = null, +}; + +const TestResult = struct { + stdout: []const u8 = &.{}, + stderr: []const u8 = &.{}, + error_union: RunError!void, +}; + +// const TestStep = struct{ +// kind: Kind, +// result: TestResult, +// +// const Kind = enum{ +// configuration, +// compilation, +// execution, +// }; +// }; + +const States = struct { + failed: TestState = .{}, + total: TestState = .{}, +}; + +const TestState = struct { + compilations: u32 = 0, + executions: u32 = 0, + tests: u32 = 0, + + fn add(total: *TestState, state: *const TestState) void { + total.compilations += state.compilations; + total.executions += state.executions; + total.tests += state.tests; + } +}; + +const TestSuite = struct { + allocator: Allocator, + state: State = .{}, + compiler_path: []const u8, + + const State = struct { + states: States = .{}, + category_map: std.EnumSet(Category) = .{}, + test_groups: std.ArrayListUnmanaged(TestGroup) = .{}, }; - std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (result.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{result.stdout}); - } - if (result.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{result.stderr}); - } - - if (compilation_success) { - const test_run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{"nat/std"}, - .max_output_bytes = std.math.maxInt(u64), - }); - const test_result: TestError!bool = switch (test_run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const test_success = test_result catch b: { - errors = true; - break :b false; - }; - std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - if (test_run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); + fn run_test_group(test_suite: *TestSuite, test_group: *TestGroup) !void { + test_suite.state.category_map.setPresent(test_group.category, true); + defer { + test_suite.state.states.total.add(&test_group.state.total); + test_suite.state.states.failed.add(&test_group.state.failed); } - if (test_run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); - } - } - if (errors) return error.fail; -} - -fn runCmakeTests(allocator: Allocator, dir_path: []const u8) !void { - var errors = false; - const original_dir = try std.fs.cwd().realpathAlloc(allocator, "."); - const cc_dir = try std.fs.cwd().openDir(dir_path, .{ - .iterate = true, - }); - - const cc_dir_path = try cc_dir.realpathAlloc(allocator, "."); - try std.posix.chdir(cc_dir_path); - - var cc_dir_iterator = cc_dir.iterate(); - while (try cc_dir_iterator.next()) |cc_entry| { - switch (cc_entry.kind) { - .directory => { - std.debug.print("{s}...\n", .{cc_entry.name}); - try std.posix.chdir(cc_entry.name); - try std.fs.cwd().deleteTree("build"); - try std.fs.cwd().makeDir("build"); - try std.posix.chdir("build"); - - const cmake = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{ - "cmake", - "..", - // "--debug-trycompile", - // "--debug-output", - // "-G", "Unix Makefiles", - // "-DCMAKE_VERBOSE_MAKEFILE=On", - try std.mem.concat(allocator, u8, &.{ "-DCMAKE_C_COMPILER=", "nat;cc" }), - try std.mem.concat(allocator, u8, &.{ "-DCMAKE_CXX_COMPILER=", "nat;c++" }), - try std.mem.concat(allocator, u8, &.{ "-DCMAKE_ASM_COMPILER=", "nat;cc" }), - }, - .max_output_bytes = std.math.maxInt(u64), + if (test_group.collect_directory_entries) |*directory_entries| { + directory_entries.* = blk: { + var dir = try std.fs.cwd().openDir(test_group.path, .{ + .iterate = true, }); - const cmake_result: TestError!bool = switch (cmake.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; + var dir_iterator = dir.iterate(); + var dir_entries = std.ArrayListUnmanaged([]const u8){}; - const cmake_success = cmake_result catch b: { - errors = true; - break :b false; - }; - - if (cmake.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{cmake.stdout}); - } - if (cmake.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{cmake.stderr}); - } - - var success = cmake_success; - if (success) { - const ninja = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{"ninja"}, - .max_output_bytes = std.math.maxInt(u64), - }); - const ninja_result: TestError!bool = switch (ninja.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const ninja_success = ninja_result catch b: { - errors = true; - break :b false; - }; - - if (!ninja_success) { - if (ninja.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{ninja.stdout}); - } - if (ninja.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{ninja.stderr}); - } - } - - success = success and ninja_success; - - if (success) { - const run = try std.ChildProcess.run(.{ - .allocator = allocator, - // TODO: delete -main_source_file? - .argv = &.{ - try std.mem.concat(allocator, u8, &.{ "./", cc_entry.name }), - }, - .max_output_bytes = std.math.maxInt(u64), - }); - const run_result: TestError!bool = switch (run.term) { - .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, - .Signal => error.signaled, - .Stopped => error.stopped, - .Unknown => error.unknown, - }; - - const run_success = run_result catch b: { - errors = true; - break :b false; - }; - - if (run.stdout.len > 0) { - std.debug.print("STDOUT:\n\n{s}\n\n", .{run.stdout}); - } - if (run.stderr.len > 0) { - std.debug.print("STDERR:\n\n{s}\n\n", .{run.stderr}); - } - - success = success and run_success; + while (try dir_iterator.next()) |entry| { + switch (entry.kind) { + .directory => try dir_entries.append(test_suite.allocator, try test_suite.allocator.dupe(u8, entry.name)), + else => return error.junk_in_test_directory, } } - std.debug.print("[TEST {s}]\n", .{if (success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); - }, - else => std.debug.panic("Entry {s} is a {s}", .{ cc_entry.name, @tagName(cc_entry.kind) }), + dir.close(); + + break :blk dir_entries.items; + }; + } else { + test_group.collect_directory_entries = &.{ test_group.path }; } - try std.posix.chdir(cc_dir_path); - } + const directory_entries = test_group.collect_directory_entries orelse unreachable; + try test_group.tests.ensureTotalCapacity(test_suite.allocator, directory_entries.len); - try std.posix.chdir(original_dir); + for (directory_entries) |directory_entry| { + var has_error = false; + test_group.state.total.tests += 1; - if (errors) { - return error.fail; + const relative_path = try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry }); + + var test_record = TestRecord{ + .name = directory_entry, + }; + + const cmake_build_dir = "build"; + switch (test_group.category) { + .cmake => { + const argv: []const []const u8 = &.{ + "cmake", + "-S", ".", + "-B", cmake_build_dir, + "--fresh", + try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_C_COMPILER={s};cc", .{test_suite.compiler_path}), + try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_CXX_COMPILER={s};c++", .{test_suite.compiler_path}), + try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_ASM_COMPILER={s};cc", .{test_suite.compiler_path}), + }; + test_record.configuration = try run_process(test_suite.allocator, argv, .{ .path = relative_path }); + test_record.configuration.?.error_union catch { + has_error = true; + test_group.state.total.compilations += 1; + test_group.state.failed.compilations += 1; + }; + }, + .build, .standalone => {}, + } + + if (!has_error) { + const compilation_cwd: Cwd = switch (test_group.category) { + .standalone => .none, + .build => .{ .path = relative_path }, + .cmake => .{ .path = try std.mem.concat(test_suite.allocator, u8, &.{ relative_path, "/" ++ cmake_build_dir }) }, + }; + + const arguments: []const []const u8 = switch (test_group.category) { + .standalone => blk: { + const source_file_path = switch (test_group.path_policy) { + .join_path => try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry, "/main.nat" }), + .take_entry => directory_entry, + }; + const compilation_argument = @tagName(test_group.compilation_kind); + break :blk &.{ test_suite.compiler_path, compilation_argument, "-main_source_file", source_file_path }; + }, + .build => &.{ test_suite.compiler_path, "build" }, + .cmake => &.{ "ninja" }, + }; + + test_record.compilation = try run_process(test_suite.allocator, arguments, compilation_cwd); + test_group.state.total.compilations += 1; + + if (test_record.compilation.?.error_union) |_| { + const executable_name = switch (test_group.path_policy) { + .join_path => directory_entry, + .take_entry => b: { + const slash_index = std.mem.lastIndexOfScalar(u8, directory_entry, '/') orelse unreachable; + const base = std.fs.path.basename(directory_entry[0..slash_index]); + break :b base; + }, + }; + const build_dir = switch (test_group.category) { + .cmake => cmake_build_dir ++ "/", + else => "nat/", + }; + const execution_cwd: Cwd = switch (test_group.category) { + .cmake => .{ .path = relative_path }, + else => compilation_cwd, + }; + test_record.execution = try run_process(test_suite.allocator, &.{try std.mem.concat(test_suite.allocator, u8, &.{ build_dir, executable_name })}, execution_cwd); + test_group.state.total.executions += 1; + + if (test_record.execution.?.error_union) |_| {} else |err| { + has_error = true; + err catch {}; + test_group.state.failed.executions += 1; + } + } else |err| { + err catch {}; + has_error = true; + test_group.state.failed.compilations += 1; + } + } + + test_group.state.failed.tests += @intFromBool(has_error); + + test_group.tests.appendAssumeCapacity(test_record); + } } +}; + +const RunError = error{ + unexpected_exit_code, + signaled, + stopped, + unknown, +}; + +const Cwd = union(enum) { + none, + path: []const u8, + descriptor: std.fs.Dir, +}; + +fn run_process(allocator: Allocator, argv: []const []const u8, cwd: Cwd) !TestResult { + var path: ?[]const u8 = null; + var descriptor: ?std.fs.Dir = null; + switch (cwd) { + .none => {}, + .path => |p| path = p, + .descriptor => |d| descriptor = d, + } + const process_result = try std.ChildProcess.run(.{ + .allocator = allocator, + .argv = argv, + .max_output_bytes = std.math.maxInt(usize), + .cwd = path, + .cwd_dir = descriptor, + }); + + return TestResult{ + .stdout = process_result.stdout, + .stderr = process_result.stderr, + .error_union = switch (process_result.term) { + .Exited => |exit_code| if (exit_code == 0) {} else error.unexpected_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }, + }; } +const Category = enum { + standalone, + build, + cmake, +}; + pub fn main() !void { - var errors = false; var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); const allocator = arena.allocator(); - runStandalone(allocator, .{ - .directory_path = "test/standalone", - .group_name = "STANDALONE", - .is_test = false, - }) catch { - errors = true; + const compiler_relative_path = "zig-out/bin/nat"; + + var test_suite = TestSuite{ + .allocator = allocator, + .compiler_path = try std.fs.realpathAlloc(allocator, compiler_relative_path), }; - runBuildTests(allocator) catch { - errors = true; - }; - runStandalone(allocator, .{ - .directory_path = "test/tests", - .group_name = "TEST EXECUTABLE", - .is_test = true, - }) catch { - errors = true; - }; - // - runStdTests(allocator) catch { - errors = true; - }; - - runCmakeTests(allocator, "test/cc") catch { - errors = true; - }; - runCmakeTests(allocator, "test/c++") catch { - errors = true; - }; - - switch (@import("builtin").os.tag) { - .macos => {}, - // .macos => {}, - .linux => switch (@import("builtin").abi) { - .gnu => runCmakeTests(allocator, "test/cc_linux") catch { - errors = true; - }, - .musl => {}, - else => @compileError("ABI not supported"), + var test_groups = [_]TestGroup{ + .{ + .category = .standalone, + .path = "test/standalone", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .exe, }, - else => @compileError("OS not supported"), + .{ + .category = .standalone, + .path = "test/tests", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .@"test", + }, + .{ + .category = .standalone, + .path = "lib/std/std.nat", + .collect_directory_entries = null, + .path_policy = .take_entry, + .compilation_kind = .@"test", + }, + .{ + .category = .build, + .path = "test/build", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .@"exe", + }, + .{ + .category = .cmake, + .path = "test/cc", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .cmake, + }, + .{ + .category = .cmake, + .path = "test/c++", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .cmake, + }, + } ++ switch (@import("builtin").os.tag) { + .linux => [_]TestGroup{ + .{ + .category = .cmake, + .path = "test/cc_linux", + .collect_directory_entries = &.{}, + .path_policy = .join_path, + .compilation_kind = .cmake, + }, + }, + else => [_]TestGroup{}, + }; + + for (&test_groups) |*test_group| { + try test_suite.run_test_group(test_group); } - if (errors) { - return error.fail; + const stdout = std.io.getStdOut(); + const stdout_writer = stdout.writer(); + var io_buffer = std.io.BufferedWriter(16 * 0x1000, @TypeOf(stdout_writer)){ .unbuffered_writer = stdout_writer }; + const io_writer = io_buffer.writer(); + try io_writer.writeByte('\n'); + + for (&test_groups) |*test_group| { + if (test_group.state.failed.tests > 0) { + for (test_group.tests.items) |*test_result| { + try io_writer.print("{s}\n", .{test_result.name}); + + if (test_result.configuration) |result| { + if (result.stdout.len > 0) { + try io_writer.writeAll(result.stdout); + try io_writer.writeByte('\n'); + } + + if (result.stderr.len > 0) { + try io_writer.writeAll(result.stderr); + try io_writer.writeByte('\n'); + } + } + + if (test_result.compilation) |result| { + if (result.stdout.len > 0) { + try io_writer.writeAll(result.stdout); + try io_writer.writeByte('\n'); + } + + if (result.stderr.len > 0) { + try io_writer.writeAll(result.stderr); + try io_writer.writeByte('\n'); + } + } + + if (test_result.execution) |result| { + if (result.stdout.len > 0) { + try io_writer.writeAll(result.stdout); + try io_writer.writeByte('\n'); + } + + if (result.stderr.len > 0) { + try io_writer.writeAll(result.stderr); + try io_writer.writeByte('\n'); + } + } + } + } + + try io_writer.print("[{s}] [{s}] Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{ + @tagName(test_group.category), + test_group.path, + test_group.state.total.tests, + test_group.state.failed.tests, + test_group.state.total.compilations, + test_group.state.failed.compilations, + test_group.state.total.executions, + test_group.state.failed.executions, + }); + } + + try io_writer.print("Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{ + test_suite.state.states.total.tests, + test_suite.state.states.failed.tests, + test_suite.state.states.total.compilations, + test_suite.state.states.failed.compilations, + test_suite.state.states.total.executions, + test_suite.state.states.failed.executions, + }); + + const success = test_suite.state.states.failed.tests == 0; + if (success) { + try io_writer.writeAll("\x1b[32mTESTS PASSED!\x1b[0m\n"); + } else { + try io_writer.writeAll("\x1b[31mTESTS FAILED!\x1b[0m\n"); + } + + try io_buffer.flush(); + + if (!success) { + std.posix.exit(1); } }