From 10fefeeb9b760e73211d41aa17eb9fbd2dd7659e Mon Sep 17 00:00:00 2001 From: David Date: Thu, 28 Mar 2024 23:57:02 -0600 Subject: [PATCH] Revert "Rewrite test runner to be easier to work with" --- bootstrap/Compilation.zig | 6 +- bootstrap/main.zig | 2 +- build.zig | 2 - build/test_runner.zig | 899 ++++++++++++++++++++++---------------- 4 files changed, 514 insertions(+), 395 deletions(-) diff --git a/bootstrap/Compilation.zig b/bootstrap/Compilation.zig index a3dbcf3..65eba9f 100644 --- a/bootstrap/Compilation.zig +++ b/bootstrap/Compilation.zig @@ -2907,11 +2907,7 @@ pub fn buildExecutable(context: *const Context, arguments: []const []const u8, o break :blk path; } else unreachable; - const executable_name = if (maybe_executable_name) |name| name else b: { - const slash_index = data_structures.last_byte(main_package_path, '/') orelse unreachable; - const p = std.fs.path.basename(main_package_path[0 .. slash_index]); - break :b p; - }; + const executable_name = if (maybe_executable_name) |name| name else std.fs.path.basename(main_package_path[0 .. main_package_path.len - "/main.nat".len]); const executable_path = maybe_executable_path orelse blk: { assert(executable_name.len > 0); diff --git a/bootstrap/main.zig b/bootstrap/main.zig index b00bb49..2221750 100644 --- a/bootstrap/main.zig +++ b/bootstrap/main.zig @@ -56,7 +56,7 @@ pub fn main() !void { try args.append(context.my_allocator, arg); } const arguments = args.slice(); - const debug_args = false; + const debug_args = true; if (debug_args) { assert(arguments.len > 0); const home_dir = std.posix.getenv("HOME") orelse unreachable; diff --git a/build.zig b/build.zig index 7aca7ae..2d73559 100644 --- a/build.zig +++ b/build.zig @@ -498,8 +498,6 @@ pub fn build(b: *std.Build) !void { b.default_step.dependOn(&test_runner.step); const test_command = b.addRunArtifact(test_runner); - const install_runner = b.addInstallArtifact(test_runner, .{}); - b.getInstallStep().dependOn(&install_runner.step); test_command.step.dependOn(&compiler.step); test_command.step.dependOn(b.getInstallStep()); diff --git a/build/test_runner.zig b/build/test_runner.zig index b0a0c1e..698e05d 100644 --- a/build/test_runner.zig +++ b/build/test_runner.zig @@ -7,407 +7,532 @@ const TestError = error{ signaled, stopped, unknown, - internal, fail, }; -const TestGroup = struct { - state: States = .{}, - tests: std.ArrayListUnmanaged(TestRecord) = .{}, - path: []const u8, - path_policy: PathPolicy, - category: Category, - compilation_kind: CompilationKind, - collect_directory_entries: ?[]const []const u8, - - const PathPolicy = enum { - join_path, - take_entry, - }; - - const CompilationKind = enum { - exe, - @"test", - build, - cmake, - }; -}; - -const TestRecord = struct { - name: []const u8, - configuration: ?TestResult = null, - compilation: ?TestResult = null, - execution: ?TestResult = null, -}; - -const TestResult = struct { - stdout: []const u8 = &.{}, - stderr: []const u8 = &.{}, - error_union: RunError!void, -}; - -// const TestStep = struct{ -// kind: Kind, -// result: TestResult, -// -// const Kind = enum{ -// configuration, -// compilation, -// execution, -// }; -// }; - -const States = struct { - failed: TestState = .{}, - total: TestState = .{}, -}; - -const TestState = struct { - compilations: u32 = 0, - executions: u32 = 0, - tests: u32 = 0, - - fn add(total: *TestState, state: *const TestState) void { - total.compilations += state.compilations; - total.executions += state.executions; - total.tests += state.tests; - } -}; - -const TestSuite = struct { - allocator: Allocator, - state: State = .{}, - compiler_path: []const u8, - - const State = struct { - states: States = .{}, - category_map: std.EnumSet(Category) = .{}, - test_groups: std.ArrayListUnmanaged(TestGroup) = .{}, - }; - - fn run_test_group(test_suite: *TestSuite, test_group: *TestGroup) !void { - test_suite.state.category_map.setPresent(test_group.category, true); - defer { - test_suite.state.states.total.add(&test_group.state.total); - test_suite.state.states.failed.add(&test_group.state.failed); - } - - if (test_group.collect_directory_entries) |*directory_entries| { - directory_entries.* = blk: { - var dir = try std.fs.cwd().openDir(test_group.path, .{ - .iterate = true, - }); - var dir_iterator = dir.iterate(); - var dir_entries = std.ArrayListUnmanaged([]const u8){}; - - while (try dir_iterator.next()) |entry| { - switch (entry.kind) { - .directory => try dir_entries.append(test_suite.allocator, try test_suite.allocator.dupe(u8, entry.name)), - else => return error.junk_in_test_directory, - } - } - - dir.close(); - - break :blk dir_entries.items; - }; - } else { - test_group.collect_directory_entries = &.{ test_group.path }; - } - - const directory_entries = test_group.collect_directory_entries orelse unreachable; - try test_group.tests.ensureTotalCapacity(test_suite.allocator, directory_entries.len); - - for (directory_entries) |directory_entry| { - var has_error = false; - test_group.state.total.tests += 1; - - const relative_path = try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry }); - - var test_record = TestRecord{ - .name = directory_entry, - }; - - const cmake_build_dir = "build"; - switch (test_group.category) { - .cmake => { - const argv: []const []const u8 = &.{ - "cmake", - "-S", ".", - "-B", cmake_build_dir, - "--fresh", - try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_C_COMPILER={s};cc", .{test_suite.compiler_path}), - try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_CXX_COMPILER={s};c++", .{test_suite.compiler_path}), - try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_ASM_COMPILER={s};cc", .{test_suite.compiler_path}), - }; - test_record.configuration = try run_process(test_suite.allocator, argv, .{ .path = relative_path }); - test_record.configuration.?.error_union catch { - has_error = true; - test_group.state.total.compilations += 1; - test_group.state.failed.compilations += 1; - }; - }, - .build, .standalone => {}, - } - - if (!has_error) { - const compilation_cwd: Cwd = switch (test_group.category) { - .standalone => .none, - .build => .{ .path = relative_path }, - .cmake => .{ .path = try std.mem.concat(test_suite.allocator, u8, &.{ relative_path, "/" ++ cmake_build_dir }) }, - }; - - const arguments: []const []const u8 = switch (test_group.category) { - .standalone => blk: { - const source_file_path = switch (test_group.path_policy) { - .join_path => try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry, "/main.nat" }), - .take_entry => directory_entry, - }; - const compilation_argument = @tagName(test_group.compilation_kind); - break :blk &.{ test_suite.compiler_path, compilation_argument, "-main_source_file", source_file_path }; - }, - .build => &.{ test_suite.compiler_path, "build" }, - .cmake => &.{ "ninja" }, - }; - - test_record.compilation = try run_process(test_suite.allocator, arguments, compilation_cwd); - test_group.state.total.compilations += 1; - - if (test_record.compilation.?.error_union) |_| { - const executable_name = switch (test_group.path_policy) { - .join_path => directory_entry, - .take_entry => b: { - const slash_index = std.mem.lastIndexOfScalar(u8, directory_entry, '/') orelse unreachable; - const base = std.fs.path.basename(directory_entry[0..slash_index]); - break :b base; - }, - }; - const build_dir = switch (test_group.category) { - .cmake => cmake_build_dir ++ "/", - else => "nat/", - }; - const execution_cwd: Cwd = switch (test_group.category) { - .cmake => .{ .path = relative_path }, - else => compilation_cwd, - }; - test_record.execution = try run_process(test_suite.allocator, &.{try std.mem.concat(test_suite.allocator, u8, &.{ build_dir, executable_name })}, execution_cwd); - test_group.state.total.executions += 1; - - if (test_record.execution.?.error_union) |_| {} else |err| { - has_error = true; - err catch {}; - test_group.state.failed.executions += 1; - } - } else |err| { - err catch {}; - has_error = true; - test_group.state.failed.compilations += 1; - } - } - - test_group.state.failed.tests += @intFromBool(has_error); - - test_group.tests.appendAssumeCapacity(test_record); - } - } -}; - -const RunError = error{ - unexpected_exit_code, - signaled, - stopped, - unknown, -}; - -const Cwd = union(enum) { - none, - path: []const u8, - descriptor: std.fs.Dir, -}; - -fn run_process(allocator: Allocator, argv: []const []const u8, cwd: Cwd) !TestResult { - var path: ?[]const u8 = null; - var descriptor: ?std.fs.Dir = null; - switch (cwd) { - .none => {}, - .path => |p| path = p, - .descriptor => |d| descriptor = d, - } - const process_result = try std.ChildProcess.run(.{ - .allocator = allocator, - .argv = argv, - .max_output_bytes = std.math.maxInt(usize), - .cwd = path, - .cwd_dir = descriptor, +fn collectDirectoryDirEntries(allocator: Allocator, path: []const u8) ![]const []const u8 { + var dir = try std.fs.cwd().openDir(path, .{ + .iterate = true, }); + var dir_iterator = dir.iterate(); + var dir_entries = std.ArrayListUnmanaged([]const u8){}; - return TestResult{ - .stdout = process_result.stdout, - .stderr = process_result.stderr, - .error_union = switch (process_result.term) { - .Exited => |exit_code| if (exit_code == 0) {} else error.unexpected_exit_code, + while (try dir_iterator.next()) |entry| { + switch (entry.kind) { + .directory => try dir_entries.append(allocator, try allocator.dupe(u8, entry.name)), + else => return error.junk_in_test_directory, + } + } + + dir.close(); + + return dir_entries.items; +} + +fn runStandalone(allocator: Allocator, args: struct { + directory_path: []const u8, + group_name: []const u8, + is_test: bool, +}) !void { + const test_names = try collectDirectoryDirEntries(allocator, args.directory_path); + + const total_compilation_count = test_names.len; + var ran_compilation_count: usize = 0; + var failed_compilation_count: usize = 0; + + var ran_test_count: usize = 0; + var failed_test_count: usize = 0; + const total_test_count = test_names.len; + + std.debug.print("\n[{s} START]\n\n", .{args.group_name}); + + for (test_names) |test_name| { + std.debug.print("{s}... ", .{test_name}); + const source_file_path = try std.mem.concat(allocator, u8, &.{ args.directory_path, "/", test_name, "/main.nat" }); + const compile_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{ "zig-out/bin/nat", if (args.is_test) "test" else "exe", "-main_source_file", source_file_path }, + .max_output_bytes = std.math.maxInt(u64), + }); + ran_compilation_count += 1; + + const compilation_result: TestError!bool = switch (compile_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, .Signal => error.signaled, .Stopped => error.stopped, .Unknown => error.unknown, - }, - }; -} + }; -const Category = enum { - standalone, - build, - cmake, -}; + const compilation_success = compilation_result catch b: { + failed_compilation_count += 1; + break :b false; + }; -pub fn main() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - const allocator = arena.allocator(); - const compiler_relative_path = "zig-out/bin/nat"; - - var test_suite = TestSuite{ - .allocator = allocator, - .compiler_path = try std.fs.realpathAlloc(allocator, compiler_relative_path), - }; - var test_groups = [_]TestGroup{ - .{ - .category = .standalone, - .path = "test/standalone", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .exe, - }, - .{ - .category = .standalone, - .path = "test/tests", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .@"test", - }, - .{ - .category = .standalone, - .path = "lib/std/std.nat", - .collect_directory_entries = null, - .path_policy = .take_entry, - .compilation_kind = .@"test", - }, - .{ - .category = .build, - .path = "test/build", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .@"exe", - }, - .{ - .category = .cmake, - .path = "test/cc", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .cmake, - }, - .{ - .category = .cmake, - .path = "test/c++", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .cmake, - }, - } ++ switch (@import("builtin").os.tag) { - .linux => [_]TestGroup{ - .{ - .category = .cmake, - .path = "test/cc_linux", - .collect_directory_entries = &.{}, - .path_policy = .join_path, - .compilation_kind = .cmake, - }, - }, - else => [_]TestGroup{}, - }; - - for (&test_groups) |*test_group| { - try test_suite.run_test_group(test_group); - } - - const stdout = std.io.getStdOut(); - const stdout_writer = stdout.writer(); - var io_buffer = std.io.BufferedWriter(16 * 0x1000, @TypeOf(stdout_writer)){ .unbuffered_writer = stdout_writer }; - const io_writer = io_buffer.writer(); - try io_writer.writeByte('\n'); - - for (&test_groups) |*test_group| { - if (test_group.state.failed.tests > 0) { - for (test_group.tests.items) |*test_result| { - try io_writer.print("{s}\n", .{test_result.name}); - - if (test_result.configuration) |result| { - if (result.stdout.len > 0) { - try io_writer.writeAll(result.stdout); - try io_writer.writeByte('\n'); - } - - if (result.stderr.len > 0) { - try io_writer.writeAll(result.stderr); - try io_writer.writeByte('\n'); - } - } - - if (test_result.compilation) |result| { - if (result.stdout.len > 0) { - try io_writer.writeAll(result.stdout); - try io_writer.writeByte('\n'); - } - - if (result.stderr.len > 0) { - try io_writer.writeAll(result.stderr); - try io_writer.writeByte('\n'); - } - } - - if (test_result.execution) |result| { - if (result.stdout.len > 0) { - try io_writer.writeAll(result.stdout); - try io_writer.writeByte('\n'); - } - - if (result.stderr.len > 0) { - try io_writer.writeAll(result.stderr); - try io_writer.writeByte('\n'); - } - } - } + std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (compile_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); + } + if (compile_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); } - try io_writer.print("[{s}] [{s}] Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{ - @tagName(test_group.category), - test_group.path, - test_group.state.total.tests, - test_group.state.failed.tests, - test_group.state.total.compilations, - test_group.state.failed.compilations, - test_group.state.total.executions, - test_group.state.failed.executions, - }); + if (compilation_success) { + const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name }); + const test_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{test_path}, + .max_output_bytes = std.math.maxInt(u64), + }); + ran_test_count += 1; + const test_result: TestError!bool = switch (test_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const test_success = test_result catch b: { + failed_test_count += 1; + break :b false; + }; + std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (test_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); + } + if (test_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); + } + } else { + std.debug.print("\n", .{}); + } } - try io_writer.print("Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{ - test_suite.state.states.total.tests, - test_suite.state.states.failed.tests, - test_suite.state.states.total.compilations, - test_suite.state.states.failed.compilations, - test_suite.state.states.total.executions, - test_suite.state.states.failed.executions, - }); + std.debug.print("\n{s} COMPILATIONS: {}. FAILED: {}\n", .{ args.group_name, total_compilation_count, failed_compilation_count }); + std.debug.print("{s} TESTS: {}. RAN: {}. FAILED: {}\n", .{ args.group_name, total_test_count, ran_test_count, failed_test_count }); - const success = test_suite.state.states.failed.tests == 0; - if (success) { - try io_writer.writeAll("\x1b[32mTESTS PASSED!\x1b[0m\n"); - } else { - try io_writer.writeAll("\x1b[31mTESTS FAILED!\x1b[0m\n"); - } - - try io_buffer.flush(); - - if (!success) { - std.posix.exit(1); + if (failed_compilation_count > 0 or failed_test_count > 0) { + return error.fail; + } +} + +fn runStandaloneTests(allocator: Allocator) !void { + const standalone_test_dir_path = "test/standalone"; + const standalone_test_names = try collectDirectoryDirEntries(allocator, standalone_test_dir_path); + + const total_compilation_count = standalone_test_names.len; + var ran_compilation_count: usize = 0; + var failed_compilation_count: usize = 0; + + var ran_test_count: usize = 0; + var failed_test_count: usize = 0; + const total_test_count = standalone_test_names.len; + + for (standalone_test_names) |standalone_test_name| { + std.debug.print("{s}... ", .{standalone_test_name}); + const source_file_path = try std.mem.concat(allocator, u8, &.{ standalone_test_dir_path, "/", standalone_test_name, "/main.nat" }); + const compile_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{ "zig-out/bin/nat", "exe", "-main_source_file", source_file_path }, + .max_output_bytes = std.math.maxInt(u64), + }); + ran_compilation_count += 1; + + const compilation_result: TestError!bool = switch (compile_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const compilation_success = compilation_result catch b: { + failed_compilation_count += 1; + break :b false; + }; + + std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (compile_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); + } + if (compile_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); + } + + if (compilation_success) { + const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", standalone_test_name }); + const test_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{test_path}, + .max_output_bytes = std.math.maxInt(u64), + }); + ran_test_count += 1; + const test_result: TestError!bool = switch (test_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const test_success = test_result catch b: { + failed_test_count += 1; + break :b false; + }; + std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (test_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); + } + if (test_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); + } + } else { + std.debug.print("\n", .{}); + } + } + + std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count }); + std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count }); + + if (failed_compilation_count > 0 or failed_test_count > 0) { + return error.fail; + } +} + +fn runBuildTests(allocator: Allocator) !void { + std.debug.print("\n[BUILD TESTS]\n\n", .{}); + const previous_cwd = try std.fs.cwd().realpathAlloc(allocator, "."); + const test_dir_path = "test/build"; + const test_names = try collectDirectoryDirEntries(allocator, test_dir_path); + const test_dir_realpath = try std.fs.cwd().realpathAlloc(allocator, test_dir_path); + const compiler_realpath = try std.fs.cwd().realpathAlloc(allocator, "zig-out/bin/nat"); + try std.posix.chdir(test_dir_realpath); + + const total_compilation_count = test_names.len; + var ran_compilation_count: usize = 0; + var failed_compilation_count: usize = 0; + + var ran_test_count: usize = 0; + var failed_test_count: usize = 0; + const total_test_count = test_names.len; + + for (test_names) |test_name| { + std.debug.print("{s}... ", .{test_name}); + try std.posix.chdir(test_name); + + const compile_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{ compiler_realpath, "build" }, + .max_output_bytes = std.math.maxInt(u64), + }); + + ran_compilation_count += 1; + + const compilation_result: TestError!bool = switch (compile_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const compilation_success = compilation_result catch b: { + failed_compilation_count += 1; + break :b false; + }; + + std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (compile_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout}); + } + if (compile_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr}); + } + + if (compilation_success) { + const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name }); + const test_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{test_path}, + .max_output_bytes = std.math.maxInt(u64), + }); + ran_test_count += 1; + const test_result: TestError!bool = switch (test_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const test_success = test_result catch b: { + failed_test_count += 1; + break :b false; + }; + std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (test_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); + } + if (test_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); + } + } else { + std.debug.print("\n", .{}); + } + + try std.posix.chdir(test_dir_realpath); + } + + std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count }); + std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count }); + + try std.posix.chdir(previous_cwd); + + if (failed_compilation_count > 0 or failed_test_count > 0) { + return error.fail; + } +} + +fn runStdTests(allocator: Allocator) !void { + var errors = false; + std.debug.print("std... ", .{}); + + const result = try std.ChildProcess.run(.{ + .allocator = allocator, + .argv = &.{ "zig-out/bin/nat", "test", "-main_source_file", "lib/std/std.nat", "-name", "std" }, + .max_output_bytes = std.math.maxInt(u64), + }); + const compilation_result: TestError!bool = switch (result.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const compilation_success = compilation_result catch b: { + errors = true; + break :b false; + }; + + std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (result.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{result.stdout}); + } + if (result.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{result.stderr}); + } + + if (compilation_success) { + const test_run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{"nat/std"}, + .max_output_bytes = std.math.maxInt(u64), + }); + const test_result: TestError!bool = switch (test_run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const test_success = test_result catch b: { + errors = true; + break :b false; + }; + std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + if (test_run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout}); + } + if (test_run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr}); + } + } + + if (errors) return error.fail; +} + +fn runCmakeTests(allocator: Allocator, dir_path: []const u8) !void { + var errors = false; + const original_dir = try std.fs.cwd().realpathAlloc(allocator, "."); + const cc_dir = try std.fs.cwd().openDir(dir_path, .{ + .iterate = true, + }); + + const cc_dir_path = try cc_dir.realpathAlloc(allocator, "."); + try std.posix.chdir(cc_dir_path); + + var cc_dir_iterator = cc_dir.iterate(); + while (try cc_dir_iterator.next()) |cc_entry| { + switch (cc_entry.kind) { + .directory => { + std.debug.print("{s}...\n", .{cc_entry.name}); + try std.posix.chdir(cc_entry.name); + try std.fs.cwd().deleteTree("build"); + try std.fs.cwd().makeDir("build"); + try std.posix.chdir("build"); + + const cmake = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{ + "cmake", + "..", + // "--debug-trycompile", + // "--debug-output", + // "-G", "Unix Makefiles", + // "-DCMAKE_VERBOSE_MAKEFILE=On", + try std.mem.concat(allocator, u8, &.{ "-DCMAKE_C_COMPILER=", "nat;cc" }), + try std.mem.concat(allocator, u8, &.{ "-DCMAKE_CXX_COMPILER=", "nat;c++" }), + try std.mem.concat(allocator, u8, &.{ "-DCMAKE_ASM_COMPILER=", "nat;cc" }), + }, + .max_output_bytes = std.math.maxInt(u64), + }); + const cmake_result: TestError!bool = switch (cmake.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const cmake_success = cmake_result catch b: { + errors = true; + break :b false; + }; + + if (cmake.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{cmake.stdout}); + } + if (cmake.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{cmake.stderr}); + } + + var success = cmake_success; + if (success) { + const ninja = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{"ninja"}, + .max_output_bytes = std.math.maxInt(u64), + }); + const ninja_result: TestError!bool = switch (ninja.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const ninja_success = ninja_result catch b: { + errors = true; + break :b false; + }; + + if (!ninja_success) { + if (ninja.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{ninja.stdout}); + } + if (ninja.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{ninja.stderr}); + } + } + + success = success and ninja_success; + + if (success) { + const run = try std.ChildProcess.run(.{ + .allocator = allocator, + // TODO: delete -main_source_file? + .argv = &.{ + try std.mem.concat(allocator, u8, &.{ "./", cc_entry.name }), + }, + .max_output_bytes = std.math.maxInt(u64), + }); + const run_result: TestError!bool = switch (run.term) { + .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code, + .Signal => error.signaled, + .Stopped => error.stopped, + .Unknown => error.unknown, + }; + + const run_success = run_result catch b: { + errors = true; + break :b false; + }; + + if (run.stdout.len > 0) { + std.debug.print("STDOUT:\n\n{s}\n\n", .{run.stdout}); + } + if (run.stderr.len > 0) { + std.debug.print("STDERR:\n\n{s}\n\n", .{run.stderr}); + } + + success = success and run_success; + } + } + + std.debug.print("[TEST {s}]\n", .{if (success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"}); + }, + else => std.debug.panic("Entry {s} is a {s}", .{ cc_entry.name, @tagName(cc_entry.kind) }), + } + + try std.posix.chdir(cc_dir_path); + } + + try std.posix.chdir(original_dir); + + if (errors) { + return error.fail; + } +} + +pub fn main() !void { + var errors = false; + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + const allocator = arena.allocator(); + runStandalone(allocator, .{ + .directory_path = "test/standalone", + .group_name = "STANDALONE", + .is_test = false, + }) catch { + errors = true; + }; + runBuildTests(allocator) catch { + errors = true; + }; + runStandalone(allocator, .{ + .directory_path = "test/tests", + .group_name = "TEST EXECUTABLE", + .is_test = true, + }) catch { + errors = true; + }; + // + runStdTests(allocator) catch { + errors = true; + }; + + runCmakeTests(allocator, "test/cc") catch { + errors = true; + }; + runCmakeTests(allocator, "test/c++") catch { + errors = true; + }; + + switch (@import("builtin").os.tag) { + .macos => {}, + // .macos => {}, + .linux => switch (@import("builtin").abi) { + .gnu => runCmakeTests(allocator, "test/cc_linux") catch { + errors = true; + }, + .musl => {}, + else => @compileError("ABI not supported"), + }, + else => @compileError("OS not supported"), + } + + if (errors) { + return error.fail; } }