Revert "Rewrite test runner to be easier to work with"

bootstrap/Compilation.zig

@@ -2907,11 +2907,7 @@ pub fn buildExecutable(context: *const Context, arguments: []const []const u8, o
         break :blk path;
     } else unreachable;
 
-    const executable_name = if (maybe_executable_name) |name| name else b: {
+    const executable_name = if (maybe_executable_name) |name| name else std.fs.path.basename(main_package_path[0 .. main_package_path.len - "/main.nat".len]);
-        const slash_index = data_structures.last_byte(main_package_path, '/') orelse unreachable;
-        const p = std.fs.path.basename(main_package_path[0 .. slash_index]);
-        break :b p;
-    };
 
     const executable_path = maybe_executable_path orelse blk: {
         assert(executable_name.len > 0);

bootstrap/main.zig

@@ -56,7 +56,7 @@ pub fn main() !void {
         try args.append(context.my_allocator, arg);
     }
     const arguments = args.slice();
-    const debug_args = false;
+    const debug_args = true;
     if (debug_args) {
         assert(arguments.len > 0);
         const home_dir = std.posix.getenv("HOME") orelse unreachable;

build.zig

@@ -498,8 +498,6 @@ pub fn build(b: *std.Build) !void {
     b.default_step.dependOn(&test_runner.step);
 
     const test_command = b.addRunArtifact(test_runner);
-    const install_runner = b.addInstallArtifact(test_runner, .{});
-    b.getInstallStep().dependOn(&install_runner.step);
     test_command.step.dependOn(&compiler.step);
     test_command.step.dependOn(b.getInstallStep());
 

build/test_runner.zig

@@ -7,407 +7,532 @@ const TestError = error{
     signaled,
     stopped,
     unknown,
-    internal,
     fail,
 };
 
-const TestGroup = struct {
+fn collectDirectoryDirEntries(allocator: Allocator, path: []const u8) ![]const []const u8 {
-    state: States = .{},
+    var dir = try std.fs.cwd().openDir(path, .{
-    tests: std.ArrayListUnmanaged(TestRecord) = .{},
+        .iterate = true,
-    path: []const u8,
-    path_policy: PathPolicy,
-    category: Category,
-    compilation_kind: CompilationKind,
-    collect_directory_entries: ?[]const []const u8,
-
-    const PathPolicy = enum {
-        join_path,
-        take_entry,
-    };
-
-    const CompilationKind = enum {
-        exe,
-        @"test",
-        build,
-        cmake,
-    };
-};
-
-const TestRecord = struct {
-    name: []const u8,
-    configuration: ?TestResult = null,
-    compilation: ?TestResult = null,
-    execution: ?TestResult = null,
-};
-
-const TestResult = struct {
-    stdout: []const u8 = &.{},
-    stderr: []const u8 = &.{},
-    error_union: RunError!void,
-};
-
-// const TestStep = struct{
-//     kind: Kind,
-//     result: TestResult,
-//
-//     const Kind = enum{
-//         configuration,
-//         compilation,
-//         execution,
-//     };
-// };
-
-const States = struct {
-    failed: TestState = .{},
-    total: TestState = .{},
-};
-
-const TestState = struct {
-    compilations: u32 = 0,
-    executions: u32 = 0,
-    tests: u32 = 0,
-
-    fn add(total: *TestState, state: *const TestState) void {
-        total.compilations += state.compilations;
-        total.executions += state.executions;
-        total.tests += state.tests;
-    }
-};
-
-const TestSuite = struct {
-    allocator: Allocator,
-    state: State = .{},
-    compiler_path: []const u8,
-
-    const State = struct {
-        states: States = .{},
-        category_map: std.EnumSet(Category) = .{},
-        test_groups: std.ArrayListUnmanaged(TestGroup) = .{},
-    };
-
-    fn run_test_group(test_suite: *TestSuite, test_group: *TestGroup) !void {
-        test_suite.state.category_map.setPresent(test_group.category, true);
-        defer {
-            test_suite.state.states.total.add(&test_group.state.total);
-            test_suite.state.states.failed.add(&test_group.state.failed);
-        }
-
-        if (test_group.collect_directory_entries) |*directory_entries| {
-            directory_entries.* = blk: {
-                var dir = try std.fs.cwd().openDir(test_group.path, .{
-                    .iterate = true,
-                });
-                var dir_iterator = dir.iterate();
-                var dir_entries = std.ArrayListUnmanaged([]const u8){};
-
-                while (try dir_iterator.next()) |entry| {
-                    switch (entry.kind) {
-                        .directory => try dir_entries.append(test_suite.allocator, try test_suite.allocator.dupe(u8, entry.name)),
-                        else => return error.junk_in_test_directory,
-                    }
-                }
-
-                dir.close();
-
-                break :blk dir_entries.items;
-            };
-        } else {
-            test_group.collect_directory_entries = &.{ test_group.path };
-        }
-
-        const directory_entries = test_group.collect_directory_entries orelse unreachable;
-        try test_group.tests.ensureTotalCapacity(test_suite.allocator, directory_entries.len);
-
-        for (directory_entries) |directory_entry| {
-            var has_error = false;
-            test_group.state.total.tests += 1;
-
-            const relative_path = try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry });
-
-            var test_record = TestRecord{
-                .name = directory_entry,
-            };
-
-            const cmake_build_dir = "build";
-            switch (test_group.category) {
-                .cmake => {
-                    const argv: []const []const u8 = &.{
-                        "cmake",
-                        "-S", ".",
-                        "-B", cmake_build_dir,
-                        "--fresh",
-                        try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_C_COMPILER={s};cc", .{test_suite.compiler_path}),
-                        try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_CXX_COMPILER={s};c++", .{test_suite.compiler_path}),
-                        try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_ASM_COMPILER={s};cc", .{test_suite.compiler_path}),
-                    };
-                    test_record.configuration = try run_process(test_suite.allocator, argv, .{ .path = relative_path });
-                    test_record.configuration.?.error_union catch {
-                        has_error = true;
-                        test_group.state.total.compilations += 1;
-                        test_group.state.failed.compilations += 1;
-                    };
-                },
-                .build, .standalone => {},
-            }
-
-            if (!has_error) {
-                const compilation_cwd: Cwd = switch (test_group.category) {
-                    .standalone => .none,
-                    .build => .{ .path = relative_path },
-                    .cmake => .{ .path = try std.mem.concat(test_suite.allocator, u8, &.{ relative_path, "/" ++ cmake_build_dir }) },
-                };
-
-                const arguments: []const []const u8 = switch (test_group.category) {
-                    .standalone => blk: {
-                        const source_file_path = switch (test_group.path_policy) {
-                            .join_path => try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry, "/main.nat" }),
-                            .take_entry => directory_entry,
-                        };
-                        const compilation_argument = @tagName(test_group.compilation_kind);
-                        break :blk &.{ test_suite.compiler_path, compilation_argument, "-main_source_file", source_file_path };
-                    },
-                    .build => &.{ test_suite.compiler_path, "build" },
-                    .cmake => &.{ "ninja" },
-                };
-
-                test_record.compilation = try run_process(test_suite.allocator, arguments, compilation_cwd);
-                test_group.state.total.compilations += 1;
-
-                if (test_record.compilation.?.error_union) |_| {
-                    const executable_name = switch (test_group.path_policy) {
-                        .join_path => directory_entry,
-                        .take_entry => b: {
-                            const slash_index = std.mem.lastIndexOfScalar(u8, directory_entry, '/') orelse unreachable;
-                            const base = std.fs.path.basename(directory_entry[0..slash_index]);
-                            break :b base;
-                        },
-                    };
-                    const build_dir = switch (test_group.category) {
-                        .cmake => cmake_build_dir ++ "/",
-                        else => "nat/",
-                    };
-                    const execution_cwd: Cwd = switch (test_group.category) {
-                        .cmake => .{ .path = relative_path },
-                        else => compilation_cwd,
-                    };
-                    test_record.execution = try run_process(test_suite.allocator, &.{try std.mem.concat(test_suite.allocator, u8, &.{ build_dir, executable_name })}, execution_cwd);
-                    test_group.state.total.executions += 1;
-
-                    if (test_record.execution.?.error_union) |_| {} else |err| {
-                        has_error = true;
-                        err catch {};
-                        test_group.state.failed.executions += 1;
-                    }
-                } else |err| {
-                    err catch {};
-                    has_error = true;
-                    test_group.state.failed.compilations += 1;
-                }
-            }
-
-                test_group.state.failed.tests += @intFromBool(has_error);
-
-                test_group.tests.appendAssumeCapacity(test_record);
-        }
-    }
-};
-
-const RunError = error{
-    unexpected_exit_code,
-    signaled,
-    stopped,
-    unknown,
-};
-
-const Cwd = union(enum) {
-    none,
-    path: []const u8,
-    descriptor: std.fs.Dir,
-};
-
-fn run_process(allocator: Allocator, argv: []const []const u8, cwd: Cwd) !TestResult {
-    var path: ?[]const u8 = null;
-    var descriptor: ?std.fs.Dir = null;
-    switch (cwd) {
-        .none => {},
-        .path => |p| path = p,
-        .descriptor => |d| descriptor = d,
-    }
-    const process_result = try std.ChildProcess.run(.{
-        .allocator = allocator,
-        .argv = argv,
-        .max_output_bytes = std.math.maxInt(usize),
-        .cwd = path,
-        .cwd_dir = descriptor,
     });
+    var dir_iterator = dir.iterate();
+    var dir_entries = std.ArrayListUnmanaged([]const u8){};
 
-    return TestResult{
+    while (try dir_iterator.next()) |entry| {
-        .stdout = process_result.stdout,
+        switch (entry.kind) {
-        .stderr = process_result.stderr,
+            .directory => try dir_entries.append(allocator, try allocator.dupe(u8, entry.name)),
-        .error_union = switch (process_result.term) {
+            else => return error.junk_in_test_directory,
-            .Exited => |exit_code| if (exit_code == 0) {} else error.unexpected_exit_code,
+        }
+    }
+
+    dir.close();
+
+    return dir_entries.items;
+}
+
+fn runStandalone(allocator: Allocator, args: struct {
+    directory_path: []const u8,
+    group_name: []const u8,
+    is_test: bool,
+}) !void {
+    const test_names = try collectDirectoryDirEntries(allocator, args.directory_path);
+
+    const total_compilation_count = test_names.len;
+    var ran_compilation_count: usize = 0;
+    var failed_compilation_count: usize = 0;
+
+    var ran_test_count: usize = 0;
+    var failed_test_count: usize = 0;
+    const total_test_count = test_names.len;
+
+    std.debug.print("\n[{s} START]\n\n", .{args.group_name});
+
+    for (test_names) |test_name| {
+        std.debug.print("{s}... ", .{test_name});
+        const source_file_path = try std.mem.concat(allocator, u8, &.{ args.directory_path, "/", test_name, "/main.nat" });
+        const compile_run = try std.ChildProcess.run(.{
+            .allocator = allocator,
+            // TODO: delete -main_source_file?
+            .argv = &.{ "zig-out/bin/nat", if (args.is_test) "test" else "exe", "-main_source_file", source_file_path },
+            .max_output_bytes = std.math.maxInt(u64),
+        });
+        ran_compilation_count += 1;
+
+        const compilation_result: TestError!bool = switch (compile_run.term) {
+            .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
             .Signal => error.signaled,
             .Stopped => error.stopped,
             .Unknown => error.unknown,
-        },
+        };
-    };
-}
 
-const Category = enum {
+        const compilation_success = compilation_result catch b: {
-    standalone,
+            failed_compilation_count += 1;
-    build,
+            break :b false;
-    cmake,
+        };
-};
 
-pub fn main() !void {
+        std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
-    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+        if (compile_run.stdout.len > 0) {
-    const allocator = arena.allocator();
+            std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
-    const compiler_relative_path = "zig-out/bin/nat";
+        }
-    
+        if (compile_run.stderr.len > 0) {
-    var test_suite = TestSuite{
+            std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
-        .allocator = allocator,
-        .compiler_path = try std.fs.realpathAlloc(allocator, compiler_relative_path),
-    };
-    var test_groups = [_]TestGroup{
-        .{
-            .category = .standalone,
-            .path = "test/standalone",
-            .collect_directory_entries = &.{},
-            .path_policy = .join_path,
-            .compilation_kind = .exe,
-        },
-        .{
-            .category = .standalone,
-            .path = "test/tests",
-            .collect_directory_entries = &.{},
-            .path_policy = .join_path,
-            .compilation_kind = .@"test",
-        },
-        .{
-            .category = .standalone,
-            .path = "lib/std/std.nat",
-            .collect_directory_entries = null,
-            .path_policy = .take_entry,
-            .compilation_kind = .@"test",
-        },
-        .{
-            .category = .build,
-            .path = "test/build",
-            .collect_directory_entries = &.{},
-            .path_policy = .join_path,
-            .compilation_kind = .@"exe",
-        },
-        .{
-            .category = .cmake,
-            .path = "test/cc",
-            .collect_directory_entries = &.{},
-            .path_policy = .join_path,
-            .compilation_kind = .cmake,
-        },
-        .{
-            .category = .cmake,
-            .path = "test/c++",
-            .collect_directory_entries = &.{},
-            .path_policy = .join_path,
-            .compilation_kind = .cmake,
-        },
-    } ++ switch (@import("builtin").os.tag) {
-        .linux => [_]TestGroup{
-            .{
-                .category = .cmake,
-                .path = "test/cc_linux",
-                .collect_directory_entries = &.{},
-                .path_policy = .join_path,
-                .compilation_kind = .cmake,
-            },
-        },
-        else => [_]TestGroup{},
-    };
-
-    for (&test_groups) |*test_group| {
-        try test_suite.run_test_group(test_group);
-    }
-
-    const stdout = std.io.getStdOut();
-    const stdout_writer = stdout.writer();
-    var io_buffer = std.io.BufferedWriter(16 * 0x1000, @TypeOf(stdout_writer)){ .unbuffered_writer = stdout_writer };
-    const io_writer = io_buffer.writer();
-    try io_writer.writeByte('\n');
-
-    for (&test_groups) |*test_group| {
-        if (test_group.state.failed.tests > 0) {
-            for (test_group.tests.items) |*test_result| {
-                try io_writer.print("{s}\n", .{test_result.name});
-
-                if (test_result.configuration) |result| {
-                    if (result.stdout.len > 0) {
-                        try io_writer.writeAll(result.stdout);
-                        try io_writer.writeByte('\n');
-                    }
-
-                    if (result.stderr.len > 0) {
-                        try io_writer.writeAll(result.stderr);
-                        try io_writer.writeByte('\n');
-                    }
-                }
-
-                if (test_result.compilation) |result| {
-                    if (result.stdout.len > 0) {
-                        try io_writer.writeAll(result.stdout);
-                        try io_writer.writeByte('\n');
-                    }
-
-                    if (result.stderr.len > 0) {
-                        try io_writer.writeAll(result.stderr);
-                        try io_writer.writeByte('\n');
-                    }
-                }
-
-                if (test_result.execution) |result| {
-                    if (result.stdout.len > 0) {
-                        try io_writer.writeAll(result.stdout);
-                        try io_writer.writeByte('\n');
-                    }
-
-                    if (result.stderr.len > 0) {
-                        try io_writer.writeAll(result.stderr);
-                        try io_writer.writeByte('\n');
-                    }
-                }
-            }
         }
 
-        try io_writer.print("[{s}] [{s}] Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{
+        if (compilation_success) {
-            @tagName(test_group.category),
+            const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name });
-            test_group.path,
+            const test_run = try std.ChildProcess.run(.{
-            test_group.state.total.tests,
+                .allocator = allocator,
-            test_group.state.failed.tests,
+                // TODO: delete -main_source_file?
-            test_group.state.total.compilations,
+                .argv = &.{test_path},
-            test_group.state.failed.compilations,
+                .max_output_bytes = std.math.maxInt(u64),
-            test_group.state.total.executions,
+            });
-            test_group.state.failed.executions,
+            ran_test_count += 1;
-        });
+            const test_result: TestError!bool = switch (test_run.term) {
+                .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                .Signal => error.signaled,
+                .Stopped => error.stopped,
+                .Unknown => error.unknown,
+            };
+
+            const test_success = test_result catch b: {
+                failed_test_count += 1;
+                break :b false;
+            };
+            std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+            if (test_run.stdout.len > 0) {
+                std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
+            }
+            if (test_run.stderr.len > 0) {
+                std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
+            }
+        } else {
+            std.debug.print("\n", .{});
+        }
     }
 
-    try io_writer.print("Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{
+    std.debug.print("\n{s} COMPILATIONS: {}. FAILED: {}\n", .{ args.group_name, total_compilation_count, failed_compilation_count });
-        test_suite.state.states.total.tests,
+    std.debug.print("{s} TESTS: {}. RAN: {}. FAILED: {}\n", .{ args.group_name, total_test_count, ran_test_count, failed_test_count });
-        test_suite.state.states.failed.tests,
-        test_suite.state.states.total.compilations,
-        test_suite.state.states.failed.compilations,
-        test_suite.state.states.total.executions,
-        test_suite.state.states.failed.executions,
-    });
 
-    const success = test_suite.state.states.failed.tests == 0;
+    if (failed_compilation_count > 0 or failed_test_count > 0) {
-    if (success) {
+        return error.fail;
-        try io_writer.writeAll("\x1b[32mTESTS PASSED!\x1b[0m\n");
+    }
-    } else {
+}
-        try io_writer.writeAll("\x1b[31mTESTS FAILED!\x1b[0m\n");
+
-    }
+fn runStandaloneTests(allocator: Allocator) !void {
-
+    const standalone_test_dir_path = "test/standalone";
-    try io_buffer.flush();
+    const standalone_test_names = try collectDirectoryDirEntries(allocator, standalone_test_dir_path);
-
+
-    if (!success) {
+    const total_compilation_count = standalone_test_names.len;
-        std.posix.exit(1);
+    var ran_compilation_count: usize = 0;
+    var failed_compilation_count: usize = 0;
+
+    var ran_test_count: usize = 0;
+    var failed_test_count: usize = 0;
+    const total_test_count = standalone_test_names.len;
+
+    for (standalone_test_names) |standalone_test_name| {
+        std.debug.print("{s}... ", .{standalone_test_name});
+        const source_file_path = try std.mem.concat(allocator, u8, &.{ standalone_test_dir_path, "/", standalone_test_name, "/main.nat" });
+        const compile_run = try std.ChildProcess.run(.{
+            .allocator = allocator,
+            // TODO: delete -main_source_file?
+            .argv = &.{ "zig-out/bin/nat", "exe", "-main_source_file", source_file_path },
+            .max_output_bytes = std.math.maxInt(u64),
+        });
+        ran_compilation_count += 1;
+
+        const compilation_result: TestError!bool = switch (compile_run.term) {
+            .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+            .Signal => error.signaled,
+            .Stopped => error.stopped,
+            .Unknown => error.unknown,
+        };
+
+        const compilation_success = compilation_result catch b: {
+            failed_compilation_count += 1;
+            break :b false;
+        };
+
+        std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+        if (compile_run.stdout.len > 0) {
+            std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
+        }
+        if (compile_run.stderr.len > 0) {
+            std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
+        }
+
+        if (compilation_success) {
+            const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", standalone_test_name });
+            const test_run = try std.ChildProcess.run(.{
+                .allocator = allocator,
+                // TODO: delete -main_source_file?
+                .argv = &.{test_path},
+                .max_output_bytes = std.math.maxInt(u64),
+            });
+            ran_test_count += 1;
+            const test_result: TestError!bool = switch (test_run.term) {
+                .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                .Signal => error.signaled,
+                .Stopped => error.stopped,
+                .Unknown => error.unknown,
+            };
+
+            const test_success = test_result catch b: {
+                failed_test_count += 1;
+                break :b false;
+            };
+            std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+            if (test_run.stdout.len > 0) {
+                std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
+            }
+            if (test_run.stderr.len > 0) {
+                std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
+            }
+        } else {
+            std.debug.print("\n", .{});
+        }
+    }
+
+    std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count });
+    std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count });
+
+    if (failed_compilation_count > 0 or failed_test_count > 0) {
+        return error.fail;
+    }
+}
+
+fn runBuildTests(allocator: Allocator) !void {
+    std.debug.print("\n[BUILD TESTS]\n\n", .{});
+    const previous_cwd = try std.fs.cwd().realpathAlloc(allocator, ".");
+    const test_dir_path = "test/build";
+    const test_names = try collectDirectoryDirEntries(allocator, test_dir_path);
+    const test_dir_realpath = try std.fs.cwd().realpathAlloc(allocator, test_dir_path);
+    const compiler_realpath = try std.fs.cwd().realpathAlloc(allocator, "zig-out/bin/nat");
+    try std.posix.chdir(test_dir_realpath);
+
+    const total_compilation_count = test_names.len;
+    var ran_compilation_count: usize = 0;
+    var failed_compilation_count: usize = 0;
+
+    var ran_test_count: usize = 0;
+    var failed_test_count: usize = 0;
+    const total_test_count = test_names.len;
+
+    for (test_names) |test_name| {
+        std.debug.print("{s}... ", .{test_name});
+        try std.posix.chdir(test_name);
+
+        const compile_run = try std.ChildProcess.run(.{
+            .allocator = allocator,
+            // TODO: delete -main_source_file?
+            .argv = &.{ compiler_realpath, "build" },
+            .max_output_bytes = std.math.maxInt(u64),
+        });
+
+        ran_compilation_count += 1;
+
+        const compilation_result: TestError!bool = switch (compile_run.term) {
+            .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+            .Signal => error.signaled,
+            .Stopped => error.stopped,
+            .Unknown => error.unknown,
+        };
+
+        const compilation_success = compilation_result catch b: {
+            failed_compilation_count += 1;
+            break :b false;
+        };
+
+        std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+        if (compile_run.stdout.len > 0) {
+            std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
+        }
+        if (compile_run.stderr.len > 0) {
+            std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
+        }
+
+        if (compilation_success) {
+            const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name });
+            const test_run = try std.ChildProcess.run(.{
+                .allocator = allocator,
+                // TODO: delete -main_source_file?
+                .argv = &.{test_path},
+                .max_output_bytes = std.math.maxInt(u64),
+            });
+            ran_test_count += 1;
+            const test_result: TestError!bool = switch (test_run.term) {
+                .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                .Signal => error.signaled,
+                .Stopped => error.stopped,
+                .Unknown => error.unknown,
+            };
+
+            const test_success = test_result catch b: {
+                failed_test_count += 1;
+                break :b false;
+            };
+            std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+            if (test_run.stdout.len > 0) {
+                std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
+            }
+            if (test_run.stderr.len > 0) {
+                std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
+            }
+        } else {
+            std.debug.print("\n", .{});
+        }
+
+        try std.posix.chdir(test_dir_realpath);
+    }
+
+    std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count });
+    std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count });
+
+    try std.posix.chdir(previous_cwd);
+
+    if (failed_compilation_count > 0 or failed_test_count > 0) {
+        return error.fail;
+    }
+}
+
+fn runStdTests(allocator: Allocator) !void {
+    var errors = false;
+    std.debug.print("std... ", .{});
+
+    const result = try std.ChildProcess.run(.{
+        .allocator = allocator,
+        .argv = &.{ "zig-out/bin/nat", "test", "-main_source_file", "lib/std/std.nat", "-name", "std" },
+        .max_output_bytes = std.math.maxInt(u64),
+    });
+    const compilation_result: TestError!bool = switch (result.term) {
+        .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+        .Signal => error.signaled,
+        .Stopped => error.stopped,
+        .Unknown => error.unknown,
+    };
+
+    const compilation_success = compilation_result catch b: {
+        errors = true;
+        break :b false;
+    };
+
+    std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+    if (result.stdout.len > 0) {
+        std.debug.print("STDOUT:\n\n{s}\n\n", .{result.stdout});
+    }
+    if (result.stderr.len > 0) {
+        std.debug.print("STDERR:\n\n{s}\n\n", .{result.stderr});
+    }
+
+    if (compilation_success) {
+        const test_run = try std.ChildProcess.run(.{
+            .allocator = allocator,
+            // TODO: delete -main_source_file?
+            .argv = &.{"nat/std"},
+            .max_output_bytes = std.math.maxInt(u64),
+        });
+        const test_result: TestError!bool = switch (test_run.term) {
+            .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+            .Signal => error.signaled,
+            .Stopped => error.stopped,
+            .Unknown => error.unknown,
+        };
+
+        const test_success = test_result catch b: {
+            errors = true;
+            break :b false;
+        };
+        std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+        if (test_run.stdout.len > 0) {
+            std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
+        }
+        if (test_run.stderr.len > 0) {
+            std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
+        }
+    }
+
+    if (errors) return error.fail;
+}
+
+fn runCmakeTests(allocator: Allocator, dir_path: []const u8) !void {
+    var errors = false;
+    const original_dir = try std.fs.cwd().realpathAlloc(allocator, ".");
+    const cc_dir = try std.fs.cwd().openDir(dir_path, .{
+        .iterate = true,
+    });
+
+    const cc_dir_path = try cc_dir.realpathAlloc(allocator, ".");
+    try std.posix.chdir(cc_dir_path);
+
+    var cc_dir_iterator = cc_dir.iterate();
+    while (try cc_dir_iterator.next()) |cc_entry| {
+        switch (cc_entry.kind) {
+            .directory => {
+                std.debug.print("{s}...\n", .{cc_entry.name});
+                try std.posix.chdir(cc_entry.name);
+                try std.fs.cwd().deleteTree("build");
+                try std.fs.cwd().makeDir("build");
+                try std.posix.chdir("build");
+
+                const cmake = try std.ChildProcess.run(.{
+                    .allocator = allocator,
+                    // TODO: delete -main_source_file?
+                    .argv = &.{
+                        "cmake",
+                        "..",
+                        // "--debug-trycompile",
+                        // "--debug-output",
+                        // "-G", "Unix Makefiles",
+                        // "-DCMAKE_VERBOSE_MAKEFILE=On",
+                        try std.mem.concat(allocator, u8, &.{ "-DCMAKE_C_COMPILER=", "nat;cc" }),
+                        try std.mem.concat(allocator, u8, &.{ "-DCMAKE_CXX_COMPILER=", "nat;c++" }),
+                        try std.mem.concat(allocator, u8, &.{ "-DCMAKE_ASM_COMPILER=", "nat;cc" }),
+                    },
+                    .max_output_bytes = std.math.maxInt(u64),
+                });
+                const cmake_result: TestError!bool = switch (cmake.term) {
+                    .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                    .Signal => error.signaled,
+                    .Stopped => error.stopped,
+                    .Unknown => error.unknown,
+                };
+
+                const cmake_success = cmake_result catch b: {
+                    errors = true;
+                    break :b false;
+                };
+
+                if (cmake.stdout.len > 0) {
+                    std.debug.print("STDOUT:\n\n{s}\n\n", .{cmake.stdout});
+                }
+                if (cmake.stderr.len > 0) {
+                    std.debug.print("STDERR:\n\n{s}\n\n", .{cmake.stderr});
+                }
+
+                var success = cmake_success;
+                if (success) {
+                    const ninja = try std.ChildProcess.run(.{
+                        .allocator = allocator,
+                        // TODO: delete -main_source_file?
+                        .argv = &.{"ninja"},
+                        .max_output_bytes = std.math.maxInt(u64),
+                    });
+                    const ninja_result: TestError!bool = switch (ninja.term) {
+                        .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                        .Signal => error.signaled,
+                        .Stopped => error.stopped,
+                        .Unknown => error.unknown,
+                    };
+
+                    const ninja_success = ninja_result catch b: {
+                        errors = true;
+                        break :b false;
+                    };
+
+                    if (!ninja_success) {
+                        if (ninja.stdout.len > 0) {
+                            std.debug.print("STDOUT:\n\n{s}\n\n", .{ninja.stdout});
+                        }
+                        if (ninja.stderr.len > 0) {
+                            std.debug.print("STDERR:\n\n{s}\n\n", .{ninja.stderr});
+                        }
+                    }
+
+                    success = success and ninja_success;
+
+                    if (success) {
+                        const run = try std.ChildProcess.run(.{
+                            .allocator = allocator,
+                            // TODO: delete -main_source_file?
+                            .argv = &.{
+                                try std.mem.concat(allocator, u8, &.{ "./", cc_entry.name }),
+                            },
+                            .max_output_bytes = std.math.maxInt(u64),
+                        });
+                        const run_result: TestError!bool = switch (run.term) {
+                            .Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
+                            .Signal => error.signaled,
+                            .Stopped => error.stopped,
+                            .Unknown => error.unknown,
+                        };
+
+                        const run_success = run_result catch b: {
+                            errors = true;
+                            break :b false;
+                        };
+
+                        if (run.stdout.len > 0) {
+                            std.debug.print("STDOUT:\n\n{s}\n\n", .{run.stdout});
+                        }
+                        if (run.stderr.len > 0) {
+                            std.debug.print("STDERR:\n\n{s}\n\n", .{run.stderr});
+                        }
+
+                        success = success and run_success;
+                    }
+                }
+
+                std.debug.print("[TEST {s}]\n", .{if (success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
+            },
+            else => std.debug.panic("Entry {s} is a {s}", .{ cc_entry.name, @tagName(cc_entry.kind) }),
+        }
+
+        try std.posix.chdir(cc_dir_path);
+    }
+
+    try std.posix.chdir(original_dir);
+
+    if (errors) {
+        return error.fail;
+    }
+}
+
+pub fn main() !void {
+    var errors = false;
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+    runStandalone(allocator, .{
+        .directory_path = "test/standalone",
+        .group_name = "STANDALONE",
+        .is_test = false,
+    }) catch {
+        errors = true;
+    };
+    runBuildTests(allocator) catch {
+        errors = true;
+    };
+    runStandalone(allocator, .{
+        .directory_path = "test/tests",
+        .group_name = "TEST EXECUTABLE",
+        .is_test = true,
+    }) catch {
+        errors = true;
+    };
+    //
+    runStdTests(allocator) catch {
+        errors = true;
+    };
+
+    runCmakeTests(allocator, "test/cc") catch {
+        errors = true;
+    };
+    runCmakeTests(allocator, "test/c++") catch {
+        errors = true;
+    };
+
+    switch (@import("builtin").os.tag) {
+        .macos => {},
+        // .macos => {},
+        .linux => switch (@import("builtin").abi) {
+            .gnu => runCmakeTests(allocator, "test/cc_linux") catch {
+                errors = true;
+            },
+            .musl => {},
+            else => @compileError("ABI not supported"),
+        },
+        else => @compileError("OS not supported"),
+    }
+
+    if (errors) {
+        return error.fail;
     }
 }