Merge pull request #122 from birth-software/rewrite-test-runner
Rewrite test runner to be easier to work with
This commit is contained in:
David
GitHub
commit
3fefe13efd
@ -2907,7 +2907,11 @@ pub fn buildExecutable(context: *const Context, arguments: []const []const u8, o
|
||||
break :blk path;
|
||||
} else unreachable;
|
||||
|
||||
const executable_name = if (maybe_executable_name) |name| name else std.fs.path.basename(main_package_path[0 .. main_package_path.len - "/main.nat".len]);
|
||||
const executable_name = if (maybe_executable_name) |name| name else b: {
|
||||
const slash_index = data_structures.last_byte(main_package_path, '/') orelse unreachable;
|
||||
const p = std.fs.path.basename(main_package_path[0 .. slash_index]);
|
||||
break :b p;
|
||||
};
|
||||
|
||||
const executable_path = maybe_executable_path orelse blk: {
|
||||
assert(executable_name.len > 0);
|
||||
|
||||
@ -56,7 +56,7 @@ pub fn main() !void {
|
||||
try args.append(context.my_allocator, arg);
|
||||
}
|
||||
const arguments = args.slice();
|
||||
const debug_args = true;
|
||||
const debug_args = false;
|
||||
if (debug_args) {
|
||||
assert(arguments.len > 0);
|
||||
const home_dir = std.posix.getenv("HOME") orelse unreachable;
|
||||
|
||||
@ -498,6 +498,8 @@ pub fn build(b: *std.Build) !void {
|
||||
b.default_step.dependOn(&test_runner.step);
|
||||
|
||||
const test_command = b.addRunArtifact(test_runner);
|
||||
const install_runner = b.addInstallArtifact(test_runner, .{});
|
||||
b.getInstallStep().dependOn(&install_runner.step);
|
||||
test_command.step.dependOn(&compiler.step);
|
||||
test_command.step.dependOn(b.getInstallStep());
|
||||
|
||||
|
||||
@ -7,532 +7,407 @@ const TestError = error{
|
||||
signaled,
|
||||
stopped,
|
||||
unknown,
|
||||
internal,
|
||||
fail,
|
||||
};
|
||||
|
||||
fn collectDirectoryDirEntries(allocator: Allocator, path: []const u8) ![]const []const u8 {
|
||||
var dir = try std.fs.cwd().openDir(path, .{
|
||||
.iterate = true,
|
||||
});
|
||||
var dir_iterator = dir.iterate();
|
||||
var dir_entries = std.ArrayListUnmanaged([]const u8){};
|
||||
const TestGroup = struct {
|
||||
state: States = .{},
|
||||
tests: std.ArrayListUnmanaged(TestRecord) = .{},
|
||||
path: []const u8,
|
||||
path_policy: PathPolicy,
|
||||
category: Category,
|
||||
compilation_kind: CompilationKind,
|
||||
collect_directory_entries: ?[]const []const u8,
|
||||
|
||||
while (try dir_iterator.next()) |entry| {
|
||||
switch (entry.kind) {
|
||||
.directory => try dir_entries.append(allocator, try allocator.dupe(u8, entry.name)),
|
||||
else => return error.junk_in_test_directory,
|
||||
}
|
||||
}
|
||||
|
||||
dir.close();
|
||||
|
||||
return dir_entries.items;
|
||||
}
|
||||
|
||||
fn runStandalone(allocator: Allocator, args: struct {
|
||||
directory_path: []const u8,
|
||||
group_name: []const u8,
|
||||
is_test: bool,
|
||||
}) !void {
|
||||
const test_names = try collectDirectoryDirEntries(allocator, args.directory_path);
|
||||
|
||||
const total_compilation_count = test_names.len;
|
||||
var ran_compilation_count: usize = 0;
|
||||
var failed_compilation_count: usize = 0;
|
||||
|
||||
var ran_test_count: usize = 0;
|
||||
var failed_test_count: usize = 0;
|
||||
const total_test_count = test_names.len;
|
||||
|
||||
std.debug.print("\n[{s} START]\n\n", .{args.group_name});
|
||||
|
||||
for (test_names) |test_name| {
|
||||
std.debug.print("{s}... ", .{test_name});
|
||||
const source_file_path = try std.mem.concat(allocator, u8, &.{ args.directory_path, "/", test_name, "/main.nat" });
|
||||
const compile_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{ "zig-out/bin/nat", if (args.is_test) "test" else "exe", "-main_source_file", source_file_path },
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
ran_compilation_count += 1;
|
||||
|
||||
const compilation_result: TestError!bool = switch (compile_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const compilation_success = compilation_result catch b: {
|
||||
failed_compilation_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (compile_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
|
||||
}
|
||||
if (compile_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
|
||||
}
|
||||
|
||||
if (compilation_success) {
|
||||
const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name });
|
||||
const test_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{test_path},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
ran_test_count += 1;
|
||||
const test_result: TestError!bool = switch (test_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const test_success = test_result catch b: {
|
||||
failed_test_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (test_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
|
||||
}
|
||||
if (test_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
|
||||
}
|
||||
} else {
|
||||
std.debug.print("\n", .{});
|
||||
}
|
||||
}
|
||||
|
||||
std.debug.print("\n{s} COMPILATIONS: {}. FAILED: {}\n", .{ args.group_name, total_compilation_count, failed_compilation_count });
|
||||
std.debug.print("{s} TESTS: {}. RAN: {}. FAILED: {}\n", .{ args.group_name, total_test_count, ran_test_count, failed_test_count });
|
||||
|
||||
if (failed_compilation_count > 0 or failed_test_count > 0) {
|
||||
return error.fail;
|
||||
}
|
||||
}
|
||||
|
||||
fn runStandaloneTests(allocator: Allocator) !void {
|
||||
const standalone_test_dir_path = "test/standalone";
|
||||
const standalone_test_names = try collectDirectoryDirEntries(allocator, standalone_test_dir_path);
|
||||
|
||||
const total_compilation_count = standalone_test_names.len;
|
||||
var ran_compilation_count: usize = 0;
|
||||
var failed_compilation_count: usize = 0;
|
||||
|
||||
var ran_test_count: usize = 0;
|
||||
var failed_test_count: usize = 0;
|
||||
const total_test_count = standalone_test_names.len;
|
||||
|
||||
for (standalone_test_names) |standalone_test_name| {
|
||||
std.debug.print("{s}... ", .{standalone_test_name});
|
||||
const source_file_path = try std.mem.concat(allocator, u8, &.{ standalone_test_dir_path, "/", standalone_test_name, "/main.nat" });
|
||||
const compile_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{ "zig-out/bin/nat", "exe", "-main_source_file", source_file_path },
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
ran_compilation_count += 1;
|
||||
|
||||
const compilation_result: TestError!bool = switch (compile_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const compilation_success = compilation_result catch b: {
|
||||
failed_compilation_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (compile_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
|
||||
}
|
||||
if (compile_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
|
||||
}
|
||||
|
||||
if (compilation_success) {
|
||||
const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", standalone_test_name });
|
||||
const test_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{test_path},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
ran_test_count += 1;
|
||||
const test_result: TestError!bool = switch (test_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const test_success = test_result catch b: {
|
||||
failed_test_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (test_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
|
||||
}
|
||||
if (test_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
|
||||
}
|
||||
} else {
|
||||
std.debug.print("\n", .{});
|
||||
}
|
||||
}
|
||||
|
||||
std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count });
|
||||
std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count });
|
||||
|
||||
if (failed_compilation_count > 0 or failed_test_count > 0) {
|
||||
return error.fail;
|
||||
}
|
||||
}
|
||||
|
||||
fn runBuildTests(allocator: Allocator) !void {
|
||||
std.debug.print("\n[BUILD TESTS]\n\n", .{});
|
||||
const previous_cwd = try std.fs.cwd().realpathAlloc(allocator, ".");
|
||||
const test_dir_path = "test/build";
|
||||
const test_names = try collectDirectoryDirEntries(allocator, test_dir_path);
|
||||
const test_dir_realpath = try std.fs.cwd().realpathAlloc(allocator, test_dir_path);
|
||||
const compiler_realpath = try std.fs.cwd().realpathAlloc(allocator, "zig-out/bin/nat");
|
||||
try std.posix.chdir(test_dir_realpath);
|
||||
|
||||
const total_compilation_count = test_names.len;
|
||||
var ran_compilation_count: usize = 0;
|
||||
var failed_compilation_count: usize = 0;
|
||||
|
||||
var ran_test_count: usize = 0;
|
||||
var failed_test_count: usize = 0;
|
||||
const total_test_count = test_names.len;
|
||||
|
||||
for (test_names) |test_name| {
|
||||
std.debug.print("{s}... ", .{test_name});
|
||||
try std.posix.chdir(test_name);
|
||||
|
||||
const compile_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{ compiler_realpath, "build" },
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
|
||||
ran_compilation_count += 1;
|
||||
|
||||
const compilation_result: TestError!bool = switch (compile_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const compilation_success = compilation_result catch b: {
|
||||
failed_compilation_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (compile_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{compile_run.stdout});
|
||||
}
|
||||
if (compile_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{compile_run.stderr});
|
||||
}
|
||||
|
||||
if (compilation_success) {
|
||||
const test_path = try std.mem.concat(allocator, u8, &.{ "nat/", test_name });
|
||||
const test_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{test_path},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
ran_test_count += 1;
|
||||
const test_result: TestError!bool = switch (test_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const test_success = test_result catch b: {
|
||||
failed_test_count += 1;
|
||||
break :b false;
|
||||
};
|
||||
std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (test_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
|
||||
}
|
||||
if (test_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
|
||||
}
|
||||
} else {
|
||||
std.debug.print("\n", .{});
|
||||
}
|
||||
|
||||
try std.posix.chdir(test_dir_realpath);
|
||||
}
|
||||
|
||||
std.debug.print("\nTOTAL COMPILATIONS: {}. FAILED: {}\n", .{ total_compilation_count, failed_compilation_count });
|
||||
std.debug.print("TOTAL TESTS: {}. RAN: {}. FAILED: {}\n", .{ total_test_count, ran_test_count, failed_test_count });
|
||||
|
||||
try std.posix.chdir(previous_cwd);
|
||||
|
||||
if (failed_compilation_count > 0 or failed_test_count > 0) {
|
||||
return error.fail;
|
||||
}
|
||||
}
|
||||
|
||||
fn runStdTests(allocator: Allocator) !void {
|
||||
var errors = false;
|
||||
std.debug.print("std... ", .{});
|
||||
|
||||
const result = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
.argv = &.{ "zig-out/bin/nat", "test", "-main_source_file", "lib/std/std.nat", "-name", "std" },
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
const compilation_result: TestError!bool = switch (result.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
const PathPolicy = enum {
|
||||
join_path,
|
||||
take_entry,
|
||||
};
|
||||
|
||||
const compilation_success = compilation_result catch b: {
|
||||
errors = true;
|
||||
break :b false;
|
||||
const CompilationKind = enum {
|
||||
exe,
|
||||
@"test",
|
||||
build,
|
||||
cmake,
|
||||
};
|
||||
};
|
||||
|
||||
const TestRecord = struct {
|
||||
name: []const u8,
|
||||
configuration: ?TestResult = null,
|
||||
compilation: ?TestResult = null,
|
||||
execution: ?TestResult = null,
|
||||
};
|
||||
|
||||
const TestResult = struct {
|
||||
stdout: []const u8 = &.{},
|
||||
stderr: []const u8 = &.{},
|
||||
error_union: RunError!void,
|
||||
};
|
||||
|
||||
// const TestStep = struct{
|
||||
// kind: Kind,
|
||||
// result: TestResult,
|
||||
//
|
||||
// const Kind = enum{
|
||||
// configuration,
|
||||
// compilation,
|
||||
// execution,
|
||||
// };
|
||||
// };
|
||||
|
||||
const States = struct {
|
||||
failed: TestState = .{},
|
||||
total: TestState = .{},
|
||||
};
|
||||
|
||||
const TestState = struct {
|
||||
compilations: u32 = 0,
|
||||
executions: u32 = 0,
|
||||
tests: u32 = 0,
|
||||
|
||||
fn add(total: *TestState, state: *const TestState) void {
|
||||
total.compilations += state.compilations;
|
||||
total.executions += state.executions;
|
||||
total.tests += state.tests;
|
||||
}
|
||||
};
|
||||
|
||||
const TestSuite = struct {
|
||||
allocator: Allocator,
|
||||
state: State = .{},
|
||||
compiler_path: []const u8,
|
||||
|
||||
const State = struct {
|
||||
states: States = .{},
|
||||
category_map: std.EnumSet(Category) = .{},
|
||||
test_groups: std.ArrayListUnmanaged(TestGroup) = .{},
|
||||
};
|
||||
|
||||
std.debug.print("[COMPILATION {s}] ", .{if (compilation_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (result.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{result.stdout});
|
||||
}
|
||||
if (result.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{result.stderr});
|
||||
}
|
||||
|
||||
if (compilation_success) {
|
||||
const test_run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{"nat/std"},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
const test_result: TestError!bool = switch (test_run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const test_success = test_result catch b: {
|
||||
errors = true;
|
||||
break :b false;
|
||||
};
|
||||
std.debug.print("[TEST {s}]\n", .{if (test_success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
if (test_run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{test_run.stdout});
|
||||
fn run_test_group(test_suite: *TestSuite, test_group: *TestGroup) !void {
|
||||
test_suite.state.category_map.setPresent(test_group.category, true);
|
||||
defer {
|
||||
test_suite.state.states.total.add(&test_group.state.total);
|
||||
test_suite.state.states.failed.add(&test_group.state.failed);
|
||||
}
|
||||
if (test_run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{test_run.stderr});
|
||||
}
|
||||
}
|
||||
|
||||
if (errors) return error.fail;
|
||||
}
|
||||
|
||||
fn runCmakeTests(allocator: Allocator, dir_path: []const u8) !void {
|
||||
var errors = false;
|
||||
const original_dir = try std.fs.cwd().realpathAlloc(allocator, ".");
|
||||
const cc_dir = try std.fs.cwd().openDir(dir_path, .{
|
||||
.iterate = true,
|
||||
});
|
||||
|
||||
const cc_dir_path = try cc_dir.realpathAlloc(allocator, ".");
|
||||
try std.posix.chdir(cc_dir_path);
|
||||
|
||||
var cc_dir_iterator = cc_dir.iterate();
|
||||
while (try cc_dir_iterator.next()) |cc_entry| {
|
||||
switch (cc_entry.kind) {
|
||||
.directory => {
|
||||
std.debug.print("{s}...\n", .{cc_entry.name});
|
||||
try std.posix.chdir(cc_entry.name);
|
||||
try std.fs.cwd().deleteTree("build");
|
||||
try std.fs.cwd().makeDir("build");
|
||||
try std.posix.chdir("build");
|
||||
|
||||
const cmake = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{
|
||||
"cmake",
|
||||
"..",
|
||||
// "--debug-trycompile",
|
||||
// "--debug-output",
|
||||
// "-G", "Unix Makefiles",
|
||||
// "-DCMAKE_VERBOSE_MAKEFILE=On",
|
||||
try std.mem.concat(allocator, u8, &.{ "-DCMAKE_C_COMPILER=", "nat;cc" }),
|
||||
try std.mem.concat(allocator, u8, &.{ "-DCMAKE_CXX_COMPILER=", "nat;c++" }),
|
||||
try std.mem.concat(allocator, u8, &.{ "-DCMAKE_ASM_COMPILER=", "nat;cc" }),
|
||||
},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
if (test_group.collect_directory_entries) |*directory_entries| {
|
||||
directory_entries.* = blk: {
|
||||
var dir = try std.fs.cwd().openDir(test_group.path, .{
|
||||
.iterate = true,
|
||||
});
|
||||
const cmake_result: TestError!bool = switch (cmake.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
var dir_iterator = dir.iterate();
|
||||
var dir_entries = std.ArrayListUnmanaged([]const u8){};
|
||||
|
||||
const cmake_success = cmake_result catch b: {
|
||||
errors = true;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
if (cmake.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{cmake.stdout});
|
||||
}
|
||||
if (cmake.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{cmake.stderr});
|
||||
}
|
||||
|
||||
var success = cmake_success;
|
||||
if (success) {
|
||||
const ninja = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{"ninja"},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
const ninja_result: TestError!bool = switch (ninja.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const ninja_success = ninja_result catch b: {
|
||||
errors = true;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
if (!ninja_success) {
|
||||
if (ninja.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{ninja.stdout});
|
||||
}
|
||||
if (ninja.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{ninja.stderr});
|
||||
}
|
||||
}
|
||||
|
||||
success = success and ninja_success;
|
||||
|
||||
if (success) {
|
||||
const run = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
// TODO: delete -main_source_file?
|
||||
.argv = &.{
|
||||
try std.mem.concat(allocator, u8, &.{ "./", cc_entry.name }),
|
||||
},
|
||||
.max_output_bytes = std.math.maxInt(u64),
|
||||
});
|
||||
const run_result: TestError!bool = switch (run.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) true else error.abnormal_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
};
|
||||
|
||||
const run_success = run_result catch b: {
|
||||
errors = true;
|
||||
break :b false;
|
||||
};
|
||||
|
||||
if (run.stdout.len > 0) {
|
||||
std.debug.print("STDOUT:\n\n{s}\n\n", .{run.stdout});
|
||||
}
|
||||
if (run.stderr.len > 0) {
|
||||
std.debug.print("STDERR:\n\n{s}\n\n", .{run.stderr});
|
||||
}
|
||||
|
||||
success = success and run_success;
|
||||
while (try dir_iterator.next()) |entry| {
|
||||
switch (entry.kind) {
|
||||
.directory => try dir_entries.append(test_suite.allocator, try test_suite.allocator.dupe(u8, entry.name)),
|
||||
else => return error.junk_in_test_directory,
|
||||
}
|
||||
}
|
||||
|
||||
std.debug.print("[TEST {s}]\n", .{if (success) "\x1b[32mOK\x1b[0m" else "\x1b[31mFAILED\x1b[0m"});
|
||||
},
|
||||
else => std.debug.panic("Entry {s} is a {s}", .{ cc_entry.name, @tagName(cc_entry.kind) }),
|
||||
dir.close();
|
||||
|
||||
break :blk dir_entries.items;
|
||||
};
|
||||
} else {
|
||||
test_group.collect_directory_entries = &.{ test_group.path };
|
||||
}
|
||||
|
||||
try std.posix.chdir(cc_dir_path);
|
||||
}
|
||||
const directory_entries = test_group.collect_directory_entries orelse unreachable;
|
||||
try test_group.tests.ensureTotalCapacity(test_suite.allocator, directory_entries.len);
|
||||
|
||||
try std.posix.chdir(original_dir);
|
||||
for (directory_entries) |directory_entry| {
|
||||
var has_error = false;
|
||||
test_group.state.total.tests += 1;
|
||||
|
||||
if (errors) {
|
||||
return error.fail;
|
||||
const relative_path = try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry });
|
||||
|
||||
var test_record = TestRecord{
|
||||
.name = directory_entry,
|
||||
};
|
||||
|
||||
const cmake_build_dir = "build";
|
||||
switch (test_group.category) {
|
||||
.cmake => {
|
||||
const argv: []const []const u8 = &.{
|
||||
"cmake",
|
||||
"-S", ".",
|
||||
"-B", cmake_build_dir,
|
||||
"--fresh",
|
||||
try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_C_COMPILER={s};cc", .{test_suite.compiler_path}),
|
||||
try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_CXX_COMPILER={s};c++", .{test_suite.compiler_path}),
|
||||
try std.fmt.allocPrint(test_suite.allocator, "-DCMAKE_ASM_COMPILER={s};cc", .{test_suite.compiler_path}),
|
||||
};
|
||||
test_record.configuration = try run_process(test_suite.allocator, argv, .{ .path = relative_path });
|
||||
test_record.configuration.?.error_union catch {
|
||||
has_error = true;
|
||||
test_group.state.total.compilations += 1;
|
||||
test_group.state.failed.compilations += 1;
|
||||
};
|
||||
},
|
||||
.build, .standalone => {},
|
||||
}
|
||||
|
||||
if (!has_error) {
|
||||
const compilation_cwd: Cwd = switch (test_group.category) {
|
||||
.standalone => .none,
|
||||
.build => .{ .path = relative_path },
|
||||
.cmake => .{ .path = try std.mem.concat(test_suite.allocator, u8, &.{ relative_path, "/" ++ cmake_build_dir }) },
|
||||
};
|
||||
|
||||
const arguments: []const []const u8 = switch (test_group.category) {
|
||||
.standalone => blk: {
|
||||
const source_file_path = switch (test_group.path_policy) {
|
||||
.join_path => try std.mem.concat(test_suite.allocator, u8, &.{ test_group.path, "/", directory_entry, "/main.nat" }),
|
||||
.take_entry => directory_entry,
|
||||
};
|
||||
const compilation_argument = @tagName(test_group.compilation_kind);
|
||||
break :blk &.{ test_suite.compiler_path, compilation_argument, "-main_source_file", source_file_path };
|
||||
},
|
||||
.build => &.{ test_suite.compiler_path, "build" },
|
||||
.cmake => &.{ "ninja" },
|
||||
};
|
||||
|
||||
test_record.compilation = try run_process(test_suite.allocator, arguments, compilation_cwd);
|
||||
test_group.state.total.compilations += 1;
|
||||
|
||||
if (test_record.compilation.?.error_union) |_| {
|
||||
const executable_name = switch (test_group.path_policy) {
|
||||
.join_path => directory_entry,
|
||||
.take_entry => b: {
|
||||
const slash_index = std.mem.lastIndexOfScalar(u8, directory_entry, '/') orelse unreachable;
|
||||
const base = std.fs.path.basename(directory_entry[0..slash_index]);
|
||||
break :b base;
|
||||
},
|
||||
};
|
||||
const build_dir = switch (test_group.category) {
|
||||
.cmake => cmake_build_dir ++ "/",
|
||||
else => "nat/",
|
||||
};
|
||||
const execution_cwd: Cwd = switch (test_group.category) {
|
||||
.cmake => .{ .path = relative_path },
|
||||
else => compilation_cwd,
|
||||
};
|
||||
test_record.execution = try run_process(test_suite.allocator, &.{try std.mem.concat(test_suite.allocator, u8, &.{ build_dir, executable_name })}, execution_cwd);
|
||||
test_group.state.total.executions += 1;
|
||||
|
||||
if (test_record.execution.?.error_union) |_| {} else |err| {
|
||||
has_error = true;
|
||||
err catch {};
|
||||
test_group.state.failed.executions += 1;
|
||||
}
|
||||
} else |err| {
|
||||
err catch {};
|
||||
has_error = true;
|
||||
test_group.state.failed.compilations += 1;
|
||||
}
|
||||
}
|
||||
|
||||
test_group.state.failed.tests += @intFromBool(has_error);
|
||||
|
||||
test_group.tests.appendAssumeCapacity(test_record);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const RunError = error{
|
||||
unexpected_exit_code,
|
||||
signaled,
|
||||
stopped,
|
||||
unknown,
|
||||
};
|
||||
|
||||
const Cwd = union(enum) {
|
||||
none,
|
||||
path: []const u8,
|
||||
descriptor: std.fs.Dir,
|
||||
};
|
||||
|
||||
fn run_process(allocator: Allocator, argv: []const []const u8, cwd: Cwd) !TestResult {
|
||||
var path: ?[]const u8 = null;
|
||||
var descriptor: ?std.fs.Dir = null;
|
||||
switch (cwd) {
|
||||
.none => {},
|
||||
.path => |p| path = p,
|
||||
.descriptor => |d| descriptor = d,
|
||||
}
|
||||
const process_result = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
.argv = argv,
|
||||
.max_output_bytes = std.math.maxInt(usize),
|
||||
.cwd = path,
|
||||
.cwd_dir = descriptor,
|
||||
});
|
||||
|
||||
return TestResult{
|
||||
.stdout = process_result.stdout,
|
||||
.stderr = process_result.stderr,
|
||||
.error_union = switch (process_result.term) {
|
||||
.Exited => |exit_code| if (exit_code == 0) {} else error.unexpected_exit_code,
|
||||
.Signal => error.signaled,
|
||||
.Stopped => error.stopped,
|
||||
.Unknown => error.unknown,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const Category = enum {
|
||||
standalone,
|
||||
build,
|
||||
cmake,
|
||||
};
|
||||
|
||||
pub fn main() !void {
|
||||
var errors = false;
|
||||
var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
|
||||
const allocator = arena.allocator();
|
||||
runStandalone(allocator, .{
|
||||
.directory_path = "test/standalone",
|
||||
.group_name = "STANDALONE",
|
||||
.is_test = false,
|
||||
}) catch {
|
||||
errors = true;
|
||||
};
|
||||
runBuildTests(allocator) catch {
|
||||
errors = true;
|
||||
};
|
||||
runStandalone(allocator, .{
|
||||
.directory_path = "test/tests",
|
||||
.group_name = "TEST EXECUTABLE",
|
||||
.is_test = true,
|
||||
}) catch {
|
||||
errors = true;
|
||||
};
|
||||
//
|
||||
runStdTests(allocator) catch {
|
||||
errors = true;
|
||||
};
|
||||
const compiler_relative_path = "zig-out/bin/nat";
|
||||
|
||||
runCmakeTests(allocator, "test/cc") catch {
|
||||
errors = true;
|
||||
var test_suite = TestSuite{
|
||||
.allocator = allocator,
|
||||
.compiler_path = try std.fs.realpathAlloc(allocator, compiler_relative_path),
|
||||
};
|
||||
runCmakeTests(allocator, "test/c++") catch {
|
||||
errors = true;
|
||||
};
|
||||
|
||||
switch (@import("builtin").os.tag) {
|
||||
.macos => {},
|
||||
// .macos => {},
|
||||
.linux => switch (@import("builtin").abi) {
|
||||
.gnu => runCmakeTests(allocator, "test/cc_linux") catch {
|
||||
errors = true;
|
||||
},
|
||||
.musl => {},
|
||||
else => @compileError("ABI not supported"),
|
||||
var test_groups = [_]TestGroup{
|
||||
.{
|
||||
.category = .standalone,
|
||||
.path = "test/standalone",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .exe,
|
||||
},
|
||||
else => @compileError("OS not supported"),
|
||||
.{
|
||||
.category = .standalone,
|
||||
.path = "test/tests",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .@"test",
|
||||
},
|
||||
.{
|
||||
.category = .standalone,
|
||||
.path = "lib/std/std.nat",
|
||||
.collect_directory_entries = null,
|
||||
.path_policy = .take_entry,
|
||||
.compilation_kind = .@"test",
|
||||
},
|
||||
.{
|
||||
.category = .build,
|
||||
.path = "test/build",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .@"exe",
|
||||
},
|
||||
.{
|
||||
.category = .cmake,
|
||||
.path = "test/cc",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .cmake,
|
||||
},
|
||||
.{
|
||||
.category = .cmake,
|
||||
.path = "test/c++",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .cmake,
|
||||
},
|
||||
} ++ switch (@import("builtin").os.tag) {
|
||||
.linux => [_]TestGroup{
|
||||
.{
|
||||
.category = .cmake,
|
||||
.path = "test/cc_linux",
|
||||
.collect_directory_entries = &.{},
|
||||
.path_policy = .join_path,
|
||||
.compilation_kind = .cmake,
|
||||
},
|
||||
},
|
||||
else => [_]TestGroup{},
|
||||
};
|
||||
|
||||
for (&test_groups) |*test_group| {
|
||||
try test_suite.run_test_group(test_group);
|
||||
}
|
||||
|
||||
if (errors) {
|
||||
return error.fail;
|
||||
const stdout = std.io.getStdOut();
|
||||
const stdout_writer = stdout.writer();
|
||||
var io_buffer = std.io.BufferedWriter(16 * 0x1000, @TypeOf(stdout_writer)){ .unbuffered_writer = stdout_writer };
|
||||
const io_writer = io_buffer.writer();
|
||||
try io_writer.writeByte('\n');
|
||||
|
||||
for (&test_groups) |*test_group| {
|
||||
if (test_group.state.failed.tests > 0) {
|
||||
for (test_group.tests.items) |*test_result| {
|
||||
try io_writer.print("{s}\n", .{test_result.name});
|
||||
|
||||
if (test_result.configuration) |result| {
|
||||
if (result.stdout.len > 0) {
|
||||
try io_writer.writeAll(result.stdout);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
|
||||
if (result.stderr.len > 0) {
|
||||
try io_writer.writeAll(result.stderr);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (test_result.compilation) |result| {
|
||||
if (result.stdout.len > 0) {
|
||||
try io_writer.writeAll(result.stdout);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
|
||||
if (result.stderr.len > 0) {
|
||||
try io_writer.writeAll(result.stderr);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (test_result.execution) |result| {
|
||||
if (result.stdout.len > 0) {
|
||||
try io_writer.writeAll(result.stdout);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
|
||||
if (result.stderr.len > 0) {
|
||||
try io_writer.writeAll(result.stderr);
|
||||
try io_writer.writeByte('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try io_writer.print("[{s}] [{s}] Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{
|
||||
@tagName(test_group.category),
|
||||
test_group.path,
|
||||
test_group.state.total.tests,
|
||||
test_group.state.failed.tests,
|
||||
test_group.state.total.compilations,
|
||||
test_group.state.failed.compilations,
|
||||
test_group.state.total.executions,
|
||||
test_group.state.failed.executions,
|
||||
});
|
||||
}
|
||||
|
||||
try io_writer.print("Ran {} tests ({} failed). Ran {} compilations ({} failed). Ran {} executions ({} failed).\n", .{
|
||||
test_suite.state.states.total.tests,
|
||||
test_suite.state.states.failed.tests,
|
||||
test_suite.state.states.total.compilations,
|
||||
test_suite.state.states.failed.compilations,
|
||||
test_suite.state.states.total.executions,
|
||||
test_suite.state.states.failed.executions,
|
||||
});
|
||||
|
||||
const success = test_suite.state.states.failed.tests == 0;
|
||||
if (success) {
|
||||
try io_writer.writeAll("\x1b[32mTESTS PASSED!\x1b[0m\n");
|
||||
} else {
|
||||
try io_writer.writeAll("\x1b[31mTESTS FAILED!\x1b[0m\n");
|
||||
}
|
||||
|
||||
try io_buffer.flush();
|
||||
|
||||
if (!success) {
|
||||
std.posix.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user