Merge pull request #185 from birth-software/test-runner-for-new-compiler

Start using the existing test runner

build/test_runner.zig

@@ -37,6 +37,7 @@ fn runStandalone(allocator: Allocator, args: struct {
     self_hosted: bool,
     is_test: bool,
     compiler_path: []const u8,
+    repetitions: usize,
 }) !void {
     const test_names = try collectDirectoryDirEntries(allocator, args.directory_path);
 
@@ -51,7 +52,8 @@ fn runStandalone(allocator: Allocator, args: struct {
     std.debug.print("\n[{s} START]\n\n", .{args.group_name});
 
     for (test_names) |test_name| {
-        std.debug.print("{s}... ", .{test_name});
+        std.debug.print("{s} [repetitions={}]\n\n", .{test_name, args.repetitions});
+        for (0..args.repetitions) |_| {
             const source_file_path = try std.mem.concat(allocator, u8, &.{ args.directory_path, "/", test_name, "/main.nat" });
             const argv: []const []const u8 = &.{ args.compiler_path, if (args.is_test) "test" else "exe", "-main_source_file", source_file_path };
             // if (std.mem.eql(u8, args.compiler_path, "nat/compiler_lightly_optimize_for_speed")) @breakpoint();
@@ -113,6 +115,7 @@ fn runStandalone(allocator: Allocator, args: struct {
                 std.debug.print("\n", .{});
             }
         }
+    }
 
     std.debug.print("\n{s} COMPILATIONS: {}. FAILED: {}\n", .{ args.group_name, total_compilation_count, failed_compilation_count });
     std.debug.print("{s} TESTS: {}. RAN: {}. FAILED: {}\n", .{ args.group_name, total_test_count, ran_test_count, failed_test_count });
@@ -583,30 +586,41 @@ pub fn main() !void {
     var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
     const allocator = arena.allocator();
 
-    var errors = run_test_suite(allocator, .{
+    try runStandalone(allocator, .{
-        .self_hosted = false,
-        .compiler_path = bootstrap_relative_path,
-    });
-
-    if (!errors) {
-        inline for (@typeInfo(Optimization).Enum.fields) |opt| {
-            const optimization = @field(Optimization, opt.name);
-            if (compile_self_hosted(allocator, .{
         .is_test = false,
-                .optimization = optimization,
+        .compiler_path = bootstrap_relative_path,
-            })) |compiler_path| {
+        .self_hosted = false,
-                errors = errors or run_test_suite(allocator, .{
+        .group_name = "STANDALONE",
-                    .self_hosted = true,
+        .directory_path = "retest/standalone",
-                    .compiler_path = compiler_path,
+        .repetitions = 100,
     });
-            } else |err| {
+
-                err catch {};
+    // var errors = run_test_suite(allocator, .{
-                errors = true;
+    //     .self_hosted = false,
-            }
+    //     .compiler_path = bootstrap_relative_path,
-        }
+    // });
+    //
+    // if (!errors) {
+    //     inline for (@typeInfo(Optimization).Enum.fields) |opt| {
+    //         const optimization = @field(Optimization, opt.name);
+    //         if (compile_self_hosted(allocator, .{
+    //             .is_test = false,
+    //             .optimization = optimization,
+    //         })) |compiler_path| {
+    //             errors = errors or run_test_suite(allocator, .{
+    //                 .self_hosted = true,
+    //                 .compiler_path = compiler_path,
+    //             });
+    //         } else |err| {
+    //             err catch {};
+    //             errors = true;
+    //         }
+    //     }
+    // }
+    //
+    // if (errors) {
+    //     return error.fail;
+    // }
 }
 
-    if (errors) {
+
-        return error.fail;
-    }
-}

ci/linux_gnu_runner.sh

@@ -14,11 +14,7 @@ download_zig master x86_64-linux
 
 # Build and test
 
-i=$((0))
+zig build test -Dthird_party_ci=true -Doptimize=Debug
-while ((i < 1000)); do
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseSafe
-    zig build run -Dthird_party_ci=true -Dprint_stack_trace=true -Doptimize=Debug -- exe -main_source_file retest/standalone/first/main.nat
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseSmall
-    zig build run -Dthird_party_ci=true -Dprint_stack_trace=true -Doptimize=ReleaseSafe -- exe -main_source_file retest/standalone/first/main.nat
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseFast
-    zig build run -Dthird_party_ci=true -Dprint_stack_trace=true -Doptimize=ReleaseSmall -- exe -main_source_file retest/standalone/first/main.nat
-    zig build run -Dthird_party_ci=true -Dprint_stack_trace=true -Doptimize=ReleaseFast -- exe -main_source_file retest/standalone/first/main.nat
-    i=$((i + 1))
-done

ci/macos_runner.sh

@@ -10,11 +10,7 @@ source ci/download_zig.sh
 download_zig master aarch64-macos
 
 # Build and test
-i=$((0))
+zig build test -Dthird_party_ci=true -Doptimize=Debug
-while ((i < 1000)); do
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseSafe
-    zig build run -Dprint_stack_trace=true -Doptimize=Debug -- exe -main_source_file retest/standalone/first/main.nat
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseSmall
-    zig build run -Dprint_stack_trace=true -Doptimize=ReleaseSafe -- exe -main_source_file retest/standalone/first/main.nat
+zig build test -Dthird_party_ci=true -Doptimize=ReleaseFast
-    zig build run -Dprint_stack_trace=true -Doptimize=ReleaseSmall -- exe -main_source_file retest/standalone/first/main.nat
-    zig build run -Dprint_stack_trace=true -Doptimize=ReleaseFast -- exe -main_source_file retest/standalone/first/main.nat
-    i=$((i + 1))
-done