Merge pull request #305 from perillo/improve-run-test

Improve running tests

.github/workflows/ci.yml

@@ -42,6 +42,5 @@ jobs:
         uses: goto-bus-stop/setup-zig@v2
         with:
           version: master
-# Temporarily disabled.
+      - name: Run unit tests
-#      - name: Run unit tests
+        run: zig build test
-#        run: zig build test

build.zig

@@ -13,6 +13,13 @@ const assert = std.debug.assert;
 const join = std.fs.path.join;
 const print = std.debug.print;
 
+const Kind = enum {
+    /// Run the artifact as a normal executable.
+    exe,
+    /// Run the artifact as a test.
+    @"test",
+};
+
 pub const Exercise = struct {
     /// main_file must have the format key_name.zig.
     /// The key will be used as a shorthand to build just one example.
@@ -34,9 +41,8 @@ pub const Exercise = struct {
     /// We need to keep track of this, so we compile with libc.
     link_libc: bool = false,
 
-    /// This exercise doesn't have a main function.
+    /// This exercise kind.
-    /// We only call the test.
+    kind: Kind = .exe,
-    run_test: bool = false,
 
     /// This exercise is not supported by the current Zig compiler.
     skip: bool = false,
@@ -225,18 +231,6 @@ const ZiglingStep = struct {
             return;
         }
 
-        // Test exercise.
-        if (self.exercise.run_test) {
-            self.is_testing = true;
-            const result_msg = self.testing(prog_node) catch {
-                std.os.exit(2);
-            };
-            const output = try trimLines(self.step.owner.allocator, result_msg);
-            print("\n{s}PASSED:\n{s}{s}\n\n", .{ green_text, output, reset_text });
-            return;
-        }
-
-        // Normal exercise.
         const exe_path = self.compile(prog_node) catch {
             if (self.exercise.hint) |hint|
                 print("\n{s}Ziglings hint: {s}{s}", .{ bold_text, hint, reset_text });
@@ -276,10 +270,14 @@ const ZiglingStep = struct {
             return err;
         };
 
-        const raw_output = if (self.exercise.check_stdout)
+        switch (self.exercise.kind) {
-            result.stdout
+            .exe => return self.check_output(result),
-        else
+            .@"test" => return self.check_test(result),
-            result.stderr;
+        }
+    }
+
+    fn check_output(self: *ZiglingStep, result: Child.ExecResult) !void {
+        const b = self.step.owner;
 
         // Make sure it exited cleanly.
         switch (result.term) {
@@ -299,6 +297,11 @@ const ZiglingStep = struct {
             },
         }
 
+        const raw_output = if (self.exercise.check_stdout)
+            result.stdout
+        else
+            result.stderr;
+
         // Validate the output.
         // NOTE: exercise.output can never contain a CR character.
         // See https://ziglang.org/documentation/master/#Source-Encoding.
@@ -323,55 +326,28 @@ const ZiglingStep = struct {
         print("{s}PASSED:\n{s}{s}\n\n", .{ green_text, output, reset_text });
     }
 
-    fn testing(self: *ZiglingStep, prog_node: *std.Progress.Node) ![]const u8 {
+    fn check_test(self: *ZiglingStep, result: Child.ExecResult) !void {
-        print("Testing {s}...\n", .{self.exercise.main_file});
+        switch (result.term) {
-
+            .Exited => |code| {
-        const b = self.step.owner;
+                if (code != 0) {
-        const exercise_path = self.exercise.main_file;
+                    // The test failed.
-        const path = join(b.allocator, &.{ self.work_path, exercise_path }) catch
+                    print("{s}{s}{s}\n", .{
-            @panic("OOM");
+                        red_text, result.stderr, reset_text,
-
-        var zig_args = std.ArrayList([]const u8).init(b.allocator);
-        defer zig_args.deinit();
-
-        zig_args.append(b.zig_exe) catch @panic("OOM");
-        zig_args.append("test") catch @panic("OOM");
-
-        zig_args.append(b.pathFromRoot(path)) catch @panic("OOM");
-
-        const argv = zig_args.items;
-        var code: u8 = undefined;
-        _ = self.eval(argv, &code, prog_node) catch |err| {
-            self.printErrors();
-
-            switch (err) {
-                error.FileNotFound => {
-                    print("{s}{s}: Unable to spawn the following command: file not found{s}\n", .{
-                        red_text, self.exercise.main_file, reset_text,
-                    });
-                    dumpArgs(argv);
-                },
-                error.ExitCodeFailure => {
-                    // Expected when test fails.
-                },
-                error.ProcessTerminated => {
-                    print("{s}{s}: The following command terminated unexpectedly:{s}\n", .{
-                        red_text, self.exercise.main_file, reset_text,
                     });
-                    dumpArgs(argv);
+
+                    return error.TestFailed;
+                }
             },
             else => {
-                    print("{s}{s}: Unexpected error: {s}{s}\n", .{
+                print("{s}{s} terminated unexpectedly{s}\n", .{
-                        red_text, self.exercise.main_file, @errorName(err), reset_text,
+                    red_text, self.exercise.main_file, reset_text,
                 });
-                    dumpArgs(argv);
+
+                return error.UnexpectedTermination;
             },
         }
 
-            return err;
+        print("{s}PASSED{s}\n\n", .{ green_text, reset_text });
-        };
-
-        return self.result_messages;
     }
 
     fn compile(self: *ZiglingStep, prog_node: *std.Progress.Node) ![]const u8 {
@@ -386,7 +362,12 @@ const ZiglingStep = struct {
         defer zig_args.deinit();
 
         zig_args.append(b.zig_exe) catch @panic("OOM");
-        zig_args.append("build-exe") catch @panic("OOM");
+
+        const cmd = switch (self.exercise.kind) {
+            .exe => "build-exe",
+            .@"test" => "test",
+        };
+        zig_args.append(cmd) catch @panic("OOM");
 
         // Enable C support for exercises that use C functions.
         if (self.exercise.link_libc) {
@@ -1220,7 +1201,7 @@ const exercises = [_]Exercise{
     .{
         .main_file = "102_testing.zig",
         .output = "",
-        .run_test = true,
+        .kind = .@"test",
     },
     .{
         .main_file = "999_the_end.zig",

test/tests.zig

@@ -93,7 +93,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step {
         const case_step = createCase(b, "case-3");
 
         for (exercises[0 .. exercises.len - 1]) |ex| {
-            if (ex.skip or ex.run_test) continue;
+            if (ex.skip) continue;
 
             if (ex.hint) |hint| {
                 const n = ex.number();
@@ -249,21 +249,6 @@ fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void {
         return;
     }
 
-    if (exercise.run_test) {
-        {
-            const actual = try readLine(reader, &buf) orelse "EOF";
-            const expect = b.fmt("Testing {s}...", .{exercise.main_file});
-            try check(step, exercise, expect, actual);
-        }
-
-        {
-            const actual = try readLine(reader, &buf) orelse "EOF";
-            try check(step, exercise, "", actual);
-        }
-
-        return;
-    }
-
     {
         const actual = try readLine(reader, &buf) orelse "EOF";
         const expect = b.fmt("Compiling {s}...", .{exercise.main_file});
@@ -278,12 +263,19 @@ fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void {
 
     {
         const actual = try readLine(reader, &buf) orelse "EOF";
-        const expect = "PASSED:";
+        const expect = switch (exercise.kind) {
+            .exe => "PASSED:",
+            .@"test" => "PASSED",
+        };
         try check(step, exercise, expect, actual);
     }
 
     // Skip the exercise output.
-    const nlines = 1 + mem.count(u8, exercise.output, "\n") + 1;
+    const nlines = switch (exercise.kind) {
+        .exe => 1 + mem.count(u8, exercise.output, "\n") + 1,
+        .@"test" => 1,
+    };
+
     var lineno: usize = 0;
     while (lineno < nlines) : (lineno += 1) {
         _ = try readLine(reader, &buf) orelse @panic("EOF");