improved testing & src cleanup

12 files changed, 118 insertions, 94 deletions

diff --git a/.gitignore b/.gitignore
index c2a342a..c49a7e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,5 +5,5 @@ Release/
 Profile/
 
 test.c
-*.json
+*MB.json
 callgrind.out
diff --git a/Makefile b/Makefile
index 9d4845a..b5489e8 100644
--- a/Makefile
+++ b/Makefile
@@ -24,5 +24,5 @@ format:
 	clang-format -i src/*.c include/*.h tests/*.c *.c
 
 check:
-	make test
 	clang-format -i --dry-run src/*.c include/*.h tests/*.c *.c --fail-on-incomplete-format
+	make test
diff --git a/README.md b/README.md
index 18d97c5..111e91d 100644
--- a/README.md
+++ b/README.md
@@ -7,8 +7,9 @@
 - web server, Json parsing, struct codegen
 - assert not in release
 - python based generate
-- test helpers to load json files
-- add test with backspaces weird
+- escape chars
+- unicode
+- fail on any control character in string
 
 ## Benchmarks
 
diff --git a/include/str.h b/include/str.h
index 153eebc..2d2eb93 100644
--- a/include/str.h
+++ b/include/str.h
@@ -6,20 +6,18 @@
 #include "../include/memory.h"
 #include "../include/types.h"
 
-#define S(s)                                                                   \
-    (Str) { s, lengthof(s) }
 typedef struct {
     byte *data;
     size len;
 } Str;
-typedef struct {
-    byte *data;
-    size len;
-} StrView;
 u64 Str_hash64(Str s);
 Str Str_copy(Arena *a, byte *data, size len);
 i32 Str_fprint(FILE *w, Str s);
 
+typedef struct {
+    byte *data;
+    size len;
+} StrView;
 StrView Str_view(byte *data, size len);
 Str StrView_copy(Arena *a, StrView s);
 
@@ -28,3 +26,5 @@ Str StrView_copy(Arena *a, StrView s);
 #define Str_equals(a, b)                                                       \
     ((a).len == (b).len) && (!(a).len || !memcmp((a).data, (b).data, (a).len))
 #define Str_fprint(w, s) fprintf(w, "%.*s", (i32)(s).len, (s).data)
+#define S(s)                                                                   \
+    (Str) { s, lengthof(s) }
diff --git a/include/testing.h b/include/testing.h
index 754733b..1ca460b 100644
--- a/include/testing.h
+++ b/include/testing.h
@@ -2,6 +2,14 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "../include/memory.h"
+
+typedef struct {
+    size size;
+    byte *content;
+} MemoryFile;
+MemoryFile load_file(Arena *a, const char *filename);
+
 #define t_assert_equal(a, b)                                                   \
     if (!(a == b)) {                                                           \
         fprintf(stderr,                                                        \
diff --git a/main.c b/main.c
index 516bb12..de186f2 100644
--- a/main.c
+++ b/main.c
@@ -7,43 +7,10 @@
 #include "include/assert.h"
 #include "include/json.h"
 #include "include/memory.h"
+#include "include/testing.h"
 #include "include/types.h"
 #include "include/vec.h"
 
-// Load entire file into memory
-byte *load_file(const char *filename, size *file_size) {
-    FILE *file = fopen(filename, "rb");
-    if (!file) {
-        fprintf(stderr, "Error: Could not open file '%s'\n", filename);
-        return NULL;
-    }
-
-    // Get file size
-    fseek(file, 0, SEEK_END);
-    *file_size = ftell(file);
-    fseek(file, 0, SEEK_SET);
-
-    // Allocate buffer and read file
-    byte *buffer = malloc(*file_size + 1);
-    if (!buffer) {
-        fprintf(stderr, "Error: Could not allocate memory for file\n");
-        fclose(file);
-        return NULL;
-    }
-
-    size bytes_read = fread(buffer, 1, *file_size, file);
-    if (bytes_read != *file_size) {
-        fprintf(stderr, "Error: Could not read entire file\n");
-        free(buffer);
-        fclose(file);
-        return NULL;
-    }
-
-    buffer[*file_size] = '\0'; // Null terminate
-    fclose(file);
-    return buffer;
-}
-
 // Get high resolution time in nanoseconds
 double get_time_ns(void) {
     struct timespec ts;
@@ -52,33 +19,32 @@ double get_time_ns(void) {
 }
 
 void run_benchmark(const char *filename, i32 iterations) {
+    Arena perm = Arena_new(1e9);
     printf("Loading file: %s\n", filename);
 
-    size file_size;
-    byte *content = load_file(filename, &file_size);
-    if (!content) return;
+    MemoryFile file = load_file(&perm, filename);
 
-    printf("File size: %zu bytes\n", file_size);
+    printf("File size: %zu bytes\n", file.size);
     printf("Running %d iterations...\n\n", iterations);
 
     // Statistics tracking
     f32 timings[iterations];
     f32 timing_avg = 0.0;
 
-    Arena arena = Arena_new(8e9);
+    Arena tmp = Arena_new(8e9);
 
     printf("Benchmarking...\n");
     for (i32 i = 0; i < iterations; i++) {
-        Arena_reset(&arena);
-        size used = Arena_get_used(arena);
+        Arena_reset(&tmp);
+        size used = Arena_get_used(tmp);
         double start_time = get_time_ns();
-        JSON_parse(&arena, content, file_size);
+        JSON_parse(&tmp, file.content, file.size);
         double end_time = get_time_ns();
 
         timings[i] = end_time - start_time;
         timing_avg += timings[i];
 
-        size new_used = Arena_get_used(arena);
+        size new_used = Arena_get_used(tmp);
         size allocated = new_used - used;
         printf("Iteration %d: %3.f ms\t%td MB\n", i + 1, timings[i] / 1000000,
                allocated / 1024 / 1024);
@@ -97,7 +63,7 @@ void run_benchmark(const char *filename, i32 iterations) {
     timing_var /= iterations;
     f32 timing_stddev = sqrt(timing_var);
     double avg_throughput_mb_s =
-        (file_size / (1024.0 * 1024.0)) / (timing_avg / 1000000000.0);
+        (file.size / (1024.0 * 1024.0)) / (timing_avg / 1000000000.0);
 
     printf("\n=== Benchmark Results ===\n");
     printf("%3.f ms ± %3.f ms per loop (mean ± std. dev. of %d runs, 1 loop "
diff --git a/src/json.c b/src/json.c
index 8531d48..d3f510f 100644
--- a/src/json.c
+++ b/src/json.c
@@ -186,22 +186,29 @@ parse_result parse_string(byte *src, uptr start_pos, uptr max_pos) {
     result.pos = start_pos + 1;
 
     // SIMD optimization: process 32 bytes at a time using AVX2
-    bytex32 quote_mask = _mm256_set1_epi8('"');
+    bytex32 quote_bytes = _mm256_set1_epi8('"');
 
     for (; result.pos + 32 <= max_pos; result.pos += 32) {
         bytex32 chunk = _mm256_loadu_si256((bytex32 *)(src + result.pos));
-        bytex32 cmp = _mm256_cmpeq_epi8(chunk, quote_mask);
-        i32 mask = _mm256_movemask_epi8(cmp);
+        i32 quote_mask =
+            _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, quote_bytes));
 
-        if (mask != 0) {
-            result.pos += stdc_trailing_zeros_ui(mask);
-            if (src[result.pos - 1] != '\\') break;
+        if (quote_mask != 0) {
+            result.pos += stdc_trailing_zeros_ui(quote_mask);
+            break;
         };
     }
 
     // Fall back to original byte-by-byte processing for the remaining bytes
-    while (src[result.pos] != '"' && src[result.pos - 1] != '\\') {
-        result.pos++;
+    for (; result.pos < max_pos; result.pos++) {
+        if (src[result.pos] == '"') {
+            uptr backslash_count = 0;
+            while (backslash_count < result.pos &&
+                   src[result.pos - backslash_count - 1] == '\\') {
+                backslash_count++;
+            }
+            if (!(backslash_count & 1)) break;
+        }
     }
 
     assert_parse(src[result.pos] == '"', "invalid string end", src, result.pos);
diff --git a/src/testing.c b/src/testing.c
new file mode 100644
index 0000000..937d2dc
--- /dev/null
+++ b/src/testing.c
@@ -0,0 +1,28 @@
+#include <stdio.h>
+
+#include "../include/assert.h"
+#include "../include/memory.h"
+#include "../include/str.h"
+#include "../include/testing.h"
+#include "../include/types.h"
+
+MemoryFile load_file(Arena *a, const char *filename) {
+    FILE *file = fopen(filename, "rb");
+    assert(file != 0, "failed to open file: %s", filename);
+    MemoryFile mem_file = {0};
+
+    // Get file size
+    fseek(file, 0, SEEK_END);
+    mem_file.size = ftell(file);
+    fseek(file, 0, SEEK_SET);
+
+    mem_file.content = new(a, byte, mem_file.size + 1);
+
+    size bytes_read = fread(mem_file.content, 1, mem_file.size, file);
+    assert(bytes_read == mem_file.size, "failed to read entire file: %s",
+           filename);
+
+    mem_file.content[mem_file.size] = '\0'; // Null terminate
+    fclose(file);
+    return mem_file;
+}
diff --git a/tests/test_json.c b/tests/test_json.c
index 82bb83e..921de64 100644
--- a/tests/test_json.c
+++ b/tests/test_json.c
@@ -151,13 +151,9 @@ i32 test_json_long_spaces() {
 
 i32 test_json_multiline() {
     Arena perm = Arena_new(128e3);
-
-    JSON doc = JSON_parse_raw(
-        &perm, "{\n"
-               "    \"generated_at\": \"2025-07-22T10:36:30.974926\",\n"
-               "    \"target_size_mb\": 1,\n"
-               "    \"generator_version\": \"1.0\"\n"
-               "}");
+    MemoryFile file =
+        load_file(&perm, "tests/test_json/test_json_multiline.json");
+    JSON doc = JSON_parse(&perm, file.content, file.size);
     JSON target_size_mb = *JSONObject_get(doc.as.object, S("target_size_mb"));
     t_assert_equal(target_size_mb.as.integer, 1);
     JSON generated_at = *JSONObject_get(doc.as.object, S("generated_at"));
@@ -191,31 +187,9 @@ i32 test_json_array_in_object() {
 
 i32 test_json_complex_multiline() {
     Arena perm = Arena_new(128e3);
-    JSON doc = JSON_parse_raw(
-        &perm, "{\n"
-               "    \"id\": 819228,\n"
-               "    \"username\": \"rCrRv4PRnUNuAy\",\n"
-               "    \"email\": \"xzbwrzj@zwod.com\",\n"
-               "    \"first_name\": \"4WH7g7PsOEm5\",\n"
-               "    \"last_name\": \"6ulndxUgjbIayl\",\n"
-               "    \"age\": 30,\n"
-               "    \"phone\": \"+1-529-712-3931\",\n"
-               "    \"is_active\": false,\n"
-               "    \"balance\": 9347.49,\n"
-               "    \"tags\": [\n"
-               "      \"TWvZi0\",\n"
-               "      \"pM0TPv9X\"\n"
-               "    ],\n"
-               "    \"metadata\": {\n"
-               "      \"last_login\": \"2024-06-13T21:22:32Z\",\n"
-               "      \"login_count\": 245,\n"
-               "      \"preferences\": {\n"
-               "        \"theme\": \"auto\",\n"
-               "        \"notifications\": true,\n"
-               "        \"language\": \"en\"\n"
-               "      }\n"
-               "    }\n"
-               "  }");
+    MemoryFile file =
+        load_file(&perm, "tests/test_json/test_json_complex_multiline.json");
+    JSON doc = JSON_parse(&perm, file.content, file.size);
 
     JSON id = *JSONObject_get(doc.as.object, S("id"));
     t_assert_equal(id.as.integer, 819228);
@@ -238,3 +212,13 @@ i32 test_negative_numbers() {
 
     return 0;
 }
+
+i32 test_json_backspaces() {
+    Arena perm = Arena_new(128e3);
+    MemoryFile file =
+        load_file(&perm, "tests/test_json/test_json_backspaces.json");
+    JSON doc = JSON_parse(&perm, file.content, file.size);
+
+    t_assert_strequal(doc.as.string, S("weird text\\"));
+    return 0;
+}
diff --git a/tests/test_json/test_json_backspaces.json b/tests/test_json/test_json_backspaces.json
new file mode 100644
index 0000000..ceb65d6
--- /dev/null
+++ b/tests/test_json/test_json_backspaces.json
@@ -0,0 +1 @@
+"weird text\\"
diff --git a/tests/test_json/test_json_complex_multiline.json b/tests/test_json/test_json_complex_multiline.json
new file mode 100644
index 0000000..a0cfc9b
--- /dev/null
+++ b/tests/test_json/test_json_complex_multiline.json
@@ -0,0 +1,24 @@
+{
+    "id": 819228,
+    "username": "rCrRv4PRnUNuAy",
+    "email": "xzbwrzj@zwod.com",
+    "first_name": "4WH7g7PsOEm5",
+    "last_name": "6ulndxUgjbIayl",
+    "age": 30,
+    "phone": "+1-529-712-3931",
+    "is_active": false,
+    "balance": 9347.49,
+    "tags": [
+        "TWvZi0",
+        "pM0TPv9X"
+    ],
+    "metadata": {
+        "last_login": "2024-06-13T21:22:32Z",
+        "login_count": 245,
+        "preferences": {
+            "theme": "auto",
+            "notifications": true,
+            "language": "en"
+        }
+    }
+}
diff --git a/tests/test_json/test_json_multiline.json b/tests/test_json/test_json_multiline.json
new file mode 100644
index 0000000..7cb5f4c
--- /dev/null
+++ b/tests/test_json/test_json_multiline.json
@@ -0,0 +1,5 @@
+{
+    "generated_at": "2025-07-22T10:36:30.974926",
+    "target_size_mb": 1,
+    "generator_version": "1.0"
+}