From 2ba3f65ecf761b39b397e9bdaa3727f2ace3779b Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Fri, 4 Sep 2026 20:48:47 +0200 Subject: rename --- .../index.md | 148 +++++++++++++++++++++ .../squeezenet1.0-12-int8.onnx.svg | 1 + 2 files changed, 149 insertions(+) create mode 100644 content/posts/machine-learning-model-formats-and-runtimes/index.md create mode 100644 content/posts/machine-learning-model-formats-and-runtimes/squeezenet1.0-12-int8.onnx.svg (limited to 'content/posts/machine-learning-model-formats-and-runtimes') diff --git a/content/posts/machine-learning-model-formats-and-runtimes/index.md b/content/posts/machine-learning-model-formats-and-runtimes/index.md new file mode 100644 index 0000000..c88a9cd --- /dev/null +++ b/content/posts/machine-learning-model-formats-and-runtimes/index.md @@ -0,0 +1,148 @@ +--- +title: Machine Learning model formats and runtimes +draft: false +date: 2026-09-04 +lastmod: +description: The Machine Learning ecosystem devised general purpose file formats and runtimes to share compute graphs and weights, and solve deployment issues since the 2010s. Now hardware or domain specific runtimes are being added to provide improved capabilities. +categories: + - Machine Learning +--- +In 2019 I first got into tinkering with machine learning. I kind of dropped out of it at the end of the year, and now that it's all over the news, I feel I can't draw a direct connection to what I used to know. Today I'd like to try and bridge some of that gap. + +## The framework era + +The classic work at the time was MNIST, hand written character recognition. This was cool because I think you could actually write the whole neural network in pure python, and still have it achieve reasonable performance within tens of minutes of training. + +With today's Torch framework, a simple solution looks like this simple module. It's a lot of jargon, but there is a single flow for data, and only two layers are stateful (`conv1` and `fc`) and actually learn. Those states are "the weights". + +```py +class SimpleMNIST(nn.Module): + def __init__(self): + super() + # Input: single-channel image (28x28) + # 1. 2D convolution: 1 channel -> 8 channels, 3x3 filters + self.conv1 = nn.Conv2d(in_channels=1, out_channels=8, kernel_size=3, padding=1) + # 2. Linear layer: 8 channels * 14x14 pixels -> 10 classes (digits 0 to 9) + self.fc = nn.Linear(8 * 14 * 14, 10) + + def forward(self, x): + x = self.conv1(x) # Shape: [batch, 8, 28, 28] + x = F.relu(x) # ReLU activation + x = F.max_pool2d(x, 2) # Shape: [batch, 8, 14, 14] + x = torch.flatten(x, 1) # Flatten: [batch, 1568] + x = self.fc(x) # Shape: [batch, 10] + return x +``` + +When I was doing research for my master's degree, each framework had a way to store and load weights for a specific module, or architecture. If you changed a step, you could not use the weights. This meant you basically had to share your script alongside your weights so someone could use it. + +## Runtimes and shared formats + +In 2019, the [ONNX](https://github.com/onnx/onnx) format became widely used. This format combines the architecture with the weights, so anyone can run a shared model. It comes with an official backend from Microsoft, [ORT - ONNX Runtime](https://github.com/microsoft/onnxruntime). + +To me, a simple definition of a runtime would be +`Runtime: model, input → output`. +If you remove the variable model, you end up with a more classic algorithm. In the same idea, Python is a runtime +`Python: code, input → output`. +The runtime itself can contain deterministic algorithms, and also provide resource management (memory allocation, thread scheduling, ...). Traditionally, in machine learning, pattern recognition benefits the most from using this trainable model slot. + +So you can basically use any training framework, export your trained model in ONNX, and a device with ORT can run the model in production. I can try this for myself. + +```py +torch.onnx.export( + model, + torch.randn(1, 1, 28, 28), + "mnist_simple.onnx", + input_names=["input_image"], + output_names=["predictions"], + opset_version=18, +) +``` + +Then, with the help of onnx lib, I can see the architecture is encoded. + +```py +import onnx + +onnx_model = onnx.load("mnist_simple.onnx") +onnx.checker.check_model(onnx_model) +for i, node in enumerate(onnx_model.graph.node): + print(f"Node {i} : {node.op_type} (Input: {node.input} → Output: {node.output})") +``` + +``` +Node 0 : Conv (Input: ['input_image', 'conv1.weight', 'conv1.bias'] → Output: ['conv2d']) +Node 1 : Relu (Input: ['conv2d'] → Output: ['relu']) +Node 2 : MaxPool (Input: ['relu'] → Output: ['max_pool2d']) +Node 3 : Reshape (Input: ['max_pool2d', 'val_4'] → Output: ['view']) +Node 4 : Gemm (Input: ['view', 'fc.weight', 'fc.bias'] → Output: ['predictions']) +``` + +I can see the ONNX format defines ops (Relu, Reshape, ...), and ORT implements kernels that the model can use. If you export a model with a recent ONNX but run it on an old runtime, the architecture would not be supported because not implemented. This allows programmatic architectures alongside highly performant, hardware specific kernels. + +This is not hypothetical: in [my recent tests around local LLMs](/posts/new-tests-around-local-llms/), llama.cpp refused to run Ling 3.0 Tiny, because of an unsupported architecture. The GGUF file declares `bailingmoe3`, a hybrid design that was brand new. The individual bricks already existed in llama.cpp, but each architecture needs its own explicit graph implementation - this one landed on August 17, 2026 ([PR #26608](https://github.com/ggml-org/llama.cpp/pull/26608)), right after my tests. Same story as ONNX: the format carries the declaration, the runtime implements it. + +Any app can now use the model, but it needs to provide the pre and post processing - transforming an image into a vector, and output probabilities into characters. + +```py +def load_image(path: str) -> np.ndarray: + # Preprocessing must mirror the training pipeline: grayscale, 28x28, [0, 1] + image = Image.open(path).convert("L").resize((28, 28)) + pixels = np.asarray(image, dtype=np.float32) / 255.0 + if pixels.mean() > 0.5: + # MNIST digits are white on black; invert dark-background images + pixels = 1.0 - pixels + return pixels.reshape(1, 1, 28, 28) + + +def detect_char(session: ort.InferenceSession, image_path: str) -> int: + input_name = session.get_inputs()[0].name + logits = np.asarray(session.run(None, {input_name: load_image(image_path)})[0]) + return int(logits[0].argmax()) + + +session = ort.InferenceSession( + "mnist_simple.onnx", providers=["CPUExecutionProvider"] +) +``` + +## Specialized runtimes + +As you can see, pre and post processing can be tedious. ORT is a very general runtime, but for specific use cases, runtimes can provide a more complete harness. + +In Computer Vision - what I'm doing here, I need image encoding, maybe resizing and normalization. Actually the few lines I wrote could become a Computer Vision runtime. The CLI could take any image in, do the transformation, and use the model of my choice as the core intelligence. + +In Speech Recognition, voice activity detection, token decoding, FFT are steps that benefit little from learning, and instead benefit from highly optimized code - FFT is done best without matrix multiplication. This is why [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) is, as its name suggests, a speech specific, hardcoded harness around an ORT pass. + +With LLMs, you hear way more often about GGUF than ONNX, and llama.cpp than ORT. LLMs need tokenization - the pre processing step, caching, quantization, parallelization, and more that I don't know. + +## Example: SqueezeNet, 1.3 MB for Computer Vision + +Let's take a small vision model and use [Netron](https://netron.app/) to display its architecture. [SqueezeNet](https://github.com/forresti/SqueezeNet) was released in 2016. + +{{< details summary="Full SqueezeNet ONNX graph" >}} +![SqueezeNet ONNX graph](squeezenet1.0-12-int8.onnx.svg) +{{< /details >}} + +Here is a small compilation of formats & runtimes couples, in case you hear about them: +- ONNX + ORT - General purpose +- ONNX + sherpa-onnx - Real time audio +- GGUF + llama.cpp / ggml - LLM, memory optimized, compression +- Safetensors + vLLM - FP16, mmap, datacenter LLM inference +- TensorRT Engine (.engine / .plan) + NVIDIA Runtime - CUDA kernels, audio / vision +- OpenVINO IR (.xml + .bin) + Runtime - Intel optimized +- Core ML (.mlpackage) + Framework - Apple Silicon +- RKNN + rknn-toolkit-lite - Rockchip / ARM Edge AI + +I hope you get the picture. Each use case has specific pre and post processing, and also processing steps that are hardcoded - and often not adapted to GPUs. In the middle, it loads matrix weights for large multiplication - that's the part that learns. sherpa-onnx for speech, [fastdeploy](https://github.com/PaddlePaddle/FastDeploy) for Vision all use ORT for this middle ONNX compute. Others have recoded a new runtime to get even better performance - llama.cpp and the GGUF format support very aggressive compression on weights for LLMs. + +{{}} +- Research +- Writing pytorch code samples +- Exploring ONNX files structure +- Acting as Publisher +{{}} + +{{}} +You can see typo changes more easily with `git diff --word-diff`. This changes my life! +{{}} diff --git a/content/posts/machine-learning-model-formats-and-runtimes/squeezenet1.0-12-int8.onnx.svg b/content/posts/machine-learning-model-formats-and-runtimes/squeezenet1.0-12-int8.onnx.svg new file mode 100644 index 0000000..af5ed48 --- /dev/null +++ b/content/posts/machine-learning-model-formats-and-runtimes/squeezenet1.0-12-int8.onnx.svg @@ -0,0 +1 @@ +1×3×224×2241×64×55×551×64×55×551×128×55×551×64×55×551×64×55×551×128×55×551×128×27×271×128×27×271×128×27×271×256×27×271×128×27×271×128×27×271×256×27×271×256×13×131×192×13×131×192×13×131×384×13×131×192×13×131×192×13×131×384×13×131×256×13×131×256×13×131×512×13×131×256×13×131×256×13×131×512×13×131×1000×1×11×1000×1×1data_0float32[1,3,224,224]QuantizeLineardata_0_QuantizeLinearfloat32y_scale = 1.07741570…uint8y_zero_point = 115QLinearConvConv_nc_rename_0_quantfloat32x_scale = 1.07741570…uint8x_zero_point = 115int8[64,3,3,3]w〈64×3×3×3〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 3.73572969…uint8y_zero_point = 0int32[64]B〈64〉MaxPoolMaxPool_nc_rename_2_quantQLinearConvConv_nc_rename_3_quantfloat32x_scale = 3.73572969…uint8x_zero_point = 0int8[16,64,1,1]w〈16×64×1×1〉float32[16]w_scale〈16〉int8[16]w_zero_point〈16〉float32y_scale = 5.96574592…uint8y_zero_point = 0int32[16]B〈16〉QLinearConvConv_nc_rename_5_quantfloat32x_scale = 5.96574592…uint8x_zero_point = 0int8[64,16,3,3]w〈64×16×3×3〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 6.75794076…uint8y_zero_point = 0int32[64]B〈64〉QLinearConvConv_nc_rename_7_quantfloat32x_scale = 5.96574592…uint8x_zero_point = 0int8[64,16,1,1]w〈64×16×1×1〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 4.27861738…uint8y_zero_point = 0int32[64]B〈64〉DequantizeLinearfire2/expand1x1_2_DequantizeLinearfloat32x_scale = 4.27861738…uint8x_zero_point = 0DequantizeLinearfire2/expand3x3_2_DequantizeLinearfloat32x_scale = 6.75794076…uint8x_zero_point = 0ConcatConcat_nc_rename_9QuantizeLinearfire2/concat_1_QuantizeLinearfloat32y_scale = 6.75794076…uint8y_zero_point = 0QLinearConvConv_nc_rename_10_quantfloat32x_scale = 6.75794076…uint8x_zero_point = 0int8[16,128,1,1]w〈16×128×1×1〉float32[16]w_scale〈16〉int8[16]w_zero_point〈16〉float32y_scale = 6.53136014…uint8y_zero_point = 0int32[16]B〈16〉QLinearConvConv_nc_rename_12_quantfloat32x_scale = 6.53136014…uint8x_zero_point = 0int8[64,16,3,3]w〈64×16×3×3〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 5.09376716…uint8y_zero_point = 0int32[64]B〈64〉QLinearConvConv_nc_rename_14_quantfloat32x_scale = 6.53136014…uint8x_zero_point = 0int8[64,16,1,1]w〈64×16×1×1〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 3.33888339…uint8y_zero_point = 0int32[64]B〈64〉DequantizeLinearfire3/expand1x1_2_DequantizeLinearfloat32x_scale = 3.33888339…uint8x_zero_point = 0DequantizeLinearfire3/expand3x3_2_DequantizeLinearfloat32x_scale = 5.09376716…uint8x_zero_point = 0ConcatConcat_nc_rename_16MaxPoolMaxPool_nc_rename_17QuantizeLinearpool3_1_QuantizeLinearfloat32y_scale = 5.09376716…uint8y_zero_point = 0QLinearConvConv_nc_rename_18_quantfloat32x_scale = 5.09376716…uint8x_zero_point = 0int8[32,128,1,1]w〈32×128×1×1〉float32[32]w_scale〈32〉int8[32]w_zero_point〈32〉float32y_scale = 8.35273838…uint8y_zero_point = 0int32[32]B〈32〉QLinearConvConv_nc_rename_20_quantfloat32x_scale = 8.35273838…uint8x_zero_point = 0int8[128,32,3,3]w〈128×32×3×3〉float32[128]w_scale〈128〉int8[128]w_zero_point〈128〉float32y_scale = 8.01775264…uint8y_zero_point = 0int32[128]B〈128〉QLinearConvConv_nc_rename_22_quantfloat32x_scale = 8.35273838…uint8x_zero_point = 0int8[128,32,1,1]w〈128×32×1×1〉float32[128]w_scale〈128〉int8[128]w_zero_point〈128〉float32y_scale = 5.12402820…uint8y_zero_point = 0int32[128]B〈128〉DequantizeLinearfire4/expand1x1_2_DequantizeLinearfloat32x_scale = 5.12402820…uint8x_zero_point = 0DequantizeLinearfire4/expand3x3_2_DequantizeLinearfloat32x_scale = 8.01775264…uint8x_zero_point = 0ConcatConcat_nc_rename_24QuantizeLinearfire4/concat_1_QuantizeLinearfloat32y_scale = 8.01775264…uint8y_zero_point = 0QLinearConvConv_nc_rename_25_quantfloat32x_scale = 8.01775264…uint8x_zero_point = 0int8[32,256,1,1]w〈32×256×1×1〉float32[32]w_scale〈32〉int8[32]w_zero_point〈32〉float32y_scale = 6.03720712…uint8y_zero_point = 0int32[32]B〈32〉QLinearConvConv_nc_rename_27_quantfloat32x_scale = 6.03720712…uint8x_zero_point = 0int8[128,32,3,3]w〈128×32×3×3〉float32[128]w_scale〈128〉int8[128]w_zero_point〈128〉float32y_scale = 5.03509283…uint8y_zero_point = 0int32[128]B〈128〉QLinearConvConv_nc_rename_29_quantfloat32x_scale = 6.03720712…uint8x_zero_point = 0int8[128,32,1,1]w〈128×32×1×1〉float32[128]w_scale〈128〉int8[128]w_zero_point〈128〉float32y_scale = 2.92680954…uint8y_zero_point = 0int32[128]B〈128〉DequantizeLinearfire5/expand1x1_2_DequantizeLinearfloat32x_scale = 2.92680954…uint8x_zero_point = 0DequantizeLinearfire5/expand3x3_2_DequantizeLinearfloat32x_scale = 5.03509283…uint8x_zero_point = 0ConcatConcat_nc_rename_31MaxPoolMaxPool_nc_rename_32QuantizeLinearpool5_1_QuantizeLinearfloat32y_scale = 5.03509283…uint8y_zero_point = 0QLinearConvConv_nc_rename_33_quantfloat32x_scale = 5.03509283…uint8x_zero_point = 0int8[48,256,1,1]w〈48×256×1×1〉float32[48]w_scale〈48〉int8[48]w_zero_point〈48〉float32y_scale = 8.55824184…uint8y_zero_point = 0int32[48]B〈48〉QLinearConvConv_nc_rename_35_quantfloat32x_scale = 8.55824184…uint8x_zero_point = 0int8[192,48,3,3]w〈192×48×3×3〉float32[192]w_scale〈192〉int8[192]w_zero_point〈192〉float32y_scale = 5.70068931…uint8y_zero_point = 0int32[192]B〈192〉QLinearConvConv_nc_rename_37_quantfloat32x_scale = 8.55824184…uint8x_zero_point = 0int8[192,48,1,1]w〈192×48×1×1〉float32[192]w_scale〈192〉int8[192]w_zero_point〈192〉float32y_scale = 4.46932649…uint8y_zero_point = 0int32[192]B〈192〉DequantizeLinearfire6/expand1x1_2_DequantizeLinearfloat32x_scale = 4.46932649…uint8x_zero_point = 0DequantizeLinearfire6/expand3x3_2_DequantizeLinearfloat32x_scale = 5.70068931…uint8x_zero_point = 0ConcatConcat_nc_rename_39QuantizeLinearfire6/concat_1_QuantizeLinearfloat32y_scale = 5.70068931…uint8y_zero_point = 0QLinearConvConv_nc_rename_40_quantfloat32x_scale = 5.70068931…uint8x_zero_point = 0int8[48,384,1,1]w〈48×384×1×1〉float32[48]w_scale〈48〉int8[48]w_zero_point〈48〉float32y_scale = 5.94832229…uint8y_zero_point = 0int32[48]B〈48〉QLinearConvConv_nc_rename_42_quantfloat32x_scale = 5.94832229…uint8x_zero_point = 0int8[192,48,3,3]w〈192×48×3×3〉float32[192]w_scale〈192〉int8[192]w_zero_point〈192〉float32y_scale = 4.35615110…uint8y_zero_point = 0int32[192]B〈192〉QLinearConvConv_nc_rename_44_quantfloat32x_scale = 5.94832229…uint8x_zero_point = 0int8[192,48,1,1]w〈192×48×1×1〉float32[192]w_scale〈192〉int8[192]w_zero_point〈192〉float32y_scale = 2.65132427…uint8y_zero_point = 0int32[192]B〈192〉DequantizeLinearfire7/expand1x1_2_DequantizeLinearfloat32x_scale = 2.65132427…uint8x_zero_point = 0DequantizeLinearfire7/expand3x3_2_DequantizeLinearfloat32x_scale = 4.35615110…uint8x_zero_point = 0ConcatConcat_nc_rename_46QuantizeLinearfire7/concat_1_QuantizeLinearfloat32y_scale = 4.35615110…uint8y_zero_point = 0QLinearConvConv_nc_rename_47_quantfloat32x_scale = 4.35615110…uint8x_zero_point = 0int8[64,384,1,1]w〈64×384×1×1〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 5.99537658…uint8y_zero_point = 0int32[64]B〈64〉QLinearConvConv_nc_rename_49_quantfloat32x_scale = 5.99537658…uint8x_zero_point = 0int8[256,64,3,3]w〈256×64×3×3〉float32[256]w_scale〈256〉int8[256]w_zero_point〈256〉float32y_scale = 5.16970443…uint8y_zero_point = 0int32[256]B〈256〉QLinearConvConv_nc_rename_51_quantfloat32x_scale = 5.99537658…uint8x_zero_point = 0int8[256,64,1,1]w〈256×64×1×1〉float32[256]w_scale〈256〉int8[256]w_zero_point〈256〉float32y_scale = 2.11400485…uint8y_zero_point = 0int32[256]B〈256〉DequantizeLinearfire8/expand1x1_2_DequantizeLinearfloat32x_scale = 2.11400485…uint8x_zero_point = 0DequantizeLinearfire8/expand3x3_2_DequantizeLinearfloat32x_scale = 5.16970443…uint8x_zero_point = 0ConcatConcat_nc_rename_53QuantizeLinearfire8/concat_1_QuantizeLinearfloat32y_scale = 5.16970443…uint8y_zero_point = 0QLinearConvConv_nc_rename_54_quantfloat32x_scale = 5.16970443…uint8x_zero_point = 0int8[64,512,1,1]w〈64×512×1×1〉float32[64]w_scale〈64〉int8[64]w_zero_point〈64〉float32y_scale = 6.17008066…uint8y_zero_point = 0int32[64]B〈64〉QLinearConvConv_nc_rename_56_quantfloat32x_scale = 6.17008066…uint8x_zero_point = 0int8[256,64,3,3]w〈256×64×3×3〉float32[256]w_scale〈256〉int8[256]w_zero_point〈256〉float32y_scale = 2.79995012…uint8y_zero_point = 0int32[256]B〈256〉QLinearConvConv_nc_rename_58_quantfloat32x_scale = 6.17008066…uint8x_zero_point = 0int8[256,64,1,1]w〈256×64×1×1〉float32[256]w_scale〈256〉int8[256]w_zero_point〈256〉float32y_scale = 2.90131521…uint8y_zero_point = 0int32[256]B〈256〉DequantizeLinearfire9/expand1x1_2_DequantizeLinearfloat32x_scale = 2.90131521…uint8x_zero_point = 0DequantizeLinearfire9/expand3x3_2_DequantizeLinearfloat32x_scale = 2.79995012…uint8x_zero_point = 0ConcatConcat_nc_rename_60QuantizeLinearfire9/concat_1_QuantizeLinearfloat32y_scale = 2.90131521…uint8y_zero_point = 0QLinearConvConv_nc_rename_61_quantfloat32x_scale = 2.90131521…uint8x_zero_point = 0int8[1000,512,1,1]w〈1000×512×1×1〉float32[1000]w_scale〈1000〉int8[1000]w_zero_point〈1000〉float32y_scale = 1.84771847…uint8y_zero_point = 0int32[1000]B〈1000〉QLinearGlo…veragePool[QLinearGlobalAveragePool]float32x_scale = 1.84771847…uint8x_zero_point = 0float32y_scale = 0.18580777…uint8y_zero_point = 0DequantizeLinearpool10_1_DequantizeLinearfloat32x_scale = 0.18580777…uint8x_zero_point = 0SoftmaxSoftmax_nc_rename_64softmaxout_1float32[1,1000,1,1] \ No newline at end of file -- cgit v1.3