From dd53768baf8cc7eb02d7ad3f804e605ae1abef60 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Fri, 21 Aug 2026 18:38:22 +0200 Subject: [PATCH] WIP OCP FP8 E4M3 support --- conversion/base.py | 125 +++++++++++++++++--- ggml/include/ggml.h | 6 +- ggml/src/ggml-cpu/ggml-cpu.c | 12 +- ggml/src/ggml-cpu/ops.cpp | 13 +- ggml/src/ggml-cpu/quants.c | 21 ++++ ggml/src/ggml-cpu/quants.h | 2 + ggml/src/ggml-impl.h | 51 ++++++++ ggml/src/ggml-quants.c | 28 +++++ ggml/src/ggml-quants.h | 3 + ggml/src/ggml.c | 10 ++ gguf-py/gguf/constants.py | 3 + gguf-py/gguf/quants.py | 17 +++ gguf-py/gguf/scripts/gguf_convert_endian.py | 1 + include/llama.h | 1 + src/llama-graph.cpp | 16 ++- src/llama-model-loader.cpp | 2 + src/llama-model.cpp | 6 +- tests/test-backend-ops.cpp | 16 ++- tests/test-quantize-fns.cpp | 62 +++++++++- 19 files changed, 353 insertions(+), 42 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index c1ecf1c651..1c5487223d 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -162,6 +162,7 @@ class ModelBase: self._is_mxfp4 = False self._fp8_as_q8 = fp8_as_q8 self._fp8_dequantized: set[str] = set() + self._fp8_e4m3_preserved: set[str] = set() # Apply heuristics to figure out typical tensor encoding based on first tensor's dtype # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie. @@ -312,7 +313,7 @@ class ModelBase: def dequant_model(self): # If all quantized tensors were already handled (e.g. pure NVFP4), skip - if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv")) for k in self.model_tensors): + if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".k_scale", ".v_scale")) for k in self.model_tensors): return tensors_to_remove: list[str] = [] @@ -487,16 +488,27 @@ class ModelBase: groups = quant_config["config_groups"] nvfp4_compressed_tensors = ( quant_format == "nvfp4-pack-quantized" - or quant_format == "mixed-precision" - and bool(groups) - and all(g.get("format") == "nvfp4-pack-quantized" for g in groups.values() if isinstance(g, dict)) + or bool(groups) + and any(g.get("format") == "nvfp4-pack-quantized" for g in groups.values() if isinstance(g, dict)) + and all( + g.get("format") == "nvfp4-pack-quantized" + or g.get("format") == "float-quantized" + and g.get("weights", {}).get("type") == "float" + and g.get("weights", {}).get("num_bits") == 8 + for g in groups.values() if isinstance(g, dict) + ) ) if len(groups) > 1 and not nvfp4_compressed_tensors: raise NotImplementedError("Can't handle multiple config groups for compressed-tensors yet") weight_config = tuple(groups.values())[0]["weights"] - if quant_format == "float-quantized" or quant_format == "int-quantized" or quant_format == "naive-quantized": + if nvfp4_compressed_tensors: + # NVFP4 tensors were handled by _generate_nvfp4_tensors and FP8 weight scales by _prepare_fp8_e4m3_tensors. + for name in self.model_tensors.keys(): + if name.endswith((".input_scale", ".k_scale", ".v_scale")): + tensors_to_remove.append(name) + elif quant_format == "float-quantized" or quant_format == "int-quantized" or quant_format == "naive-quantized": block_size = weight_config.get("block_structure", None) strategy = weight_config.get("strategy") assert strategy == "channel" or strategy == "block" @@ -537,9 +549,6 @@ class ModelBase: tensors_to_remove += [base_name + n for n in ("_packed", "_shape", "_scale")] if (base_name + "_zero_point") in self.model_tensors: tensors_to_remove.append(base_name + "_zero_point") - elif nvfp4_compressed_tensors: - # Don't error from compressed-tensors, we'll handle them in _generate_nvfp4_tensors - pass else: raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported") elif quant_method == "modelopt": @@ -573,6 +582,64 @@ class ModelBase: for name, value in new_tensors.items(): self.model_tensors[name] = value + def _prepare_fp8_e4m3_tensors(self): + if self._fp8_as_q8: + return + + scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {} + consumed: list[str] = [] + + for scale_name in list(self.model_tensors.keys()): + weight_name = None + if scale_name.endswith(".weight_scale"): + weight_name = scale_name.removesuffix("_scale") + elif scale_name.endswith("_scale_inv"): + weight_name = scale_name.removesuffix("_scale_inv") + elif scale_name.endswith(".qscale_weight"): + weight_name = scale_name.removesuffix("qscale_weight") + "weight" + + if weight_name is None or weight_name not in self.model_tensors: + continue + + weight = LazyTorchTensor.to_eager(self.model_tensors[weight_name]()) + if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2: + continue + + scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]()).float().flatten() + if scale.numel() != 1: + continue + + self._fp8_e4m3_preserved.add(weight_name) + consumed.append(scale_name) + + expert_match = re.search(r"\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$", weight_name) + if expert_match: + bid = int(expert_match.group(1)) + expert_id = int(expert_match.group(2)) + proj_type = expert_match.group(3) + merged_name = f"model.layers.{bid}.mlp.experts.{proj_type}.weight" + new_name = self.map_tensor_name(merged_name) + target_name = new_name.replace(".weight", ".scale") + entries = scale_tensors.setdefault(target_name, []) + assert isinstance(entries, list) + entries.append((expert_id, float(scale[0]))) + else: + new_name = self.map_tensor_name(weight_name) + scale_tensors[new_name.replace(".weight", ".scale")] = scale.numpy() + + for name in consumed: + self.model_tensors.pop(name, None) + + for name, values in scale_tensors.items(): + if isinstance(values, list): + values.sort(key=lambda item: item[0]) + scale = np.array([item[1] for item in values], dtype=np.float32) + else: + scale = values.astype(np.float32) + if not np.allclose(scale, 1.0, atol=1e-6): + logger.info(f" + {name} (FP8 scale, shape [{scale.size}])") + self.gguf_writer.add_tensor(name, scale) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item @@ -649,6 +716,8 @@ class ModelBase: def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: del new_name, bid # unused + if name in self._fp8_e4m3_preserved and n_dims >= 2: + return gguf.GGMLQuantizationType.F8_E4M3 # Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16. if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2: return gguf.GGMLQuantizationType.Q8_0 @@ -856,9 +925,15 @@ class ModelBase: # per-layer NVFP4/FP8) instead of a single global "NVFP4" value. nvfp4_compressed_tensors = quant_method == "compressed-tensors" and ( quant_format == "nvfp4-pack-quantized" - or quant_format == "mixed-precision" - and bool(quant_groups) - and all(g.get("format") == "nvfp4-pack-quantized" for g in quant_groups.values() if isinstance(g, dict)) + or bool(quant_groups) + and any(g.get("format") == "nvfp4-pack-quantized" for g in quant_groups.values() if isinstance(g, dict)) + and all( + g.get("format") == "nvfp4-pack-quantized" + or g.get("format") == "float-quantized" + and g.get("weights", {}).get("type") == "float" + and g.get("weights", {}).get("num_bits") == 8 + for g in quant_groups.values() if isinstance(g, dict) + ) ) if quant_algo != "NVFP4": if nvfp4_compressed_tensors: @@ -897,6 +972,7 @@ class ModelBase: self.model_tensors[input_scale_name] = inverse_scale(self.model_tensors.pop(name)) self._generate_nvfp4_tensors() + self._prepare_fp8_e4m3_tensors() self.dequant_model() # Handle empty tensor_map for models with block_count=0 (like MobileNetV5) @@ -910,10 +986,15 @@ class ModelBase: if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")): continue + preserve_fp8_e4m3 = not self._fp8_as_q8 and data_torch.dtype == torch.float8_e4m3fn + if preserve_fp8_e4m3: + self._fp8_e4m3_preserved.add(name) + data_torch = LazyTorchTensor.to_eager(data_torch) + old_dtype = data_torch.dtype # convert any unsupported data types to float32 - if data_torch.dtype not in (torch.float16, torch.float32): + if data_torch.dtype not in (torch.float16, torch.float32) and name not in self._fp8_e4m3_preserved: data_torch = data_torch.to(torch.float32) # use the first number-like part of the tensor name as the block id @@ -926,7 +1007,10 @@ class ModelBase: for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)): # TODO: why do we squeeze here? # data = data_torch.squeeze().numpy() - data = data_torch.numpy() + if preserve_fp8_e4m3: + data = data_torch.view(torch.uint8).numpy() + else: + data = data_torch.numpy() n_dims = len(data.shape) data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims) @@ -1010,12 +1094,13 @@ class ModelBase: quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else ( lambda qtype, d=data: gguf.quants.quantize(d, qtype)) - try: - data = quantize(data_qtype) - except gguf.QuantError as e: - logger.warning("%s, %s", e, "falling back to F16") - data_qtype = gguf.GGMLQuantizationType.F16 - data = quantize(data_qtype) + if not (data_qtype == gguf.GGMLQuantizationType.F8_E4M3 and data.dtype == np.uint8): + try: + data = quantize(data_qtype) + except gguf.QuantError as e: + logger.warning("%s, %s", e, "falling back to F16") + data_qtype = gguf.GGMLQuantizationType.F16 + data = quantize(data_qtype) shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape @@ -1049,6 +1134,8 @@ class ModelBase: self.ftype = gguf.LlamaFileType.MOSTLY_NVFP4 elif self._is_mxfp4: self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE + elif self._fp8_e4m3_preserved: + self.ftype = gguf.LlamaFileType.MOSTLY_F8_E4M3 # Generate parameter weight class (useful for leader boards) if not yet determined if self.metadata.size_label is None and total_params > 0: diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index b88b7e54a5..6e677d60ae 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -381,6 +381,8 @@ extern "C" { GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + typedef struct { uint8_t bits; } ggml_fp8_e4m3_t; + struct ggml_object; struct ggml_context; struct ggml_cgraph; @@ -430,7 +432,8 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + GGML_TYPE_F8_E4M3 = 43, + GGML_TYPE_COUNT = 44, }; // precision @@ -475,6 +478,7 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors }; // available tensor operations: diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 87a329f269..9777d8dca0 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -296,6 +296,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_F8_E4M3] = { + .from_float = quantize_row_f8_e4m3, + .vec_dot = ggml_vec_dot_f8_e4m3_f32, + .vec_dot_type = GGML_TYPE_F32, + .nrows = 1, + }, [GGML_TYPE_Q2_K] = { .from_float = quantize_row_q2_K, .vec_dot = ggml_vec_dot_q2_K_q8_K, @@ -2848,7 +2854,7 @@ struct ggml_cplan ggml_graph_plan( case GGML_OP_CPY: case GGML_OP_DUP: { - if (ggml_is_quantized(node->type) || + if (ggml_is_quantized(node->type) || node->type == GGML_TYPE_F8_E4M3 || // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32 (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) || (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) || @@ -2862,7 +2868,7 @@ struct ggml_cplan ggml_graph_plan( case GGML_OP_ADD_ID: case GGML_OP_ADD1: { - if (ggml_is_quantized(node->src[0]->type)) { + if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; @@ -2914,7 +2920,7 @@ struct ggml_cplan ggml_graph_plan( } break; case GGML_OP_OUT_PROD: { - if (ggml_is_quantized(node->src[0]->type) || + if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3 || node->src[0]->type == GGML_TYPE_F16) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 266261c5e5..362b455ee9 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -564,7 +564,7 @@ void ggml_compute_forward_dup( } break; default: { - if (ggml_is_quantized(src0->type) && dst->type == GGML_TYPE_F32) { + if ((ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3) && dst->type == GGML_TYPE_F32) { ggml_compute_forward_dup_from_q(params, dst); break; } @@ -605,7 +605,7 @@ static void ggml_compute_forward_add_q_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - GGML_ASSERT(ggml_is_quantized(src0->type)); + GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3); GGML_ASSERT(src1->type == GGML_TYPE_F32); // rows per thread @@ -673,6 +673,7 @@ void ggml_compute_forward_add( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -950,7 +951,7 @@ static void ggml_compute_forward_add1_q_f32( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - GGML_ASSERT(ggml_is_quantized(src0->type)); + GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3); GGML_ASSERT(dst->type == src0->type); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -1125,6 +1126,7 @@ void ggml_compute_forward_add1( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1256,6 +1258,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -4658,6 +4661,7 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_Q8_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -4935,6 +4939,7 @@ void ggml_compute_forward_set( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5160,6 +5165,7 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -5917,6 +5923,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 5e36459f8c..112211bbe9 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -62,6 +62,10 @@ void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i quantize_row_nvfp4_ref(x, y, k); } +void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_f8_e4m3_ref(x, y, k); +} + // // 2-6 bit quantization in super-blocks // @@ -362,6 +366,23 @@ void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } +void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const ggml_fp8_e4m3_t * GGML_RESTRICT x = vx; + const float * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + for (int i = 0; i < n; ++i) { + sumf += ggml_f8_e4m3_to_fp32(x[i].bits) * y[i]; + } + *s = sumf; +} + void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index 93ea7eeffe..f04be52f54 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -23,6 +23,7 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -40,6 +41,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, // Dot product void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 62b76abbce..cce45caaf2 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -552,6 +552,57 @@ static inline uint8_t ggml_fp32_to_ue4m3(float x) { return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man); } +static inline float ggml_f8_e4m3_to_fp32(uint8_t x) { + const uint8_t ax = x & 0x7F; + if (ax == 0x7F) { + return NAN; + } + + const int exp = (ax >> 3) & 0xF; + const int man = ax & 0x7; + const float value = exp == 0 + ? ldexpf((float) man, -9) + : ldexpf(1.0f + (float) man / 8.0f, exp - 7); + return x & 0x80 ? -value : value; +} + +static inline int ggml_round_to_nearest_even(float x) { + const int value = (int) floorf(x); + const float fraction = x - value; + return fraction > 0.5f || (fraction == 0.5f && (value & 1)) ? value + 1 : value; +} + +static inline uint8_t ggml_fp32_to_f8_e4m3(float x) { + const uint8_t sign = signbit(x) ? 0x80 : 0; + x = fabsf(x); + + if (isnan(x)) { + return sign | 0x7F; + } + if (x == 0.0f) { + return sign; + } + if (isinf(x) || x >= 448.0f) { + return sign | 0x7E; + } + if (x < 0.015625f) { + return sign | (uint8_t) ggml_round_to_nearest_even(x * 512.0f); + } + + int exp; + const float mantissa = frexpf(x, &exp) * 2.0f; + int encoded_exp = exp + 6; + int encoded_man = ggml_round_to_nearest_even((mantissa - 1.0f) * 8.0f); + if (encoded_man == 8) { + encoded_man = 0; + encoded_exp++; + } + if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) { + return sign | 0x7E; + } + return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man; +} + /** * Converts brain16 to float32. * diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 1ebc50a763..06912a001a 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -416,6 +416,12 @@ void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RE } } +void quantize_row_f8_e4m3_ref(const float * GGML_RESTRICT x, ggml_fp8_e4m3_t * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i].bits = ggml_fp32_to_f8_e4m3(x[i]); + } +} + void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK1_0; @@ -611,6 +617,12 @@ void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_REST } } +void dequantize_row_f8_e4m3(const ggml_fp8_e4m3_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + for (int64_t i = 0; i < k; ++i) { + y[i] = ggml_f8_e4m3_to_fp32(x[i].bits); + } +} + // // 2-6 bit quantization in super-blocks // @@ -2311,6 +2323,12 @@ size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * ggml_row_size(GGML_TYPE_NVFP4, n_per_row); } +size_t quantize_f8_e4m3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_UNUSED(quant_weights); + quantize_row_f8_e4m3_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_F8_E4M3, n_per_row); +} + // ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs) void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k) { @@ -5567,6 +5585,16 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte GGML_UNUSED(data); GGML_UNUSED(nb); } break; + case GGML_TYPE_F8_E4M3: + { + const ggml_fp8_e4m3_t * q = (const ggml_fp8_e4m3_t *) data; + for (size_t i = 0; i < nb; ++i) { + if ((q[i].bits & 0x7F) == 0x7F) { + fprintf(stderr, "%s: found NaN at index %zu in row of %zu F8_E4M3 values\n", __func__, i, nb); + return false; + } + } + } break; case GGML_TYPE_Q2_K: { VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af1..ca36d13695 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -25,6 +25,7 @@ GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_f8_e4m3_ref(const float * GGML_RESTRICT x, ggml_fp8_e4m3_t * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k); @@ -54,6 +55,7 @@ GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GG GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_f8_e4m3(const ggml_fp8_e4m3_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -104,6 +106,7 @@ GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_f8_e4m3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API void iq2xs_init_impl(enum ggml_type type); GGML_API void iq2xs_free_impl(enum ggml_type type); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 6257cdbe58..6bae112d6d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -765,6 +765,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_nvfp4, .from_float_ref = (ggml_from_float_t)quantize_row_nvfp4_ref, }, + [GGML_TYPE_F8_E4M3] = { + .type_name = "f8_e4m3", + .blck_size = 1, + .type_size = sizeof(ggml_fp8_e4m3_t), + .is_quantized = false, + .to_float = (ggml_to_float_t) dequantize_row_f8_e4m3, + .from_float_ref = (ggml_from_float_t) quantize_row_f8_e4m3_ref, + }, [GGML_TYPE_Q2_K] = { .type_name = "q2_K", .blck_size = QK_K, @@ -1440,6 +1448,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break; case GGML_FTYPE_MOSTLY_MXFP4: wtype = GGML_TYPE_MXFP4; break; case GGML_FTYPE_MOSTLY_NVFP4: wtype = GGML_TYPE_NVFP4; break; + case GGML_FTYPE_MOSTLY_F8_E4M3: wtype = GGML_TYPE_F8_E4M3; break; case GGML_FTYPE_MOSTLY_Q2_K: wtype = GGML_TYPE_Q2_K; break; case GGML_FTYPE_MOSTLY_Q3_K: wtype = GGML_TYPE_Q3_K; break; case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break; @@ -8009,6 +8018,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_F8_E4M3: result = quantize_f8_e4m3(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q2_K: result = quantize_q2_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q3_K: result = quantize_q3_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_K: result = quantize_q4_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 399d31f1d5..84f36d0eaa 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -5573,6 +5573,7 @@ class GGMLQuantizationType(IntEnum): NVFP4 = 40 Q1_0 = 41 Q2_0 = 42 + F8_E4M3 = 43 class ExpertGatingFuncType(IntEnum): @@ -5629,6 +5630,7 @@ class LlamaFileType(IntEnum): MOSTLY_NVFP4 = 39 # except 1d tensors MOSTLY_Q1_0 = 40 # except 1d tensors MOSTLY_Q2_0 = 41 # except 1d tensors + MOSTLY_F8_E4M3 = 42 # except 1d tensors GUESSED = 1024 # not specified in the model file @@ -5766,6 +5768,7 @@ GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = { GGMLQuantizationType.NVFP4: (64, 4 + 32), GGMLQuantizationType.Q1_0: (128, 2 + 16), GGMLQuantizationType.Q2_0: (64, 2 + 16), + GGMLQuantizationType.F8_E4M3: (1, 1), } diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py index 80966b6ef1..fe03f52f64 100644 --- a/gguf-py/gguf/quants.py +++ b/gguf-py/gguf/quants.py @@ -764,6 +764,23 @@ class NVFP4(__Quant, qtype=GGMLQuantizationType.NVFP4): return (d * vals.astype(np.float32)).reshape(n_super, 64) +class F8_E4M3(__Quant, qtype=GGMLQuantizationType.F8_E4M3): + @classmethod + def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + bits = blocks.astype(np.uint8) + sign = np.where(bits & 0x80, -1.0, 1.0) + magnitude = bits & 0x7F + exponent = magnitude >> 3 + mantissa = magnitude & 0x07 + values = np.where( + exponent == 0, + np.ldexp(mantissa.astype(np.float32), -9), + np.ldexp(1.0 + mantissa.astype(np.float32) / 8.0, exponent.astype(np.int32) - 7), + ) + values = np.where(magnitude == 0x7F, np.nan, values) + return sign * values + + class IQ2_XXS(__Quant, qtype=GGMLQuantizationType.IQ2_XXS): ksigns: bytes = ( b"\x00\x81\x82\x03\x84\x05\x06\x87\x88\x09\x0a\x8b\x0c\x8d\x8e\x0f" diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.py index 31618acfc7..08d03f20d5 100755 --- a/gguf-py/gguf/scripts/gguf_convert_endian.py +++ b/gguf-py/gguf/scripts/gguf_convert_endian.py @@ -84,6 +84,7 @@ byteswap_tensors = { gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0, gguf.GGMLQuantizationType.MXFP4: byteswap_noop, gguf.GGMLQuantizationType.NVFP4: byteswap_noop, + gguf.GGMLQuantizationType.F8_E4M3: byteswap_noop, } diff --git a/include/llama.h b/include/llama.h index ef7a012c43..1903717132 100644 --- a/include/llama.h +++ b/include/llama.h @@ -156,6 +156,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors + LLAMA_FTYPE_MOSTLY_F8_E4M3 = 42, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 8ea441f441..a75f632b8c 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1724,12 +1724,16 @@ ggml_tensor * llm_graph_context::build_ffn( return false; }; - GGML_ASSERT(!up_s || !up_b || !up || up->type != GGML_TYPE_NVFP4); - GGML_ASSERT(!gate_s || !gate_b || !gate || gate->type != GGML_TYPE_NVFP4); - GGML_ASSERT(!down_s || !down_b || !down || down->type != GGML_TYPE_NVFP4); - GGML_ASSERT(!up_s || !up || up->type != GGML_TYPE_NVFP4 || !has_lora(up)); - GGML_ASSERT(!gate_s || !gate || gate->type != GGML_TYPE_NVFP4 || !has_lora(gate)); - GGML_ASSERT(!down_s || !down || down->type != GGML_TYPE_NVFP4 || !has_lora(down)); + auto is_scaled_low_precision = [](ggml_tensor * w) { + return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3); + }; + + GGML_ASSERT(!up_s || !up_b || !is_scaled_low_precision(up)); + GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate)); + GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down)); + GGML_ASSERT(!up_s || !is_scaled_low_precision(up) || !has_lora(up)); + GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate)); + GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down)); ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur; cb(tmp, "ffn_up", il); diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 91bb5e7cc8..161f36c7a3 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -39,6 +39,7 @@ const char * llama_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break; case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break; case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break; + case LLAMA_FTYPE_MOSTLY_F8_E4M3: name = LLAMA_FTYPE_PREFIX "F8_E4M3"; break; case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break; case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break; case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break; @@ -772,6 +773,7 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break; + case GGML_TYPE_F8_E4M3: ftype = LLAMA_FTYPE_MOSTLY_F8_E4M3; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index b837e27654..14078fde52 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1661,7 +1661,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } // output scales - if (output && output->type == GGML_TYPE_NVFP4) { + if (output && (output->type == GGML_TYPE_NVFP4 || output->type == GGML_TYPE_F8_E4M3)) { // weight scale if (!output_s) { output_s = create_tensor(tn(LLM_TENSOR_OUTPUT, "scale"), {1}, TENSOR_NOT_REQUIRED); @@ -1674,11 +1674,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } ml.done_getting_tensors(); - // Tied NVFP4 output is valid when no separate LM-head scale tensors are present. + // Tied low-precision output is valid when no separate LM-head scale tensors are present. // If sidecar scales exist, the output weight must be an actual output tensor. GGML_ASSERT(!(output && tok_embd && strcmp(output->name, tok_embd->name) == 0 && - output->type == GGML_TYPE_NVFP4 && + (output->type == GGML_TYPE_NVFP4 || output->type == GGML_TYPE_F8_E4M3) && (output_s || output_in_s))); // populate tensors_by_name for (auto & [_, ctx_ptr] : ml.ctx_map) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 2cf9d9caf0..be6ee77161 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -84,7 +84,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float)); - } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { + } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F8_E4M3 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0); // dummy importance matrix @@ -258,7 +258,7 @@ static std::vector tensor_to_float(const ggml_tensor * t) { const auto * tt = ggml_get_type_traits(t->type); size_t bs = ggml_blck_size(t->type); std::vector vq(ggml_blck_size(t->type)); - bool quantized = ggml_is_quantized(t->type); + bool uses_type_conversion = ggml_is_quantized(t->type) || t->type == GGML_TYPE_F8_E4M3; // access elements by index to avoid gaps in views for (int64_t i3 = 0; i3 < t->ne[3]; i3++) { @@ -280,7 +280,7 @@ static std::vector tensor_to_float(const ggml_tensor * t) { tv.push_back((float)*(int16_t *) &buf[i]); } else if (t->type == GGML_TYPE_I8) { tv.push_back((float)*(int8_t *) &buf[i]); - } else if (quantized) { + } else if (uses_type_conversion) { tt->to_float(&buf[i], vq.data(), bs); tv.insert(tv.end(), vq.begin(), vq.end()); } else { @@ -4639,6 +4639,9 @@ struct test_mul_mat : public test_case { if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { return 2e-2; } + if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) { + return 5e-3; + } return max_nmse_err(); } @@ -4840,6 +4843,9 @@ struct test_mul_mat_id : public test_case { if ((type_a == GGML_TYPE_MXFP4 || type_a == GGML_TYPE_NVFP4) && backend_has_feature(backend, "BLACKWELL_NATIVE_FP4")) { return 2e-2; } + if (type_a == GGML_TYPE_F8_E4M3 && backend_has_feature(backend, "NATIVE_FP8")) { + return 5e-3; + } return max_nmse_err(); } @@ -8559,6 +8565,7 @@ static const ggml_type all_types[] = { GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, + GGML_TYPE_F8_E4M3, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, @@ -8578,6 +8585,7 @@ static const ggml_type base_types[] = { GGML_TYPE_Q4_1, // for I8MM tests GGML_TYPE_Q4_K, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, // TODO: or "other" + GGML_TYPE_F8_E4M3, GGML_TYPE_IQ2_XXS }; @@ -10434,7 +10442,7 @@ static std::vector> make_test_cases_eval() { continue; } for (bool with_lane_scale : {false, true}) { - if (with_lane_scale && type != GGML_TYPE_NVFP4) { + if (with_lane_scale && type != GGML_TYPE_NVFP4 && type != GGML_TYPE_F8_E4M3) { continue; } test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256, diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14ce..5d57ef650e 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -86,10 +86,9 @@ static float dot_product(const float * a1, const float * a2, size_t test_size) { static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) { GGML_UNUSED(qfns); - std::vector tmp_q1(2*test_size); - std::vector tmp_q2(2*test_size); - const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type); + std::vector tmp_q1(2*test_size); + std::vector tmp_q2(ggml_row_size(qfns_cpu->vec_dot_type, test_size)); qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size); vdot->from_float(test_data2, tmp_q2.data(), test_size); @@ -126,6 +125,62 @@ static int test_vec_dot_f32(bool verbose) { return num_failed; } +static float f8_e4m3_to_fp32_ref(uint8_t value) { + const uint8_t magnitude = value & 0x7F; + if (magnitude == 0x7F) { + return NAN; + } + const int exponent = magnitude >> 3; + const int mantissa = magnitude & 0x07; + const float result = exponent == 0 + ? ldexpf((float) mantissa, -9) + : ldexpf(1.0f + (float) mantissa / 8.0f, exponent - 7); + return value & 0x80 ? -result : result; +} + +static int test_f8_e4m3(bool verbose) { + static_assert(sizeof(ggml_fp8_e4m3_t) == 1); + + const auto * traits = ggml_get_type_traits(GGML_TYPE_F8_E4M3); + const auto * traits_cpu = ggml_get_type_traits_cpu(GGML_TYPE_F8_E4M3); + std::vector encoded(256); + std::vector roundtrip(256); + std::vector decoded(256); + int num_failed = 0; + + const bool traits_failed = traits->blck_size != 1 || traits->type_size != sizeof(ggml_fp8_e4m3_t) || traits->is_quantized; + num_failed += traits_failed; + if (verbose || traits_failed) { + printf("f8_e4m3 scalar type traits: %s\n", RESULT_STR[traits_failed]); + } + + for (int i = 0; i < 256; ++i) { + encoded[i] = (uint8_t) i; + } + traits->to_float(encoded.data(), decoded.data(), decoded.size()); + traits_cpu->from_float(decoded.data(), roundtrip.data(), decoded.size()); + + for (int i = 0; i < 256; ++i) { + const float expected = f8_e4m3_to_fp32_ref((uint8_t) i); + const bool is_nan = (i & 0x7F) == 0x7F; + const bool decode_failed = is_nan ? !isnan(decoded[i]) + : decoded[i] != expected || (expected == 0.0f && signbit(decoded[i]) != signbit(expected)); + const bool encode_failed = !is_nan && roundtrip[i] != encoded[i]; + if (decode_failed || encode_failed) { + num_failed++; + if (verbose) { + printf("f8_e4m3 code 0x%02x failed: decoded=%f expected=%f roundtrip=0x%02x\n", + i, decoded[i], expected, roundtrip[i]); + } + } + } + + if (verbose || num_failed) { + printf("f8_e4m3 exhaustive conversion: %s (%d failures)\n", RESULT_STR[num_failed != 0], num_failed); + } + return num_failed; +} + static int test_vec_dot_q(bool verbose) { int num_failed = 0; @@ -220,6 +275,7 @@ int main(int argc, char * argv[]) { int num_failed = 0; num_failed += test_vec_dot_f32(verbose); + num_failed += test_f8_e4m3(verbose); num_failed += test_vec_dot_q(verbose); if (num_failed || verbose) {