From 3a3f76f2e8b4ea855ab119e5af28f5f4fb207c9a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 11 Sep 2026 16:05:37 -0500 Subject: [PATCH] cont : rename common_schema to common_chat_schema --- common/chat-auto-parser-generator.cpp | 2 +- common/json-schema-to-grammar.cpp | 112 +++++++------- common/json-schema-to-grammar.h | 4 +- common/json-schema.cpp | 204 ++++++++++++------------- common/json-schema.h | 76 +++++----- common/parsers/deepseek.cpp | 2 +- common/parsers/minicpm5.cpp | 2 +- common/parsers/minimax-m3.cpp | 28 ++-- common/parsers/muse-glimmer.cpp | 2 +- common/parsers/parsers.cpp | 6 +- common/parsers/parsers.h | 2 +- common/parsers/qwen3-coder.cpp | 2 +- common/peg-parser.cpp | 6 +- common/peg-parser.h | 6 +- docs/development/parsing.md | 4 +- tests/test-json-schema-to-grammar.cpp | 6 +- tests/test-json-schema.cpp | 208 +++++++++++++------------- 17 files changed, 336 insertions(+), 336 deletions(-) diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index 17db540be3..b78789d8c0 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -371,7 +371,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte // Build parser for each argument, separating required and optional std::vector required_parsers; std::vector optional_parsers; - foreach_parameter(func, [&](const common_schema_property & param, const common_schema_document_ptr & doc) { + foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { auto arg = p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) + arguments.name_suffix) + diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index e2c5e7c038..e0426098c0 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -338,7 +338,7 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { return 2 + n_hex; } -class common_schema_converter { +class common_chat_schema_converter { private: friend std::string build_grammar(const std::function & cb, const common_grammar_options & options); bool _dotall; @@ -348,7 +348,7 @@ private: std::vector _warnings; template - static const T & as(const common_schema & node) { + static const T & as(const common_chat_schema & node) { return static_cast(node); } @@ -367,7 +367,7 @@ private: return key; } - std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { + std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { std::vector rules; rules.reserve(alt_schemas.size()); for (size_t i = 0; i < alt_schemas.size(); i++) { @@ -678,7 +678,7 @@ private: return out.str(); } - std::string _resolve_ref(const common_schema_ref & schema) { + std::string _resolve_ref(const common_chat_schema_ref & schema) { auto it = schema.ref.find('#'); std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref; static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)"); @@ -696,10 +696,10 @@ private: } std::string _build_object_rule( - const std::vector> & properties, + const std::vector> & properties, const std::unordered_set & required, const std::string & name, - const common_schema * additional_properties) + const common_chat_schema * additional_properties) { std::vector required_props; std::vector optional_props; @@ -724,7 +724,7 @@ private: if (additional_properties) { std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; std::string value_rule = - additional_properties->kind() != common_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value") + additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value") : _add_primitive("value", PRIMITIVE_RULES.at("value")); auto key_rule = @@ -812,11 +812,11 @@ private: } public: - explicit common_schema_converter(bool dotall) : _dotall(dotall) { + explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) { _rules["space"] = SPACE_RULE; } - std::string add_schema(const std::string & name, const common_schema & schema) { + std::string add_schema(const std::string & name, const common_chat_schema & schema) { return visit(schema, name); } @@ -828,31 +828,31 @@ public: return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type)); } - std::string _visit_all_of(const common_schema_all_of & schema, const std::string & name, const std::string & rule_name) { + std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) { std::unordered_set required; - std::vector> properties; + std::vector> properties; std::map enum_values; - std::function add_component = [&](const common_schema & comp, bool is_required) { - if (comp.kind() == common_schema::KIND_REF) { - if (const auto * target = as(comp).target) { + std::function add_component = [&](const common_chat_schema & comp, bool is_required) { + if (comp.kind() == common_chat_schema::KIND_REF) { + if (const auto * target = as(comp).target) { add_component(*target, is_required); } - } else if (comp.kind() == common_schema::KIND_OBJECT) { - for (const auto & prop : as(comp).properties) { + } else if (comp.kind() == common_chat_schema::KIND_OBJECT) { + for (const auto & prop : as(comp).properties) { properties.emplace_back(prop.name, prop.schema.get()); if (is_required) { required.insert(prop.name); } } - } else if (comp.kind() == common_schema::KIND_ENUM) { - for (const auto & v : as(comp).values) { + } else if (comp.kind() == common_chat_schema::KIND_ENUM) { + for (const auto & v : as(comp).values) { enum_values[_generate_constant_rule(v)] += 1; } } }; for (const auto & child : schema.children) { - if (child->kind() == common_schema::KIND_ANY_OF) { - for (const auto & alt : as(*child).children) { + if (child->kind() == common_chat_schema::KIND_ANY_OF) { + for (const auto & alt : as(*child).children) { add_component(*alt, false); } } else { @@ -873,32 +873,32 @@ public: return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr)); } - std::string visit(const common_schema & schema, const std::string & name) { + std::string visit(const common_chat_schema & schema, const std::string & name) { std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; std::string sub_name = name + (name.empty() ? "" : "-"); switch (schema.kind()) { - case common_schema::KIND_REF: - return _add_rule(rule_name, _resolve_ref(as(schema))); - case common_schema::KIND_ANY_OF: - return _add_rule(rule_name, _generate_union_rule(name, as(schema).children)); - case common_schema::KIND_ALL_OF: - return _visit_all_of(as(schema), name, rule_name); - case common_schema::KIND_CONST: - return _add_rule(rule_name, _generate_constant_rule(as(schema).value)); - case common_schema::KIND_ENUM: { + case common_chat_schema::KIND_REF: + return _add_rule(rule_name, _resolve_ref(as(schema))); + case common_chat_schema::KIND_ANY_OF: + return _add_rule(rule_name, _generate_union_rule(name, as(schema).children)); + case common_chat_schema::KIND_ALL_OF: + return _visit_all_of(as(schema), name, rule_name); + case common_chat_schema::KIND_CONST: + return _add_rule(rule_name, _generate_constant_rule(as(schema).value)); + case common_chat_schema::KIND_ENUM: { std::vector enum_values; - for (const auto & v : as(schema).values) { + for (const auto & v : as(schema).values) { enum_values.push_back(_generate_constant_rule(v)); } return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); } - case common_schema::KIND_OBJECT: { - const auto & obj = as(schema); - if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_schema::KIND_ANY) { + case common_chat_schema::KIND_OBJECT: { + const auto & obj = as(schema); + if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) { return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); } - std::vector> properties; + std::vector> properties; std::unordered_set required; for (const auto & prop : obj.properties) { properties.emplace_back(prop.name, prop.schema.get()); @@ -908,8 +908,8 @@ public: } return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get())); } - case common_schema::KIND_TUPLE: { - const auto & items = as(schema).items; + case common_chat_schema::KIND_TUPLE: { + const auto & items = as(schema).items; std::string rule = "\"[\" space "; for (size_t i = 0; i < items.size(); i++) { if (i > 0) { @@ -920,25 +920,25 @@ public: rule += " space \"]\""; return _add_rule(rule_name, rule); } - case common_schema::KIND_ARRAY: { - const auto & arr = as(schema); - if (arr.items->kind() == common_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) { + case common_chat_schema::KIND_ARRAY: { + const auto & arr = as(schema); + if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) { return _visit_primitive(rule_name, "array"); } std::string item_rule_name = visit(*arr.items, sub_name + "item"); int max_items = arr.max_items < 0 ? std::numeric_limits::max() : arr.max_items; return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\""); } - case common_schema::KIND_STRING: { - const auto & str = as(schema); + case common_chat_schema::KIND_STRING: { + const auto & str = as(schema); if (!str.pattern.empty()) { return _visit_pattern(str.pattern, rule_name); } - if (str.format == common_schema::FORMAT_UUID) { + if (str.format == common_chat_schema::FORMAT_UUID) { return _visit_primitive(rule_name, "uuid"); } - if (str.format != common_schema::FORMAT_NONE) { - std::string prim_name = std::string(str.format == common_schema::FORMAT_DATE ? "date" : str.format == common_schema::FORMAT_TIME ? "time" : "date-time") + "-string"; + if (str.format != common_chat_schema::FORMAT_NONE) { + std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string"; return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); } if (str.min_length > 0 || str.max_length >= 0) { @@ -948,8 +948,8 @@ public: } return _visit_primitive(rule_name, "string"); } - case common_schema::KIND_INTEGER: { - const auto & i = as(schema); + case common_chat_schema::KIND_INTEGER: { + const auto & i = as(schema); if (i.minimum == std::numeric_limits::min() && i.maximum == std::numeric_limits::max()) { return _visit_primitive(rule_name, "integer"); } @@ -959,13 +959,13 @@ public: out << ")"; return _add_rule(rule_name, out.str()); } - case common_schema::KIND_NUMBER: + case common_chat_schema::KIND_NUMBER: return _visit_primitive(rule_name, "number"); - case common_schema::KIND_BOOLEAN: + case common_chat_schema::KIND_BOOLEAN: return _visit_primitive(rule_name, "boolean"); - case common_schema::KIND_NULL: + case common_chat_schema::KIND_NULL: return _visit_primitive(rule_name, "null"); - case common_schema::KIND_ANY: + case common_chat_schema::KIND_ANY: return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value"))); } return ""; @@ -998,26 +998,26 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) (void)force_gbnf; #endif // LLAMA_USE_LLGUIDANCE try { - return json_schema_to_grammar(common_schema_from_json(schema)); + return json_schema_to_grammar(common_chat_schema_from_json(schema)); } catch (const std::runtime_error & e) { throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what()); } } -std::string json_schema_to_grammar(const common_schema_document & schema) { - common_schema_converter converter(false); +std::string json_schema_to_grammar(const common_chat_schema_document & schema) { + common_chat_schema_converter converter(false); converter.visit(*schema.root, ""); converter.check_errors(); return converter.format_grammar(); } std::string build_grammar(const std::function & cb, const common_grammar_options & options) { - common_schema_converter converter(options.dotall); + common_chat_schema_converter converter(options.dotall); common_grammar_builder builder { /* .add_rule = */ [&](const std::string & name, const std::string & rule) { return converter._add_rule(name, rule); }, - /* .add_schema = */ [&](const std::string & name, const common_schema & schema) { + /* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) { return converter.add_schema(name == "root" ? "" : name, schema); }, }; diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h index 4ef811c19e..b928c250bb 100644 --- a/common/json-schema-to-grammar.h +++ b/common/json-schema-to-grammar.h @@ -7,11 +7,11 @@ #include std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false); -std::string json_schema_to_grammar(const common_schema_document & schema); +std::string json_schema_to_grammar(const common_chat_schema_document & schema); struct common_grammar_builder { std::function add_rule; - std::function add_schema; + std::function add_schema; }; struct common_grammar_options { diff --git a/common/json-schema.cpp b/common/json-schema.cpp index 3bf87d5d5a..6898840e7d 100644 --- a/common/json-schema.cpp +++ b/common/json-schema.cpp @@ -9,15 +9,15 @@ #include #include -class common_schema_builder { +class common_chat_schema_builder { const common_json & root_; - common_schema_document & doc_; + common_chat_schema_document & doc_; // the targets built here, moved into doc_ once the whole schema is built - std::map refs_; + std::map refs_; // ref nodes get their target once every $ref is built, a cycle would otherwise need it too early - std::vector pending_; + std::vector pending_; [[noreturn]] static void fail(const std::string & path, const std::string & msg) { throw std::runtime_error("JSON schema error at " + path + ": " + msg); @@ -47,9 +47,9 @@ class common_schema_builder { return (int64_t) (round_up ? std::ceil(d) : std::floor(d)); } - static common_schema::string_format get_format(const common_json & schema, const std::string & path) { + static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) { if (!schema.contains("format")) { - return common_schema::FORMAT_NONE; + return common_chat_schema::FORMAT_NONE; } const common_json & value = schema.at("format"); if (!value.is_string()) { @@ -57,18 +57,18 @@ class common_schema_builder { } std::string format = value.get(); if (format == "date") { - return common_schema::FORMAT_DATE; + return common_chat_schema::FORMAT_DATE; } if (format == "time") { - return common_schema::FORMAT_TIME; + return common_chat_schema::FORMAT_TIME; } if (format == "date-time") { - return common_schema::FORMAT_DATE_TIME; + return common_chat_schema::FORMAT_DATE_TIME; } if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) { - return common_schema::FORMAT_UUID; + return common_chat_schema::FORMAT_UUID; } - return common_schema::FORMAT_NONE; + return common_chat_schema::FORMAT_NONE; } const common_json & resolve_ref(const std::string & ref, const std::string & path) { @@ -96,7 +96,7 @@ class common_schema_builder { return *target; } - common_schema_ptr build_ref(const common_json & value, const std::string & path) { + common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) { if (!value.is_string()) { fail(path, "$ref must be a string"); } @@ -109,13 +109,13 @@ class common_schema_builder { refs_[ref] = nullptr; refs_[ref] = build_node(resolve_ref(ref, path), ref); } - auto node = std::make_unique(ref); + auto node = std::make_unique(ref); pending_.push_back(node.get()); return node; } template - common_schema_ptr build_alternatives(const common_json & alts, const std::string & path) { + common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) { if (!alts.is_array()) { fail(path, "must be an array of schemas"); } @@ -130,8 +130,8 @@ class common_schema_builder { return node; } - common_schema_ptr build_object(const common_json & schema, const std::string & path) { - auto node = std::make_unique(); + common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); std::unordered_set required; if (schema.contains("required") && schema.at("required").is_array()) { @@ -156,7 +156,7 @@ class common_schema_builder { const common_json & additional = schema.at("additionalProperties"); if (additional.is_boolean()) { if (additional.get()) { - node->additional_properties = std::make_unique(); + node->additional_properties = std::make_unique(); } } else if (additional.is_object()) { node->additional_properties = build_node(additional, path + "/additionalProperties"); @@ -165,20 +165,20 @@ class common_schema_builder { } } else if (!schema.contains("properties")) { // {"type": "object"} on its own accepts any object - node->additional_properties = std::make_unique(); + node->additional_properties = std::make_unique(); } return node; } - common_schema_ptr build_array(const common_json & schema, const std::string & path) { - auto node = std::make_unique(); + common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); if (schema.contains("items") || schema.contains("prefixItems")) { // "items" wins when both are present; as in the converter, a schema instead of an array is the item schema const std::string key = schema.contains("items") ? "items" : "prefixItems"; const common_json & items = schema.at(key); if (items.is_array()) { - auto tuple = std::make_unique(); + auto tuple = std::make_unique(); size_t i = 0; for (const auto & item : items) { tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++))); @@ -187,15 +187,15 @@ class common_schema_builder { } node->items = build_node(items, path + "/" + key); } else { - node->items = std::make_unique(); + node->items = std::make_unique(); } node->min_items = get_count(schema, "minItems", path, 0); node->max_items = get_count(schema, "maxItems", path, -1); return node; } - common_schema_ptr build_string(const common_json & schema, const std::string & path) { - auto node = std::make_unique(); + common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); if (schema.contains("pattern")) { const common_json & pattern = schema.at("pattern"); if (!pattern.is_string()) { @@ -209,8 +209,8 @@ class common_schema_builder { return node; } - common_schema_ptr build_integer(const common_json & schema, const std::string & path) { - auto node = std::make_unique(); + common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); if (schema.contains("minimum")) { node->minimum = get_bound(schema, "minimum", path, /* round_up */ true); } else if (schema.contains("exclusiveMinimum")) { @@ -224,7 +224,7 @@ class common_schema_builder { return node; } - common_schema_ptr build_node(const common_json & schema, const std::string & path) { + common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) { if (!schema.is_object()) { fail(path, "schema must be an object"); } @@ -233,7 +233,7 @@ class common_schema_builder { } if (schema.contains("oneOf") || schema.contains("anyOf")) { const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf"; - return build_alternatives(schema.at(key), path + "/" + key); + return build_alternatives(schema.at(key), path + "/" + key); } common_json type; @@ -245,7 +245,7 @@ class common_schema_builder { if (type.empty()) { fail(path, "type must not be empty"); } - auto node = std::make_unique(); + auto node = std::make_unique(); size_t i = 0; for (const auto & t : type) { common_json alt = schema; @@ -255,14 +255,14 @@ class common_schema_builder { return node; } if (schema.contains("const")) { - return std::make_unique(schema.at("const")); + return std::make_unique(schema.at("const")); } if (schema.contains("enum")) { const common_json & values = schema.at("enum"); if (!values.is_array() || values.empty()) { fail(path, "enum must be a non-empty array"); } - auto node = std::make_unique(); + auto node = std::make_unique(); for (const auto & value : values) { node->values.push_back(value); } @@ -282,25 +282,25 @@ class common_schema_builder { return build_object(schema, path); } if (schema.contains("allOf")) { - return build_alternatives(schema.at("allOf"), path + "/allOf"); + return build_alternatives(schema.at("allOf"), path + "/allOf"); } if (schema.contains("items") || schema.contains("prefixItems")) { return build_array(schema, path); } - if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_schema::FORMAT_NONE) { + if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) { return build_string(schema, path); } - return std::make_unique(); + return std::make_unique(); } if (type_name == "object") { if (!has_properties && schema.contains("allOf")) { - return build_alternatives(schema.at("allOf"), path + "/allOf"); + return build_alternatives(schema.at("allOf"), path + "/allOf"); } return build_object(schema, path); } if (type_name == "string") { if (schema.contains("allOf")) { - return build_alternatives(schema.at("allOf"), path + "/allOf"); + return build_alternatives(schema.at("allOf"), path + "/allOf"); } return build_string(schema, path); } @@ -311,21 +311,21 @@ class common_schema_builder { return build_integer(schema, path); } if (type_name == "number") { - return std::make_unique(); + return std::make_unique(); } if (type_name == "boolean") { - return std::make_unique(); + return std::make_unique(); } if (type_name == "null") { - return std::make_unique(); + return std::make_unique(); } fail(path, "unrecognized type " + type_name); } public: - common_schema_builder(const common_json & root, common_schema_document & doc) : root_(root), doc_(doc) {} + common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {} - common_schema_ptr build() { + common_chat_schema_ptr build() { auto node = build_node(root_, "#"); for (auto & entry : refs_) { doc_.refs[entry.first] = std::move(entry.second); @@ -337,64 +337,64 @@ class common_schema_builder { } }; -common_schema_document common_schema_from_json(const common_json & schema) { - common_schema_document doc; - doc.root = common_schema_builder(schema, doc).build(); +common_chat_schema_document common_chat_schema_from_json(const common_json & schema) { + common_chat_schema_document doc; + doc.root = common_chat_schema_builder(schema, doc).build(); return doc; } -static common_schema::value_type json_type(const common_json & value) { +static common_chat_schema::value_type json_type(const common_json & value) { if (value.is_null()) { - return common_schema::TYPE_NULL; + return common_chat_schema::TYPE_NULL; } if (value.is_boolean()) { - return common_schema::TYPE_BOOLEAN; + return common_chat_schema::TYPE_BOOLEAN; } if (value.is_number_integer()) { - return common_schema::TYPE_INTEGER; + return common_chat_schema::TYPE_INTEGER; } if (value.is_number()) { - return common_schema::TYPE_NUMBER; + return common_chat_schema::TYPE_NUMBER; } if (value.is_string()) { - return common_schema::TYPE_STRING; + return common_chat_schema::TYPE_STRING; } if (value.is_array()) { - return common_schema::TYPE_ARRAY; + return common_chat_schema::TYPE_ARRAY; } - return common_schema::TYPE_OBJECT; + return common_chat_schema::TYPE_OBJECT; } -static common_schema::type_set value_types_impl(const common_schema & s, std::unordered_set & visited) { +static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set & visited) { switch (s.kind()) { - case common_schema::KIND_ANY: - return common_schema::type_set::all(); - case common_schema::KIND_NULL: - return { common_schema::TYPE_NULL }; - case common_schema::KIND_BOOLEAN: - return { common_schema::TYPE_BOOLEAN }; - case common_schema::KIND_NUMBER: - return { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER }; - case common_schema::KIND_INTEGER: - return { common_schema::TYPE_INTEGER }; - case common_schema::KIND_STRING: - return { common_schema::TYPE_STRING }; - case common_schema::KIND_ARRAY: - case common_schema::KIND_TUPLE: - return { common_schema::TYPE_ARRAY }; - case common_schema::KIND_OBJECT: - return { common_schema::TYPE_OBJECT }; - case common_schema::KIND_CONST: - return { json_type(static_cast(s).value) }; - case common_schema::KIND_ENUM: { - common_schema::type_set types; - for (const auto & value : static_cast(s).values) { + case common_chat_schema::KIND_ANY: + return common_chat_schema::type_set::all(); + case common_chat_schema::KIND_NULL: + return { common_chat_schema::TYPE_NULL }; + case common_chat_schema::KIND_BOOLEAN: + return { common_chat_schema::TYPE_BOOLEAN }; + case common_chat_schema::KIND_NUMBER: + return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_INTEGER: + return { common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_STRING: + return { common_chat_schema::TYPE_STRING }; + case common_chat_schema::KIND_ARRAY: + case common_chat_schema::KIND_TUPLE: + return { common_chat_schema::TYPE_ARRAY }; + case common_chat_schema::KIND_OBJECT: + return { common_chat_schema::TYPE_OBJECT }; + case common_chat_schema::KIND_CONST: + return { json_type(static_cast(s).value) }; + case common_chat_schema::KIND_ENUM: { + common_chat_schema::type_set types; + for (const auto & value : static_cast(s).values) { types.add(json_type(value)); } return types; } - case common_schema::KIND_REF: { - const auto * target = static_cast(s).target; + case common_chat_schema::KIND_REF: { + const auto * target = static_cast(s).target; if (!target || !visited.insert(target).second) { // a cycle contributes no type, to be safe return {}; @@ -403,16 +403,16 @@ static common_schema::type_set value_types_impl(const common_schema & s, std::un visited.erase(target); return types; } - case common_schema::KIND_ANY_OF: { - common_schema::type_set types; - for (const auto & child : static_cast(s).children) { + case common_chat_schema::KIND_ANY_OF: { + common_chat_schema::type_set types; + for (const auto & child : static_cast(s).children) { types |= value_types_impl(*child, visited); } return types; } - case common_schema::KIND_ALL_OF: { - auto types = common_schema::type_set::all(); - for (const auto & child : static_cast(s).children) { + case common_chat_schema::KIND_ALL_OF: { + auto types = common_chat_schema::type_set::all(); + for (const auto & child : static_cast(s).children) { types &= value_types_impl(*child, visited); } return types; @@ -421,27 +421,27 @@ static common_schema::type_set value_types_impl(const common_schema & s, std::un return {}; } -common_schema::type_set common_schema::value_types() const { - std::unordered_set visited; +common_chat_schema::type_set common_chat_schema::value_types() const { + std::unordered_set visited; return value_types_impl(*this, visited); } -static bool may_be_string_impl(const common_schema & s, std::unordered_set & visited) { +static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set & visited) { switch (s.kind()) { - case common_schema::KIND_STRING: + case common_chat_schema::KIND_STRING: return true; - case common_schema::KIND_CONST: - return static_cast(s).value.is_string(); - case common_schema::KIND_ENUM: - for (const auto & v : static_cast(s).values) { + case common_chat_schema::KIND_CONST: + return static_cast(s).value.is_string(); + case common_chat_schema::KIND_ENUM: + for (const auto & v : static_cast(s).values) { if (v.is_string()) { return true; } } return false; - case common_schema::KIND_REF: { + case common_chat_schema::KIND_REF: { // a cycle is taken as not a string, to be safe - const auto * target = static_cast(s).target; + const auto * target = static_cast(s).target; if (!target || !visited.insert(target).second) { return false; } @@ -449,18 +449,18 @@ static bool may_be_string_impl(const common_schema & s, std::unordered_set(s).children) { + case common_chat_schema::KIND_ANY_OF: + for (const auto & child : static_cast(s).children) { if (may_be_string_impl(*child, visited)) { return true; } } return false; - case common_schema::KIND_ALL_OF: { + case common_chat_schema::KIND_ALL_OF: { // every child must allow a string, an any child constrains nothing bool any_string = false; - for (const auto & child : static_cast(s).children) { - if (child->kind() == common_schema::KIND_ANY) { + for (const auto & child : static_cast(s).children) { + if (child->kind() == common_chat_schema::KIND_ANY) { continue; } if (!may_be_string_impl(*child, visited)) { @@ -475,12 +475,12 @@ static bool may_be_string_impl(const common_schema & s, std::unordered_set visited; +bool common_chat_schema::may_be_string() const { + std::unordered_set visited; return may_be_string_impl(*this, visited); } -const char * common_schema::kind_name(node_kind kind) { +const char * common_chat_schema::kind_name(node_kind kind) { switch (kind) { case KIND_ANY: return "any"; case KIND_REF: return "ref"; @@ -500,7 +500,7 @@ const char * common_schema::kind_name(node_kind kind) { return "?"; } -const char * common_schema::type_name(value_type type) { +const char * common_chat_schema::type_name(value_type type) { switch (type) { case TYPE_NULL: return "null"; case TYPE_BOOLEAN: return "boolean"; diff --git a/common/json-schema.h b/common/json-schema.h index cee229e6ca..084208c962 100644 --- a/common/json-schema.h +++ b/common/json-schema.h @@ -11,7 +11,7 @@ // JSON schema, covering the subset that json_schema_to_grammar() can convert. -struct common_schema { +struct common_chat_schema { enum node_kind { KIND_ANY, KIND_REF, @@ -75,7 +75,7 @@ struct common_schema { bool operator!=(const type_set & other) const { return mask_ != other.mask_; } }; - virtual ~common_schema() = default; + virtual ~common_chat_schema() = default; virtual node_kind kind() const = 0; type_set value_types() const; @@ -87,70 +87,70 @@ struct common_schema { static const char * type_name(value_type type); }; -using common_schema_ptr = std::unique_ptr; +using common_chat_schema_ptr = std::unique_ptr; -struct common_schema_any : common_schema { +struct common_chat_schema_any : common_chat_schema { node_kind kind() const override { return KIND_ANY; } }; // {"$ref": "#/..."}, only references into the same document are supported -struct common_schema_ref : common_schema { - std::string ref; - const common_schema * target = nullptr; // owned by common_schema_document::refs +struct common_chat_schema_ref : common_chat_schema { + std::string ref; + const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs - explicit common_schema_ref(std::string ref) : ref(std::move(ref)) {} + explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {} node_kind kind() const override { return KIND_REF; } }; // oneOf / anyOf, or a "type" array expanded to one alternative per type -struct common_schema_any_of : common_schema { - std::vector children; +struct common_chat_schema_any_of : common_chat_schema { + std::vector children; node_kind kind() const override { return KIND_ANY_OF; } }; -struct common_schema_all_of : common_schema { - std::vector children; +struct common_chat_schema_all_of : common_chat_schema { + std::vector children; node_kind kind() const override { return KIND_ALL_OF; } }; -struct common_schema_const : common_schema { +struct common_chat_schema_const : common_chat_schema { common_json value; - explicit common_schema_const(common_json value) : value(std::move(value)) {} + explicit common_chat_schema_const(common_json value) : value(std::move(value)) {} node_kind kind() const override { return KIND_CONST; } }; -struct common_schema_enum : common_schema { +struct common_chat_schema_enum : common_chat_schema { std::vector values; node_kind kind() const override { return KIND_ENUM; } }; -struct common_schema_null : common_schema { +struct common_chat_schema_null : common_chat_schema { node_kind kind() const override { return KIND_NULL; } }; -struct common_schema_boolean : common_schema { +struct common_chat_schema_boolean : common_chat_schema { node_kind kind() const override { return KIND_BOOLEAN; } }; -struct common_schema_number : common_schema { +struct common_chat_schema_number : common_chat_schema { node_kind kind() const override { return KIND_NUMBER; } }; // bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in -struct common_schema_integer : common_schema { +struct common_chat_schema_integer : common_chat_schema { int64_t minimum = INT64_MIN; // INT64_MIN for unbounded int64_t maximum = INT64_MAX; // INT64_MAX for unbounded node_kind kind() const override { return KIND_INTEGER; } }; -struct common_schema_string : common_schema { +struct common_chat_schema_string : common_chat_schema { std::string pattern; // empty when absent string_format format = FORMAT_NONE; int min_length = 0; @@ -159,40 +159,40 @@ struct common_schema_string : common_schema { node_kind kind() const override { return KIND_STRING; } }; -struct common_schema_array : common_schema { - common_schema_ptr items; // a common_schema_any when "items" is absent - int min_items = 0; - int max_items = -1; // -1 for unbounded +struct common_chat_schema_array : common_chat_schema { + common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent + int min_items = 0; + int max_items = -1; // -1 for unbounded node_kind kind() const override { return KIND_ARRAY; } }; -struct common_schema_tuple : common_schema { - std::vector items; +struct common_chat_schema_tuple : common_chat_schema { + std::vector items; node_kind kind() const override { return KIND_TUPLE; } }; -struct common_schema_property { - std::string name; - common_schema_ptr schema; - bool required = false; +struct common_chat_schema_property { + std::string name; + common_chat_schema_ptr schema; + bool required = false; }; -struct common_schema_object : common_schema { - std::vector properties; // in schema order - common_schema_ptr additional_properties; // null when not allowed +struct common_chat_schema_object : common_chat_schema { + std::vector properties; // in schema order + common_chat_schema_ptr additional_properties; // null when not allowed node_kind kind() const override { return KIND_OBJECT; } }; -struct common_schema_document { - common_schema_ptr root; - std::map refs; +struct common_chat_schema_document { + common_chat_schema_ptr root; + std::map refs; }; // A document shared by the PEG parsers built from its nodes, which it keeps alive -using common_schema_document_ptr = std::shared_ptr; +using common_chat_schema_document_ptr = std::shared_ptr; // Throws std::runtime_error when the schema falls outside the supported subset. -common_schema_document common_schema_from_json(const common_json & schema); +common_chat_schema_document common_chat_schema_from_json(const common_json & schema); diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 933958c3b3..640fa9e156 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -152,7 +152,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ std::vector required_parsers; std::vector optional_parsers; - foreach_parameter(function, [&](const common_schema_property & param, const common_schema_document_ptr & doc) { + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { bool is_string = param.schema->may_be_string(); auto arg = p.tool_arg( diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp index 6734056716..4d18d3d960 100644 --- a/common/parsers/minicpm5.cpp +++ b/common/parsers/minicpm5.cpp @@ -73,7 +73,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & const std::string name = function.at("name"); std::vector arg_rules; - foreach_parameter(function, [&](const common_schema_property & prop, const common_schema_document_ptr & doc) { + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { auto value_parser = p.eps(); if (prop.schema->may_be_string()) { value_parser = string_value; diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp index abd2e9ec97..7ea9bfe5a0 100644 --- a/common/parsers/minimax-m3.cpp +++ b/common/parsers/minimax-m3.cpp @@ -89,13 +89,13 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template const auto & function = tool.at("function"); std::string name = function.at("name"); auto params = common_chat_tool_parameters(function); - auto doc = std::make_shared(common_schema_from_json(params)); + auto doc = std::make_shared(common_chat_schema_from_json(params)); // The template expands argument values recursively in XML (see the to_xml() macro) - std::function value_of; - std::function members_of; + std::function value_of; + std::function members_of; - auto element_of = [&](const std::string & tag, const common_schema & schema, const std::string & rule_name) { + auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) { const std::string close = NS + ""; return p.rule(rule_name, p.tool_arg( @@ -106,7 +106,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template value_of(schema, rule_name, close))); }; - value_of = [&](const common_schema & schema, + value_of = [&](const common_chat_schema & schema, const std::string & rule_name, const std::string & close) -> common_peg_parser { auto close_tag = p.tool_arg_close(p.literal(close)); @@ -116,11 +116,11 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); } - if (schema.kind() == common_schema::KIND_ANY_OF) { + if (schema.kind() == common_chat_schema::KIND_ANY_OF) { std::vector choices; size_t index = 0; - for (const auto & alternative : static_cast(schema).children) { + for (const auto & alternative : static_cast(schema).children) { const std::string alt_name = rule_name + "-" + std::to_string(index++); // There is a risk that this breaks streaming deltas, but that's a risk we @@ -131,19 +131,19 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template return p.choice(choices); } - if (schema.kind() == common_schema::KIND_OBJECT) { - const auto & object = static_cast(schema); + if (schema.kind() == common_chat_schema::KIND_OBJECT) { + const auto & object = static_cast(schema); if (!object.properties.empty()) { return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag; } } - if (schema.kind() == common_schema::KIND_ARRAY) { + if (schema.kind() == common_chat_schema::KIND_ARRAY) { const std::string item_close = NS + ""; auto item = p.rule(rule_name + "-item", p.tag(mm3::TOOL_ARG_ITEM, p.literal(NS + "") + - value_of(*static_cast(schema).items, rule_name + "-item", item_close))); + value_of(*static_cast(schema).items, rule_name + "-item", item_close))); return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; } @@ -151,7 +151,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template }; // Required properties in schema order, then any number of optional ones in any order. - members_of = [&](const common_schema_object & object, const std::string & rule_prefix) -> common_peg_parser { + members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser { std::vector required_elements; std::vector optional_elements; for (const auto & prop : object.properties) { @@ -179,8 +179,8 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template }; common_peg_parser invoke_body = p.eps(); - if (doc->root->kind() == common_schema::KIND_OBJECT) { - invoke_body = members_of(static_cast(*doc->root), "tool-" + name + "-arg"); + if (doc->root->kind() == common_chat_schema::KIND_OBJECT) { + invoke_body = members_of(static_cast(*doc->root), "tool-" + name + "-arg"); } auto func_parser = p.tool( diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp index 71df641587..add95697eb 100644 --- a/common/parsers/muse-glimmer.cpp +++ b/common/parsers/muse-glimmer.cpp @@ -76,7 +76,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa const std::string name = function.at("name"); std::vector arg_rules; - foreach_parameter(function, [&](const common_schema_property & prop, const common_schema_document_ptr & doc) { + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { auto value_parser = p.eps(); if (prop.schema->may_be_string()) { value_parser = string_value; diff --git a/common/parsers/parsers.cpp b/common/parsers/parsers.cpp index a34fd786e5..643186c34c 100644 --- a/common/parsers/parsers.cpp +++ b/common/parsers/parsers.cpp @@ -12,10 +12,10 @@ void foreach_function(const json & tools, const std::function & fn) { +void foreach_parameter(const json & function, const std::function & fn) { auto params = common_chat_tool_parameters(function); - auto doc = std::make_shared(common_schema_from_json(params)); - const auto * object = dynamic_cast(doc->root.get()); + auto doc = std::make_shared(common_chat_schema_from_json(params)); + const auto * object = dynamic_cast(doc->root.get()); if (!object) { return; } diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h index 1af26be576..73fc719fdd 100644 --- a/common/parsers/parsers.h +++ b/common/parsers/parsers.h @@ -21,7 +21,7 @@ using json = common_json; void foreach_function(const json & tools, const std::function & fn); // iterate over the parameters of a function tool, with the document that owns them -void foreach_parameter(const json & function, const std::function & fn); +void foreach_parameter(const json & function, const std::function & fn); // render a template; the override arguments let a parser feed in messages, tools or context it has rewritten std::string common_chat_template_direct_apply_impl( diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index 99e524aa05..dfc7440847 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -99,7 +99,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat std::vector required_args; std::vector optional_args; - foreach_parameter(function, [&](const common_schema_property & param, const common_schema_document_ptr & doc) { + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { auto rule_name = "tool-" + name + "-arg-" + param.name; auto arg_open = p.tool_arg_open("\n"); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 6fc793220d..10735389ea 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -953,7 +953,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id } else if constexpr (std::is_same_v) { return "Until(" + string_join(p.delimiters, " | ") + ")"; } else if constexpr (std::is_same_v) { - return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_schema::kind_name(p.node->kind()) : "null") + ")"; + return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")"; } else if constexpr (std::is_same_v) { return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { @@ -1119,12 +1119,12 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes, return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max})); } -common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_schema_document_ptr doc, const common_schema & node, bool raw) { +common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) { return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw})); } common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) { - auto doc = std::make_shared(common_schema_from_json(schema)); + auto doc = std::make_shared(common_chat_schema_from_json(schema)); return this->schema(p, name, doc, *doc->root, raw); } diff --git a/common/peg-parser.h b/common/peg-parser.h index b80132eb33..fb5d82b30f 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -246,8 +246,8 @@ struct common_peg_until_parser { struct common_peg_schema_parser { common_peg_parser_id child; std::string name; - common_schema_document_ptr doc; // owns node - const common_schema * node = nullptr; + common_chat_schema_document_ptr doc; // owns node + const common_chat_schema * node = nullptr; // Indicates if the GBNF should accept a raw string that matches the schema. bool raw; @@ -491,7 +491,7 @@ class common_peg_parser_builder { common_peg_parser marker(); // Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it - common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_schema_document_ptr doc, const common_schema & node, bool raw = false); + common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false); // Parses the JSON schema into a document of its own common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false); diff --git a/docs/development/parsing.md b/docs/development/parsing.md index 14c4588000..0cb372eca5 100644 --- a/docs/development/parsing.md +++ b/docs/development/parsing.md @@ -108,7 +108,7 @@ For a more complete example, see `test_example_native()` in - **`rule(name, p, trigger)`** - Creates a named rule and returns a reference - **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation) - **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation -- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_schema_document` built earlier, e.g. one tool parameter +- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter ### AST Control @@ -150,7 +150,7 @@ implementation to generate the grammar instead of the underlying parser. The `raw` option emits a grammar suitable for a raw string instead of a JSON string. In other words, it won't be wrapped in quotes or require escaping quotes. It only takes effect when the schema may be a string, as reported by -`common_schema::may_be_string()`, otherwise the JSON grammar is used. +`common_chat_schema::may_be_string()`, otherwise the JSON grammar is used. The downside is that it can potentially lead to ambiguous grammars. For example, if a user provides the pattern `^.*$`, the following grammar may be diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index b3f29a3c5a..4c4206c6e6 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -1534,7 +1534,7 @@ int main() { } } })"""); - assert(json_schema_to_grammar(common_schema_from_json(schema)) == json_schema_to_grammar(schema, true)); + assert(json_schema_to_grammar(common_chat_schema_from_json(schema)) == json_schema_to_grammar(schema, true)); } // a property node carries its $ref target, so its grammar names the ref rule @@ -1565,9 +1565,9 @@ int main() { string ::= "\"" char* "\"" )""", }; - auto doc = common_schema_from_json(parameters); + auto doc = common_chat_schema_from_json(parameters); tc.verify(build_grammar([&](const common_grammar_builder & builder) { - const auto & item = static_cast(*doc.root).properties.at(0); + const auto & item = static_cast(*doc.root).properties.at(0); builder.add_schema("root", *item.schema); })); } diff --git a/tests/test-json-schema.cpp b/tests/test-json-schema.cpp index cf08513963..fb8cee18b9 100644 --- a/tests/test-json-schema.cpp +++ b/tests/test-json-schema.cpp @@ -8,13 +8,13 @@ #include #include -static common_schema_document parse(const std::string & schema) { - return common_schema_from_json(common_json::parse(schema)); +static common_chat_schema_document parse(const std::string & schema) { + return common_chat_schema_from_json(common_json::parse(schema)); } // the node as T, aborting the current test when it is some other kind template -static const T & as(testing & t, const common_schema * node, const char * what) { +static const T & as(testing & t, const common_chat_schema * node, const char * what) { const T * typed = dynamic_cast(node); if (!t.assert_true(std::string(what) + " has the expected kind", typed != nullptr)) { throw std::runtime_error(std::string(what) + " has the wrong kind"); @@ -23,7 +23,7 @@ static const T & as(testing & t, const common_schema * node, const char * what) } template -static const T & root(testing & t, const common_schema_document & doc) { +static const T & root(testing & t, const common_chat_schema_document & doc) { return as(t, doc.root.get(), "root"); } @@ -40,52 +40,52 @@ static void assert_error(testing & t, const std::string & schema, const std::str static void test_any(testing & t) { t.test("empty schema", [](testing & t) { auto doc = parse("{}"); - root(t, doc); + root(t, doc); t.assert_true("no refs", doc.refs.empty()); }); t.test("keywords that do not imply a type", [](testing & t) { auto doc = parse(R"({"description": "x", "format": "email", "additionalProperties": true})"); - root(t, doc); + root(t, doc); }); } static void test_primitives(testing & t) { t.test("null, boolean, number", [](testing & t) { auto doc_null = parse(R"({"type": "null"})"); - root(t, doc_null); + root(t, doc_null); auto doc_bool = parse(R"({"type": "boolean"})"); - root(t, doc_bool); + root(t, doc_bool); auto doc_num = parse(R"({"type": "number", "minimum": 1, "maximum": 2})"); - root(t, doc_num); + root(t, doc_num); }); } static void test_integer(testing & t) { t.test("unbounded", [](testing & t) { auto doc = parse(R"({"type": "integer"})"); - const auto & i = root(t, doc); + const auto & i = root(t, doc); t.assert_equal("minimum", INT64_MIN, i.minimum); t.assert_equal("maximum", INT64_MAX, i.maximum); }); t.test("inclusive bounds", [](testing & t) { auto doc = parse(R"({"type": "integer", "minimum": -5, "maximum": 10})"); - const auto & i = root(t, doc); + const auto & i = root(t, doc); t.assert_equal("minimum", -5, i.minimum); t.assert_equal("maximum", 10, i.maximum); }); t.test("exclusive bounds are folded", [](testing & t) { auto doc = parse(R"({"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10})"); - const auto & i = root(t, doc); + const auto & i = root(t, doc); t.assert_equal("minimum", 1, i.minimum); t.assert_equal("maximum", 9, i.maximum); }); t.test("fractional bounds round inwards", [](testing & t) { auto doc = parse(R"({"type": "integer", "minimum": 1.5, "exclusiveMaximum": 9.5})"); - const auto & i = root(t, doc); + const auto & i = root(t, doc); t.assert_equal("minimum", 2, i.minimum); t.assert_equal("maximum", 9, i.maximum); }); @@ -94,92 +94,92 @@ static void test_integer(testing & t) { static void test_string(testing & t) { t.test("defaults", [](testing & t) { auto doc = parse(R"({"type": "string"})"); - const auto & s = root(t, doc); + const auto & s = root(t, doc); t.assert_equal("pattern", "", s.pattern); - t.assert_equal("format", common_schema::FORMAT_NONE, s.format); + t.assert_equal("format", common_chat_schema::FORMAT_NONE, s.format); t.assert_equal("min_length", 0, s.min_length); t.assert_equal("max_length", -1, s.max_length); }); t.test("all keywords are kept", [](testing & t) { auto doc = parse(R"({"type": "string", "pattern": "^[a-z]+$", "format": "date", "minLength": 2, "maxLength": 8})"); - const auto & s = root(t, doc); + const auto & s = root(t, doc); t.assert_equal("pattern", "^[a-z]+$", s.pattern); - t.assert_equal("format", common_schema::FORMAT_DATE, s.format); + t.assert_equal("format", common_chat_schema::FORMAT_DATE, s.format); t.assert_equal("min_length", 2, s.min_length); t.assert_equal("max_length", 8, s.max_length); }); t.test("formats", [](testing & t) { - auto expect = [&](const char * format, common_schema::string_format expected) { + auto expect = [&](const char * format, common_chat_schema::string_format expected) { auto doc = parse(std::string(R"({"type": "string", "format": ")") + format + "\"}"); - t.assert_equal(format, expected, root(t, doc).format); + t.assert_equal(format, expected, root(t, doc).format); }; - expect("time", common_schema::FORMAT_TIME); - expect("date-time", common_schema::FORMAT_DATE_TIME); - expect("uuid", common_schema::FORMAT_UUID); - expect("uuid5", common_schema::FORMAT_UUID); - expect("email", common_schema::FORMAT_NONE); + expect("time", common_chat_schema::FORMAT_TIME); + expect("date-time", common_chat_schema::FORMAT_DATE_TIME); + expect("uuid", common_chat_schema::FORMAT_UUID); + expect("uuid5", common_chat_schema::FORMAT_UUID); + expect("email", common_chat_schema::FORMAT_NONE); }); t.test("pattern, length and known format imply a string", [](testing & t) { auto doc_pattern = parse(R"({"pattern": "^a$"})"); - t.assert_equal("pattern", "^a$", root(t, doc_pattern).pattern); + t.assert_equal("pattern", "^a$", root(t, doc_pattern).pattern); auto doc_length = parse(R"({"minLength": 1, "maxLength": 3})"); - t.assert_equal("min_length", 1, root(t, doc_length).min_length); - t.assert_equal("max_length", 3, root(t, doc_length).max_length); + t.assert_equal("min_length", 1, root(t, doc_length).min_length); + t.assert_equal("max_length", 3, root(t, doc_length).max_length); auto doc_format = parse(R"({"format": "uuid"})"); - t.assert_equal("format", common_schema::FORMAT_UUID, root(t, doc_format).format); + t.assert_equal("format", common_chat_schema::FORMAT_UUID, root(t, doc_format).format); }); } static void test_array(testing & t) { t.test("items with bounds", [](testing & t) { auto doc = parse(R"({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3})"); - const auto & a = root(t, doc); - as(t, a.items.get(), "items"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); t.assert_equal("min_items", 1, a.min_items); t.assert_equal("max_items", 3, a.max_items); }); t.test("no items", [](testing & t) { auto doc = parse(R"({"type": "array"})"); - const auto & a = root(t, doc); - as(t, a.items.get(), "items"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); t.assert_equal("min_items", 0, a.min_items); t.assert_equal("max_items", -1, a.max_items); }); t.test("items imply an array", [](testing & t) { auto doc = parse(R"({"items": {"type": "string"}})"); - const auto & a = root(t, doc); - as(t, a.items.get(), "items"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); }); } static void test_tuple(testing & t) { t.test("prefixItems", [](testing & t) { auto doc = parse(R"({"prefixItems": [{"type": "string"}, {"type": "number"}]})"); - const auto & tup = root(t, doc); + const auto & tup = root(t, doc); t.assert_equal("size", (size_t) 2, tup.items.size()); - as(t, tup.items[0].get(), "items[0]"); - as(t, tup.items[1].get(), "items[1]"); + as(t, tup.items[0].get(), "items[0]"); + as(t, tup.items[1].get(), "items[1]"); }); t.test("items as an array", [](testing & t) { auto doc = parse(R"({"type": "array", "items": [{"type": "boolean"}]})"); - const auto & tup = root(t, doc); + const auto & tup = root(t, doc); t.assert_equal("size", (size_t) 1, tup.items.size()); - as(t, tup.items[0].get(), "items[0]"); + as(t, tup.items[0].get(), "items[0]"); }); } static void test_object(testing & t) { t.test("type alone accepts any object", [](testing & t) { auto doc = parse(R"({"type": "object"})"); - const auto & o = root(t, doc); + const auto & o = root(t, doc); t.assert_true("no properties", o.properties.empty()); - as(t, o.additional_properties.get(), "additional_properties"); + as(t, o.additional_properties.get(), "additional_properties"); }); t.test("properties", [](testing & t) { @@ -192,7 +192,7 @@ static void test_object(testing & t) { }, "required": ["a", "c"] })"); - const auto & o = root(t, doc); + const auto & o = root(t, doc); t.assert_equal("size", (size_t) 3, o.properties.size()); t.assert_equal("order", "b", o.properties[0].name); t.assert_equal("order", "a", o.properties[1].name); @@ -200,53 +200,53 @@ static void test_object(testing & t) { t.assert_true("b optional", !o.properties[0].required); t.assert_true("a required", o.properties[1].required); t.assert_true("c required", o.properties[2].required); - as(t, o.properties[0].schema.get(), "b"); - as(t, o.properties[1].schema.get(), "a"); - as(t, o.properties[2].schema.get(), "c"); + as(t, o.properties[0].schema.get(), "b"); + as(t, o.properties[1].schema.get(), "a"); + as(t, o.properties[2].schema.get(), "c"); t.assert_true("closed", o.additional_properties == nullptr); }); t.test("unknown required entries are ignored", [](testing & t) { auto doc = parse(R"({"properties": {"a": {}}, "required": ["a", "zzz", 1]})"); - const auto & o = root(t, doc); + const auto & o = root(t, doc); t.assert_equal("size", (size_t) 1, o.properties.size()); t.assert_true("a required", o.properties[0].required); }); t.test("additionalProperties false implies an object", [](testing & t) { auto doc = parse(R"({"additionalProperties": false})"); - const auto & o = root(t, doc); + const auto & o = root(t, doc); t.assert_true("no properties", o.properties.empty()); t.assert_true("closed", o.additional_properties == nullptr); }); t.test("additionalProperties schema", [](testing & t) { auto doc = parse(R"({"properties": {"a": {}}, "additionalProperties": {"type": "integer", "minimum": 0}})"); - const auto & o = root(t, doc); + const auto & o = root(t, doc); t.assert_equal("size", (size_t) 1, o.properties.size()); - const auto & v = as(t, o.additional_properties.get(), "additional_properties"); + const auto & v = as(t, o.additional_properties.get(), "additional_properties"); t.assert_equal("minimum", 0, v.minimum); }); t.test("nested", [](testing & t) { auto doc = parse(R"({"properties": {"inner": {"properties": {"leaf": {"type": "null"}}, "required": ["leaf"]}}})"); - const auto & o = root(t, doc); - const auto & inner = as(t, o.properties[0].schema.get(), "inner"); + const auto & o = root(t, doc); + const auto & inner = as(t, o.properties[0].schema.get(), "inner"); t.assert_equal("leaf name", "leaf", inner.properties[0].name); t.assert_true("leaf required", inner.properties[0].required); - as(t, inner.properties[0].schema.get(), "leaf"); + as(t, inner.properties[0].schema.get(), "leaf"); }); } static void test_const_enum(testing & t) { t.test("const", [](testing & t) { auto doc = parse(R"({"const": {"a": [1, null]}})"); - t.assert_equal("value", R"({"a":[1,null]})", root(t, doc).value.dump()); + t.assert_equal("value", R"({"a":[1,null]})", root(t, doc).value.dump()); }); t.test("enum", [](testing & t) { auto doc = parse(R"({"enum": ["a", 1, null, true]})"); - const auto & e = root(t, doc); + const auto & e = root(t, doc); t.assert_equal("size", (size_t) 4, e.values.size()); t.assert_equal("values[0]", "\"a\"", e.values[0].dump()); t.assert_equal("values[1]", "1", e.values[1].dump()); @@ -256,81 +256,81 @@ static void test_const_enum(testing & t) { t.test("const wins over enum, enum wins over type", [](testing & t) { auto doc_enum = parse(R"({"type": "integer", "enum": [1, 2]})"); - root(t, doc_enum); + root(t, doc_enum); auto doc_const = parse(R"({"type": "string", "const": "x", "enum": ["y"]})"); - t.assert_equal("value", "\"x\"", root(t, doc_const).value.dump()); + t.assert_equal("value", "\"x\"", root(t, doc_const).value.dump()); }); } static void test_any_of(testing & t) { t.test("anyOf and oneOf", [](testing & t) { auto doc_any = parse(R"({"anyOf": [{"type": "string"}, {"type": "number"}]})"); - const auto & u = root(t, doc_any); + const auto & u = root(t, doc_any); t.assert_equal("size", (size_t) 2, u.children.size()); - as(t, u.children[0].get(), "children[0]"); - as(t, u.children[1].get(), "children[1]"); + as(t, u.children[0].get(), "children[0]"); + as(t, u.children[1].get(), "children[1]"); auto doc_one = parse(R"({"oneOf": [{"type": "null"}]})"); - const auto & o = root(t, doc_one); + const auto & o = root(t, doc_one); t.assert_equal("size", (size_t) 1, o.children.size()); - as(t, o.children[0].get(), "children[0]"); + as(t, o.children[0].get(), "children[0]"); }); t.test("oneOf wins over anyOf and type", [](testing & t) { auto doc = parse(R"({"type": "string", "oneOf": [{"type": "null"}], "anyOf": [{"type": "number"}, {"type": "boolean"}]})"); - const auto & u = root(t, doc); + const auto & u = root(t, doc); t.assert_equal("size", (size_t) 1, u.children.size()); - as(t, u.children[0].get(), "children[0]"); + as(t, u.children[0].get(), "children[0]"); }); t.test("type array expands with sibling keywords", [](testing & t) { auto doc = parse(R"({"type": ["string", "null", "integer"], "minLength": 2, "minimum": 5})"); - const auto & u = root(t, doc); + const auto & u = root(t, doc); t.assert_equal("size", (size_t) 3, u.children.size()); - t.assert_equal("min_length", 2, as(t, u.children[0].get(), "children[0]").min_length); - as(t, u.children[1].get(), "children[1]"); - t.assert_equal("minimum", 5, as(t, u.children[2].get(), "children[2]").minimum); + t.assert_equal("min_length", 2, as(t, u.children[0].get(), "children[0]").min_length); + as(t, u.children[1].get(), "children[1]"); + t.assert_equal("minimum", 5, as(t, u.children[2].get(), "children[2]").minimum); }); } static void test_all_of(testing & t) { t.test("components", [](testing & t) { auto doc = parse(R"({"allOf": [{"properties": {"a": {}}}, {"anyOf": [{"properties": {"b": {}}}, {"type": "null"}]}]})"); - const auto & all = root(t, doc); + const auto & all = root(t, doc); t.assert_equal("size", (size_t) 2, all.children.size()); - as(t, all.children[0].get(), "children[0]"); - as(t, all.children[1].get(), "children[1]"); + as(t, all.children[0].get(), "children[0]"); + as(t, all.children[1].get(), "children[1]"); auto doc_typed = parse(R"({"type": "object", "allOf": [{"properties": {"a": {}}}]})"); - root(t, doc_typed); + root(t, doc_typed); }); t.test("properties win over allOf", [](testing & t) { auto doc = parse(R"({"type": "object", "properties": {"a": {}}, "allOf": [{"properties": {"b": {}}}]})"); - t.assert_equal("size", (size_t) 1, root(t, doc).properties.size()); + t.assert_equal("size", (size_t) 1, root(t, doc).properties.size()); }); t.test("other types ignore allOf", [](testing & t) { auto doc = parse(R"({"type": "integer", "allOf": [{"minimum": 1}]})"); - root(t, doc); + root(t, doc); }); } static void test_ref(testing & t) { t.test("target is owned by the document", [](testing & t) { auto doc = parse(R"({"$ref": "#/$defs/t", "type": "string", "$defs": {"t": {"type": "boolean"}}})"); - const auto & r = root(t, doc); + const auto & r = root(t, doc); t.assert_equal("ref", "#/$defs/t", r.ref); t.assert_equal("refs", (size_t) 1, doc.refs.size()); t.assert_true("target", r.target != nullptr && r.target == doc.refs.at("#/$defs/t").get()); - as(t, r.target, "target"); + as(t, r.target, "target"); }); t.test("definitions", [](testing & t) { auto doc = parse(R"({"properties": {"a": {"$ref": "#/definitions/t"}}, "definitions": {"t": {"type": "number"}}})"); - const auto & o = root(t, doc); - const auto & r = as(t, o.properties[0].schema.get(), "a"); - as(t, r.target, "target"); + const auto & o = root(t, doc); + const auto & r = as(t, o.properties[0].schema.get(), "a"); + as(t, r.target, "target"); }); t.test("recursive", [](testing & t) { @@ -347,28 +347,28 @@ static void test_ref(testing & t) { } } })"); - const auto & r = root(t, doc); - const auto & node = as(t, r.target, "node"); + const auto & r = root(t, doc); + const auto & node = as(t, r.target, "node"); t.assert_equal("properties", (size_t) 2, node.properties.size()); - const auto & next = as(t, node.properties[1].schema.get(), "next"); + const auto & next = as(t, node.properties[1].schema.get(), "next"); t.assert_true("cycle", next.target == r.target); t.assert_equal("refs", (size_t) 1, doc.refs.size()); }); t.test("pointer through an array", [](testing & t) { auto doc = parse(R"({"oneOf": [{"type": "null"}, {"$ref": "#/oneOf/0"}]})"); - const auto & u = root(t, doc); - const auto & r = as(t, u.children[1].get(), "children[1]"); - as(t, r.target, "target"); + const auto & u = root(t, doc); + const auto & r = as(t, u.children[1].get(), "children[1]"); + as(t, r.target, "target"); }); t.test("targets survive moving the document", [](testing & t) { auto parsed = parse(R"({"items": {"$ref": "#/$defs/t"}, "$defs": {"t": {"type": "null"}}})"); - common_schema_document doc = std::move(parsed); - const auto & a = root(t, doc); - const auto & r = as(t, a.items.get(), "items"); + common_chat_schema_document doc = std::move(parsed); + const auto & a = root(t, doc); + const auto & r = as(t, a.items.get(), "items"); t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get()); - as(t, r.target, "target"); + as(t, r.target, "target"); }); } @@ -404,43 +404,43 @@ static void test_may_be_string(testing & t) { } // e.g. {number, integer}, in type order -static std::string dump(const common_schema::type_set & types) { - static const common_schema::value_type order[] = { common_schema::TYPE_NULL, common_schema::TYPE_BOOLEAN, common_schema::TYPE_NUMBER, - common_schema::TYPE_INTEGER, common_schema::TYPE_STRING, common_schema::TYPE_ARRAY, - common_schema::TYPE_OBJECT }; +static std::string dump(const common_chat_schema::type_set & types) { + static const common_chat_schema::value_type order[] = { common_chat_schema::TYPE_NULL, common_chat_schema::TYPE_BOOLEAN, common_chat_schema::TYPE_NUMBER, + common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_ARRAY, + common_chat_schema::TYPE_OBJECT }; std::string out; for (auto type : order) { if (types.has(type)) { - out += (out.empty() ? "" : ", ") + std::string(common_schema::type_name(type)); + out += (out.empty() ? "" : ", ") + std::string(common_chat_schema::type_name(type)); } } return "{" + out + "}"; } static void test_value_types(testing & t) { - auto check = [](testing & t, const std::string & schema, const common_schema::type_set & expected) { + auto check = [](testing & t, const std::string & schema, const common_chat_schema::type_set & expected) { t.assert_equal(schema, dump(expected), dump(parse(schema).root->value_types())); }; t.test("leaves", [&](testing & t) { - check(t, R"({"type": "string"})", { common_schema::TYPE_STRING }); - check(t, R"({"type": "number"})", { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER }); - check(t, R"({"description": "x"})", common_schema::type_set::all()); - check(t, R"({"properties": {"a": {"type": "string"}}})", { common_schema::TYPE_OBJECT }); - check(t, R"({"items": {"type": "string"}})", { common_schema::TYPE_ARRAY }); - check(t, R"({"const": 1.5})", { common_schema::TYPE_NUMBER }); - check(t, R"({"enum": [1, "a", null]})", { common_schema::TYPE_INTEGER, common_schema::TYPE_STRING, common_schema::TYPE_NULL }); + check(t, R"({"type": "string"})", { common_chat_schema::TYPE_STRING }); + check(t, R"({"type": "number"})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); + check(t, R"({"description": "x"})", common_chat_schema::type_set::all()); + check(t, R"({"properties": {"a": {"type": "string"}}})", { common_chat_schema::TYPE_OBJECT }); + check(t, R"({"items": {"type": "string"}})", { common_chat_schema::TYPE_ARRAY }); + check(t, R"({"const": 1.5})", { common_chat_schema::TYPE_NUMBER }); + check(t, R"({"enum": [1, "a", null]})", { common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); }); t.test("any_of is the union, all_of is the intersection", [&](testing & t) { - check(t, R"({"type": ["string", "null"]})", { common_schema::TYPE_STRING, common_schema::TYPE_NULL }); - check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER }); + check(t, R"({"type": ["string", "null"]})", { common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); + check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", {}); }); t.test("ref", [&](testing & t) { check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", - { common_schema::TYPE_STRING }); + { common_chat_schema::TYPE_STRING }); }); }