wylab/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-14 18:02:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

cont : rename common_schema to common_chat_schema

Browse Source
This commit is contained in:
Alde Rojas
2026-09-12 13:23:12 -05:00
parent a1e2fc9e8b
commit 3a3f76f2e8
17 changed files with 336 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat-auto-parser-generator.cpp
+1 -1
View File
@@ -371,7 +371,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
// Build parser for each argument, separating required and optional
std::vector<common_peg_parser> required_parsers;
std::vector<common_peg_parser> optional_parsers;
foreach_parameter(func, [&](const common_schema_property & param, const common_schema_document_ptr & doc) {
foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
auto arg =
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
arguments.name_suffix) +
common/json-schema-to-grammar.cpp
+56 -56
View File
@@ -338,7 +338,7 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
return 2 + n_hex;
}
class common_schema_converter {
class common_chat_schema_converter {
private:
friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
bool _dotall;
@@ -348,7 +348,7 @@ private:
std::vector<std::string> _warnings;
template <typename T>
static const T & as(const common_schema & node) {
static const T & as(const common_chat_schema & node) {
return static_cast<const T &>(node);
}
@@ -367,7 +367,7 @@ private:
return key;
}
std::string _generate_union_rule(const std::string & name, const std::vector<common_schema_ptr> & alt_schemas) {
std::string _generate_union_rule(const std::string & name, const std::vector<common_chat_schema_ptr> & alt_schemas) {
std::vector<std::string> rules;
rules.reserve(alt_schemas.size());
for (size_t i = 0; i < alt_schemas.size(); i++) {
@@ -678,7 +678,7 @@ private:
return out.str();
}
std::string _resolve_ref(const common_schema_ref & schema) {
std::string _resolve_ref(const common_chat_schema_ref & schema) {
auto it = schema.ref.find('#');
std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
@@ -696,10 +696,10 @@ private:
}
std::string _build_object_rule(
const std::vector<std::pair<std::string, const common_schema *>> & properties,
const std::vector<std::pair<std::string, const common_chat_schema *>> & properties,
const std::unordered_set<std::string> & required,
const std::string & name,
const common_schema * additional_properties)
const common_chat_schema * additional_properties)
{
std::vector<std::string> required_props;
std::vector<std::string> optional_props;
@@ -724,7 +724,7 @@ private:
if (additional_properties) {
std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
std::string value_rule =
additional_properties->kind() != common_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value")
additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value")
: _add_primitive("value", PRIMITIVE_RULES.at("value"));
auto key_rule =
@@ -812,11 +812,11 @@ private:
}
public:
explicit common_schema_converter(bool dotall) : _dotall(dotall) {
explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) {
_rules["space"] = SPACE_RULE;
}
std::string add_schema(const std::string & name, const common_schema & schema) {
std::string add_schema(const std::string & name, const common_chat_schema & schema) {
return visit(schema, name);
}
@@ -828,31 +828,31 @@ public:
return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type));
}
std::string _visit_all_of(const common_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
std::unordered_set<std::string> required;
std::vector<std::pair<std::string, const common_schema *>> properties;
std::vector<std::pair<std::string, const common_chat_schema *>> properties;
std::map<std::string, size_t> enum_values;
std::function<void(const common_schema &, bool)> add_component = [&](const common_schema & comp, bool is_required) {
if (comp.kind() == common_schema::KIND_REF) {
if (const auto * target = as<common_schema_ref>(comp).target) {
std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
if (comp.kind() == common_chat_schema::KIND_REF) {
if (const auto * target = as<common_chat_schema_ref>(comp).target) {
add_component(*target, is_required);
}
} else if (comp.kind() == common_schema::KIND_OBJECT) {
for (const auto & prop : as<common_schema_object>(comp).properties) {
} else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
properties.emplace_back(prop.name, prop.schema.get());
if (is_required) {
required.insert(prop.name);
}
}
} else if (comp.kind() == common_schema::KIND_ENUM) {
for (const auto & v : as<common_schema_enum>(comp).values) {
} else if (comp.kind() == common_chat_schema::KIND_ENUM) {
for (const auto & v : as<common_chat_schema_enum>(comp).values) {
enum_values[_generate_constant_rule(v)] += 1;
}
}
};
for (const auto & child : schema.children) {
if (child->kind() == common_schema::KIND_ANY_OF) {
for (const auto & alt : as<common_schema_any_of>(*child).children) {
if (child->kind() == common_chat_schema::KIND_ANY_OF) {
for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
add_component(*alt, false);
}
} else {
@@ -873,32 +873,32 @@ public:
return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
}
std::string visit(const common_schema & schema, const std::string & name) {
std::string visit(const common_chat_schema & schema, const std::string & name) {
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
std::string sub_name = name + (name.empty() ? "" : "-");
switch (schema.kind()) {
case common_schema::KIND_REF:
return _add_rule(rule_name, _resolve_ref(as<common_schema_ref>(schema)));
case common_schema::KIND_ANY_OF:
return _add_rule(rule_name, _generate_union_rule(name, as<common_schema_any_of>(schema).children));
case common_schema::KIND_ALL_OF:
return _visit_all_of(as<common_schema_all_of>(schema), name, rule_name);
case common_schema::KIND_CONST:
return _add_rule(rule_name, _generate_constant_rule(as<common_schema_const>(schema).value));
case common_schema::KIND_ENUM: {
case common_chat_schema::KIND_REF:
return _add_rule(rule_name, _resolve_ref(as<common_chat_schema_ref>(schema)));
case common_chat_schema::KIND_ANY_OF:
return _add_rule(rule_name, _generate_union_rule(name, as<common_chat_schema_any_of>(schema).children));
case common_chat_schema::KIND_ALL_OF:
return _visit_all_of(as<common_chat_schema_all_of>(schema), name, rule_name);
case common_chat_schema::KIND_CONST:
return _add_rule(rule_name, _generate_constant_rule(as<common_chat_schema_const>(schema).value));
case common_chat_schema::KIND_ENUM: {
std::vector<std::string> enum_values;
for (const auto & v : as<common_schema_enum>(schema).values) {
for (const auto & v : as<common_chat_schema_enum>(schema).values) {
enum_values.push_back(_generate_constant_rule(v));
}
return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
}
case common_schema::KIND_OBJECT: {
const auto & obj = as<common_schema_object>(schema);
if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_schema::KIND_ANY) {
case common_chat_schema::KIND_OBJECT: {
const auto & obj = as<common_chat_schema_object>(schema);
if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) {
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
}
std::vector<std::pair<std::string, const common_schema *>> properties;
std::vector<std::pair<std::string, const common_chat_schema *>> properties;
std::unordered_set<std::string> required;
for (const auto & prop : obj.properties) {
properties.emplace_back(prop.name, prop.schema.get());
@@ -908,8 +908,8 @@ public:
}
return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get()));
}
case common_schema::KIND_TUPLE: {
const auto & items = as<common_schema_tuple>(schema).items;
case common_chat_schema::KIND_TUPLE: {
const auto & items = as<common_chat_schema_tuple>(schema).items;
std::string rule = "\"[\" space ";
for (size_t i = 0; i < items.size(); i++) {
if (i > 0) {
@@ -920,25 +920,25 @@ public:
rule += " space \"]\"";
return _add_rule(rule_name, rule);
}
case common_schema::KIND_ARRAY: {
const auto & arr = as<common_schema_array>(schema);
if (arr.items->kind() == common_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
case common_chat_schema::KIND_ARRAY: {
const auto & arr = as<common_chat_schema_array>(schema);
if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
return _visit_primitive(rule_name, "array");
}
std::string item_rule_name = visit(*arr.items, sub_name + "item");
int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items;
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\"");
}
case common_schema::KIND_STRING: {
const auto & str = as<common_schema_string>(schema);
case common_chat_schema::KIND_STRING: {
const auto & str = as<common_chat_schema_string>(schema);
if (!str.pattern.empty()) {
return _visit_pattern(str.pattern, rule_name);
}
if (str.format == common_schema::FORMAT_UUID) {
if (str.format == common_chat_schema::FORMAT_UUID) {
return _visit_primitive(rule_name, "uuid");
}
if (str.format != common_schema::FORMAT_NONE) {
std::string prim_name = std::string(str.format == common_schema::FORMAT_DATE ? "date" : str.format == common_schema::FORMAT_TIME ? "time" : "date-time") + "-string";
if (str.format != common_chat_schema::FORMAT_NONE) {
std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string";
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
}
if (str.min_length > 0 || str.max_length >= 0) {
@@ -948,8 +948,8 @@ public:
}
return _visit_primitive(rule_name, "string");
}
case common_schema::KIND_INTEGER: {
const auto & i = as<common_schema_integer>(schema);
case common_chat_schema::KIND_INTEGER: {
const auto & i = as<common_chat_schema_integer>(schema);
if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) {
return _visit_primitive(rule_name, "integer");
}
@@ -959,13 +959,13 @@ public:
out << ")";
return _add_rule(rule_name, out.str());
}
case common_schema::KIND_NUMBER:
case common_chat_schema::KIND_NUMBER:
return _visit_primitive(rule_name, "number");
case common_schema::KIND_BOOLEAN:
case common_chat_schema::KIND_BOOLEAN:
return _visit_primitive(rule_name, "boolean");
case common_schema::KIND_NULL:
case common_chat_schema::KIND_NULL:
return _visit_primitive(rule_name, "null");
case common_schema::KIND_ANY:
case common_chat_schema::KIND_ANY:
return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
}
return "";
@@ -998,26 +998,26 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf)
(void)force_gbnf;
#endif // LLAMA_USE_LLGUIDANCE
try {
return json_schema_to_grammar(common_schema_from_json(schema));
return json_schema_to_grammar(common_chat_schema_from_json(schema));
} catch (const std::runtime_error & e) {
throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what());
}
}
std::string json_schema_to_grammar(const common_schema_document & schema) {
common_schema_converter converter(false);
std::string json_schema_to_grammar(const common_chat_schema_document & schema) {
common_chat_schema_converter converter(false);
converter.visit(*schema.root, "");
converter.check_errors();
return converter.format_grammar();
}
std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
common_schema_converter converter(options.dotall);
common_chat_schema_converter converter(options.dotall);
common_grammar_builder builder {
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
return converter._add_rule(name, rule);
},
/* .add_schema = */ [&](const std::string & name, const common_schema & schema) {
/* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) {
return converter.add_schema(name == "root" ? "" : name, schema);
},
};
common/json-schema-to-grammar.h
+2 -2
View File
@@ -7,11 +7,11 @@
#include <string>
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false);
std::string json_schema_to_grammar(const common_schema_document & schema);
std::string json_schema_to_grammar(const common_chat_schema_document & schema);
struct common_grammar_builder {
std::function<std::string(const std::string &, const std::string &)> add_rule;
std::function<std::string(const std::string &, const common_schema &)> add_schema;
std::function<std::string(const std::string &, const common_chat_schema &)> add_schema;
};
struct common_grammar_options {
common/json-schema.cpp
+102 -102
View File
@@ -9,15 +9,15 @@
#include <utility>
#include <vector>
class common_schema_builder {
class common_chat_schema_builder {
const common_json & root_;
common_schema_document & doc_;
common_chat_schema_document & doc_;
// the targets built here, moved into doc_ once the whole schema is built
std::map<std::string, common_schema_ptr> refs_;
std::map<std::string, common_chat_schema_ptr> refs_;
// ref nodes get their target once every $ref is built, a cycle would otherwise need it too early
std::vector<common_schema_ref *> pending_;
std::vector<common_chat_schema_ref *> pending_;
[[noreturn]] static void fail(const std::string & path, const std::string & msg) {
throw std::runtime_error("JSON schema error at " + path + ": " + msg);
@@ -47,9 +47,9 @@ class common_schema_builder {
return (int64_t) (round_up ? std::ceil(d) : std::floor(d));
}
static common_schema::string_format get_format(const common_json & schema, const std::string & path) {
static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) {
if (!schema.contains("format")) {
return common_schema::FORMAT_NONE;
return common_chat_schema::FORMAT_NONE;
}
const common_json & value = schema.at("format");
if (!value.is_string()) {
@@ -57,18 +57,18 @@ class common_schema_builder {
}
std::string format = value.get<std::string>();
if (format == "date") {
return common_schema::FORMAT_DATE;
return common_chat_schema::FORMAT_DATE;
}
if (format == "time") {
return common_schema::FORMAT_TIME;
return common_chat_schema::FORMAT_TIME;
}
if (format == "date-time") {
return common_schema::FORMAT_DATE_TIME;
return common_chat_schema::FORMAT_DATE_TIME;
}
if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) {
return common_schema::FORMAT_UUID;
return common_chat_schema::FORMAT_UUID;
}
return common_schema::FORMAT_NONE;
return common_chat_schema::FORMAT_NONE;
}
const common_json & resolve_ref(const std::string & ref, const std::string & path) {
@@ -96,7 +96,7 @@ class common_schema_builder {
return *target;
}
common_schema_ptr build_ref(const common_json & value, const std::string & path) {
common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) {
if (!value.is_string()) {
fail(path, "$ref must be a string");
}
@@ -109,13 +109,13 @@ class common_schema_builder {
refs_[ref] = nullptr;
refs_[ref] = build_node(resolve_ref(ref, path), ref);
}
auto node = std::make_unique<common_schema_ref>(ref);
auto node = std::make_unique<common_chat_schema_ref>(ref);
pending_.push_back(node.get());
return node;
}
template <typename T>
common_schema_ptr build_alternatives(const common_json & alts, const std::string & path) {
common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) {
if (!alts.is_array()) {
fail(path, "must be an array of schemas");
}
@@ -130,8 +130,8 @@ class common_schema_builder {
return node;
}
common_schema_ptr build_object(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_schema_object>();
common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_chat_schema_object>();
std::unordered_set<std::string> required;
if (schema.contains("required") && schema.at("required").is_array()) {
@@ -156,7 +156,7 @@ class common_schema_builder {
const common_json & additional = schema.at("additionalProperties");
if (additional.is_boolean()) {
if (additional.get<bool>()) {
node->additional_properties = std::make_unique<common_schema_any>();
node->additional_properties = std::make_unique<common_chat_schema_any>();
}
} else if (additional.is_object()) {
node->additional_properties = build_node(additional, path + "/additionalProperties");
@@ -165,20 +165,20 @@ class common_schema_builder {
}
} else if (!schema.contains("properties")) {
// {"type": "object"} on its own accepts any object
node->additional_properties = std::make_unique<common_schema_any>();
node->additional_properties = std::make_unique<common_chat_schema_any>();
}
return node;
}
common_schema_ptr build_array(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_schema_array>();
common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_chat_schema_array>();
if (schema.contains("items") || schema.contains("prefixItems")) {
// "items" wins when both are present; as in the converter, a schema instead of an array is the item schema
const std::string key = schema.contains("items") ? "items" : "prefixItems";
const common_json & items = schema.at(key);
if (items.is_array()) {
auto tuple = std::make_unique<common_schema_tuple>();
auto tuple = std::make_unique<common_chat_schema_tuple>();
size_t i = 0;
for (const auto & item : items) {
tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++)));
@@ -187,15 +187,15 @@ class common_schema_builder {
}
node->items = build_node(items, path + "/" + key);
} else {
node->items = std::make_unique<common_schema_any>();
node->items = std::make_unique<common_chat_schema_any>();
}
node->min_items = get_count(schema, "minItems", path, 0);
node->max_items = get_count(schema, "maxItems", path, -1);
return node;
}
common_schema_ptr build_string(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_schema_string>();
common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_chat_schema_string>();
if (schema.contains("pattern")) {
const common_json & pattern = schema.at("pattern");
if (!pattern.is_string()) {
@@ -209,8 +209,8 @@ class common_schema_builder {
return node;
}
common_schema_ptr build_integer(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_schema_integer>();
common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) {
auto node = std::make_unique<common_chat_schema_integer>();
if (schema.contains("minimum")) {
node->minimum = get_bound(schema, "minimum", path, /* round_up */ true);
} else if (schema.contains("exclusiveMinimum")) {
@@ -224,7 +224,7 @@ class common_schema_builder {
return node;
}
common_schema_ptr build_node(const common_json & schema, const std::string & path) {
common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
if (!schema.is_object()) {
fail(path, "schema must be an object");
}
@@ -233,7 +233,7 @@ class common_schema_builder {
}
if (schema.contains("oneOf") || schema.contains("anyOf")) {
const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf";
return build_alternatives<common_schema_any_of>(schema.at(key), path + "/" + key);
return build_alternatives<common_chat_schema_any_of>(schema.at(key), path + "/" + key);
}
common_json type;
@@ -245,7 +245,7 @@ class common_schema_builder {
if (type.empty()) {
fail(path, "type must not be empty");
}
auto node = std::make_unique<common_schema_any_of>();
auto node = std::make_unique<common_chat_schema_any_of>();
size_t i = 0;
for (const auto & t : type) {
common_json alt = schema;
@@ -255,14 +255,14 @@ class common_schema_builder {
return node;
}
if (schema.contains("const")) {
return std::make_unique<common_schema_const>(schema.at("const"));
return std::make_unique<common_chat_schema_const>(schema.at("const"));
}
if (schema.contains("enum")) {
const common_json & values = schema.at("enum");
if (!values.is_array() || values.empty()) {
fail(path, "enum must be a non-empty array");
}
auto node = std::make_unique<common_schema_enum>();
auto node = std::make_unique<common_chat_schema_enum>();
for (const auto & value : values) {
node->values.push_back(value);
}
@@ -282,25 +282,25 @@ class common_schema_builder {
return build_object(schema, path);
}
if (schema.contains("allOf")) {
return build_alternatives<common_schema_all_of>(schema.at("allOf"), path + "/allOf");
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
}
if (schema.contains("items") || schema.contains("prefixItems")) {
return build_array(schema, path);
}
if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_schema::FORMAT_NONE) {
if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) {
return build_string(schema, path);
}
return std::make_unique<common_schema_any>();
return std::make_unique<common_chat_schema_any>();
}
if (type_name == "object") {
if (!has_properties && schema.contains("allOf")) {
return build_alternatives<common_schema_all_of>(schema.at("allOf"), path + "/allOf");
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
}
return build_object(schema, path);
}
if (type_name == "string") {
if (schema.contains("allOf")) {
return build_alternatives<common_schema_all_of>(schema.at("allOf"), path + "/allOf");
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
}
return build_string(schema, path);
}
@@ -311,21 +311,21 @@ class common_schema_builder {
return build_integer(schema, path);
}
if (type_name == "number") {
return std::make_unique<common_schema_number>();
return std::make_unique<common_chat_schema_number>();
}
if (type_name == "boolean") {
return std::make_unique<common_schema_boolean>();
return std::make_unique<common_chat_schema_boolean>();
}
if (type_name == "null") {
return std::make_unique<common_schema_null>();
return std::make_unique<common_chat_schema_null>();
}
fail(path, "unrecognized type " + type_name);
}
public:
common_schema_builder(const common_json & root, common_schema_document & doc) : root_(root), doc_(doc) {}
common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {}
common_schema_ptr build() {
common_chat_schema_ptr build() {
auto node = build_node(root_, "#");
for (auto & entry : refs_) {
doc_.refs[entry.first] = std::move(entry.second);
@@ -337,64 +337,64 @@ class common_schema_builder {
}
};
common_schema_document common_schema_from_json(const common_json & schema) {
common_schema_document doc;
doc.root = common_schema_builder(schema, doc).build();
common_chat_schema_document common_chat_schema_from_json(const common_json & schema) {
common_chat_schema_document doc;
doc.root = common_chat_schema_builder(schema, doc).build();
return doc;
}
static common_schema::value_type json_type(const common_json & value) {
static common_chat_schema::value_type json_type(const common_json & value) {
if (value.is_null()) {
return common_schema::TYPE_NULL;
return common_chat_schema::TYPE_NULL;
}
if (value.is_boolean()) {
return common_schema::TYPE_BOOLEAN;
return common_chat_schema::TYPE_BOOLEAN;
}
if (value.is_number_integer()) {
return common_schema::TYPE_INTEGER;
return common_chat_schema::TYPE_INTEGER;
}
if (value.is_number()) {
return common_schema::TYPE_NUMBER;
return common_chat_schema::TYPE_NUMBER;
}
if (value.is_string()) {
return common_schema::TYPE_STRING;
return common_chat_schema::TYPE_STRING;
}
if (value.is_array()) {
return common_schema::TYPE_ARRAY;
return common_chat_schema::TYPE_ARRAY;
}
return common_schema::TYPE_OBJECT;
return common_chat_schema::TYPE_OBJECT;
}
static common_schema::type_set value_types_impl(const common_schema & s, std::unordered_set<const common_schema *> & visited) {
static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
switch (s.kind()) {
case common_schema::KIND_ANY:
return common_schema::type_set::all();
case common_schema::KIND_NULL:
return { common_schema::TYPE_NULL };
case common_schema::KIND_BOOLEAN:
return { common_schema::TYPE_BOOLEAN };
case common_schema::KIND_NUMBER:
return { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER };
case common_schema::KIND_INTEGER:
return { common_schema::TYPE_INTEGER };
case common_schema::KIND_STRING:
return { common_schema::TYPE_STRING };
case common_schema::KIND_ARRAY:
case common_schema::KIND_TUPLE:
return { common_schema::TYPE_ARRAY };
case common_schema::KIND_OBJECT:
return { common_schema::TYPE_OBJECT };
case common_schema::KIND_CONST:
return { json_type(static_cast<const common_schema_const &>(s).value) };
case common_schema::KIND_ENUM: {
common_schema::type_set types;
for (const auto & value : static_cast<const common_schema_enum &>(s).values) {
case common_chat_schema::KIND_ANY:
return common_chat_schema::type_set::all();
case common_chat_schema::KIND_NULL:
return { common_chat_schema::TYPE_NULL };
case common_chat_schema::KIND_BOOLEAN:
return { common_chat_schema::TYPE_BOOLEAN };
case common_chat_schema::KIND_NUMBER:
return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER };
case common_chat_schema::KIND_INTEGER:
return { common_chat_schema::TYPE_INTEGER };
case common_chat_schema::KIND_STRING:
return { common_chat_schema::TYPE_STRING };
case common_chat_schema::KIND_ARRAY:
case common_chat_schema::KIND_TUPLE:
return { common_chat_schema::TYPE_ARRAY };
case common_chat_schema::KIND_OBJECT:
return { common_chat_schema::TYPE_OBJECT };
case common_chat_schema::KIND_CONST:
return { json_type(static_cast<const common_chat_schema_const &>(s).value) };
case common_chat_schema::KIND_ENUM: {
common_chat_schema::type_set types;
for (const auto & value : static_cast<const common_chat_schema_enum &>(s).values) {
types.add(json_type(value));
}
return types;
}
case common_schema::KIND_REF: {
const auto * target = static_cast<const common_schema_ref &>(s).target;
case common_chat_schema::KIND_REF: {
const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
if (!target || !visited.insert(target).second) {
// a cycle contributes no type, to be safe
return {};
@@ -403,16 +403,16 @@ static common_schema::type_set value_types_impl(const common_schema & s, std::un
visited.erase(target);
return types;
}
case common_schema::KIND_ANY_OF: {
common_schema::type_set types;
for (const auto & child : static_cast<const common_schema_any_of &>(s).children) {
case common_chat_schema::KIND_ANY_OF: {
common_chat_schema::type_set types;
for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
types |= value_types_impl(*child, visited);
}
return types;
}
case common_schema::KIND_ALL_OF: {
auto types = common_schema::type_set::all();
for (const auto & child : static_cast<const common_schema_all_of &>(s).children) {
case common_chat_schema::KIND_ALL_OF: {
auto types = common_chat_schema::type_set::all();
for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
types &= value_types_impl(*child, visited);
}
return types;
@@ -421,27 +421,27 @@ static common_schema::type_set value_types_impl(const common_schema & s, std::un
return {};
}
common_schema::type_set common_schema::value_types() const {
std::unordered_set<const common_schema *> visited;
common_chat_schema::type_set common_chat_schema::value_types() const {
std::unordered_set<const common_chat_schema *> visited;
return value_types_impl(*this, visited);
}
static bool may_be_string_impl(const common_schema & s, std::unordered_set<const common_schema *> & visited) {
static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
switch (s.kind()) {
case common_schema::KIND_STRING:
case common_chat_schema::KIND_STRING:
return true;
case common_schema::KIND_CONST:
return static_cast<const common_schema_const &>(s).value.is_string();
case common_schema::KIND_ENUM:
for (const auto & v : static_cast<const common_schema_enum &>(s).values) {
case common_chat_schema::KIND_CONST:
return static_cast<const common_chat_schema_const &>(s).value.is_string();
case common_chat_schema::KIND_ENUM:
for (const auto & v : static_cast<const common_chat_schema_enum &>(s).values) {
if (v.is_string()) {
return true;
}
}
return false;
case common_schema::KIND_REF: {
case common_chat_schema::KIND_REF: {
// a cycle is taken as not a string, to be safe
const auto * target = static_cast<const common_schema_ref &>(s).target;
const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
if (!target || !visited.insert(target).second) {
return false;
}
@@ -449,18 +449,18 @@ static bool may_be_string_impl(const common_schema & s, std::unordered_set<const
visited.erase(target);
return result;
}
case common_schema::KIND_ANY_OF:
for (const auto & child : static_cast<const common_schema_any_of &>(s).children) {
case common_chat_schema::KIND_ANY_OF:
for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
if (may_be_string_impl(*child, visited)) {
return true;
}
}
return false;
case common_schema::KIND_ALL_OF: {
case common_chat_schema::KIND_ALL_OF: {
// every child must allow a string, an any child constrains nothing
bool any_string = false;
for (const auto & child : static_cast<const common_schema_all_of &>(s).children) {
if (child->kind() == common_schema::KIND_ANY) {
for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
if (child->kind() == common_chat_schema::KIND_ANY) {
continue;
}
if (!may_be_string_impl(*child, visited)) {
@@ -475,12 +475,12 @@ static bool may_be_string_impl(const common_schema & s, std::unordered_set<const
}
}
bool common_schema::may_be_string() const {
std::unordered_set<const common_schema *> visited;
bool common_chat_schema::may_be_string() const {
std::unordered_set<const common_chat_schema *> visited;
return may_be_string_impl(*this, visited);
}
const char * common_schema::kind_name(node_kind kind) {
const char * common_chat_schema::kind_name(node_kind kind) {
switch (kind) {
case KIND_ANY: return "any";
case KIND_REF: return "ref";
@@ -500,7 +500,7 @@ const char * common_schema::kind_name(node_kind kind) {
return "?";
}
const char * common_schema::type_name(value_type type) {
const char * common_chat_schema::type_name(value_type type) {
switch (type) {
case TYPE_NULL: return "null";
case TYPE_BOOLEAN: return "boolean";
common/json-schema.h
+38 -38
View File
@@ -11,7 +11,7 @@
// JSON schema, covering the subset that json_schema_to_grammar() can convert.
struct common_schema {
struct common_chat_schema {
enum node_kind {
KIND_ANY,
KIND_REF,
@@ -75,7 +75,7 @@ struct common_schema {
bool operator!=(const type_set & other) const { return mask_ != other.mask_; }
};
virtual ~common_schema() = default;
virtual ~common_chat_schema() = default;
virtual node_kind kind() const = 0;
type_set value_types() const;
@@ -87,70 +87,70 @@ struct common_schema {
static const char * type_name(value_type type);
};
using common_schema_ptr = std::unique_ptr<common_schema>;
using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
struct common_schema_any : common_schema {
struct common_chat_schema_any : common_chat_schema {
node_kind kind() const override { return KIND_ANY; }
};
// {"$ref": "#/..."}, only references into the same document are supported
struct common_schema_ref : common_schema {
std::string ref;
const common_schema * target = nullptr; // owned by common_schema_document::refs
struct common_chat_schema_ref : common_chat_schema {
std::string ref;
const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs
explicit common_schema_ref(std::string ref) : ref(std::move(ref)) {}
explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
node_kind kind() const override { return KIND_REF; }
};
// oneOf / anyOf, or a "type" array expanded to one alternative per type
struct common_schema_any_of : common_schema {
std::vector<common_schema_ptr> children;
struct common_chat_schema_any_of : common_chat_schema {
std::vector<common_chat_schema_ptr> children;
node_kind kind() const override { return KIND_ANY_OF; }
};
struct common_schema_all_of : common_schema {
std::vector<common_schema_ptr> children;
struct common_chat_schema_all_of : common_chat_schema {
std::vector<common_chat_schema_ptr> children;
node_kind kind() const override { return KIND_ALL_OF; }
};
struct common_schema_const : common_schema {
struct common_chat_schema_const : common_chat_schema {
common_json value;
explicit common_schema_const(common_json value) : value(std::move(value)) {}
explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
node_kind kind() const override { return KIND_CONST; }
};
struct common_schema_enum : common_schema {
struct common_chat_schema_enum : common_chat_schema {
std::vector<common_json> values;
node_kind kind() const override { return KIND_ENUM; }
};
struct common_schema_null : common_schema {
struct common_chat_schema_null : common_chat_schema {
node_kind kind() const override { return KIND_NULL; }
};
struct common_schema_boolean : common_schema {
struct common_chat_schema_boolean : common_chat_schema {
node_kind kind() const override { return KIND_BOOLEAN; }
};
struct common_schema_number : common_schema {
struct common_chat_schema_number : common_chat_schema {
node_kind kind() const override { return KIND_NUMBER; }
};
// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
struct common_schema_integer : common_schema {
struct common_chat_schema_integer : common_chat_schema {
int64_t minimum = INT64_MIN; // INT64_MIN for unbounded
int64_t maximum = INT64_MAX; // INT64_MAX for unbounded
node_kind kind() const override { return KIND_INTEGER; }
};
struct common_schema_string : common_schema {
struct common_chat_schema_string : common_chat_schema {
std::string pattern; // empty when absent
string_format format = FORMAT_NONE;
int min_length = 0;
@@ -159,40 +159,40 @@ struct common_schema_string : common_schema {
node_kind kind() const override { return KIND_STRING; }
};
struct common_schema_array : common_schema {
common_schema_ptr items; // a common_schema_any when "items" is absent
int min_items = 0;
int max_items = -1; // -1 for unbounded
struct common_chat_schema_array : common_chat_schema {
common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent
int min_items = 0;
int max_items = -1; // -1 for unbounded
node_kind kind() const override { return KIND_ARRAY; }
};
struct common_schema_tuple : common_schema {
std::vector<common_schema_ptr> items;
struct common_chat_schema_tuple : common_chat_schema {
std::vector<common_chat_schema_ptr> items;
node_kind kind() const override { return KIND_TUPLE; }
};
struct common_schema_property {
std::string name;
common_schema_ptr schema;
bool required = false;
struct common_chat_schema_property {
std::string name;
common_chat_schema_ptr schema;
bool required = false;
};
struct common_schema_object : common_schema {
std::vector<common_schema_property> properties; // in schema order
common_schema_ptr additional_properties; // null when not allowed
struct common_chat_schema_object : common_chat_schema {
std::vector<common_chat_schema_property> properties; // in schema order
common_chat_schema_ptr additional_properties; // null when not allowed
node_kind kind() const override { return KIND_OBJECT; }
};
struct common_schema_document {
common_schema_ptr root;
std::map<std::string, common_schema_ptr> refs;
struct common_chat_schema_document {
common_chat_schema_ptr root;
std::map<std::string, common_chat_schema_ptr> refs;
};
// A document shared by the PEG parsers built from its nodes, which it keeps alive
using common_schema_document_ptr = std::shared_ptr<const common_schema_document>;
using common_chat_schema_document_ptr = std::shared_ptr<const common_chat_schema_document>;
// Throws std::runtime_error when the schema falls outside the supported subset.
common_schema_document common_schema_from_json(const common_json & schema);
common_chat_schema_document common_chat_schema_from_json(const common_json & schema);
common/parsers/deepseek.cpp
+1 -1
View File
@@ -152,7 +152,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
std::vector<common_peg_parser> required_parsers;
std::vector<common_peg_parser> optional_parsers;
foreach_parameter(function, [&](const common_schema_property & param, const common_schema_document_ptr & doc) {
foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
bool is_string = param.schema->may_be_string();
auto arg = p.tool_arg(
common/parsers/minicpm5.cpp
+1 -1
View File
@@ -73,7 +73,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
const std::string name = function.at("name");
std::vector<common_peg_parser> arg_rules;
foreach_parameter(function, [&](const common_schema_property & prop, const common_schema_document_ptr & doc) {
foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
auto value_parser = p.eps();
if (prop.schema->may_be_string()) {
value_parser = string_value;
common/parsers/minimax-m3.cpp
+14 -14
View File
@@ -89,13 +89,13 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
const auto & function = tool.at("function");
std::string name = function.at("name");
auto params = common_chat_tool_parameters(function);
auto doc = std::make_shared<const common_schema_document>(common_schema_from_json(params));
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
// The template expands argument values recursively in XML (see the to_xml() macro)
std::function<common_peg_parser(const common_schema &, const std::string &, const std::string &)> value_of;
std::function<common_peg_parser(const common_schema_object &, const std::string &)> members_of;
std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of;
std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)> members_of;
auto element_of = [&](const std::string & tag, const common_schema & schema, const std::string & rule_name) {
auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) {
const std::string close = NS + "</" + tag + ">";
return p.rule(rule_name,
p.tool_arg(
@@ -106,7 +106,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
value_of(schema, rule_name, close)));
};
value_of = [&](const common_schema & schema,
value_of = [&](const common_chat_schema & schema,
const std::string & rule_name,
const std::string & close) -> common_peg_parser {
auto close_tag = p.tool_arg_close(p.literal(close));
@@ -116,11 +116,11 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
}
if (schema.kind() == common_schema::KIND_ANY_OF) {
if (schema.kind() == common_chat_schema::KIND_ANY_OF) {
std::vector<common_peg_parser> choices;
size_t index = 0;
for (const auto & alternative : static_cast<const common_schema_any_of &>(schema).children) {
for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) {
const std::string alt_name = rule_name + "-" + std::to_string(index++);
// There is a risk that this breaks streaming deltas, but that's a risk we
@@ -131,19 +131,19 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
return p.choice(choices);
}
if (schema.kind() == common_schema::KIND_OBJECT) {
const auto & object = static_cast<const common_schema_object &>(schema);
if (schema.kind() == common_chat_schema::KIND_OBJECT) {
const auto & object = static_cast<const common_chat_schema_object &>(schema);
if (!object.properties.empty()) {
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag;
}
}
if (schema.kind() == common_schema::KIND_ARRAY) {
if (schema.kind() == common_chat_schema::KIND_ARRAY) {
const std::string item_close = NS + "</item>";
auto item = p.rule(rule_name + "-item",
p.tag(mm3::TOOL_ARG_ITEM,
p.literal(NS + "<item>") +
value_of(*static_cast<const common_schema_array &>(schema).items, rule_name + "-item", item_close)));
value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close)));
return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
}
@@ -151,7 +151,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
};
// Required properties in schema order, then any number of optional ones in any order.
members_of = [&](const common_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
std::vector<common_peg_parser> required_elements;
std::vector<common_peg_parser> optional_elements;
for (const auto & prop : object.properties) {
@@ -179,8 +179,8 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
};
common_peg_parser invoke_body = p.eps();
if (doc->root->kind() == common_schema::KIND_OBJECT) {
invoke_body = members_of(static_cast<const common_schema_object &>(*doc->root), "tool-" + name + "-arg");
if (doc->root->kind() == common_chat_schema::KIND_OBJECT) {
invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + name + "-arg");
}
auto func_parser = p.tool(
common/parsers/muse-glimmer.cpp
+1 -1
View File
@@ -76,7 +76,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
const std::string name = function.at("name");
std::vector<common_peg_parser> arg_rules;
foreach_parameter(function, [&](const common_schema_property & prop, const common_schema_document_ptr & doc) {
foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
auto value_parser = p.eps();
if (prop.schema->may_be_string()) {
value_parser = string_value;
common/parsers/parsers.cpp
+3 -3
View File
@@ -12,10 +12,10 @@ void foreach_function(const json & tools, const std::function<void(const json &)
}
}
void foreach_parameter(const json & function, const std::function<void(const common_schema_property &, const common_schema_document_ptr &)> & fn) {
void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn) {
auto params = common_chat_tool_parameters(function);
auto doc = std::make_shared<const common_schema_document>(common_schema_from_json(params));
const auto * object = dynamic_cast<const common_schema_object *>(doc->root.get());
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
const auto * object = dynamic_cast<const common_chat_schema_object *>(doc->root.get());
if (!object) {
return;
}
common/parsers/parsers.h
+1 -1
View File
@@ -21,7 +21,7 @@ using json = common_json;
void foreach_function(const json & tools, const std::function<void(const json &)> & fn);
// iterate over the parameters of a function tool, with the document that owns them
void foreach_parameter(const json & function, const std::function<void(const common_schema_property &, const common_schema_document_ptr &)> & fn);
void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn);
// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten
std::string common_chat_template_direct_apply_impl(
common/parsers/qwen3-coder.cpp
+1 -1
View File
@@ -99,7 +99,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
std::vector<common_peg_parser> required_args;
std::vector<common_peg_parser> optional_args;
foreach_parameter(function, [&](const common_schema_property & param, const common_schema_document_ptr & doc) {
foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
auto rule_name = "tool-" + name + "-arg-" + param.name;
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n");
common/peg-parser.cpp
+3 -3
View File
@@ -953,7 +953,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
return "Until(" + string_join(p.delimiters, " | ") + ")";
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_schema::kind_name(p.node->kind()) : "null") + ")";
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")";
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
@@ -1119,12 +1119,12 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_schema_document_ptr doc, const common_schema & node, bool raw) {
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
auto doc = std::make_shared<const common_schema_document>(common_schema_from_json(schema));
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(schema));
return this->schema(p, name, doc, *doc->root, raw);
}
common/peg-parser.h
+3 -3
View File
@@ -246,8 +246,8 @@ struct common_peg_until_parser {
struct common_peg_schema_parser {
common_peg_parser_id child;
std::string name;
common_schema_document_ptr doc; // owns node
const common_schema * node = nullptr;
common_chat_schema_document_ptr doc; // owns node
const common_chat_schema * node = nullptr;
// Indicates if the GBNF should accept a raw string that matches the schema.
bool raw;
@@ -491,7 +491,7 @@ class common_peg_parser_builder {
common_peg_parser marker();
// Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it
common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_schema_document_ptr doc, const common_schema & node, bool raw = false);
common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false);
// Parses the JSON schema into a document of its own
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
docs/development/parsing.md
+2 -2
View File
@@ -108,7 +108,7 @@ For a more complete example, see `test_example_native()` in
- **`rule(name, p, trigger)`** - Creates a named rule and returns a reference
- **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation)
- **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation
- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_schema_document` built earlier, e.g. one tool parameter
- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter
### AST Control
@@ -150,7 +150,7 @@ implementation to generate the grammar instead of the underlying parser.
The `raw` option emits a grammar suitable for a raw string instead of a JSON
string. In other words, it won't be wrapped in quotes or require escaping
quotes. It only takes effect when the schema may be a string, as reported by
`common_schema::may_be_string()`, otherwise the JSON grammar is used.
`common_chat_schema::may_be_string()`, otherwise the JSON grammar is used.
The downside is that it can potentially lead to ambiguous grammars. For
example, if a user provides the pattern `^.*$`, the following grammar may be
tests/test-json-schema-to-grammar.cpp
+3 -3
View File
@@ -1534,7 +1534,7 @@ int main() {
}
}
})""");
assert(json_schema_to_grammar(common_schema_from_json(schema)) == json_schema_to_grammar(schema, true));
assert(json_schema_to_grammar(common_chat_schema_from_json(schema)) == json_schema_to_grammar(schema, true));
}
// a property node carries its $ref target, so its grammar names the ref rule
@@ -1565,9 +1565,9 @@ int main() {
string ::= "\"" char* "\""
)""",
};
auto doc = common_schema_from_json(parameters);
auto doc = common_chat_schema_from_json(parameters);
tc.verify(build_grammar([&](const common_grammar_builder & builder) {
const auto & item = static_cast<const common_schema_object &>(*doc.root).properties.at(0);
const auto & item = static_cast<const common_chat_schema_object &>(*doc.root).properties.at(0);
builder.add_schema("root", *item.schema);
}));
}
tests/test-json-schema.cpp
+104 -104
View File
@@ -8,13 +8,13 @@
#include <string>
#include <utility>
static common_schema_document parse(const std::string & schema) {
return common_schema_from_json(common_json::parse(schema));
static common_chat_schema_document parse(const std::string & schema) {
return common_chat_schema_from_json(common_json::parse(schema));
}
// the node as T, aborting the current test when it is some other kind
template <typename T>
static const T & as(testing & t, const common_schema * node, const char * what) {
static const T & as(testing & t, const common_chat_schema * node, const char * what) {
const T * typed = dynamic_cast<const T *>(node);
if (!t.assert_true(std::string(what) + " has the expected kind", typed != nullptr)) {
throw std::runtime_error(std::string(what) + " has the wrong kind");
@@ -23,7 +23,7 @@ static const T & as(testing & t, const common_schema * node, const char * what)
}
template <typename T>
static const T & root(testing & t, const common_schema_document & doc) {
static const T & root(testing & t, const common_chat_schema_document & doc) {
return as<T>(t, doc.root.get(), "root");
}
@@ -40,52 +40,52 @@ static void assert_error(testing & t, const std::string & schema, const std::str
static void test_any(testing & t) {
t.test("empty schema", [](testing & t) {
auto doc = parse("{}");
root<common_schema_any>(t, doc);
root<common_chat_schema_any>(t, doc);
t.assert_true("no refs", doc.refs.empty());
});
t.test("keywords that do not imply a type", [](testing & t) {
auto doc = parse(R"({"description": "x", "format": "email", "additionalProperties": true})");
root<common_schema_any>(t, doc);
root<common_chat_schema_any>(t, doc);
});
}
static void test_primitives(testing & t) {
t.test("null, boolean, number", [](testing & t) {
auto doc_null = parse(R"({"type": "null"})");
root<common_schema_null>(t, doc_null);
root<common_chat_schema_null>(t, doc_null);
auto doc_bool = parse(R"({"type": "boolean"})");
root<common_schema_boolean>(t, doc_bool);
root<common_chat_schema_boolean>(t, doc_bool);
auto doc_num = parse(R"({"type": "number", "minimum": 1, "maximum": 2})");
root<common_schema_number>(t, doc_num);
root<common_chat_schema_number>(t, doc_num);
});
}
static void test_integer(testing & t) {
t.test("unbounded", [](testing & t) {
auto doc = parse(R"({"type": "integer"})");
const auto & i = root<common_schema_integer>(t, doc);
const auto & i = root<common_chat_schema_integer>(t, doc);
t.assert_equal("minimum", INT64_MIN, i.minimum);
t.assert_equal("maximum", INT64_MAX, i.maximum);
});
t.test("inclusive bounds", [](testing & t) {
auto doc = parse(R"({"type": "integer", "minimum": -5, "maximum": 10})");
const auto & i = root<common_schema_integer>(t, doc);
const auto & i = root<common_chat_schema_integer>(t, doc);
t.assert_equal("minimum", -5, i.minimum);
t.assert_equal("maximum", 10, i.maximum);
});
t.test("exclusive bounds are folded", [](testing & t) {
auto doc = parse(R"({"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10})");
const auto & i = root<common_schema_integer>(t, doc);
const auto & i = root<common_chat_schema_integer>(t, doc);
t.assert_equal("minimum", 1, i.minimum);
t.assert_equal("maximum", 9, i.maximum);
});
t.test("fractional bounds round inwards", [](testing & t) {
auto doc = parse(R"({"type": "integer", "minimum": 1.5, "exclusiveMaximum": 9.5})");
const auto & i = root<common_schema_integer>(t, doc);
const auto & i = root<common_chat_schema_integer>(t, doc);
t.assert_equal("minimum", 2, i.minimum);
t.assert_equal("maximum", 9, i.maximum);
});
@@ -94,92 +94,92 @@ static void test_integer(testing & t) {
static void test_string(testing & t) {
t.test("defaults", [](testing & t) {
auto doc = parse(R"({"type": "string"})");
const auto & s = root<common_schema_string>(t, doc);
const auto & s = root<common_chat_schema_string>(t, doc);
t.assert_equal("pattern", "", s.pattern);
t.assert_equal("format", common_schema::FORMAT_NONE, s.format);
t.assert_equal("format", common_chat_schema::FORMAT_NONE, s.format);
t.assert_equal("min_length", 0, s.min_length);
t.assert_equal("max_length", -1, s.max_length);
});
t.test("all keywords are kept", [](testing & t) {
auto doc = parse(R"({"type": "string", "pattern": "^[a-z]+$", "format": "date", "minLength": 2, "maxLength": 8})");
const auto & s = root<common_schema_string>(t, doc);
const auto & s = root<common_chat_schema_string>(t, doc);
t.assert_equal("pattern", "^[a-z]+$", s.pattern);
t.assert_equal("format", common_schema::FORMAT_DATE, s.format);
t.assert_equal("format", common_chat_schema::FORMAT_DATE, s.format);
t.assert_equal("min_length", 2, s.min_length);
t.assert_equal("max_length", 8, s.max_length);
});
t.test("formats", [](testing & t) {
auto expect = [&](const char * format, common_schema::string_format expected) {
auto expect = [&](const char * format, common_chat_schema::string_format expected) {
auto doc = parse(std::string(R"({"type": "string", "format": ")") + format + "\"}");
t.assert_equal(format, expected, root<common_schema_string>(t, doc).format);
t.assert_equal(format, expected, root<common_chat_schema_string>(t, doc).format);
};
expect("time", common_schema::FORMAT_TIME);
expect("date-time", common_schema::FORMAT_DATE_TIME);
expect("uuid", common_schema::FORMAT_UUID);
expect("uuid5", common_schema::FORMAT_UUID);
expect("email", common_schema::FORMAT_NONE);
expect("time", common_chat_schema::FORMAT_TIME);
expect("date-time", common_chat_schema::FORMAT_DATE_TIME);
expect("uuid", common_chat_schema::FORMAT_UUID);
expect("uuid5", common_chat_schema::FORMAT_UUID);
expect("email", common_chat_schema::FORMAT_NONE);
});
t.test("pattern, length and known format imply a string", [](testing & t) {
auto doc_pattern = parse(R"({"pattern": "^a$"})");
t.assert_equal("pattern", "^a$", root<common_schema_string>(t, doc_pattern).pattern);
t.assert_equal("pattern", "^a$", root<common_chat_schema_string>(t, doc_pattern).pattern);
auto doc_length = parse(R"({"minLength": 1, "maxLength": 3})");
t.assert_equal("min_length", 1, root<common_schema_string>(t, doc_length).min_length);
t.assert_equal("max_length", 3, root<common_schema_string>(t, doc_length).max_length);
t.assert_equal("min_length", 1, root<common_chat_schema_string>(t, doc_length).min_length);
t.assert_equal("max_length", 3, root<common_chat_schema_string>(t, doc_length).max_length);
auto doc_format = parse(R"({"format": "uuid"})");
t.assert_equal("format", common_schema::FORMAT_UUID, root<common_schema_string>(t, doc_format).format);
t.assert_equal("format", common_chat_schema::FORMAT_UUID, root<common_chat_schema_string>(t, doc_format).format);
});
}
static void test_array(testing & t) {
t.test("items with bounds", [](testing & t) {
auto doc = parse(R"({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3})");
const auto & a = root<common_schema_array>(t, doc);
as<common_schema_integer>(t, a.items.get(), "items");
const auto & a = root<common_chat_schema_array>(t, doc);
as<common_chat_schema_integer>(t, a.items.get(), "items");
t.assert_equal("min_items", 1, a.min_items);
t.assert_equal("max_items", 3, a.max_items);
});
t.test("no items", [](testing & t) {
auto doc = parse(R"({"type": "array"})");
const auto & a = root<common_schema_array>(t, doc);
as<common_schema_any>(t, a.items.get(), "items");
const auto & a = root<common_chat_schema_array>(t, doc);
as<common_chat_schema_any>(t, a.items.get(), "items");
t.assert_equal("min_items", 0, a.min_items);
t.assert_equal("max_items", -1, a.max_items);
});
t.test("items imply an array", [](testing & t) {
auto doc = parse(R"({"items": {"type": "string"}})");
const auto & a = root<common_schema_array>(t, doc);
as<common_schema_string>(t, a.items.get(), "items");
const auto & a = root<common_chat_schema_array>(t, doc);
as<common_chat_schema_string>(t, a.items.get(), "items");
});
}
static void test_tuple(testing & t) {
t.test("prefixItems", [](testing & t) {
auto doc = parse(R"({"prefixItems": [{"type": "string"}, {"type": "number"}]})");
const auto & tup = root<common_schema_tuple>(t, doc);
const auto & tup = root<common_chat_schema_tuple>(t, doc);
t.assert_equal("size", (size_t) 2, tup.items.size());
as<common_schema_string>(t, tup.items[0].get(), "items[0]");
as<common_schema_number>(t, tup.items[1].get(), "items[1]");
as<common_chat_schema_string>(t, tup.items[0].get(), "items[0]");
as<common_chat_schema_number>(t, tup.items[1].get(), "items[1]");
});
t.test("items as an array", [](testing & t) {
auto doc = parse(R"({"type": "array", "items": [{"type": "boolean"}]})");
const auto & tup = root<common_schema_tuple>(t, doc);
const auto & tup = root<common_chat_schema_tuple>(t, doc);
t.assert_equal("size", (size_t) 1, tup.items.size());
as<common_schema_boolean>(t, tup.items[0].get(), "items[0]");
as<common_chat_schema_boolean>(t, tup.items[0].get(), "items[0]");
});
}
static void test_object(testing & t) {
t.test("type alone accepts any object", [](testing & t) {
auto doc = parse(R"({"type": "object"})");
const auto & o = root<common_schema_object>(t, doc);
const auto & o = root<common_chat_schema_object>(t, doc);
t.assert_true("no properties", o.properties.empty());
as<common_schema_any>(t, o.additional_properties.get(), "additional_properties");
as<common_chat_schema_any>(t, o.additional_properties.get(), "additional_properties");
});
t.test("properties", [](testing & t) {
@@ -192,7 +192,7 @@ static void test_object(testing & t) {
},
"required": ["a", "c"]
})");
const auto & o = root<common_schema_object>(t, doc);
const auto & o = root<common_chat_schema_object>(t, doc);
t.assert_equal("size", (size_t) 3, o.properties.size());
t.assert_equal("order", "b", o.properties[0].name);
t.assert_equal("order", "a", o.properties[1].name);
@@ -200,53 +200,53 @@ static void test_object(testing & t) {
t.assert_true("b optional", !o.properties[0].required);
t.assert_true("a required", o.properties[1].required);
t.assert_true("c required", o.properties[2].required);
as<common_schema_string>(t, o.properties[0].schema.get(), "b");
as<common_schema_integer>(t, o.properties[1].schema.get(), "a");
as<common_schema_boolean>(t, o.properties[2].schema.get(), "c");
as<common_chat_schema_string>(t, o.properties[0].schema.get(), "b");
as<common_chat_schema_integer>(t, o.properties[1].schema.get(), "a");
as<common_chat_schema_boolean>(t, o.properties[2].schema.get(), "c");
t.assert_true("closed", o.additional_properties == nullptr);
});
t.test("unknown required entries are ignored", [](testing & t) {
auto doc = parse(R"({"properties": {"a": {}}, "required": ["a", "zzz", 1]})");
const auto & o = root<common_schema_object>(t, doc);
const auto & o = root<common_chat_schema_object>(t, doc);
t.assert_equal("size", (size_t) 1, o.properties.size());
t.assert_true("a required", o.properties[0].required);
});
t.test("additionalProperties false implies an object", [](testing & t) {
auto doc = parse(R"({"additionalProperties": false})");
const auto & o = root<common_schema_object>(t, doc);
const auto & o = root<common_chat_schema_object>(t, doc);
t.assert_true("no properties", o.properties.empty());
t.assert_true("closed", o.additional_properties == nullptr);
});
t.test("additionalProperties schema", [](testing & t) {
auto doc = parse(R"({"properties": {"a": {}}, "additionalProperties": {"type": "integer", "minimum": 0}})");
const auto & o = root<common_schema_object>(t, doc);
const auto & o = root<common_chat_schema_object>(t, doc);
t.assert_equal("size", (size_t) 1, o.properties.size());
const auto & v = as<common_schema_integer>(t, o.additional_properties.get(), "additional_properties");
const auto & v = as<common_chat_schema_integer>(t, o.additional_properties.get(), "additional_properties");
t.assert_equal("minimum", 0, v.minimum);
});
t.test("nested", [](testing & t) {
auto doc = parse(R"({"properties": {"inner": {"properties": {"leaf": {"type": "null"}}, "required": ["leaf"]}}})");
const auto & o = root<common_schema_object>(t, doc);
const auto & inner = as<common_schema_object>(t, o.properties[0].schema.get(), "inner");
const auto & o = root<common_chat_schema_object>(t, doc);
const auto & inner = as<common_chat_schema_object>(t, o.properties[0].schema.get(), "inner");
t.assert_equal("leaf name", "leaf", inner.properties[0].name);
t.assert_true("leaf required", inner.properties[0].required);
as<common_schema_null>(t, inner.properties[0].schema.get(), "leaf");
as<common_chat_schema_null>(t, inner.properties[0].schema.get(), "leaf");
});
}
static void test_const_enum(testing & t) {
t.test("const", [](testing & t) {
auto doc = parse(R"({"const": {"a": [1, null]}})");
t.assert_equal("value", R"({"a":[1,null]})", root<common_schema_const>(t, doc).value.dump());
t.assert_equal("value", R"({"a":[1,null]})", root<common_chat_schema_const>(t, doc).value.dump());
});
t.test("enum", [](testing & t) {
auto doc = parse(R"({"enum": ["a", 1, null, true]})");
const auto & e = root<common_schema_enum>(t, doc);
const auto & e = root<common_chat_schema_enum>(t, doc);
t.assert_equal("size", (size_t) 4, e.values.size());
t.assert_equal("values[0]", "\"a\"", e.values[0].dump());
t.assert_equal("values[1]", "1", e.values[1].dump());
@@ -256,81 +256,81 @@ static void test_const_enum(testing & t) {
t.test("const wins over enum, enum wins over type", [](testing & t) {
auto doc_enum = parse(R"({"type": "integer", "enum": [1, 2]})");
root<common_schema_enum>(t, doc_enum);
root<common_chat_schema_enum>(t, doc_enum);
auto doc_const = parse(R"({"type": "string", "const": "x", "enum": ["y"]})");
t.assert_equal("value", "\"x\"", root<common_schema_const>(t, doc_const).value.dump());
t.assert_equal("value", "\"x\"", root<common_chat_schema_const>(t, doc_const).value.dump());
});
}
static void test_any_of(testing & t) {
t.test("anyOf and oneOf", [](testing & t) {
auto doc_any = parse(R"({"anyOf": [{"type": "string"}, {"type": "number"}]})");
const auto & u = root<common_schema_any_of>(t, doc_any);
const auto & u = root<common_chat_schema_any_of>(t, doc_any);
t.assert_equal("size", (size_t) 2, u.children.size());
as<common_schema_string>(t, u.children[0].get(), "children[0]");
as<common_schema_number>(t, u.children[1].get(), "children[1]");
as<common_chat_schema_string>(t, u.children[0].get(), "children[0]");
as<common_chat_schema_number>(t, u.children[1].get(), "children[1]");
auto doc_one = parse(R"({"oneOf": [{"type": "null"}]})");
const auto & o = root<common_schema_any_of>(t, doc_one);
const auto & o = root<common_chat_schema_any_of>(t, doc_one);
t.assert_equal("size", (size_t) 1, o.children.size());
as<common_schema_null>(t, o.children[0].get(), "children[0]");
as<common_chat_schema_null>(t, o.children[0].get(), "children[0]");
});
t.test("oneOf wins over anyOf and type", [](testing & t) {
auto doc = parse(R"({"type": "string", "oneOf": [{"type": "null"}], "anyOf": [{"type": "number"}, {"type": "boolean"}]})");
const auto & u = root<common_schema_any_of>(t, doc);
const auto & u = root<common_chat_schema_any_of>(t, doc);
t.assert_equal("size", (size_t) 1, u.children.size());
as<common_schema_null>(t, u.children[0].get(), "children[0]");
as<common_chat_schema_null>(t, u.children[0].get(), "children[0]");
});
t.test("type array expands with sibling keywords", [](testing & t) {
auto doc = parse(R"({"type": ["string", "null", "integer"], "minLength": 2, "minimum": 5})");
const auto & u = root<common_schema_any_of>(t, doc);
const auto & u = root<common_chat_schema_any_of>(t, doc);
t.assert_equal("size", (size_t) 3, u.children.size());
t.assert_equal("min_length", 2, as<common_schema_string>(t, u.children[0].get(), "children[0]").min_length);
as<common_schema_null>(t, u.children[1].get(), "children[1]");
t.assert_equal("minimum", 5, as<common_schema_integer>(t, u.children[2].get(), "children[2]").minimum);
t.assert_equal("min_length", 2, as<common_chat_schema_string>(t, u.children[0].get(), "children[0]").min_length);
as<common_chat_schema_null>(t, u.children[1].get(), "children[1]");
t.assert_equal("minimum", 5, as<common_chat_schema_integer>(t, u.children[2].get(), "children[2]").minimum);
});
}
static void test_all_of(testing & t) {
t.test("components", [](testing & t) {
auto doc = parse(R"({"allOf": [{"properties": {"a": {}}}, {"anyOf": [{"properties": {"b": {}}}, {"type": "null"}]}]})");
const auto & all = root<common_schema_all_of>(t, doc);
const auto & all = root<common_chat_schema_all_of>(t, doc);
t.assert_equal("size", (size_t) 2, all.children.size());
as<common_schema_object>(t, all.children[0].get(), "children[0]");
as<common_schema_any_of>(t, all.children[1].get(), "children[1]");
as<common_chat_schema_object>(t, all.children[0].get(), "children[0]");
as<common_chat_schema_any_of>(t, all.children[1].get(), "children[1]");
auto doc_typed = parse(R"({"type": "object", "allOf": [{"properties": {"a": {}}}]})");
root<common_schema_all_of>(t, doc_typed);
root<common_chat_schema_all_of>(t, doc_typed);
});
t.test("properties win over allOf", [](testing & t) {
auto doc = parse(R"({"type": "object", "properties": {"a": {}}, "allOf": [{"properties": {"b": {}}}]})");
t.assert_equal("size", (size_t) 1, root<common_schema_object>(t, doc).properties.size());
t.assert_equal("size", (size_t) 1, root<common_chat_schema_object>(t, doc).properties.size());
});
t.test("other types ignore allOf", [](testing & t) {
auto doc = parse(R"({"type": "integer", "allOf": [{"minimum": 1}]})");
root<common_schema_integer>(t, doc);
root<common_chat_schema_integer>(t, doc);
});
}
static void test_ref(testing & t) {
t.test("target is owned by the document", [](testing & t) {
auto doc = parse(R"({"$ref": "#/$defs/t", "type": "string", "$defs": {"t": {"type": "boolean"}}})");
const auto & r = root<common_schema_ref>(t, doc);
const auto & r = root<common_chat_schema_ref>(t, doc);
t.assert_equal("ref", "#/$defs/t", r.ref);
t.assert_equal("refs", (size_t) 1, doc.refs.size());
t.assert_true("target", r.target != nullptr && r.target == doc.refs.at("#/$defs/t").get());
as<common_schema_boolean>(t, r.target, "target");
as<common_chat_schema_boolean>(t, r.target, "target");
});
t.test("definitions", [](testing & t) {
auto doc = parse(R"({"properties": {"a": {"$ref": "#/definitions/t"}}, "definitions": {"t": {"type": "number"}}})");
const auto & o = root<common_schema_object>(t, doc);
const auto & r = as<common_schema_ref>(t, o.properties[0].schema.get(), "a");
as<common_schema_number>(t, r.target, "target");
const auto & o = root<common_chat_schema_object>(t, doc);
const auto & r = as<common_chat_schema_ref>(t, o.properties[0].schema.get(), "a");
as<common_chat_schema_number>(t, r.target, "target");
});
t.test("recursive", [](testing & t) {
@@ -347,28 +347,28 @@ static void test_ref(testing & t) {
}
}
})");
const auto & r = root<common_schema_ref>(t, doc);
const auto & node = as<common_schema_object>(t, r.target, "node");
const auto & r = root<common_chat_schema_ref>(t, doc);
const auto & node = as<common_chat_schema_object>(t, r.target, "node");
t.assert_equal("properties", (size_t) 2, node.properties.size());
const auto & next = as<common_schema_ref>(t, node.properties[1].schema.get(), "next");
const auto & next = as<common_chat_schema_ref>(t, node.properties[1].schema.get(), "next");
t.assert_true("cycle", next.target == r.target);
t.assert_equal("refs", (size_t) 1, doc.refs.size());
});
t.test("pointer through an array", [](testing & t) {
auto doc = parse(R"({"oneOf": [{"type": "null"}, {"$ref": "#/oneOf/0"}]})");
const auto & u = root<common_schema_any_of>(t, doc);
const auto & r = as<common_schema_ref>(t, u.children[1].get(), "children[1]");
as<common_schema_null>(t, r.target, "target");
const auto & u = root<common_chat_schema_any_of>(t, doc);
const auto & r = as<common_chat_schema_ref>(t, u.children[1].get(), "children[1]");
as<common_chat_schema_null>(t, r.target, "target");
});
t.test("targets survive moving the document", [](testing & t) {
auto parsed = parse(R"({"items": {"$ref": "#/$defs/t"}, "$defs": {"t": {"type": "null"}}})");
common_schema_document doc = std::move(parsed);
const auto & a = root<common_schema_array>(t, doc);
const auto & r = as<common_schema_ref>(t, a.items.get(), "items");
common_chat_schema_document doc = std::move(parsed);
const auto & a = root<common_chat_schema_array>(t, doc);
const auto & r = as<common_chat_schema_ref>(t, a.items.get(), "items");
t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get());
as<common_schema_null>(t, r.target, "target");
as<common_chat_schema_null>(t, r.target, "target");
});
}
@@ -404,43 +404,43 @@ static void test_may_be_string(testing & t) {
}
// e.g. {number, integer}, in type order
static std::string dump(const common_schema::type_set & types) {
static const common_schema::value_type order[] = { common_schema::TYPE_NULL, common_schema::TYPE_BOOLEAN, common_schema::TYPE_NUMBER,
common_schema::TYPE_INTEGER, common_schema::TYPE_STRING, common_schema::TYPE_ARRAY,
common_schema::TYPE_OBJECT };
static std::string dump(const common_chat_schema::type_set & types) {
static const common_chat_schema::value_type order[] = { common_chat_schema::TYPE_NULL, common_chat_schema::TYPE_BOOLEAN, common_chat_schema::TYPE_NUMBER,
common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_ARRAY,
common_chat_schema::TYPE_OBJECT };
std::string out;
for (auto type : order) {
if (types.has(type)) {
out += (out.empty() ? "" : ", ") + std::string(common_schema::type_name(type));
out += (out.empty() ? "" : ", ") + std::string(common_chat_schema::type_name(type));
}
}
return "{" + out + "}";
}
static void test_value_types(testing & t) {
auto check = [](testing & t, const std::string & schema, const common_schema::type_set & expected) {
auto check = [](testing & t, const std::string & schema, const common_chat_schema::type_set & expected) {
t.assert_equal(schema, dump(expected), dump(parse(schema).root->value_types()));
};
t.test("leaves", [&](testing & t) {
check(t, R"({"type": "string"})", { common_schema::TYPE_STRING });
check(t, R"({"type": "number"})", { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER });
check(t, R"({"description": "x"})", common_schema::type_set::all());
check(t, R"({"properties": {"a": {"type": "string"}}})", { common_schema::TYPE_OBJECT });
check(t, R"({"items": {"type": "string"}})", { common_schema::TYPE_ARRAY });
check(t, R"({"const": 1.5})", { common_schema::TYPE_NUMBER });
check(t, R"({"enum": [1, "a", null]})", { common_schema::TYPE_INTEGER, common_schema::TYPE_STRING, common_schema::TYPE_NULL });
check(t, R"({"type": "string"})", { common_chat_schema::TYPE_STRING });
check(t, R"({"type": "number"})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER });
check(t, R"({"description": "x"})", common_chat_schema::type_set::all());
check(t, R"({"properties": {"a": {"type": "string"}}})", { common_chat_schema::TYPE_OBJECT });
check(t, R"({"items": {"type": "string"}})", { common_chat_schema::TYPE_ARRAY });
check(t, R"({"const": 1.5})", { common_chat_schema::TYPE_NUMBER });
check(t, R"({"enum": [1, "a", null]})", { common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL });
});
t.test("any_of is the union, all_of is the intersection", [&](testing & t) {
check(t, R"({"type": ["string", "null"]})", { common_schema::TYPE_STRING, common_schema::TYPE_NULL });
check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_schema::TYPE_NUMBER, common_schema::TYPE_INTEGER });
check(t, R"({"type": ["string", "null"]})", { common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL });
check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER });
check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", {});
});
t.test("ref", [&](testing & t) {
check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})",
{ common_schema::TYPE_STRING });
{ common_chat_schema::TYPE_STRING });
});
}