mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-17 20:31:47 +02:00
common : refactor json-schema-to-grammar to use common_schema
This commit is contained in:
Alde Rojas
+2
-2
@@ -2277,14 +2277,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-j", "--json-schema"}, "SCHEMA",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))};
|
||||
}
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-jf", "--json-schema-file"}, "FILE",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
[](common_params & params, const std::string & value) {
|
||||
std::ifstream file(value);
|
||||
if (!file) {
|
||||
|
||||
+236
-368
@@ -338,16 +338,21 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
|
||||
|
||||
class common_schema_converter {
|
||||
private:
|
||||
friend class common_schema_info;
|
||||
friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
|
||||
std::function<json(const std::string &)> _fetch_json;
|
||||
bool _dotall;
|
||||
std::map<std::string, std::string> _rules;
|
||||
std::unordered_map<std::string, json> _refs;
|
||||
std::unordered_set<std::string> _refs_being_resolved;
|
||||
std::vector<std::string> _errors;
|
||||
std::vector<std::string> _warnings;
|
||||
|
||||
// every schema given to resolve_refs() or add_schema() is parsed into this document, so their $refs resolve through each other
|
||||
common_schema_document _doc;
|
||||
|
||||
template <typename T>
|
||||
static const T & as(const common_schema & node) {
|
||||
return static_cast<const T &>(node);
|
||||
}
|
||||
|
||||
std::string _add_rule(const std::string & name, const std::string & rule) {
|
||||
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
|
||||
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
|
||||
@@ -363,11 +368,11 @@ private:
|
||||
return key;
|
||||
}
|
||||
|
||||
std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
|
||||
std::string _generate_union_rule(const std::string & name, const std::vector<common_schema_ptr> & alt_schemas) {
|
||||
std::vector<std::string> rules;
|
||||
rules.reserve(alt_schemas.size());
|
||||
for (size_t i = 0; i < alt_schemas.size(); i++) {
|
||||
rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
|
||||
rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
|
||||
}
|
||||
return string_join(rules, " | ");
|
||||
}
|
||||
@@ -694,25 +699,28 @@ private:
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string _resolve_ref(const std::string & ref) {
|
||||
auto it = ref.find('#');
|
||||
std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref;
|
||||
std::string _resolve_ref(const common_schema_ref & ref) {
|
||||
auto it = ref.ref.find('#');
|
||||
std::string ref_fragment = it != std::string::npos ? ref.ref.substr(it + 1) : ref.ref;
|
||||
static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
|
||||
std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
|
||||
if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
|
||||
_refs_being_resolved.insert(ref);
|
||||
json resolved = _refs[ref];
|
||||
ref_name = visit(resolved, ref_name);
|
||||
_refs_being_resolved.erase(ref);
|
||||
if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref.ref) == _refs_being_resolved.end()) {
|
||||
if (!ref.target) {
|
||||
_errors.push_back("Unresolved $ref " + ref.ref);
|
||||
return "";
|
||||
}
|
||||
_refs_being_resolved.insert(ref.ref);
|
||||
ref_name = visit(*ref.target, ref_name);
|
||||
_refs_being_resolved.erase(ref.ref);
|
||||
}
|
||||
return ref_name;
|
||||
}
|
||||
|
||||
std::string _build_object_rule(
|
||||
const std::vector<std::pair<std::string, json>> & properties,
|
||||
const std::vector<std::pair<std::string, const common_schema *>> & properties,
|
||||
const std::unordered_set<std::string> & required,
|
||||
const std::string & name,
|
||||
const json & additional_properties)
|
||||
const common_schema * additional_properties)
|
||||
{
|
||||
std::vector<std::string> required_props;
|
||||
std::vector<std::string> optional_props;
|
||||
@@ -722,7 +730,7 @@ private:
|
||||
const auto &prop_name = kv.first;
|
||||
const auto &prop_schema = kv.second;
|
||||
|
||||
std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
|
||||
std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name);
|
||||
prop_kv_rule_names[prop_name] = _add_rule(
|
||||
name + (name.empty() ? "" : "-") + prop_name + "-kv",
|
||||
format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
|
||||
@@ -734,10 +742,10 @@ private:
|
||||
}
|
||||
prop_names.push_back(prop_name);
|
||||
}
|
||||
if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) {
|
||||
if (additional_properties) {
|
||||
std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
|
||||
std::string value_rule =
|
||||
additional_properties.is_object() ? visit(additional_properties, sub_name + "-value")
|
||||
additional_properties->kind() != COMMON_SCHEMA_KIND_ANY ? visit(*additional_properties, sub_name + "-value")
|
||||
: _add_primitive("value", PRIMITIVE_RULES.at("value"));
|
||||
|
||||
auto key_rule =
|
||||
@@ -825,267 +833,184 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
common_schema_converter(
|
||||
const std::function<json(const std::string &)> & fetch_json,
|
||||
bool dotall)
|
||||
: _fetch_json(fetch_json), _dotall(dotall)
|
||||
{
|
||||
explicit common_schema_converter(bool dotall) : _dotall(dotall) {
|
||||
_rules["space"] = SPACE_RULE;
|
||||
}
|
||||
|
||||
void resolve_refs(json & schema, const std::string & url) {
|
||||
/*
|
||||
* Resolves all $ref fields in the given schema, fetching any remote schemas,
|
||||
* replacing each $ref with absolute reference URL and populates _refs with the
|
||||
* respective referenced (sub)schema dictionaries.
|
||||
*/
|
||||
std::function<void(json &)> visit_refs = [&](json & n) {
|
||||
if (n.is_array()) {
|
||||
for (auto & x : n) {
|
||||
visit_refs(x);
|
||||
}
|
||||
} else if (n.is_object()) {
|
||||
if (n.contains("$ref")) {
|
||||
std::string ref = n["$ref"];
|
||||
if (_refs.find(ref) == _refs.end()) {
|
||||
json target;
|
||||
if (ref.find("https://") == 0) {
|
||||
std::string base_url = ref.substr(0, ref.find('#'));
|
||||
auto it = _refs.find(base_url);
|
||||
if (it != _refs.end()) {
|
||||
target = it->second;
|
||||
} else {
|
||||
// Fetch the referenced schema and resolve its refs
|
||||
auto referenced = _fetch_json(ref);
|
||||
resolve_refs(referenced, base_url);
|
||||
_refs[base_url] = referenced;
|
||||
}
|
||||
if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
|
||||
return;
|
||||
}
|
||||
} else if (ref.find("#/") == 0) {
|
||||
target = schema;
|
||||
n["$ref"] = url + ref;
|
||||
ref = url + ref;
|
||||
} else {
|
||||
_errors.push_back("Unsupported ref: " + ref);
|
||||
return;
|
||||
}
|
||||
std::string pointer = ref.substr(ref.find('#') + 1);
|
||||
std::vector<std::string> tokens = string_split(pointer, "/");
|
||||
for (size_t i = 1; i < tokens.size(); ++i) {
|
||||
const std::string& sel = tokens[i];
|
||||
if (target.is_object() && target.contains(sel)) {
|
||||
target = target[sel];
|
||||
} else if (target.is_array()) {
|
||||
size_t sel_index;
|
||||
try {
|
||||
sel_index = std::stoull(sel);
|
||||
} catch (const std::invalid_argument & e) {
|
||||
sel_index = target.size();
|
||||
}
|
||||
if (sel_index >= target.size()) {
|
||||
_errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
|
||||
return;
|
||||
}
|
||||
target = target[sel_index];
|
||||
} else {
|
||||
_errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
|
||||
return;
|
||||
}
|
||||
}
|
||||
_refs[ref] = target;
|
||||
}
|
||||
} else {
|
||||
for (const auto & kv : n.items()) {
|
||||
visit_refs(kv.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// Parses the document, so that the $refs of the schemas added after it resolve
|
||||
void resolve_refs(const common_json & schema) {
|
||||
try {
|
||||
common_schema_parse(schema, _doc);
|
||||
} catch (const std::runtime_error & e) {
|
||||
_errors.push_back(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
visit_refs(schema);
|
||||
std::string add_schema(const std::string & name, const common_json & schema) {
|
||||
common_schema_ptr node;
|
||||
try {
|
||||
node = common_schema_parse(schema, _doc);
|
||||
} catch (const std::runtime_error & e) {
|
||||
_errors.push_back(e.what());
|
||||
return "";
|
||||
}
|
||||
return visit(*node, name);
|
||||
}
|
||||
|
||||
static std::string _generate_constant_rule(const json & value) {
|
||||
return format_literal(value.dump());
|
||||
}
|
||||
|
||||
std::string visit(const json & schema, const std::string & name) {
|
||||
json schema_type = schema.contains("type") ? schema["type"] : json();
|
||||
std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
|
||||
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
|
||||
// the primitive under its own name, or inlined when it is the root
|
||||
std::string _visit_primitive(const std::string & rule_name, const std::string & type) {
|
||||
return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type));
|
||||
}
|
||||
|
||||
if (schema.contains("$ref")) {
|
||||
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
|
||||
}
|
||||
if (schema.contains("oneOf") || schema.contains("anyOf")) {
|
||||
const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
|
||||
std::vector<json> alt_schemas;
|
||||
for (const auto & alt : alts) {
|
||||
alt_schemas.push_back(alt);
|
||||
}
|
||||
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
|
||||
}
|
||||
if (schema_type.is_array()) {
|
||||
std::vector<json> schema_types;
|
||||
for (const auto & t : schema_type) {
|
||||
json schema_copy(schema);
|
||||
schema_copy["type"] = t;
|
||||
schema_types.push_back(schema_copy);
|
||||
}
|
||||
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
|
||||
}
|
||||
if (schema.contains("const")) {
|
||||
return _add_rule(rule_name, _generate_constant_rule(schema["const"]));
|
||||
}
|
||||
if (schema.contains("enum")) {
|
||||
std::vector<std::string> enum_values;
|
||||
for (const auto & v : schema["enum"]) {
|
||||
enum_values.push_back(_generate_constant_rule(v));
|
||||
}
|
||||
return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "object")
|
||||
&& (schema.contains("properties") ||
|
||||
(schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
|
||||
std::unordered_set<std::string> required;
|
||||
if (schema.contains("required") && schema["required"].is_array()) {
|
||||
for (const auto & item : schema["required"]) {
|
||||
if (item.is_string()) {
|
||||
required.insert(item.get<std::string>());
|
||||
// An allOf merged the way the Python converter does it: the properties of a direct component are required, those of a nested anyOf are optional, enums are intersected.
|
||||
std::string _visit_all_of(const common_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
|
||||
std::unordered_set<std::string> required;
|
||||
std::vector<std::pair<std::string, const common_schema *>> properties;
|
||||
std::map<std::string, size_t> enum_values;
|
||||
std::function<void(const common_schema &, bool)> add_component = [&](const common_schema & comp, bool is_required) {
|
||||
if (comp.kind() == COMMON_SCHEMA_KIND_REF) {
|
||||
if (const auto * target = as<common_schema_ref>(comp).target) {
|
||||
add_component(*target, is_required);
|
||||
}
|
||||
} else if (comp.kind() == COMMON_SCHEMA_KIND_OBJECT) {
|
||||
for (const auto & prop : as<common_schema_object>(comp).properties) {
|
||||
properties.emplace_back(prop.name, prop.schema.get());
|
||||
if (is_required) {
|
||||
required.insert(prop.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::vector<std::pair<std::string, json>> properties;
|
||||
if (schema.contains("properties")) {
|
||||
for (const auto & prop : schema["properties"].items()) {
|
||||
properties.emplace_back(prop.key(), prop.value());
|
||||
} else if (comp.kind() == COMMON_SCHEMA_KIND_ENUM) {
|
||||
for (const auto & v : as<common_schema_enum>(comp).values) {
|
||||
enum_values[_generate_constant_rule(v)] += 1;
|
||||
}
|
||||
}
|
||||
return _add_rule(rule_name,
|
||||
_build_object_rule(
|
||||
properties, required, name,
|
||||
schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
|
||||
};
|
||||
for (const auto & child : schema.children) {
|
||||
if (child->kind() == COMMON_SCHEMA_KIND_ANY_OF) {
|
||||
for (const auto & alt : as<common_schema_any_of>(*child).children) {
|
||||
add_component(*alt, false);
|
||||
}
|
||||
} else {
|
||||
add_component(*child, true);
|
||||
}
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) {
|
||||
std::unordered_set<std::string> required;
|
||||
std::vector<std::pair<std::string, json>> properties;
|
||||
std::map<std::string, size_t> enum_values;
|
||||
const std::string& hybrid_name = name;
|
||||
std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
|
||||
if (comp_schema.contains("$ref")) {
|
||||
add_component(_refs[comp_schema["$ref"]], is_required);
|
||||
} else if (comp_schema.contains("properties")) {
|
||||
for (const auto & prop : comp_schema["properties"].items()) {
|
||||
properties.emplace_back(prop.key(), prop.value());
|
||||
if (is_required) {
|
||||
required.insert(prop.key());
|
||||
}
|
||||
}
|
||||
} else if (comp_schema.contains("enum")) {
|
||||
for (const auto & v : comp_schema["enum"]) {
|
||||
const auto rule = _generate_constant_rule(v);
|
||||
if (enum_values.find(rule) == enum_values.end()) {
|
||||
enum_values[rule] = 0;
|
||||
}
|
||||
enum_values[rule] += 1;
|
||||
}
|
||||
} else {
|
||||
// todo warning
|
||||
}
|
||||
};
|
||||
for (const auto & t : schema["allOf"]) {
|
||||
if (t.contains("anyOf")) {
|
||||
for (const auto & tt : t["anyOf"]) {
|
||||
add_component(tt, false);
|
||||
}
|
||||
} else {
|
||||
add_component(t, true);
|
||||
if (!enum_values.empty()) {
|
||||
std::vector<std::string> enum_intersection;
|
||||
for (const auto & p : enum_values) {
|
||||
if (p.second == schema.children.size()) {
|
||||
enum_intersection.push_back(p.first);
|
||||
}
|
||||
}
|
||||
if (!enum_values.empty()) {
|
||||
std::vector<std::string> enum_intersection;
|
||||
for (const auto & p : enum_values) {
|
||||
if (p.second == schema["allOf"].size()) {
|
||||
enum_intersection.push_back(p.first);
|
||||
}
|
||||
}
|
||||
if (!enum_intersection.empty()) {
|
||||
return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
|
||||
}
|
||||
if (!enum_intersection.empty()) {
|
||||
return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
|
||||
}
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
|
||||
json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
|
||||
if (items.is_array()) {
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
|
||||
}
|
||||
|
||||
std::string visit(const common_schema & schema, const std::string & name) {
|
||||
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
|
||||
std::string sub_name = name + (name.empty() ? "" : "-");
|
||||
|
||||
switch (schema.kind()) {
|
||||
case COMMON_SCHEMA_KIND_REF:
|
||||
return _add_rule(rule_name, _resolve_ref(as<common_schema_ref>(schema)));
|
||||
case COMMON_SCHEMA_KIND_ANY_OF:
|
||||
return _add_rule(rule_name, _generate_union_rule(name, as<common_schema_any_of>(schema).children));
|
||||
case COMMON_SCHEMA_KIND_ALL_OF:
|
||||
return _visit_all_of(as<common_schema_all_of>(schema), name, rule_name);
|
||||
case COMMON_SCHEMA_KIND_CONST:
|
||||
return _add_rule(rule_name, _generate_constant_rule(as<common_schema_const>(schema).value));
|
||||
case COMMON_SCHEMA_KIND_ENUM: {
|
||||
std::vector<std::string> enum_values;
|
||||
for (const auto & v : as<common_schema_enum>(schema).values) {
|
||||
enum_values.push_back(_generate_constant_rule(v));
|
||||
}
|
||||
return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
|
||||
}
|
||||
case COMMON_SCHEMA_KIND_OBJECT: {
|
||||
const auto & obj = as<common_schema_object>(schema);
|
||||
if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == COMMON_SCHEMA_KIND_ANY) {
|
||||
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
|
||||
}
|
||||
std::vector<std::pair<std::string, const common_schema *>> properties;
|
||||
std::unordered_set<std::string> required;
|
||||
for (const auto & prop : obj.properties) {
|
||||
properties.emplace_back(prop.name, prop.schema.get());
|
||||
if (prop.required) {
|
||||
required.insert(prop.name);
|
||||
}
|
||||
}
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get()));
|
||||
}
|
||||
case COMMON_SCHEMA_KIND_TUPLE: {
|
||||
const auto & items = as<common_schema_tuple>(schema).items;
|
||||
std::string rule = "\"[\" space ";
|
||||
for (size_t i = 0; i < items.size(); i++) {
|
||||
if (i > 0) {
|
||||
rule += " \",\" space ";
|
||||
}
|
||||
rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
|
||||
rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i));
|
||||
}
|
||||
rule += " space \"]\"";
|
||||
return _add_rule(rule_name, rule);
|
||||
}
|
||||
std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
|
||||
int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
|
||||
json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
|
||||
int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
|
||||
|
||||
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\"");
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
|
||||
return _visit_pattern(schema["pattern"], rule_name);
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
|
||||
return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
|
||||
auto prim_name = schema_format + "-string";
|
||||
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
|
||||
}
|
||||
if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
|
||||
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||
int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
|
||||
int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
|
||||
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\"");
|
||||
}
|
||||
if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
|
||||
int64_t min_value = std::numeric_limits<int64_t>::min();
|
||||
int64_t max_value = std::numeric_limits<int64_t>::max();
|
||||
if (schema.contains("minimum")) {
|
||||
min_value = schema["minimum"].get<int64_t>();
|
||||
} else if (schema.contains("exclusiveMinimum")) {
|
||||
min_value = schema["exclusiveMinimum"].get<int64_t>() + 1;
|
||||
case COMMON_SCHEMA_KIND_ARRAY: {
|
||||
const auto & arr = as<common_schema_array>(schema);
|
||||
if (arr.items->kind() == COMMON_SCHEMA_KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
|
||||
return _visit_primitive(rule_name, "array");
|
||||
}
|
||||
std::string item_rule_name = visit(*arr.items, sub_name + "item");
|
||||
int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items;
|
||||
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\"");
|
||||
}
|
||||
if (schema.contains("maximum")) {
|
||||
max_value = schema["maximum"].get<int64_t>();
|
||||
} else if (schema.contains("exclusiveMaximum")) {
|
||||
max_value = schema["exclusiveMaximum"].get<int64_t>() - 1;
|
||||
case COMMON_SCHEMA_KIND_STRING: {
|
||||
const auto & str = as<common_schema_string>(schema);
|
||||
if (!str.pattern.empty()) {
|
||||
return _visit_pattern(str.pattern, rule_name);
|
||||
}
|
||||
if (str.format == COMMON_SCHEMA_FORMAT_UUID) {
|
||||
return _visit_primitive(rule_name, "uuid");
|
||||
}
|
||||
if (str.format != COMMON_SCHEMA_FORMAT_NONE) {
|
||||
std::string prim_name = std::string(str.format == COMMON_SCHEMA_FORMAT_DATE ? "date" : str.format == COMMON_SCHEMA_FORMAT_TIME ? "time" : "date-time") + "-string";
|
||||
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
|
||||
}
|
||||
if (str.min_length > 0 || str.max_length >= 0) {
|
||||
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||
int max_len = str.max_length < 0 ? std::numeric_limits<int>::max() : str.max_length;
|
||||
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\"");
|
||||
}
|
||||
return _visit_primitive(rule_name, "string");
|
||||
}
|
||||
std::stringstream out;
|
||||
out << "(";
|
||||
build_min_max_int(min_value, max_value, out);
|
||||
out << ")";
|
||||
return _add_rule(rule_name, out.str());
|
||||
case COMMON_SCHEMA_KIND_INTEGER: {
|
||||
const auto & i = as<common_schema_integer>(schema);
|
||||
if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) {
|
||||
return _visit_primitive(rule_name, "integer");
|
||||
}
|
||||
std::stringstream out;
|
||||
out << "(";
|
||||
build_min_max_int(i.minimum, i.maximum, out);
|
||||
out << ")";
|
||||
return _add_rule(rule_name, out.str());
|
||||
}
|
||||
case COMMON_SCHEMA_KIND_NUMBER:
|
||||
return _visit_primitive(rule_name, "number");
|
||||
case COMMON_SCHEMA_KIND_BOOLEAN:
|
||||
return _visit_primitive(rule_name, "boolean");
|
||||
case COMMON_SCHEMA_KIND_NULL:
|
||||
return _visit_primitive(rule_name, "null");
|
||||
case COMMON_SCHEMA_KIND_ANY:
|
||||
return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
|
||||
case COMMON_SCHEMA_KIND_NONE:
|
||||
_errors.push_back("No value satisfies the schema of " + rule_name);
|
||||
return "";
|
||||
}
|
||||
if (schema.empty() || schema_type == "object") {
|
||||
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
|
||||
}
|
||||
if (schema_type.is_null() && schema.is_object()) {
|
||||
// No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
|
||||
// Per JSON Schema semantics this is equivalent to {} and accepts any value.
|
||||
return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
|
||||
}
|
||||
if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
|
||||
_errors.push_back("Unrecognized schema: " + schema.dump());
|
||||
return "";
|
||||
}
|
||||
// TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
|
||||
return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
|
||||
return "";
|
||||
}
|
||||
|
||||
void check_errors() {
|
||||
@@ -1106,134 +1031,72 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
// common_schema_info implementation (pimpl)
|
||||
bool common_schema_resolves_to_string(const common_schema & schema) {
|
||||
std::unordered_set<const common_schema *> visited;
|
||||
|
||||
common_schema_info::common_schema_info()
|
||||
: impl_(std::make_unique<common_schema_converter>(
|
||||
[](const std::string &) { return json(); },
|
||||
false)) {}
|
||||
|
||||
common_schema_info::~common_schema_info() = default;
|
||||
|
||||
common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
|
||||
common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
|
||||
|
||||
void common_schema_info::resolve_refs(common_json & schema) {
|
||||
impl_->resolve_refs(schema, "");
|
||||
}
|
||||
|
||||
// Determines if a JSON schema can resolve to a string type through any path.
|
||||
// Some models emit raw string values rather than JSON-encoded strings for string parameters.
|
||||
// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
|
||||
// true, allowing callers to handle the value as a raw string for simplicity.
|
||||
bool common_schema_info::resolves_to_string(const common_json & schema) {
|
||||
std::unordered_set<std::string> visited_refs;
|
||||
|
||||
std::function<bool(const json &)> check = [&](const json & s) -> bool {
|
||||
if (!s.is_object()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Handle $ref
|
||||
if (s.contains("$ref")) {
|
||||
const std::string & ref = s["$ref"];
|
||||
if (visited_refs.find(ref) != visited_refs.end()) {
|
||||
// Circular reference, assume not a string to be safe
|
||||
return false;
|
||||
}
|
||||
visited_refs.insert(ref);
|
||||
auto it = impl_->_refs.find(ref);
|
||||
if (it != impl_->_refs.end()) {
|
||||
return check(it->second);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check type field
|
||||
if (s.contains("type")) {
|
||||
const json & schema_type = s["type"];
|
||||
if (schema_type.is_string()) {
|
||||
if (schema_type == "string") {
|
||||
return true;
|
||||
}
|
||||
} else if (schema_type.is_array()) {
|
||||
// Type can be an array like ["string", "null"]
|
||||
for (const auto & t : schema_type) {
|
||||
if (t == "string") {
|
||||
std::function<bool(const common_schema &)> check = [&](const common_schema & s) -> bool {
|
||||
switch (s.kind()) {
|
||||
case COMMON_SCHEMA_KIND_STRING:
|
||||
return true;
|
||||
case COMMON_SCHEMA_KIND_CONST:
|
||||
return static_cast<const common_schema_const &>(s).value.is_string();
|
||||
case COMMON_SCHEMA_KIND_ENUM:
|
||||
for (const auto & v : static_cast<const common_schema_enum &>(s).values) {
|
||||
if (v.is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
case COMMON_SCHEMA_KIND_REF: {
|
||||
// a cycle is taken as not a string, to be safe
|
||||
const auto * target = static_cast<const common_schema_ref &>(s).target;
|
||||
return target && visited.insert(target).second && check(*target);
|
||||
}
|
||||
}
|
||||
|
||||
// Check oneOf/anyOf - if any alternative can be a string
|
||||
if (s.contains("oneOf")) {
|
||||
for (const auto & alt : s["oneOf"]) {
|
||||
if (check(alt)) {
|
||||
return true;
|
||||
case COMMON_SCHEMA_KIND_ANY_OF:
|
||||
for (const auto & child : static_cast<const common_schema_any_of &>(s).children) {
|
||||
if (check(*child)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (s.contains("anyOf")) {
|
||||
for (const auto & alt : s["anyOf"]) {
|
||||
if (check(alt)) {
|
||||
return true;
|
||||
return false;
|
||||
case COMMON_SCHEMA_KIND_ALL_OF: {
|
||||
// every child must allow a string, an any child constrains nothing
|
||||
bool any_string = false;
|
||||
for (const auto & child : static_cast<const common_schema_all_of &>(s).children) {
|
||||
if (child->kind() == COMMON_SCHEMA_KIND_ANY) {
|
||||
continue;
|
||||
}
|
||||
if (!check(*child)) {
|
||||
return false;
|
||||
}
|
||||
any_string = true;
|
||||
}
|
||||
return any_string;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check allOf - all components must be compatible with string type
|
||||
if (s.contains("allOf")) {
|
||||
bool all_string = true;
|
||||
for (const auto & component : s["allOf"]) {
|
||||
if (!check(component)) {
|
||||
all_string = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (all_string) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check const - if the constant value is a string
|
||||
if (s.contains("const")) {
|
||||
if (s["const"].is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check enum - if any enum value is a string
|
||||
if (s.contains("enum")) {
|
||||
for (const auto & val : s["enum"]) {
|
||||
if (val.is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// String-specific keywords imply string type
|
||||
if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check format - many formats imply string
|
||||
if (s.contains("format")) {
|
||||
const std::string & fmt = s["format"];
|
||||
if (fmt == "date" || fmt == "time" || fmt == "date-time" ||
|
||||
fmt == "uri" || fmt == "email" || fmt == "hostname" ||
|
||||
fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" ||
|
||||
fmt.find("uuid") == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
return check(schema);
|
||||
}
|
||||
|
||||
void common_schema_info::resolve_refs(const common_json & schema) {
|
||||
// a schema that does not parse is reported when its grammar is built, here it only answers no
|
||||
try {
|
||||
common_schema_parse(schema, doc_);
|
||||
} catch (const std::runtime_error &) {
|
||||
}
|
||||
}
|
||||
|
||||
bool common_schema_info::resolves_to_string(const common_json & schema) {
|
||||
try {
|
||||
return common_schema_resolves_to_string(*common_schema_parse(schema, doc_));
|
||||
} catch (const std::runtime_error &) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
|
||||
#ifdef LLAMA_USE_LLGUIDANCE
|
||||
if (!force_gbnf) {
|
||||
@@ -1243,23 +1106,28 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf)
|
||||
(void)force_gbnf;
|
||||
#endif // LLAMA_USE_LLGUIDANCE
|
||||
return build_grammar([&](const common_grammar_builder & callbacks) {
|
||||
auto copy = schema;
|
||||
callbacks.resolve_refs(copy);
|
||||
callbacks.add_schema("", copy);
|
||||
callbacks.add_schema("", schema);
|
||||
});
|
||||
}
|
||||
|
||||
std::string json_schema_to_grammar(const common_schema_document & schema) {
|
||||
common_schema_converter converter(false);
|
||||
converter.visit(*schema.root, "");
|
||||
converter.check_errors();
|
||||
return converter.format_grammar();
|
||||
}
|
||||
|
||||
std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
|
||||
common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall);
|
||||
common_schema_converter converter(options.dotall);
|
||||
common_grammar_builder builder {
|
||||
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
|
||||
return converter._add_rule(name, rule);
|
||||
},
|
||||
/* .add_schema = */ [&](const std::string & name, const common_json & schema) {
|
||||
return converter.visit(schema, name == "root" ? "" : name);
|
||||
return converter.add_schema(name == "root" ? "" : name, schema);
|
||||
},
|
||||
/* .resolve_refs = */ [&](common_json & schema) {
|
||||
converter.resolve_refs(schema, "");
|
||||
/* .resolve_refs = */ [&](const common_json & schema) {
|
||||
converter.resolve_refs(schema);
|
||||
}
|
||||
};
|
||||
cb(builder);
|
||||
|
||||
@@ -1,37 +1,35 @@
|
||||
#pragma once
|
||||
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
std::string json_schema_to_grammar(const common_json & schema,
|
||||
bool force_gbnf = false);
|
||||
// The JSON overload goes through common_schema_parse() first: JSON -> common_schema -> GBNF
|
||||
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false);
|
||||
std::string json_schema_to_grammar(const common_schema_document & schema);
|
||||
|
||||
class common_schema_converter;
|
||||
// Whether a value matching the schema may be a string, through any branch of it.
|
||||
// Some models emit raw string values rather than JSON-encoded strings for string parameters.
|
||||
bool common_schema_resolves_to_string(const common_schema & schema);
|
||||
|
||||
// Probes a JSON schema to extract information about its structure and type constraints.
|
||||
// Probes the sub-schemas of one JSON schema, e.g. the parameters of a tool
|
||||
class common_schema_info {
|
||||
std::unique_ptr<common_schema_converter> impl_;
|
||||
common_schema_document doc_;
|
||||
|
||||
public:
|
||||
common_schema_info();
|
||||
~common_schema_info();
|
||||
// Parses the schema, so that the $refs of its sub-schemas resolve
|
||||
void resolve_refs(const common_json & schema);
|
||||
|
||||
common_schema_info(const common_schema_info &) = delete;
|
||||
common_schema_info & operator=(const common_schema_info &) = delete;
|
||||
common_schema_info(common_schema_info &&) noexcept;
|
||||
common_schema_info & operator=(common_schema_info &&) noexcept;
|
||||
|
||||
void resolve_refs(common_json & schema);
|
||||
// common_schema_resolves_to_string() for a sub-schema of a schema given to resolve_refs(), false when it does not parse
|
||||
bool resolves_to_string(const common_json & schema);
|
||||
};
|
||||
|
||||
struct common_grammar_builder {
|
||||
std::function<std::string(const std::string &, const std::string &)> add_rule;
|
||||
std::function<std::string(const std::string &, const common_json &)> add_schema;
|
||||
std::function<void(common_json &)> resolve_refs;
|
||||
std::function<void(const common_json &)> resolve_refs;
|
||||
};
|
||||
|
||||
struct common_grammar_options {
|
||||
|
||||
+26
-16
@@ -3,6 +3,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
@@ -11,8 +12,11 @@
|
||||
#include <vector>
|
||||
|
||||
class common_schema_parser {
|
||||
const common_json & root_;
|
||||
common_schema_document doc_;
|
||||
const common_json & root_;
|
||||
common_schema_document & doc_;
|
||||
|
||||
// the targets parsed here, moved into doc_ once the whole schema parsed
|
||||
std::map<std::string, common_schema_ptr> refs_;
|
||||
|
||||
// ref nodes get their target once every $ref is parsed, a cycle would otherwise need it too early
|
||||
std::vector<common_schema_ref *> pending_;
|
||||
@@ -103,10 +107,10 @@ class common_schema_parser {
|
||||
if (ref.compare(0, 2, "#/") != 0) {
|
||||
fail(path, "unsupported $ref " + ref + ", only references into the same document are supported");
|
||||
}
|
||||
if (doc_.refs.find(ref) == doc_.refs.end()) {
|
||||
if (doc_.refs.find(ref) == doc_.refs.end() && refs_.find(ref) == refs_.end()) {
|
||||
// reserve the key first, so that a cycle back to this $ref stops here
|
||||
doc_.refs[ref] = nullptr;
|
||||
doc_.refs[ref] = parse_schema(resolve_ref(ref, path), ref);
|
||||
refs_[ref] = nullptr;
|
||||
refs_[ref] = parse_schema(resolve_ref(ref, path), ref);
|
||||
}
|
||||
auto node = std::make_unique<common_schema_ref>(ref);
|
||||
pending_.push_back(node.get());
|
||||
@@ -173,7 +177,7 @@ class common_schema_parser {
|
||||
common_schema_ptr parse_array(const common_json & schema, const std::string & path) {
|
||||
auto node = std::make_unique<common_schema_array>();
|
||||
if (schema.contains("items") || schema.contains("prefixItems")) {
|
||||
// "items" given as an array is the older spelling of "prefixItems", and wins when both are present
|
||||
// "items" wins when both are present; as in the converter, a schema instead of an array is the item schema
|
||||
const std::string key = schema.contains("items") ? "items" : "prefixItems";
|
||||
const common_json & items = schema.at(key);
|
||||
if (items.is_array()) {
|
||||
@@ -184,9 +188,6 @@ class common_schema_parser {
|
||||
}
|
||||
return tuple;
|
||||
}
|
||||
if (key == "prefixItems") {
|
||||
fail(path, "prefixItems must be an array");
|
||||
}
|
||||
node->items = parse_schema(items, path + "/" + key);
|
||||
} else {
|
||||
node->items = std::make_unique<common_schema_any>();
|
||||
@@ -325,19 +326,28 @@ class common_schema_parser {
|
||||
}
|
||||
|
||||
public:
|
||||
explicit common_schema_parser(const common_json & root) : root_(root) {}
|
||||
common_schema_parser(const common_json & root, common_schema_document & doc) : root_(root), doc_(doc) {}
|
||||
|
||||
common_schema_document parse() {
|
||||
doc_.root = parse_schema(root_, "#");
|
||||
for (auto * node : pending_) {
|
||||
node->target = doc_.refs.at(node->ref).get();
|
||||
common_schema_ptr parse() {
|
||||
auto node = parse_schema(root_, "#");
|
||||
for (auto & entry : refs_) {
|
||||
doc_.refs[entry.first] = std::move(entry.second);
|
||||
}
|
||||
return std::move(doc_);
|
||||
for (auto * ref : pending_) {
|
||||
ref->target = doc_.refs.at(ref->ref).get();
|
||||
}
|
||||
return node;
|
||||
}
|
||||
};
|
||||
|
||||
common_schema_document common_schema_parse(const common_json & schema) {
|
||||
return common_schema_parser(schema).parse();
|
||||
common_schema_document doc;
|
||||
doc.root = common_schema_parser(schema, doc).parse();
|
||||
return doc;
|
||||
}
|
||||
|
||||
common_schema_ptr common_schema_parse(const common_json & schema, common_schema_document & doc) {
|
||||
return common_schema_parser(schema, doc).parse();
|
||||
}
|
||||
|
||||
class common_schema_optimizer {
|
||||
|
||||
@@ -159,6 +159,11 @@ struct common_schema_document {
|
||||
// Throws std::runtime_error when the schema falls outside the supported subset.
|
||||
common_schema_document common_schema_parse(const common_json & schema);
|
||||
|
||||
// Parses a schema that belongs to a document parsed earlier, e.g. one property of it.
|
||||
// A $ref it cannot resolve on its own is looked up in doc.refs, the targets it resolves itself are added there.
|
||||
// doc is unchanged when the schema is rejected.
|
||||
common_schema_ptr common_schema_parse(const common_json & schema, common_schema_document & doc);
|
||||
|
||||
// Rewrites a document in place into an equivalent one with less redundancy: allOf becomes the intersection
|
||||
// of its children, nested anyOf are flattened, branches that can match nothing are pruned, up to a
|
||||
// common_schema_none root, and $refs nothing reaches anymore are dropped.
|
||||
|
||||
@@ -1,842 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, List, Optional, Set, Tuple, Union
|
||||
|
||||
def _build_repetition(item_rule, min_items, max_items, separator_rule=None):
|
||||
|
||||
if max_items == 0:
|
||||
return ""
|
||||
|
||||
if min_items == 0 and max_items == 1:
|
||||
return f'{item_rule}?'
|
||||
|
||||
if not separator_rule:
|
||||
if min_items == 1 and max_items is None:
|
||||
return f'{item_rule}+'
|
||||
elif min_items == 0 and max_items is None:
|
||||
return f'{item_rule}*'
|
||||
else:
|
||||
return f'{item_rule}{{{min_items},{max_items if max_items is not None else ""}}}'
|
||||
|
||||
result = item_rule + ' ' + _build_repetition(f'({separator_rule} {item_rule})', min_items - 1 if min_items > 0 else 0, max_items - 1 if max_items is not None else None)
|
||||
return f'({result})?' if min_items == 0 else result
|
||||
|
||||
def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True):
|
||||
def digit_range(from_char: str, to_char: str):
|
||||
out.append("[")
|
||||
if from_char == to_char:
|
||||
out.append(from_char)
|
||||
else:
|
||||
out.append(from_char)
|
||||
out.append("-")
|
||||
out.append(to_char)
|
||||
out.append("]")
|
||||
|
||||
def more_digits(min_digits: int, max_digits: int):
|
||||
out.append("[0-9]")
|
||||
if min_digits == max_digits and min_digits == 1:
|
||||
return
|
||||
out.append("{")
|
||||
out.append(str(min_digits))
|
||||
if max_digits != min_digits:
|
||||
out.append(",")
|
||||
if max_digits != sys.maxsize:
|
||||
out.append(str(max_digits))
|
||||
out.append("}")
|
||||
|
||||
def uniform_range(from_str: str, to_str: str):
|
||||
i = 0
|
||||
while i < len(from_str) and from_str[i] == to_str[i]:
|
||||
i += 1
|
||||
if i > 0:
|
||||
out.append("\"")
|
||||
out.append(from_str[:i])
|
||||
out.append("\"")
|
||||
if i < len(from_str):
|
||||
if i > 0:
|
||||
out.append(" ")
|
||||
sub_len = len(from_str) - i - 1
|
||||
if sub_len > 0:
|
||||
from_sub = from_str[i+1:]
|
||||
to_sub = to_str[i+1:]
|
||||
sub_zeros = "0" * sub_len
|
||||
sub_nines = "9" * sub_len
|
||||
|
||||
to_reached = False
|
||||
out.append("(")
|
||||
if from_sub == sub_zeros:
|
||||
digit_range(from_str[i], chr(ord(to_str[i]) - 1))
|
||||
out.append(" ")
|
||||
more_digits(sub_len, sub_len)
|
||||
else:
|
||||
out.append("[")
|
||||
out.append(from_str[i])
|
||||
out.append("] ")
|
||||
out.append("(")
|
||||
uniform_range(from_sub, sub_nines)
|
||||
out.append(")")
|
||||
if ord(from_str[i]) < ord(to_str[i]) - 1:
|
||||
out.append(" | ")
|
||||
if to_sub == sub_nines:
|
||||
digit_range(chr(ord(from_str[i]) + 1), to_str[i])
|
||||
to_reached = True
|
||||
else:
|
||||
digit_range(chr(ord(from_str[i]) + 1), chr(ord(to_str[i]) - 1))
|
||||
out.append(" ")
|
||||
more_digits(sub_len, sub_len)
|
||||
if not to_reached:
|
||||
out.append(" | ")
|
||||
digit_range(to_str[i], to_str[i])
|
||||
out.append(" ")
|
||||
uniform_range(sub_zeros, to_sub)
|
||||
out.append(")")
|
||||
else:
|
||||
out.append("[")
|
||||
out.append(from_str[i])
|
||||
out.append("-")
|
||||
out.append(to_str[i])
|
||||
out.append("]")
|
||||
|
||||
if min_value is not None and max_value is not None:
|
||||
if min_value < 0 and max_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True)
|
||||
out.append(")")
|
||||
return
|
||||
|
||||
if min_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(0, -min_value, out, decimals_left, top_level=True)
|
||||
out.append(") | ")
|
||||
min_value = 0
|
||||
|
||||
min_s = str(min_value)
|
||||
max_s = str(max_value)
|
||||
min_digits = len(min_s)
|
||||
max_digits = len(max_s)
|
||||
|
||||
for digits in range(min_digits, max_digits):
|
||||
uniform_range(min_s, "9" * digits)
|
||||
min_s = "1" + "0" * digits
|
||||
out.append(" | ")
|
||||
uniform_range(min_s, max_s)
|
||||
return
|
||||
|
||||
less_decimals = max(decimals_left - 1, 1)
|
||||
|
||||
if min_value is not None:
|
||||
if min_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(None, -min_value, out, decimals_left, top_level=False)
|
||||
out.append(") | [0] | [1-9] ")
|
||||
more_digits(0, decimals_left - 1)
|
||||
elif min_value == 0:
|
||||
if top_level:
|
||||
out.append("[0] | [1-9] ")
|
||||
more_digits(0, less_decimals)
|
||||
else:
|
||||
more_digits(1, decimals_left)
|
||||
elif min_value <= 9:
|
||||
c = str(min_value)
|
||||
range_start = '1' if top_level else '0'
|
||||
if c > range_start:
|
||||
digit_range(range_start, chr(ord(c) - 1))
|
||||
out.append(" ")
|
||||
more_digits(1, less_decimals)
|
||||
out.append(" | ")
|
||||
digit_range(c, "9")
|
||||
out.append(" ")
|
||||
more_digits(0, less_decimals)
|
||||
else:
|
||||
min_s = str(min_value)
|
||||
length = len(min_s)
|
||||
c = min_s[0]
|
||||
|
||||
if c > "1":
|
||||
digit_range("1" if top_level else "0", chr(ord(c) - 1))
|
||||
out.append(" ")
|
||||
more_digits(length, less_decimals)
|
||||
out.append(" | ")
|
||||
digit_range(c, c)
|
||||
out.append(" (")
|
||||
_generate_min_max_int(int(min_s[1:]), None, out, less_decimals, top_level=False)
|
||||
out.append(")")
|
||||
if c < "9":
|
||||
out.append(" | ")
|
||||
digit_range(chr(ord(c) + 1), "9")
|
||||
out.append(" ")
|
||||
more_digits(length - 1, less_decimals)
|
||||
return
|
||||
|
||||
if max_value is not None:
|
||||
if max_value >= 0:
|
||||
if top_level:
|
||||
out.append("\"-\" [1-9] ")
|
||||
more_digits(0, less_decimals)
|
||||
out.append(" | ")
|
||||
_generate_min_max_int(0, max_value, out, decimals_left, top_level=True)
|
||||
else:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(-max_value, None, out, decimals_left, top_level=False)
|
||||
out.append(")")
|
||||
return
|
||||
|
||||
raise RuntimeError("At least one of min_value or max_value must be set")
|
||||
|
||||
class BuiltinRule:
|
||||
def __init__(self, content: str, deps: list | None = None):
|
||||
self.content = content
|
||||
self.deps = deps or []
|
||||
|
||||
# Constraining spaces to prevent model "running away".
|
||||
SPACE_RULE = '| " " | "\\n"{1,2} [ \\t]{0,20}'
|
||||
|
||||
PRIMITIVE_RULES = {
|
||||
'boolean' : BuiltinRule('("true" | "false")', []),
|
||||
'decimal-part' : BuiltinRule('[0-9]{1,16}', []),
|
||||
'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []),
|
||||
'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?', ['integral-part', 'decimal-part']),
|
||||
'integer' : BuiltinRule('("-"? integral-part)', ['integral-part']),
|
||||
'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']),
|
||||
'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? space "}"', ['string', 'value']),
|
||||
'array' : BuiltinRule('"[" space ( value ("," space value)* )? space "]"', ['value']),
|
||||
'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\""', []),
|
||||
'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []),
|
||||
'string' : BuiltinRule(r'"\"" char* "\""', ['char']),
|
||||
'null' : BuiltinRule('"null"', []),
|
||||
}
|
||||
|
||||
# TODO: support "uri", "email" string formats
|
||||
STRING_FORMAT_RULES = {
|
||||
'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []),
|
||||
'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []),
|
||||
'date-time' : BuiltinRule('date "T" time', ['date', 'time']),
|
||||
'date-string' : BuiltinRule('"\\"" date "\\""', ['date']),
|
||||
'time-string' : BuiltinRule('"\\"" time "\\""', ['time']),
|
||||
'date-time-string': BuiltinRule('"\\"" date-time "\\""', ['date-time']),
|
||||
}
|
||||
|
||||
DOTALL = '[\\U00000000-\\U0010FFFF]'
|
||||
DOT = '[^\\x0A\\x0D]'
|
||||
|
||||
RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()])
|
||||
|
||||
INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+')
|
||||
GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\\]')
|
||||
GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]')
|
||||
GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]', '\\': '\\\\'}
|
||||
|
||||
NON_LITERAL_SET = set('|.()[]{}*+?')
|
||||
ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?')
|
||||
|
||||
|
||||
class SchemaConverter:
|
||||
def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern):
|
||||
self._prop_order = prop_order
|
||||
self._allow_fetch = allow_fetch
|
||||
self._dotall = dotall
|
||||
self._raw_pattern = raw_pattern
|
||||
self._rules = {
|
||||
'space': SPACE_RULE,
|
||||
}
|
||||
self._refs = {}
|
||||
self._refs_being_resolved = set()
|
||||
|
||||
def _format_literal(self, literal):
|
||||
escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub(
|
||||
lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)) or m.group(0), literal
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
|
||||
def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str:
|
||||
'''
|
||||
not_literal('a') -> '[^a]'
|
||||
not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?'
|
||||
'''
|
||||
assert len(literal) > 0, 'Empty literal not supported'
|
||||
def recurse(i: int):
|
||||
c = literal[i]
|
||||
if maybe_escaped_underscores and c == '_':
|
||||
yield f'[^{c}\\\\]'
|
||||
yield ' | '
|
||||
yield f'"\\\\"? "{c}"'
|
||||
else:
|
||||
yield f'[^{c}]'
|
||||
if i < len(literal) - 1:
|
||||
yield ' | '
|
||||
yield self._format_literal(c)
|
||||
yield ' ('
|
||||
yield from recurse(i + 1)
|
||||
yield ')?'
|
||||
|
||||
return ''.join(('(', *recurse(0), ')'))
|
||||
|
||||
def _not_strings(self, strings):
|
||||
class TrieNode:
|
||||
def __init__(self):
|
||||
self.children = {}
|
||||
self.is_end_of_string = False
|
||||
|
||||
def insert(self, string):
|
||||
node = self
|
||||
for c in string:
|
||||
node = node.children.setdefault(c, TrieNode())
|
||||
node.is_end_of_string = True
|
||||
|
||||
trie = TrieNode()
|
||||
for s in strings:
|
||||
trie.insert(s)
|
||||
|
||||
char_rule = self._add_primitive('char', PRIMITIVE_RULES['char'])
|
||||
out = ['["] ( ']
|
||||
|
||||
def visit(node):
|
||||
rejects = []
|
||||
first = True
|
||||
for c in sorted(node.children.keys()):
|
||||
child = node.children[c]
|
||||
rejects.append(c)
|
||||
if first:
|
||||
first = False
|
||||
else:
|
||||
out.append(' | ')
|
||||
out.append(f'[{c}]')
|
||||
if child.children:
|
||||
out.append(f' (')
|
||||
visit(child)
|
||||
out.append(')')
|
||||
elif child.is_end_of_string:
|
||||
out.append(f' {char_rule}+')
|
||||
if node.children:
|
||||
if not first:
|
||||
out.append(' | ')
|
||||
out.append(f'[^"{"".join(rejects)}] {char_rule}*')
|
||||
visit(trie)
|
||||
|
||||
out.append(f' ){"" if trie.is_end_of_string else "?"} ["]')
|
||||
return ''.join(out)
|
||||
|
||||
def _add_rule(self, name, rule):
|
||||
esc_name = INVALID_RULE_CHARS_RE.sub('-', name)
|
||||
if esc_name not in self._rules or self._rules[esc_name] == rule:
|
||||
key = esc_name
|
||||
else:
|
||||
i = 0
|
||||
while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule:
|
||||
i += 1
|
||||
key = f'{esc_name}{i}'
|
||||
self._rules[key] = rule
|
||||
return key
|
||||
|
||||
def resolve_refs(self, schema: dict, url: str):
|
||||
'''
|
||||
Resolves all $ref fields in the given schema, fetching any remote schemas,
|
||||
replacing $ref with absolute reference URL and populating self._refs with the
|
||||
respective referenced (sub)schema dictionaries.
|
||||
'''
|
||||
def visit(n: dict):
|
||||
if isinstance(n, list):
|
||||
return [visit(x) for x in n]
|
||||
elif isinstance(n, dict):
|
||||
ref = n.get('$ref')
|
||||
if ref is not None and ref not in self._refs:
|
||||
if ref.startswith('https://'):
|
||||
assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)'
|
||||
import requests
|
||||
|
||||
frag_split = ref.split('#')
|
||||
base_url = frag_split[0]
|
||||
|
||||
target = self._refs.get(base_url)
|
||||
if target is None:
|
||||
target = self.resolve_refs(requests.get(ref).json(), base_url)
|
||||
self._refs[base_url] = target
|
||||
|
||||
if len(frag_split) == 1 or frag_split[-1] == '':
|
||||
return target
|
||||
elif ref.startswith('#/'):
|
||||
target = schema
|
||||
ref = f'{url}{ref}'
|
||||
n['$ref'] = ref
|
||||
else:
|
||||
raise ValueError(f'Unsupported ref {ref}')
|
||||
|
||||
for sel in ref.split('#')[-1].split('/')[1:]:
|
||||
assert target is not None, f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
if isinstance(target, list):
|
||||
try:
|
||||
sel_index = int(sel)
|
||||
except ValueError:
|
||||
raise ValueError(f'Error resolving ref {ref}: {sel} not in {target}')
|
||||
assert 0 <= sel_index < len(target), f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
target = target[sel_index]
|
||||
else:
|
||||
assert sel in target, f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
target = target[sel]
|
||||
|
||||
self._refs[ref] = target
|
||||
else:
|
||||
for v in n.values():
|
||||
visit(v)
|
||||
|
||||
return n
|
||||
return visit(schema)
|
||||
|
||||
def _generate_union_rule(self, name, alt_schemas):
|
||||
return ' | '.join((
|
||||
self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}')
|
||||
for i, alt_schema in enumerate(alt_schemas)
|
||||
))
|
||||
|
||||
def _visit_pattern(self, pattern, name):
|
||||
'''
|
||||
Transforms a regular expression pattern into a GBNF rule.
|
||||
|
||||
Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions
|
||||
Output: https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md
|
||||
|
||||
Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers.
|
||||
|
||||
Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which
|
||||
we define sub-rules to keep the output lean.
|
||||
'''
|
||||
|
||||
assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"'
|
||||
pattern = pattern[1:-1]
|
||||
sub_rule_ids = {}
|
||||
|
||||
i = 0
|
||||
length = len(pattern)
|
||||
|
||||
def to_rule(s: tuple[str, bool]) -> str:
|
||||
(txt, is_literal) = s
|
||||
return "\"" + txt + "\"" if is_literal else txt
|
||||
|
||||
def transform() -> tuple[str, bool]:
|
||||
'''
|
||||
Parse a unit at index i (advancing it), and return its string representation + whether it's a literal.
|
||||
'''
|
||||
nonlocal i
|
||||
nonlocal pattern
|
||||
nonlocal sub_rule_ids
|
||||
|
||||
start = i
|
||||
# For each component of this sequence, store its string representation and whether it's a literal.
|
||||
# We only need a flat structure here to apply repetition operators to the last item, and
|
||||
# to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially
|
||||
# (GBNF's syntax is luckily very close to regular expressions!)
|
||||
seq: list[tuple[str, bool]] = []
|
||||
|
||||
def get_dot():
|
||||
if self._dotall:
|
||||
rule = DOTALL
|
||||
else:
|
||||
# Accept any character... except \n and \r line break chars (\x0A and \xOD)
|
||||
rule = DOT
|
||||
return self._add_rule(f'dot', rule)
|
||||
|
||||
def join_seq():
|
||||
nonlocal seq
|
||||
ret = []
|
||||
for is_literal, g in itertools.groupby(seq, lambda x: x[1]):
|
||||
if is_literal:
|
||||
ret.append((''.join(x[0] for x in g), True))
|
||||
else:
|
||||
ret.extend(g)
|
||||
if len(ret) == 1:
|
||||
return ret[0]
|
||||
return (' '.join(to_rule(x) for x in seq), False)
|
||||
|
||||
while i < length:
|
||||
c = pattern[i]
|
||||
if c == '.':
|
||||
seq.append((get_dot(), False))
|
||||
i += 1
|
||||
elif c == '(':
|
||||
i += 1
|
||||
if i < length:
|
||||
assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/'
|
||||
seq.append((f'({to_rule(transform())})', False))
|
||||
elif c == ')':
|
||||
i += 1
|
||||
assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}'
|
||||
return join_seq()
|
||||
elif c == '[':
|
||||
square_brackets = c
|
||||
i += 1
|
||||
while i < length and pattern[i] != ']':
|
||||
if pattern[i] == '\\':
|
||||
square_brackets += pattern[i:i+2]
|
||||
i += 2
|
||||
else:
|
||||
square_brackets += pattern[i]
|
||||
i += 1
|
||||
assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}'
|
||||
square_brackets += ']'
|
||||
i += 1
|
||||
seq.append((square_brackets, False))
|
||||
elif c == '|':
|
||||
seq.append(('|', False))
|
||||
i += 1
|
||||
elif c in ('*', '+', '?'):
|
||||
seq[-1] = (to_rule(seq[-1]) + c, False)
|
||||
i += 1
|
||||
elif c == '{':
|
||||
curly_brackets = c
|
||||
i += 1
|
||||
while i < length and pattern[i] != '}':
|
||||
curly_brackets += pattern[i]
|
||||
i += 1
|
||||
assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}'
|
||||
curly_brackets += '}'
|
||||
i += 1
|
||||
nums = [s.strip() for s in curly_brackets[1:-1].split(',')]
|
||||
min_times = 0
|
||||
max_times = None
|
||||
try:
|
||||
if len(nums) == 1:
|
||||
min_times = int(nums[0])
|
||||
max_times = min_times
|
||||
else:
|
||||
assert len(nums) == 2
|
||||
min_times = int(nums[0]) if nums[0] else 0
|
||||
max_times = int(nums[1]) if nums[1] else None
|
||||
except ValueError:
|
||||
raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/')
|
||||
|
||||
(sub, sub_is_literal) = seq[-1]
|
||||
|
||||
if not sub_is_literal:
|
||||
id = sub_rule_ids.get(sub)
|
||||
if id is None:
|
||||
id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub)
|
||||
sub_rule_ids[sub] = id
|
||||
sub = id
|
||||
|
||||
seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times), False)
|
||||
else:
|
||||
literal = ''
|
||||
while i < length:
|
||||
if pattern[i] == '\\' and i < length - 1:
|
||||
next = pattern[i + 1]
|
||||
if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS:
|
||||
i += 1
|
||||
literal += pattern[i]
|
||||
i += 1
|
||||
else:
|
||||
literal += pattern[i:i+2]
|
||||
i += 2
|
||||
elif pattern[i] == '"' and not self._raw_pattern:
|
||||
literal += '\\"'
|
||||
i += 1
|
||||
elif pattern[i] not in NON_LITERAL_SET and \
|
||||
(i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET):
|
||||
literal += pattern[i]
|
||||
i += 1
|
||||
else:
|
||||
break
|
||||
if literal:
|
||||
seq.append((literal, True))
|
||||
|
||||
return join_seq()
|
||||
|
||||
return self._add_rule(
|
||||
name,
|
||||
to_rule(transform()) if self._raw_pattern \
|
||||
else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"")
|
||||
|
||||
|
||||
def _resolve_ref(self, ref):
|
||||
ref_fragment = ref.split('#')[-1]
|
||||
ref_name = 'ref' + re.sub(r'[^a-zA-Z0-9-]+', '-', ref_fragment)
|
||||
if ref_name not in self._rules and ref not in self._refs_being_resolved:
|
||||
self._refs_being_resolved.add(ref)
|
||||
resolved = self._refs[ref]
|
||||
ref_name = self.visit(resolved, ref_name)
|
||||
self._refs_being_resolved.remove(ref)
|
||||
return ref_name
|
||||
|
||||
def _generate_constant_rule(self, value):
|
||||
return self._format_literal(json.dumps(value))
|
||||
|
||||
def visit(self, schema, name):
|
||||
schema_type = schema.get('type')
|
||||
schema_format = schema.get('format')
|
||||
rule_name = name + '-' if name in RESERVED_NAMES else name or 'root'
|
||||
|
||||
if (ref := schema.get('$ref')) is not None:
|
||||
return self._add_rule(rule_name, self._resolve_ref(ref))
|
||||
|
||||
elif 'oneOf' in schema or 'anyOf' in schema:
|
||||
return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf']))
|
||||
|
||||
elif isinstance(schema_type, list):
|
||||
return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type]))
|
||||
|
||||
elif 'const' in schema:
|
||||
return self._add_rule(rule_name, self._generate_constant_rule(schema['const']))
|
||||
|
||||
elif 'enum' in schema:
|
||||
rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ')'
|
||||
return self._add_rule(rule_name, rule)
|
||||
|
||||
elif schema_type in (None, 'object') and \
|
||||
('properties' in schema or \
|
||||
('additionalProperties' in schema and schema['additionalProperties'] is not True)):
|
||||
required = set(schema.get('required', []))
|
||||
properties = list(schema.get('properties', {}).items())
|
||||
return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties')))
|
||||
|
||||
elif schema_type in (None, 'object', 'string') and 'allOf' in schema:
|
||||
required = set()
|
||||
properties = []
|
||||
enum_sets = []
|
||||
hybrid_name = name
|
||||
def add_component(comp_schema, is_required):
|
||||
if (ref := comp_schema.get('$ref')) is not None:
|
||||
comp_schema = self._refs[ref]
|
||||
|
||||
if 'properties' in comp_schema:
|
||||
for prop_name, prop_schema in comp_schema['properties'].items():
|
||||
properties.append((prop_name, prop_schema))
|
||||
if is_required:
|
||||
required.add(prop_name)
|
||||
|
||||
if 'enum' in comp_schema:
|
||||
enum_sets.append(set(comp_schema['enum']))
|
||||
|
||||
for t in schema['allOf']:
|
||||
if 'anyOf' in t:
|
||||
for tt in t['anyOf']:
|
||||
add_component(tt, is_required=False)
|
||||
else:
|
||||
add_component(t, is_required=True)
|
||||
|
||||
if enum_sets:
|
||||
enum_intersection = enum_sets[0]
|
||||
for s in enum_sets[1:]:
|
||||
enum_intersection &= s
|
||||
|
||||
if enum_intersection:
|
||||
rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ')'
|
||||
return self._add_rule(rule_name, rule)
|
||||
|
||||
return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None))
|
||||
|
||||
elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema):
|
||||
items = schema.get('items', schema.get('prefixItems'))
|
||||
if isinstance(items, list):
|
||||
return self._add_rule(
|
||||
rule_name,
|
||||
'"[" space ' +
|
||||
' "," space '.join(
|
||||
self.visit(item, f'{name}{"-" if name else ""}tuple-{i}')
|
||||
for i, item in enumerate(items)) +
|
||||
' space "]"')
|
||||
else:
|
||||
item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item')
|
||||
min_items = schema.get("minItems", 0)
|
||||
max_items = schema.get("maxItems")
|
||||
return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' space "]"')
|
||||
|
||||
elif schema_type in (None, 'string') and 'pattern' in schema:
|
||||
return self._visit_pattern(schema['pattern'], rule_name)
|
||||
|
||||
elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''):
|
||||
return self._add_primitive(
|
||||
'root' if rule_name == 'root' else schema_format,
|
||||
PRIMITIVE_RULES['uuid']
|
||||
)
|
||||
|
||||
elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES:
|
||||
prim_name = f'{schema_format}-string'
|
||||
return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name]))
|
||||
|
||||
elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema):
|
||||
char_rule = self._add_primitive('char', PRIMITIVE_RULES['char'])
|
||||
min_len = schema.get('minLength', 0)
|
||||
max_len = schema.get('maxLength')
|
||||
|
||||
return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\""')
|
||||
|
||||
elif schema_type in (None, 'integer') and \
|
||||
('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema):
|
||||
min_value = None
|
||||
max_value = None
|
||||
if 'minimum' in schema:
|
||||
min_value = schema['minimum']
|
||||
elif 'exclusiveMinimum' in schema:
|
||||
min_value = schema['exclusiveMinimum'] + 1
|
||||
if 'maximum' in schema:
|
||||
max_value = schema['maximum']
|
||||
elif 'exclusiveMaximum' in schema:
|
||||
max_value = schema['exclusiveMaximum'] - 1
|
||||
|
||||
out = ["("]
|
||||
_generate_min_max_int(min_value, max_value, out)
|
||||
out.append(")")
|
||||
return self._add_rule(rule_name, ''.join(out))
|
||||
|
||||
elif (schema_type == 'object') or (len(schema) == 0):
|
||||
return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object']))
|
||||
|
||||
elif schema_type is None and isinstance(schema, dict):
|
||||
# No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
|
||||
# Per JSON Schema semantics this is equivalent to {} and accepts any value.
|
||||
return self._add_rule(rule_name, self._add_primitive('value', PRIMITIVE_RULES['value']))
|
||||
|
||||
else:
|
||||
assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}'
|
||||
# TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
|
||||
return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type])
|
||||
|
||||
def _add_primitive(self, name: str, rule: BuiltinRule):
|
||||
n = self._add_rule(name, rule.content)
|
||||
|
||||
for dep in rule.deps:
|
||||
dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep)
|
||||
assert dep_rule, f'Rule {dep} not known'
|
||||
if dep not in self._rules:
|
||||
self._add_primitive(dep, dep_rule)
|
||||
return n
|
||||
|
||||
def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Optional[Union[bool, Any]]):
|
||||
prop_order = self._prop_order
|
||||
# sort by position in prop_order (if specified) then by original order
|
||||
sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))]
|
||||
|
||||
prop_kv_rule_names = {}
|
||||
for prop_name, prop_schema in properties:
|
||||
prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}')
|
||||
prop_kv_rule_names[prop_name] = self._add_rule(
|
||||
f'{name}{"-" if name else ""}{prop_name}-kv',
|
||||
fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}'
|
||||
)
|
||||
required_props = [k for k in sorted_props if k in required]
|
||||
optional_props = [k for k in sorted_props if k not in required]
|
||||
|
||||
if additional_properties is not None and additional_properties != False:
|
||||
sub_name = f'{name}{"-" if name else ""}additional'
|
||||
value_rule = self.visit(additional_properties, f'{sub_name}-value') if isinstance(additional_properties, dict) else \
|
||||
self._add_primitive('value', PRIMITIVE_RULES['value'])
|
||||
key_rule = self._add_primitive('string', PRIMITIVE_RULES['string']) if not sorted_props \
|
||||
else self._add_rule(f'{sub_name}-k', self._not_strings(sorted_props))
|
||||
|
||||
prop_kv_rule_names["*"] = self._add_rule(
|
||||
f'{sub_name}-kv',
|
||||
f'{key_rule} ":" space {value_rule}'
|
||||
)
|
||||
optional_props.append("*")
|
||||
|
||||
if not required_props and not optional_props:
|
||||
return '"{" space "}"'
|
||||
|
||||
rule = '"{" space '
|
||||
rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props)
|
||||
|
||||
if optional_props:
|
||||
rule += ' ('
|
||||
if required_props:
|
||||
rule += ' "," space ( '
|
||||
|
||||
def get_recursive_refs(ks, first_is_optional):
|
||||
[k, *rest] = ks
|
||||
kv_rule_name = prop_kv_rule_names[k]
|
||||
comma_ref = f'( "," space {kv_rule_name} )'
|
||||
if first_is_optional:
|
||||
res = comma_ref + ('*' if k == '*' else '?')
|
||||
else:
|
||||
res = kv_rule_name + (' ' + comma_ref + "*" if k == '*' else '')
|
||||
if len(rest) > 0:
|
||||
res += ' ' + self._add_rule(
|
||||
f'{name}{"-" if name else ""}{k}-rest',
|
||||
get_recursive_refs(rest, first_is_optional=True)
|
||||
)
|
||||
return res
|
||||
|
||||
rule += ' | '.join(
|
||||
get_recursive_refs(optional_props[i:], first_is_optional=False)
|
||||
for i in range(len(optional_props))
|
||||
)
|
||||
if required_props:
|
||||
rule += ' )'
|
||||
rule += ' )?'
|
||||
|
||||
rule += ' space "}"'
|
||||
|
||||
return rule
|
||||
|
||||
def format_grammar(self):
|
||||
return '\n'.join(
|
||||
f'{name} ::= {rule}'
|
||||
for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0])
|
||||
)
|
||||
|
||||
|
||||
def main(args_in = None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description='''
|
||||
Generates a grammar (suitable for use in ./llama-cli) that produces JSON conforming to a
|
||||
given JSON schema. Only a subset of JSON schema features are supported; more may be
|
||||
added in the future.
|
||||
''',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--prop-order',
|
||||
default=[],
|
||||
type=lambda s: s.split(','),
|
||||
help='''
|
||||
comma-separated property names defining the order of precedence for object properties;
|
||||
properties not specified here are given lower precedence than those that are, and
|
||||
are kept in their original order from the schema. Required properties are always
|
||||
given precedence over optional properties.
|
||||
'''
|
||||
)
|
||||
parser.add_argument(
|
||||
'--allow-fetch',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether to allow fetching referenced schemas over HTTPS')
|
||||
parser.add_argument(
|
||||
'--dotall',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns')
|
||||
parser.add_argument(
|
||||
'--raw-pattern',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Treats string patterns as raw patterns w/o quotes (or quote escapes)')
|
||||
|
||||
parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)')
|
||||
args = parser.parse_args(args_in)
|
||||
|
||||
if args.schema.startswith('https://'):
|
||||
url = args.schema
|
||||
import requests
|
||||
schema = requests.get(url).json()
|
||||
elif args.schema == '-':
|
||||
url = 'stdin'
|
||||
schema = json.load(sys.stdin)
|
||||
else:
|
||||
url = f'file://{args.schema}'
|
||||
with open(args.schema) as f:
|
||||
schema = json.load(f)
|
||||
converter = SchemaConverter(
|
||||
prop_order={name: idx for idx, name in enumerate(args.prop_order)},
|
||||
allow_fetch=args.allow_fetch,
|
||||
dotall=args.dotall,
|
||||
raw_pattern=args.raw_pattern)
|
||||
schema = converter.resolve_refs(schema, url)
|
||||
converter.visit(schema, '')
|
||||
print(converter.format_grammar())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,20 +0,0 @@
|
||||
import json, subprocess, sys, os
|
||||
|
||||
assert len(sys.argv) >= 2
|
||||
[_, pattern, *rest] = sys.argv
|
||||
|
||||
print(subprocess.check_output(
|
||||
[
|
||||
"python",
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"json_schema_to_grammar.py"),
|
||||
*rest,
|
||||
"-",
|
||||
"--raw-pattern",
|
||||
],
|
||||
text=True,
|
||||
input=json.dumps({
|
||||
"type": "string",
|
||||
"pattern": pattern,
|
||||
}, indent=2)))
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# ./examples/ts-type-to-grammar.sh "{a:string,b:string,c?:string}"
|
||||
# python examples/json_schema_to_grammar.py https://json.schemastore.org/tsconfig.json
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
readonly type="$1"
|
||||
|
||||
# Create a temporary directory
|
||||
TMPDIR=""
|
||||
trap 'rm -fR "$TMPDIR"' EXIT
|
||||
TMPDIR=$(mktemp -d)
|
||||
|
||||
DTS_FILE="$TMPDIR/type.d.ts"
|
||||
SCHEMA_FILE="$TMPDIR/schema.json"
|
||||
|
||||
echo "export type MyType = $type" > "$DTS_FILE"
|
||||
|
||||
# This is a fork of typescript-json-schema, actively maintained as of March 2024:
|
||||
# https://github.com/vega/ts-json-schema-generator
|
||||
npx ts-json-schema-generator --unstable --no-top-ref --path "$DTS_FILE" --type MyType -e none > "$SCHEMA_FILE"
|
||||
|
||||
# Alternative, not actively maintained as of March 2024:
|
||||
# https://github.com/YousefED/typescript-json-schema
|
||||
# npx typescript-json-schema --defaultProps --required "$DTS_FILE" MyType | tee "$SCHEMA_FILE" >&2
|
||||
|
||||
./examples/json_schema_to_grammar.py "$SCHEMA_FILE"
|
||||
+1
-7
@@ -146,8 +146,6 @@ You can use GBNF grammars:
|
||||
- For any completion endpoints, passed as the `json_schema` body field
|
||||
- For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`)
|
||||
- In [llama-cli](../tools/cli) and [llama-completion](../tools/completion), passed as the `--json` / `-j` flag
|
||||
- To convert to a grammar ahead of time:
|
||||
- in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
|
||||
|
||||
> [!NOTE]
|
||||
> The JSON schema is only used to constrain the model output and is not injected into the prompt. The model has no visibility into the schema, so if you want it to understand the expected structure, describe it explicitly in your prompt. This does not apply to tool calling, where schemas are injected into the prompt.
|
||||
@@ -187,11 +185,7 @@ llama-cli \
|
||||
|
||||
<summary>Show grammar</summary>
|
||||
|
||||
You can convert any schema in command-line with:
|
||||
|
||||
```bash
|
||||
examples/json_schema_to_grammar.py name-age-schema.json
|
||||
```
|
||||
The schema above converts to:
|
||||
|
||||
```
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
|
||||
@@ -163,11 +163,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
|
||||
llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
|
||||
target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
|
||||
target_link_libraries(test-chat PRIVATE server-context)
|
||||
# TODO: disabled on loongarch64 because the ggml-ci node lacks Python 3.8
|
||||
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
|
||||
llama_build_and_test(test-json-schema-to-grammar.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
|
||||
target_include_directories(test-json-schema-to-grammar PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
|
||||
endif()
|
||||
llama_build_and_test(test-json-schema-to-grammar.cpp)
|
||||
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
llama_build(test-quantize-stats.cpp)
|
||||
|
||||
@@ -918,7 +918,7 @@ static void test_json_schema() {
|
||||
// Otherwise, this test structure is the same.
|
||||
|
||||
test_schema(
|
||||
"empty schema (object)",
|
||||
"empty schema (any value)",
|
||||
// Schema
|
||||
R"""(
|
||||
{}
|
||||
@@ -927,14 +927,16 @@ static void test_json_schema() {
|
||||
{
|
||||
R"""({})""",
|
||||
R"""({"foo": "bar"})""",
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
"",
|
||||
"[]",
|
||||
"null",
|
||||
R"""("")""",
|
||||
"true",
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
"",
|
||||
R"""({"foo"})""",
|
||||
"foo",
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -9,8 +9,6 @@
|
||||
#include "json.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <regex>
|
||||
|
||||
static std::string trim(const std::string & source) {
|
||||
@@ -64,21 +62,8 @@ struct TestCase {
|
||||
}
|
||||
};
|
||||
|
||||
static void write(const std::string & file, const std::string & content) {
|
||||
std::ofstream f;
|
||||
f.open(file.c_str());
|
||||
f << content.c_str();
|
||||
f.close();
|
||||
}
|
||||
|
||||
static std::string read(const std::string & file) {
|
||||
std::ostringstream actuals;
|
||||
actuals << std::ifstream(file.c_str()).rdbuf();
|
||||
return actuals.str();
|
||||
}
|
||||
|
||||
static void test_all(const std::string & lang, std::function<void(const TestCase &)> runner) {
|
||||
fprintf(stderr, "#\n# Testing JSON schema conversion (%s)\n#\n", lang.c_str());
|
||||
static void test_all(const std::string & title, std::function<void(const TestCase &)> runner) {
|
||||
fprintf(stderr, "#\n# %s\n#\n", title.c_str());
|
||||
auto test = [&](const TestCase & tc) {
|
||||
fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : "");
|
||||
runner(tc);
|
||||
@@ -330,7 +315,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"empty schema (object)",
|
||||
"empty schema (any value)",
|
||||
"{}",
|
||||
R"""(
|
||||
array ::= "[" space ( value ("," space value)* )? space "]"
|
||||
@@ -341,7 +326,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
null ::= "null"
|
||||
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?
|
||||
object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}"
|
||||
root ::= object
|
||||
root ::= value
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
value ::= object | array | string | number | boolean | null
|
||||
@@ -569,6 +554,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
)"""
|
||||
});
|
||||
|
||||
// items {} constrains nothing, the same as no items at all
|
||||
test({
|
||||
SUCCESS,
|
||||
"array with empty items",
|
||||
@@ -582,11 +568,10 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
decimal-part ::= [0-9]{1,16}
|
||||
integral-part ::= [0] | [1-9] [0-9]{0,15}
|
||||
item ::= object
|
||||
null ::= "null"
|
||||
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?
|
||||
object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}"
|
||||
root ::= "[" space (item ("," space item)*)? space "]"
|
||||
root ::= "[" space ( value ("," space value)* )? space "]"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
value ::= object | array | string | number | boolean | null
|
||||
@@ -607,11 +592,10 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
decimal-part ::= [0-9]{1,16}
|
||||
integral-part ::= [0] | [1-9] [0-9]{0,15}
|
||||
item ::= object
|
||||
null ::= "null"
|
||||
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?
|
||||
object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? space "}"
|
||||
root ::= "[" space (item ("," space item)*)? space "]"
|
||||
root ::= "[" space ( value ("," space value)* )? space "]"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
value ::= object | array | string | number | boolean | null
|
||||
@@ -1434,6 +1418,96 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)"""
|
||||
});
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"regexp with non-capturing group",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(?:foo|bar)baz$"
|
||||
})""",
|
||||
R"""(
|
||||
root ::= "\"" (("foo" | "bar") "baz") "\""
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)"""
|
||||
});
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"regexp with nested non-capturing groups",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(?:(?:ab)+c)?d$"
|
||||
})""",
|
||||
R"""(
|
||||
root ::= "\"" ((("ab")+ "c")? "d") "\""
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)"""
|
||||
});
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"unanchored regexp",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "[0-9]+"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)"""
|
||||
});
|
||||
|
||||
// the rules of the partial conversion (here "root-0") must not leak into the grammar
|
||||
test({
|
||||
SUCCESS,
|
||||
"regexp with unsupported shorthand",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]{3}\\w$"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)"""
|
||||
});
|
||||
|
||||
// a regexp that is invalid under any flavor is still an error
|
||||
test({
|
||||
FAILURE,
|
||||
"regexp with unbalanced parentheses",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(a$"
|
||||
})""",
|
||||
""
|
||||
});
|
||||
|
||||
// only the property with the bad pattern degrades
|
||||
test({
|
||||
SUCCESS,
|
||||
"unsupported regexp in a property",
|
||||
R"""({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": { "type": "string", "pattern": "^[a-z\\-]+$" }
|
||||
},
|
||||
"required": ["a"],
|
||||
"additionalProperties": false
|
||||
})""",
|
||||
R"""(
|
||||
a ::= string
|
||||
a-kv ::= "\"a\"" space ":" space a
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= "{" space a-kv space "}"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)"""
|
||||
});
|
||||
}
|
||||
|
||||
static void test_resolves_to_string() {
|
||||
@@ -1465,15 +1539,15 @@ static void test_resolves_to_string() {
|
||||
test("type array with string", R"({"type": ["string", "null"]})", true);
|
||||
test("type array without string", R"({"type": ["integer", "null"]})", false);
|
||||
|
||||
// String-specific keywords
|
||||
test("minLength implies string", R"({"minLength": 1})", true);
|
||||
test("maxLength implies string", R"({"maxLength": 10})", true);
|
||||
// String-specific keywords, a length alone is not one as the converter still accepts any value there
|
||||
test("minLength alone", R"({"minLength": 1})", false);
|
||||
test("maxLength alone", R"({"maxLength": 10})", false);
|
||||
test("pattern implies string", R"({"pattern": "^[a-z]+$"})", true);
|
||||
|
||||
// Format
|
||||
// Format, only the ones the converter knows
|
||||
test("format date", R"({"format": "date"})", true);
|
||||
test("format uuid", R"({"format": "uuid"})", true);
|
||||
test("format email", R"({"format": "email"})", true);
|
||||
test("format email", R"({"format": "email"})", false);
|
||||
|
||||
// Const
|
||||
test("const string", R"({"const": "hello"})", true);
|
||||
@@ -1510,12 +1584,9 @@ static void test_resolves_to_string() {
|
||||
}
|
||||
|
||||
int main() {
|
||||
fprintf(stderr, "LLAMA_NODE_AVAILABLE = %s\n", getenv("LLAMA_NODE_AVAILABLE") ? "true" : "false");
|
||||
fprintf(stderr, "LLAMA_PYTHON_AVAILABLE = %s\n", getenv("LLAMA_PYTHON_AVAILABLE") ? "true" : "false");
|
||||
|
||||
test_resolves_to_string();
|
||||
|
||||
test_all("C++", [](const TestCase & tc) {
|
||||
test_all("JSON schema conversion", [](const TestCase & tc) {
|
||||
try {
|
||||
tc.verify(json_schema_to_grammar(common_json::parse(tc.schema), true));
|
||||
tc.verify_status(SUCCESS);
|
||||
@@ -1525,127 +1596,57 @@ int main() {
|
||||
}
|
||||
});
|
||||
|
||||
// C++ only tests (features not yet supported in JS/Python implementations)
|
||||
// a document parsed up front gives the same grammar as the JSON, recursion included
|
||||
{
|
||||
fprintf(stderr, "#\n# Testing C++ only features\n#\n");
|
||||
auto run = [](const TestCase & tc) {
|
||||
fprintf(stderr, "- %s\n", tc.name.c_str());
|
||||
try {
|
||||
tc.verify(json_schema_to_grammar(common_json::parse(tc.schema), true));
|
||||
tc.verify_status(SUCCESS);
|
||||
} catch (const std::invalid_argument & ex) {
|
||||
fprintf(stderr, "Error: %s\n", ex.what());
|
||||
tc.verify_status(FAILURE);
|
||||
fprintf(stderr, "- parsed document\n");
|
||||
auto schema = common_json::parse(R"""({
|
||||
"$ref": "#/$defs/node",
|
||||
"$defs": {
|
||||
"node": {
|
||||
"type": "object",
|
||||
"properties": {"next": {"$ref": "#/$defs/node"}, "leaf": {}},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
})""");
|
||||
assert(json_schema_to_grammar(common_schema_parse(schema)) == json_schema_to_grammar(schema, true));
|
||||
}
|
||||
|
||||
// a sub-schema added on its own resolves its $refs through the document given to resolve_refs()
|
||||
{
|
||||
fprintf(stderr, "- sub-schema $ref\n");
|
||||
auto parameters = common_json::parse(R"""({
|
||||
"type": "object",
|
||||
"properties": {"item": {"$ref": "#/$defs/item"}},
|
||||
"$defs": {
|
||||
"item": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "string"}},
|
||||
"required": ["a"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
})""");
|
||||
TestCase tc {
|
||||
SUCCESS,
|
||||
"sub-schema $ref",
|
||||
"",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
ref-defs-item ::= "{" space ref-defs-item-a-kv space "}"
|
||||
ref-defs-item-a-kv ::= "\"a\"" space ":" space string
|
||||
root ::= ref-defs-item
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
};
|
||||
|
||||
run({
|
||||
SUCCESS,
|
||||
"regexp with non-capturing group",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(?:foo|bar)baz$"
|
||||
})""",
|
||||
R"""(
|
||||
root ::= "\"" (("foo" | "bar") "baz") "\""
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""",
|
||||
});
|
||||
|
||||
run({
|
||||
SUCCESS,
|
||||
"regexp with nested non-capturing groups",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(?:(?:ab)+c)?d$"
|
||||
})""",
|
||||
R"""(
|
||||
root ::= "\"" ((("ab")+ "c")? "d") "\""
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""",
|
||||
});
|
||||
|
||||
run({
|
||||
SUCCESS,
|
||||
"unanchored regexp",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "[0-9]+"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
|
||||
// the rules of the partial conversion (here "root-0") must not leak into the grammar
|
||||
run({
|
||||
SUCCESS,
|
||||
"regexp with unsupported shorthand",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]{3}\\w$"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
|
||||
// a regexp that is invalid under any flavor is still an error
|
||||
run({
|
||||
FAILURE,
|
||||
"regexp with unbalanced parentheses",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(a$"
|
||||
})""",
|
||||
""
|
||||
});
|
||||
|
||||
// only the property with the bad pattern degrades
|
||||
run({
|
||||
SUCCESS,
|
||||
"unsupported regexp in a property",
|
||||
R"""({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": { "type": "string", "pattern": "^[a-z\\-]+$" }
|
||||
},
|
||||
"required": ["a"],
|
||||
"additionalProperties": false
|
||||
})""",
|
||||
R"""(
|
||||
a ::= string
|
||||
a-kv ::= "\"a\"" space ":" space a
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= "{" space a-kv space "}"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
tc.verify(build_grammar([&](const common_grammar_builder & builder) {
|
||||
builder.resolve_refs(parameters);
|
||||
builder.add_schema("root", parameters.at("properties").at("item"));
|
||||
}));
|
||||
}
|
||||
|
||||
if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) {
|
||||
fprintf(stderr, "\033[33mWARNING: Skipping slow tests on emulator.\n\033[0m");
|
||||
} else {
|
||||
if (getenv("LLAMA_PYTHON_AVAILABLE") || (std::system("python -c \"import sys; exit(1) if sys.version_info < (3, 8) else print('Python version is sufficient')\"") == 0)) {
|
||||
test_all("Python", [](const TestCase & tc) {
|
||||
write("test-json-schema-input.tmp", tc.schema);
|
||||
tc.verify_status(std::system(
|
||||
"python ./examples/json_schema_to_grammar.py test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE);
|
||||
tc.verify(read("test-grammar-output.tmp"));
|
||||
});
|
||||
} else {
|
||||
fprintf(stderr, "\033[33mWARNING: Python not found (min version required is 3.8), skipping Python JSON schema -> grammar tests.\n\033[0m");
|
||||
}
|
||||
}
|
||||
|
||||
test_all("Check Expectations Validity", [](const TestCase & tc) {
|
||||
test_all("Check the expectations parse", [](const TestCase & tc) {
|
||||
if (tc.expected_status == SUCCESS) {
|
||||
tc.verify_expectation_parseable();
|
||||
}
|
||||
|
||||
@@ -292,6 +292,12 @@ static void test_array(testing & t) {
|
||||
const auto & a = root<common_schema_array>(t, doc);
|
||||
as<common_schema_string>(t, a.items.get(), "items");
|
||||
});
|
||||
|
||||
t.test("prefixItems given as a schema is items", [](testing & t) {
|
||||
auto doc = parse(R"({"prefixItems": {"type": "string"}})");
|
||||
const auto & a = root<common_schema_array>(t, doc);
|
||||
as<common_schema_string>(t, a.items.get(), "items");
|
||||
});
|
||||
}
|
||||
|
||||
static void test_tuple(testing & t) {
|
||||
@@ -610,6 +616,32 @@ static void test_ref(testing & t) {
|
||||
t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get());
|
||||
as<common_schema_null>(t, r.target, "target");
|
||||
});
|
||||
|
||||
t.test("a schema parsed into a document reuses its refs", [](testing & t) {
|
||||
auto doc = parse(R"({"properties": {"a": {"$ref": "#/$defs/t"}}, "$defs": {"t": {"type": "boolean"}}})");
|
||||
auto node = common_schema_parse(common_json::parse(R"({"items": {"$ref": "#/$defs/t"}})"), doc);
|
||||
const auto & a = as<common_schema_array>(t, node.get(), "node");
|
||||
const auto & r = as<common_schema_ref>(t, a.items.get(), "items");
|
||||
t.assert_true("shared target", r.target == doc.refs.at("#/$defs/t").get());
|
||||
t.assert_equal("refs", (size_t) 1, doc.refs.size());
|
||||
});
|
||||
|
||||
t.test("a schema parsed into a document adds its refs", [](testing & t) {
|
||||
common_schema_document doc;
|
||||
auto node = common_schema_parse(common_json::parse(R"({"$ref": "#/$defs/t", "$defs": {"t": {"type": "null"}}})"), doc);
|
||||
as<common_schema_null>(t, as<common_schema_ref>(t, node.get(), "node").target, "target");
|
||||
t.assert_equal("refs", (size_t) 1, doc.refs.size());
|
||||
});
|
||||
|
||||
t.test("a rejected schema leaves the document unchanged", [](testing & t) {
|
||||
common_schema_document doc;
|
||||
try {
|
||||
common_schema_parse(common_json::parse(R"({"allOf": [{"$ref": "#/$defs/t"}, {"type": "x"}], "$defs": {"t": {"type": "null"}}})"), doc);
|
||||
t.assert_true("rejected", false);
|
||||
} catch (const std::runtime_error &) {
|
||||
t.assert_true("no refs", doc.refs.empty());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static void test_errors(testing & t) {
|
||||
@@ -663,7 +695,6 @@ static void test_errors(testing & t) {
|
||||
t.test("array", [](testing & t) {
|
||||
assert_error(t, R"({"type": "array", "maxItems": 1.5})", "#: maxItems must be a non-negative integer");
|
||||
assert_error(t, R"({"type": "array", "items": {"type": "x"}})", "#/items: unrecognized type x");
|
||||
assert_error(t, R"({"prefixItems": {}})", "#: prefixItems must be an array");
|
||||
assert_error(t, R"({"prefixItems": [{"type": "x"}]})", "#/prefixItems/0: unrecognized type x");
|
||||
});
|
||||
|
||||
|
||||
@@ -1188,7 +1188,10 @@ json oaicompat_chat_params_parse(
|
||||
std::string response_type = json_value(response_format, "type", std::string());
|
||||
if (response_type == "json_object") {
|
||||
if (response_format.contains("schema") || json_schema.empty()) {
|
||||
json_schema = json_value(response_format, "schema", json::object());
|
||||
// any object without a schema, {} would be any value
|
||||
json any_object = json::object();
|
||||
any_object["type"] = "object";
|
||||
json_schema = json_value(response_format, "schema", any_object);
|
||||
}
|
||||
} else if (response_type == "json_schema") {
|
||||
auto schema_wrapper = json_value(response_format, "json_schema", json::object());
|
||||
|
||||
Reference in New Issue
Block a user