Compare commits

...
5 Commits
Author SHA1 Message Date
7acdbb1f19 mtmd: fix LFM2 image tiling threshold (#27057)
* mtmd: fix LFM2 image tiling threshold

* refactor testing

* fix

* fix on windows

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
2026-08-18 11:11:19 +02:00
Georgi Gerganov 1511ce3bc3 sync : ggml 2026-08-18 11:30:03 +03:00
Georgi Gerganov da786dc23e ggml : bump version to 0.20.2 (ggml/1589) 2026-08-18 11:30:03 +03:00
Georgi GerganovandGitHub 27e345b574 build : fix xcframework + cmake clean-up (#27304)
* xcframework : fix build

* mtmd : remove unused include path

* vendor : use vendor::hash alias target in cmake

CMake reserves "::" in target names for imported/alias targets, so the real
target keeps the name vendor-hash and a vendor::hash ALIAS target is added.
Consumers (mtmd, llama-gguf-hash) now link against the namespaced alias.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* vendor : add cmake targets for all vendored libs with vendor:: aliases

Add INTERFACE targets for the header-only vendor libs (miniaudio, nlohmann,
sheredom, stb) and ALIAS targets named vendor::<lib> for all of them,
including cpp-httplib and hash. Each exposes the vendor/ root so includes
are namespaced, e.g. <nlohmann/json.hpp>.

Consolidate the per-lib add_subdirectory calls into a single
add_subdirectory(vendor), keeping the cpp-httplib gate on LLAMA_BUILD_COMMON.
Consumers (llama-common, mtmd) now link the aliases instead of relying on
raw vendor/ include paths.

hash: consumers now include via "hash/hash.h"; the vendor/hash dir is kept
as a PRIVATE include so the synced upstream sources compile unmodified.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* readme : use foo/bar names in acknowledgements

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ocd : fix valign
2026-08-18 11:16:51 +03:00
Georgi GerganovandGitHub 8b86400975 ci : create pre-release with change log and nightly link in make-release (#27302)
* ci : create pre-release with change log and nightly link in make-release

After pushing the tag, create a pre-release using
ggml-org/action-create-release. The release description is generated by
scripts/make-release-desc.sh: the change log between the current and
previous version (one line per commit), a link to the corresponding
nightly build when it exists, and a note that semantic versioning is
still work in progress.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* cmake : bump version to 0.1.2

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ci : find the nightly tag by commit in make-release-desc.sh

The nightly release is guaranteed by the release checks to point at HEAD,
so instead of reconstructing its name (commit count, branch, hash) just
pick the b* tag pointing at HEAD. This also drops the RELEASE_BRANCH env
var from the workflow.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ci : resolve the release commit from the version tag in make-release-desc.sh

The change log and nightly lookup now use the commit the version tag
points at (HEAD when the tag does not exist), instead of always HEAD.
This makes the script usable locally for older versions, e.g.
./scripts/make-release-desc.sh v0.1.1. The tag is resolved to a SHA
first, since --points-at does not peel annotated tags.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ci : normalize the version argument in make-release-desc.sh

Accept the version with or without the leading v (0.1.1 == v0.1.1) and
reject anything else, instead of silently treating a bare version as a
non-existent tag name.

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* cont : clean-up
2026-08-18 10:14:41 +03:00
24 changed files with 293 additions and 22 deletions
+27
View File
@@ -49,6 +49,33 @@ jobs:
git push origin "${VERSION}"
echo "Created and pushed tag ${VERSION}"
- name: Generate release description
id: desc
run: bash scripts/make-release-desc.sh "${{ steps.checks.outputs.version }}"
env:
GITHUB_REPOSITORY: ${{ github.repository }}
- name: Create release
if: ${{ github.event.inputs.dry_run == 'false' }}
uses: ggml-org/action-create-release@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
tag_name: ${{ steps.checks.outputs.version }}
# TODO: remove the prerelease flag once the semantic versioning workflow is ready
# ref: https://github.com/ggml-org/ggml/discussions/1579
prerelease: true
body: |
> [!NOTE]
> Semantic versioning is still work in progress.
> More info can be found in https://github.com/ggml-org/ggml/discussions/1579
${{ steps.desc.outputs.nightly }}
## ${{ steps.desc.outputs.changelog_title }}
${{ steps.desc.outputs.changelog }}
- name: Dry run summary
if: ${{ github.event.inputs.dry_run == 'true' }}
run: |
+2 -4
View File
@@ -5,7 +5,7 @@ include(CheckIncludeFileCXX)
### llama.cpp version
set(LLAMA_VERSION_MAJOR 0)
set(LLAMA_VERSION_MINOR 1)
set(LLAMA_VERSION_PATCH 1)
set(LLAMA_VERSION_PATCH 2)
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
# whether this is a development/nightly build
@@ -224,12 +224,10 @@ add_subdirectory(src)
# utils, programs, examples and tests
#
# mtmd needs this even when common is not built
add_subdirectory(vendor/hash)
add_subdirectory(vendor)
if (LLAMA_BUILD_COMMON)
add_subdirectory(common)
add_subdirectory(vendor/cpp-httplib)
endif()
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
+3 -3
View File
@@ -119,7 +119,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
## Acknowledgements
- [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license
- [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
- [nothings/stb](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
- [nlohmann/json](https://github.com/nlohmann/json) - Single-header JSON library, used by various tools/examples - MIT License
- [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
- [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
- [mackron/miniaudio](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
- [sheredom/subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
+1
View File
@@ -290,6 +290,7 @@ combine_static_libraries() {
"${base_dir}/${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a"
"${base_dir}/${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a"
"${base_dir}/${build_dir}/tools/mtmd/${release_dir}/libmtmd.a"
"${base_dir}/${build_dir}/vendor/hash/${release_dir}/libvendor-hash.a"
)
# Create temporary directory for processing
+2 -1
View File
@@ -126,7 +126,8 @@ set_target_properties(${TARGET} PROPERTIES
MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number
)
target_include_directories(${TARGET} PUBLIC . ../vendor)
target_include_directories(${TARGET} PUBLIC .)
target_link_libraries (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom)
target_compile_features (${TARGET} PUBLIC cxx_std_17)
if (LLAMA_SUBPROCESS)
+1 -1
View File
@@ -2,5 +2,5 @@ set(TARGET llama-gguf-hash)
add_executable(${TARGET} gguf-hash.cpp)
install(TARGETS ${TARGET} RUNTIME)
target_link_libraries(${TARGET} PRIVATE vendor-hash ggml ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${TARGET} PRIVATE vendor::hash ggml ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
+3 -3
View File
@@ -17,15 +17,15 @@
extern "C" {
#endif
#include "xxhash/xxhash.h"
#include "sha256/sha256.h"
#include "hash/xxhash/xxhash.h"
#include "hash/sha256/sha256.h"
#ifdef __cplusplus
}
#endif
// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py
#include "sha1/sha1.h"
#include "hash/sha1/sha1.h"
using namespace vendor_hash;
+1 -1
View File
@@ -5,7 +5,7 @@ project("ggml" C CXX ASM)
### GGML Version
set(GGML_VERSION_MAJOR 0)
set(GGML_VERSION_MINOR 20)
set(GGML_VERSION_PATCH 1)
set(GGML_VERSION_PATCH 2)
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
+87
View File
@@ -0,0 +1,87 @@
#!/bin/bash
# Generate the description of a release: the previous release version, the
# change log and the link to the nightly release corresponding to the commit being released.
#
# Usage: make-release-desc.sh <version>
# <version>: current release version (v<maj>.<min>.<pat>, the leading v is optional)
#
# The previous version is the highest plain semver tag (v<maj>.<min>.<pat>)
# strictly below <version>. The change log lists all commits between the
# previous version tag and the release commit, one line per commit.
#
# The release commit is the commit <version> points at when the tag exists,
# HEAD otherwise. The nightly release is the b* tag pointing at that commit
# (release.yml tags the same commit); the link is only generated when that
# tag exists.
#
# Env (when running in GitHub Actions):
# GITHUB_OUTPUT: previous_tag, changelog_title, changelog and nightly are written here
# GITHUB_REPOSITORY: owner/repo, used to build the nightly release URL (skipped when unset)
set -euo pipefail
if [[ $# -ne 1 ]]; then
echo "Usage: $(basename "$0") <version>"
exit 1
fi
VERSION="$1"
# Accept the version with or without the leading v, reject anything else
if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
VERSION="v${VERSION}"
elif [[ ! "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: invalid version '${VERSION}' (expected v<maj>.<min>.<pat>)"
exit 1
fi
# Make sure all remote tags are available locally (skipped on local runs without origin)
if ! git fetch --tags origin 2>/dev/null; then
echo "Warning: could not fetch tags from origin (local run?)"
fi
# Release commit: the commit <version> points at when the tag exists, HEAD otherwise.
if ! RELEASE_COMMIT="$(git rev-parse -q --verify "refs/tags/${VERSION}^{commit}" 2>/dev/null)"; then
RELEASE_COMMIT="$(git rev-parse HEAD)"
fi
echo "Release commit: $(git rev-parse --short "${RELEASE_COMMIT}")"
PREV="$( { git tag --list; echo "${VERSION}"; } \
| grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
| sort -V \
| awk -v cur="${VERSION}" '$0 == cur { exit } { prev = $0 } END { print prev }')"
if [[ -n "${PREV}" ]]; then
CHANGELOG="$(git log --oneline "${PREV}..${RELEASE_COMMIT}")"
CHANGELOG_TITLE="Change log since ${PREV}"
else
CHANGELOG="(no previous release tag found)"
CHANGELOG_TITLE="Change log"
fi
# Nightly release: the b* tag pointing at the release commit (|| true: no match is not an error)
NIGHTLY_TAG="$(git tag --points-at "${RELEASE_COMMIT}" | grep -E '(^|-)b[0-9]+(-[0-9a-f]{7})?$' | head -n 1 || true)"
NIGHTLY=""
if [[ -n "${NIGHTLY_TAG}" ]]; then
if [[ -n "${GITHUB_REPOSITORY:-}" ]]; then
NIGHTLY_URL="https://github.com/${GITHUB_REPOSITORY}/releases/tag/${NIGHTLY_TAG}"
NIGHTLY="**Nightly build:** [${NIGHTLY_TAG}](${NIGHTLY_URL})"
echo "Nightly release: ${NIGHTLY_URL}"
fi
else
echo "No nightly release found for commit $(git rev-parse --short "${RELEASE_COMMIT}")"
fi
echo "Previous version: ${PREV:-none}"
echo "${CHANGELOG}"
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
{
echo "previous_tag=${PREV}"
echo "changelog_title=${CHANGELOG_TITLE}"
echo "nightly=${NIGHTLY}"
echo "changelog<<CHANGELOG_EOF"
echo "${CHANGELOG}"
echo "CHANGELOG_EOF"
} >> "${GITHUB_OUTPUT}"
fi
+1 -1
View File
@@ -1 +1 @@
3834fd814e74e8af277939dabd69ecc780affd21
8c63e70982c95ceb862e3a1073a2c1beef75d60a
+3
View File
@@ -310,6 +310,9 @@ llama_build_and_test(test-mtmd-c-api.c)
target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
unset(LLAMA_TEST_NAME)
llama_build_and_test(test-mtmd-impl.cpp)
target_link_libraries(test-mtmd-impl PRIVATE mtmd)
# GGUF model data fetcher library for tests that need real model metadata
# Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT)
if (TARGET cpp-httplib)
+88
View File
@@ -0,0 +1,88 @@
#include "testing.h"
#include "mtmd-image.h"
#include <iostream>
#include <string>
#include <utility>
#include <vector>
// this test file contains:
// 1. test cases for mtmd helpers
// 2. test cases for internal mtmd components
// internal headers can be included here
struct test_registry {
using fn_t = void (*)(testing &);
struct entry {
std::string name;
fn_t fn;
};
static std::vector<entry> & all() {
static std::vector<entry> entries;
return entries;
}
test_registry(const char * name, fn_t fn) {
all().push_back({ name, fn });
}
};
#define MAKE_TEST(name) \
static void name(testing & t); \
static const test_registry test_registry_ ## name(#name, &name); \
static void name(testing & t)
//
// mtmd_image
//
MAKE_TEST(test_image_preprocessor_lfm2) {
clip_hparams hparams;
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
// { image size, expected tiling }
const std::vector<std::pair<clip_image_size, bool>> cases = {
{ { 704, 704 }, false },
// 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF
// if round_by_factor rounds half to even (22) instead of away from zero (23)
{ { 720, 720 }, false },
{ { 736, 736 }, true },
{ { 1024, 977 }, true },
{ { 1056, 384 }, false },
};
for (const auto & [size, expected] : cases) {
const bool actual = mtmd_image_preprocessor_lfm2::should_tile(hparams, size);
t.assert_equal(
"tiling for " + std::to_string(size.width) + "x" + std::to_string(size.height),
std::string(expected ? "tiled" : "single"),
std::string(actual ? "tiled" : "single"));
}
}
//
// main
//
int main(int argc, char ** argv) {
testing t(std::cout);
t.verbose = true;
// usage: test-mtmd-impl [filter_regex]
for (int i = 1; i < argc; i++) {
t.set_filter(argv[i]);
}
for (const auto & e : test_registry::all()) {
t.test(e.name, e.fn);
}
return t.summary();
}
+4 -3
View File
@@ -78,10 +78,8 @@ set_target_properties(mtmd PROPERTIES
)
target_link_libraries (mtmd PUBLIC ggml llama)
target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash)
target_link_libraries (mtmd PRIVATE Threads::Threads vendor::hash vendor::miniaudio vendor::stb vendor::sheredom)
target_include_directories(mtmd PUBLIC .)
target_include_directories(mtmd PRIVATE ../..)
target_include_directories(mtmd PRIVATE ../../vendor)
target_compile_features (mtmd PRIVATE cxx_std_17)
if (MTMD_VIDEO)
@@ -92,6 +90,9 @@ if (BUILD_SHARED_LIBS)
set_target_properties (mtmd PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_compile_definitions(mtmd PRIVATE LLAMA_BUILD)
target_compile_definitions(mtmd PUBLIC LLAMA_SHARED)
# export all symbols so that internal components can be tested by test-mtmd-impl
set_target_properties (mtmd PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif()
set(MTMD_PUBLIC_HEADERS
+4
View File
@@ -858,6 +858,9 @@ static std::ifstream open_ifstream_binary(const std::string & fname) {
}
#endif
// in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated
// this is a quick fix to avoid compilation errors
#ifndef DIRECTORY_SEPARATOR
static std::string string_format(const char * fmt, ...) {
va_list ap;
va_list ap2;
@@ -915,6 +918,7 @@ inline bool string_ends_with(std::string_view str, std::string_view suffix) {
return str.size() >= suffix.size() &&
str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
}
#endif
//
// gguf utils
+1 -1
View File
@@ -12,7 +12,7 @@
#include "mtmd-helper-common.h"
#include "llama.h"
#include "hash.h"
#include "hash/hash.h"
#include <algorithm>
#include <cinttypes>
+19 -2
View File
@@ -1013,14 +1013,31 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image
return output;
}
bool mtmd_image_preprocessor_lfm2::should_tile(
const clip_hparams & hparams,
const clip_image_size & original_size) {
const int align_size = hparams.patch_size * hparams.n_merge;
const auto round_by_factor = [align_size](float x) {
// see https://github.com/ggml-org/llama.cpp/pull/27057#discussion_r3796264887
return static_cast<int>(std::nearbyint(static_cast<double>(x) / align_size)) * align_size;
};
const int h_bar = std::max(hparams.patch_size, round_by_factor(original_size.height));
const int w_bar = std::max(hparams.patch_size, round_by_factor(original_size.width));
return static_cast<double>(h_bar) * static_cast<double>(w_bar) >
static_cast<double>(hparams.image_max_pixels) * max_pixels_tolerance;
}
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
inst.overview_size = img_tool::calc_size_preserved_ratio(
original_size,
{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
// tile if either dimension exceeds tile_size with tolerance
const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
const bool needs_tiling = should_tile(hparams, original_size);
if (!needs_tiling) {
inst.refined_size = clip_image_size{0, 0};
+2
View File
@@ -148,6 +148,8 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) override;
static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
private:
clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
+11
View File
@@ -0,0 +1,11 @@
# mtmd needs these even when common is not built
add_subdirectory(hash)
add_subdirectory(miniaudio)
add_subdirectory(nlohmann)
add_subdirectory(sheredom)
add_subdirectory(stb)
# only used by common
if (LLAMA_BUILD_COMMON)
add_subdirectory(cpp-httplib)
endif()
+2
View File
@@ -9,6 +9,8 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
add_library(${TARGET} STATIC httplib.cpp httplib.h)
add_library(vendor::cpp-httplib ALIAS ${TARGET})
# disable warnings in 3rd party code
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
target_compile_options(${TARGET} PRIVATE /w)
+7 -2
View File
@@ -16,6 +16,8 @@ add_library(${TARGET} STATIC
${VENDOR_SRCS}
)
add_library(vendor::hash ALIAS ${TARGET})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
# disable warnings in 3rd party code, but keep them for hash.cpp
@@ -29,5 +31,8 @@ set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_
# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py
set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX)
# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too
target_include_directories(${TARGET} PUBLIC .)
# expose the vendor/ root so consumers can include via "hash/hash.h"
target_include_directories(${TARGET} PUBLIC ..)
# internal includes of the vendored sources, e.g. sha256.c -> "rotate-bits/rotate-bits.h"
target_include_directories(${TARGET} PRIVATE .)
+6
View File
@@ -0,0 +1,6 @@
# header-only: interface target exposing the vendor/ root so consumers
# can include via <miniaudio/miniaudio.h>
add_library(miniaudio INTERFACE)
add_library(vendor::miniaudio ALIAS miniaudio)
target_include_directories(miniaudio INTERFACE ..)
+6
View File
@@ -0,0 +1,6 @@
# header-only: interface target exposing the vendor/ root so consumers
# can include via <nlohmann/json.hpp>
add_library(nlohmann INTERFACE)
add_library(vendor::nlohmann ALIAS nlohmann)
target_include_directories(nlohmann INTERFACE ..)
+6
View File
@@ -0,0 +1,6 @@
# header-only: interface target exposing the vendor/ root so consumers
# can include via <sheredom/subprocess.h>
add_library(sheredom INTERFACE)
add_library(vendor::sheredom ALIAS sheredom)
target_include_directories(sheredom INTERFACE ..)
+6
View File
@@ -0,0 +1,6 @@
# header-only: interface target exposing the vendor/ root so consumers
# can include via <stb/stb_image.h>
add_library(stb INTERFACE)
add_library(vendor::stb ALIAS stb)
target_include_directories(stb INTERFACE ..)