ClaudeBot/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-17 20:31:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

ui : cache hugging face hub model data per session

Browse Source 
Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash
This commit is contained in:
Aleksander Grygier
2026-09-16 19:07:37 +02:00
parent 445ec8f13d
commit 8c2a222edb
2 changed files with 103 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ui/src/lib/services/huggingface.service.ts
+94 -32
View File
@@ -88,6 +88,13 @@ export class HuggingFaceService {
Promise<{ org: string; name: string } | null>
>();
// Model details and file trees fetched this session, keyed by repo id. The
// Hub rate limits aggressively, so each repo costs at most one request per
// app load; failed fetches are not cached so the next mount can retry.
private static detailsCache = new Map<string, HfModelDetailInfo | null>();
private static detailsPending = new Map<string, Promise<HfModelDetailInfo | null>>();
/**
* Map of quant token to its average bit-depth in bits-per-weight (bpw).
*/
@@ -127,6 +134,10 @@ export class HuggingFaceService {
Q8_0: 8
};
private static treeCache = new Map<string, HfModelSibling[]>();
private static treePending = new Map<string, Promise<HfModelSibling[]>>();
/**
* Collapse split GGUF shard sets (`-00001-of-00015.gguf`, ...) to their first
* shard, summing every shard's size so the kept entry reflects the whole
@@ -389,16 +400,20 @@ export class HuggingFaceService {
* base model. Results are cached per repo.
*/
static getBaseModel(repoId: string): Promise<{ org: string; name: string } | null> {
const cached = this.baseModelCache.get(repoId);
// llama.cpp model ids carry the quant tag after a colon
// (`org/repo:Q4_K_XL`); the HF repo id is the part before it, and all
// quants of one repo share the cached lookup
const [hfRepoId] = repoId.split(':');
const cached = this.baseModelCache.get(hfRepoId);
if (cached !== undefined) return Promise.resolve(cached);
const pending = this.baseModelPending.get(repoId);
const pending = this.baseModelPending.get(hfRepoId);
if (pending) return pending;
const promise = (async () => {
const details = await this.getDetails(repoId);
const details = await this.getDetails(hfRepoId);
const base = this.getBaseModels(details)[0];
if (!base) return null;
@@ -408,11 +423,11 @@ export class HuggingFaceService {
return { name: rest.join(PATH_SEPARATOR), org };
})();
this.baseModelPending.set(repoId, promise);
this.baseModelPending.set(hfRepoId, promise);
promise
.then((result) => this.baseModelCache.set(repoId, result))
.finally(() => this.baseModelPending.delete(repoId));
.then((result) => this.baseModelCache.set(hfRepoId, result))
.finally(() => this.baseModelPending.delete(hfRepoId));
return promise;
}
@@ -476,26 +491,51 @@ export class HuggingFaceService {
return (await response.json()) as HfCatalogEntry[];
}
static async getDetails(modelId: string): Promise<HfModelDetailInfo | null> {
// Do not encode the modelId, it contains slashes for author/name.
// `full=true` includes cardData (description, base_model) and safetensors.
const url = `${HF_API_MODELS_URL}${PATH_SEPARATOR}${modelId}?${HF_FULL_DETAIL_PARAM}`;
static getDetails(modelId: string): Promise<HfModelDetailInfo | null> {
const cached = HuggingFaceService.detailsCache.get(modelId);
try {
const response = await fetch(url);
if (cached !== undefined) return Promise.resolve(cached);
if (response.status === HF_HTTP_NOT_FOUND) return null;
const pending = HuggingFaceService.detailsPending.get(modelId);
if (!response.ok) throw new Error(`Failed to fetch model details: ${response.status}`);
if (pending) return pending;
const data = (await response.json()) as HfModelDetailInfo;
const promise = (async () => {
// Do not encode the modelId, it contains slashes for author/name.
// `full=true` includes cardData (description, base_model) and safetensors.
const url = `${HF_API_MODELS_URL}${PATH_SEPARATOR}${modelId}?${HF_FULL_DETAIL_PARAM}`;
return data;
} catch (error) {
console.error(`Error fetching details for ${modelId}:`, error);
try {
const response = await fetch(url);
return null;
}
// a missing model is a definitive answer, cache it
if (response.status === HF_HTTP_NOT_FOUND) {
HuggingFaceService.detailsCache.set(modelId, null);
return null;
}
if (!response.ok) throw new Error(`Failed to fetch model details: ${response.status}`);
const data = (await response.json()) as HfModelDetailInfo;
HuggingFaceService.detailsCache.set(modelId, data);
return data;
} catch (error) {
// not cached: a rate limited or failed fetch should retry on the
// next mount instead of hiding the model for the whole session
console.error(`Error fetching details for ${modelId}:`, error);
return null;
} finally {
HuggingFaceService.detailsPending.delete(modelId);
}
})();
HuggingFaceService.detailsPending.set(modelId, promise);
return promise;
}
/**
@@ -555,19 +595,28 @@ export class HuggingFaceService {
* repos that keep quants in per-quant subdirectories (e.g. `UD-Q4_K_XL/`)
* are included; follows cursor pagination for repos over one page.
*/
static async getTree(modelId: string): Promise<HfModelSibling[]> {
const files: HfModelSibling[] = [];
const firstUrl =
`${HF_API_MODELS_URL}${PATH_SEPARATOR}${modelId}${PATH_SEPARATOR}${HF_TREE_PATH}` +
`${PATH_SEPARATOR}${HF_MAIN_BRANCH}?${HF_RECURSIVE_TREE_PARAM}`;
static getTree(modelId: string): Promise<HfModelSibling[]> {
const cached = HuggingFaceService.treeCache.get(modelId);
let url: string | null = firstUrl;
if (cached) return Promise.resolve(cached);
const pending = HuggingFaceService.treePending.get(modelId);
if (pending) return pending;
const promise = (async () => {
const files: HfModelSibling[] = [];
const firstUrl =
`${HF_API_MODELS_URL}${PATH_SEPARATOR}${modelId}${PATH_SEPARATOR}${HF_TREE_PATH}` +
`${PATH_SEPARATOR}${HF_MAIN_BRANCH}?${HF_RECURSIVE_TREE_PARAM}`;
let url: string | null = firstUrl;
try {
for (let page = 0; url && page < HF_TREE_MAX_PAGES; page++) {
const response: Response = await fetch(url);
if (!response.ok) return files;
if (!response.ok)
throw new Error(`Failed to fetch tree for ${modelId}: ${response.status}`);
const data = (await response.json()) as HfModelSibling[];
@@ -575,11 +624,24 @@ export class HuggingFaceService {
url = HuggingFaceService.parseNextPageUrl(response.headers.get(HF_LINK_HEADER));
}
} catch {
// Return whatever was fetched before the failure.
}
return files;
HuggingFaceService.treeCache.set(modelId, files);
return files;
})()
.catch((error: unknown) => {
// not cached: a rate limited or failed fetch should retry on the
// next mount; an empty tree makes the store fall back to the
// catalog's advertised sizes
console.error(`Error fetching tree for ${modelId}:`, error);
return [] as HfModelSibling[];
})
.finally(() => HuggingFaceService.treePending.delete(modelId));
HuggingFaceService.treePending.set(modelId, promise);
return promise;
}
/**
tools/ui/src/lib/stores/models-discover/index.svelte.ts
+9 -2
View File
@@ -80,6 +80,7 @@ class ModelsDiscoverStore {
this.catalog = catalog;
const builds = this.catalogBuilds();
const failedBuilds: HfCatalogBuild[] = [];
this.defaultModels = [];
@@ -98,7 +99,11 @@ class ModelsDiscoverStore {
);
for (const { build, info, tree } of batch) {
if (!info) continue;
if (!info) {
failedBuilds.push(build);
continue;
}
this.catalogSizeRanges.set(build.repo, this.sizeRangeFor(build, tree));
@@ -112,7 +117,9 @@ class ModelsDiscoverStore {
}
}
this.fetched = true;
// stay incomplete when some repos failed, so the next mount retries
// them; succeeded repos are served from the service cache for free
this.fetched = failedBuilds.length === 0;
} catch (err) {
this.error = err instanceof Error ? err.message : 'Failed to fetch models';
} finally {