ggml: Fix data race in ggml threadpool (#11736)

After the barrier in last iteration is executed, still the loop termination
condition will be executed. However main thread can destroy the cgraph object
and its nodes already, then another thread will access it, but the thing is already gone.
Also trouble can happen when n_nodes == 0 or abort is called, but I'm not sure if the
prior situation is possible.

Last syncronization should be done after the loop to ensure the cgraph/cplan won't be
accessed after the main thread exits from the function.

examples/run/run.cpp

@@ -346,7 +346,7 @@ class HttpClient {
         if (!output_file.empty()) {
             output_file_partial = output_file + ".partial";
             if (!out.open(output_file_partial, "ab")) {
-                printe("Failed to open file\n");
+                printe("Failed to open file for writing\n");
 
                 return 1;
             }

examples/server/README.md

@@ -220,7 +220,7 @@ services:
 The project includes a web-based user interface that enables interaction with the model through the `/chat/completions` endpoint.
 
 The web UI is developed using:
-- `vue` framework for frontend development
+- `react` framework for frontend development
 - `tailwindcss` and `daisyui` for styling
 - `vite` for build tooling
 

examples/server/webui/src/components/MarkdownDisplay.tsx

@@ -23,6 +23,7 @@ export default function MarkdownDisplay({ content }: { content: string }) {
         button: (props) => (
           <CopyCodeButton {...props} origContent={preprocessedContent} />
         ),
+        // note: do not use "pre", "p" or other basic html elements here, it will cause the node to re-render when the message is being generated (this should be a bug with react-markdown, not sure how to fix it)
       }}
     >
       {preprocessedContent}

examples/server/webui/src/components/SettingDialog.tsx

@@ -3,6 +3,7 @@ import { useAppContext } from '../utils/app.context';
 import { CONFIG_DEFAULT, CONFIG_INFO } from '../Config';
 import { isDev } from '../Config';
 import StorageUtils from '../utils/storage';
+import { isBoolean, isNumeric, isString } from '../utils/misc';
 
 type SettKey = keyof typeof CONFIG_DEFAULT;
 
@@ -52,7 +53,42 @@ export default function SettingDialog({
   };
 
   const handleSave = () => {
-    saveConfig(localConfig);
+    // copy the local config to prevent direct mutation
+    const newConfig: typeof CONFIG_DEFAULT = JSON.parse(
+      JSON.stringify(localConfig)
+    );
+    // validate the config
+    for (const key in newConfig) {
+      const value = newConfig[key as SettKey];
+      const mustBeBoolean = isBoolean(CONFIG_DEFAULT[key as SettKey]);
+      const mustBeString = isString(CONFIG_DEFAULT[key as SettKey]);
+      const mustBeNumeric = isNumeric(CONFIG_DEFAULT[key as SettKey]);
+      if (mustBeString) {
+        if (!isString(value)) {
+          alert(`Value for ${key} must be string`);
+          return;
+        }
+      } else if (mustBeNumeric) {
+        const trimedValue = value.toString().trim();
+        const numVal = Number(trimedValue);
+        if (isNaN(numVal) || !isNumeric(numVal) || trimedValue.length === 0) {
+          alert(`Value for ${key} must be numeric`);
+          return;
+        }
+        // force conversion to number
+        // @ts-expect-error this is safe
+        newConfig[key] = numVal;
+      } else if (mustBeBoolean) {
+        if (!isBoolean(value)) {
+          alert(`Value for ${key} must be boolean`);
+          return;
+        }
+      } else {
+        console.error(`Unknown default type for key ${key}`);
+      }
+    }
+    if (isDev) console.log('Saving config', newConfig);
+    saveConfig(newConfig);
     onClose();
   };
 
@@ -66,6 +102,11 @@ export default function SettingDialog({
     onClose();
   };
 
+  const onChange = (key: SettKey) => (value: string | boolean) => {
+    // note: we do not perform validation here, because we may get incomplete value as user is still typing it
+    setLocalConfig({ ...localConfig, [key]: value });
+  };
+
   return (
     <dialog className={`modal ${show ? 'modal-open' : ''}`}>
       <div className="modal-box">
@@ -79,9 +120,7 @@ export default function SettingDialog({
             configKey="apiKey"
             configDefault={CONFIG_DEFAULT}
             value={localConfig.apiKey}
-            onChange={(value) =>
-              setLocalConfig({ ...localConfig, apiKey: value })
-            }
+            onChange={onChange('apiKey')}
           />
 
           <label className="form-control mb-2">
@@ -92,12 +131,7 @@ export default function SettingDialog({
               className="textarea textarea-bordered h-24"
               placeholder={`Default: ${CONFIG_DEFAULT.systemMessage}`}
               value={localConfig.systemMessage}
-              onChange={(e) =>
-                setLocalConfig({
-                  ...localConfig,
-                  systemMessage: e.target.value,
-                })
-              }
+              onChange={(e) => onChange('systemMessage')(e.target.value)}
             />
           </label>
 
@@ -107,9 +141,7 @@ export default function SettingDialog({
               configKey={key}
               configDefault={CONFIG_DEFAULT}
               value={localConfig[key]}
-              onChange={(value) =>
-                setLocalConfig({ ...localConfig, [key]: value })
-              }
+              onChange={onChange(key)}
             />
           ))}
 
@@ -123,9 +155,7 @@ export default function SettingDialog({
                 configKey="samplers"
                 configDefault={CONFIG_DEFAULT}
                 value={localConfig.samplers}
-                onChange={(value) =>
-                  setLocalConfig({ ...localConfig, samplers: value })
-                }
+                onChange={onChange('samplers')}
               />
               {OTHER_SAMPLER_KEYS.map((key) => (
                 <SettingsModalShortInput
@@ -133,9 +163,7 @@ export default function SettingDialog({
                   configKey={key}
                   configDefault={CONFIG_DEFAULT}
                   value={localConfig[key]}
-                  onChange={(value) =>
-                    setLocalConfig({ ...localConfig, [key]: value })
-                  }
+                  onChange={onChange(key)}
                 />
               ))}
             </div>
@@ -152,9 +180,7 @@ export default function SettingDialog({
                   configKey={key}
                   configDefault={CONFIG_DEFAULT}
                   value={localConfig[key]}
-                  onChange={(value) =>
-                    setLocalConfig({ ...localConfig, [key]: value })
-                  }
+                  onChange={onChange(key)}
                 />
               ))}
             </div>
@@ -171,10 +197,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.showThoughtInProgress}
                   onChange={(e) =>
-                    setLocalConfig({
-                      ...localConfig,
-                      showThoughtInProgress: e.target.checked,
-                    })
+                    onChange('showThoughtInProgress')(e.target.checked)
                   }
                 />
                 <span className="ml-4">
@@ -187,10 +210,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.excludeThoughtOnReq}
                   onChange={(e) =>
-                    setLocalConfig({
-                      ...localConfig,
-                      excludeThoughtOnReq: e.target.checked,
-                    })
+                    onChange('excludeThoughtOnReq')(e.target.checked)
                   }
                 />
                 <span className="ml-4">
@@ -220,10 +240,7 @@ export default function SettingDialog({
                   className="checkbox"
                   checked={localConfig.showTokensPerSecond}
                   onChange={(e) =>
-                    setLocalConfig({
-                      ...localConfig,
-                      showTokensPerSecond: e.target.checked,
-                    })
+                    onChange('showTokensPerSecond')(e.target.checked)
                   }
                 />
                 <span className="ml-4">Show tokens per second</span>
@@ -245,9 +262,7 @@ export default function SettingDialog({
                   className="textarea textarea-bordered h-24"
                   placeholder='Example: { "mirostat": 1, "min_p": 0.1 }'
                   value={localConfig.custom}
-                  onChange={(e) =>
-                    setLocalConfig({ ...localConfig, custom: e.target.value })
-                  }
+                  onChange={(e) => onChange('custom')(e.target.value)}
                 />
               </label>
             </div>

ggml/src/ggml-cpu/ggml-cpu.c

@@ -13856,9 +13856,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
             tp->ec    = GGML_STATUS_ABORTED;
         }
 
-        ggml_barrier(state->threadpool);
+        if (node_n + 1 < cgraph->n_nodes) {
+            ggml_barrier(state->threadpool);
+        }
     }
 
+    ggml_barrier(state->threadpool);
+
     return 0;
 }
 

ggml/src/ggml-cuda/mma.cuh

@@ -16,7 +16,7 @@
 #include "common.cuh"
 
 
-#if CUDART_VERSION >= 11800
+#if CUDART_VERSION >= 11080
 
 static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
     int ret = 0;
@@ -50,7 +50,7 @@ static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
     return ret_low | ret_high;
 }
 
-#endif // CUDART_VERSION >= 11800
+#endif // CUDART_VERSION >= 11080
 
 
 template <typename T>