Skip to content

Commit 59a3d0c

Browse files
ggerganovggerganov
authored
ggml : sync (ggml-alloc, GPU, eps, etc.) (#1220)
* ggml : sync (ggml-alloc, GPU, eps, etc.) * ggml : fix build * wasm : fix build
1 parent 6780c98 commit 59a3d0c

17 files changed

+11732
-4573
lines changed

‎bindings/javascript/libwhisper.worker.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bindings/javascript/whisper.js

+5-7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎examples/common.cpp

+73-21
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#define _USE_MATH_DEFINES // for M_PI
2+
13
#include "common.h"
24

35
// third-party utilities
@@ -13,53 +15,59 @@
1315
#include <codecvt>
1416
#include <sstream>
1517

16-
#ifndef M_PI
17-
#define M_PI 3.14159265358979323846
18-
#endif
19-
2018
#if defined(_MSC_VER)
2119
#pragma warning(disable: 4244 4267) // possible loss of data
2220
#endif
2321

22+
// Function to check if the next argument exists
23+
std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
24+
if (i + 1 < argc && argv[i + 1][0] != '-') {
25+
return argv[++i];
26+
} else {
27+
fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
28+
gpt_print_usage(argc, argv, params);
29+
exit(0);
30+
}
31+
}
32+
2433
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
2534
for (int i = 1; i < argc; i++) {
2635
std::string arg = argv[i];
2736

2837
if (arg == "-s" || arg == "--seed") {
29-
params.seed = std::stoi(argv[++i]);
38+
params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
3039
} else if (arg == "-t" || arg == "--threads") {
31-
params.n_threads = std::stoi(argv[++i]);
40+
params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
41+
} else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
42+
params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
3243
} else if (arg == "-p" || arg == "--prompt") {
33-
params.prompt = argv[++i];
44+
params.prompt = get_next_arg(i, argc, argv, arg, params);
3445
} else if (arg == "-n" || arg == "--n_predict") {
35-
params.n_predict = std::stoi(argv[++i]);
46+
params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
3647
} else if (arg == "--top_k") {
37-
params.top_k = std::max(1, std::stoi(argv[++i]));
48+
params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
3849
} else if (arg == "--top_p") {
39-
params.top_p = std::stof(argv[++i]);
50+
params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
4051
} else if (arg == "--temp") {
41-
params.temp = std::stof(argv[++i]);
52+
params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
4253
} else if (arg == "--repeat-last-n") {
43-
params.repeat_last_n = std::stof(argv[++i]);
54+
params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
4455
} else if (arg == "--repeat-penalty") {
45-
params.repeat_penalty = std::stof(argv[++i]);
56+
params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
4657
} else if (arg == "-b" || arg == "--batch_size") {
47-
params.n_batch = std::stoi(argv[++i]);
58+
params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
4859
} else if (arg == "-m" || arg == "--model") {
49-
params.model = argv[++i];
60+
params.model = get_next_arg(i, argc, argv, arg, params);
5061
} else if (arg == "-i" || arg == "--interactive") {
5162
params.interactive = true;
5263
} else if (arg == "-ip" || arg == "--interactive-port") {
5364
params.interactive = true;
54-
params.interactive_port = std::stoi(argv[++i]);
65+
params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
5566
} else if (arg == "-h" || arg == "--help") {
5667
gpt_print_usage(argc, argv, params);
5768
exit(0);
5869
} else if (arg == "-f" || arg == "--file") {
59-
if (++i > argc) {
60-
fprintf(stderr, "Invalid file param");
61-
break;
62-
}
70+
get_next_arg(i, argc, argv, arg, params);
6371
std::ifstream file(argv[i]);
6472
if (!file) {
6573
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
@@ -70,7 +78,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
7078
params.prompt.pop_back();
7179
}
7280
} else if (arg == "-tt" || arg == "--token_test") {
73-
params.token_test = argv[++i];
81+
params.token_test = get_next_arg(i, argc, argv, arg, params);
7482
}
7583
else {
7684
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
@@ -89,6 +97,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
8997
fprintf(stderr, " -h, --help show this help message and exit\n");
9098
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
9199
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
100+
fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
92101
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
93102
fprintf(stderr, " prompt to start generation with (default: random)\n");
94103
fprintf(stderr, " -f FNAME, --file FNAME\n");
@@ -755,3 +764,46 @@ float similarity(const std::string & s0, const std::string & s1) {
755764

756765
return 1.0f - (dist / std::max(s0.size(), s1.size()));
757766
}
767+
768+
bool sam_params_parse(int argc, char ** argv, sam_params & params) {
769+
for (int i = 1; i < argc; i++) {
770+
std::string arg = argv[i];
771+
772+
if (arg == "-s" || arg == "--seed") {
773+
params.seed = std::stoi(argv[++i]);
774+
} else if (arg == "-t" || arg == "--threads") {
775+
params.n_threads = std::stoi(argv[++i]);
776+
} else if (arg == "-m" || arg == "--model") {
777+
params.model = argv[++i];
778+
} else if (arg == "-i" || arg == "--inp") {
779+
params.fname_inp = argv[++i];
780+
} else if (arg == "-o" || arg == "--out") {
781+
params.fname_out = argv[++i];
782+
} else if (arg == "-h" || arg == "--help") {
783+
sam_print_usage(argc, argv, params);
784+
exit(0);
785+
} else {
786+
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
787+
sam_print_usage(argc, argv, params);
788+
exit(0);
789+
}
790+
}
791+
792+
return true;
793+
}
794+
795+
void sam_print_usage(int argc, char ** argv, const sam_params & params) {
796+
fprintf(stderr, "usage: %s [options]\n", argv[0]);
797+
fprintf(stderr, "\n");
798+
fprintf(stderr, "options:\n");
799+
fprintf(stderr, " -h, --help show this help message and exit\n");
800+
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
801+
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
802+
fprintf(stderr, " -m FNAME, --model FNAME\n");
803+
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
804+
fprintf(stderr, " -i FNAME, --inp FNAME\n");
805+
fprintf(stderr, " input file (default: %s)\n", params.fname_inp.c_str());
806+
fprintf(stderr, " -o FNAME, --out FNAME\n");
807+
fprintf(stderr, " output file (default: %s)\n", params.fname_out.c_str());
808+
fprintf(stderr, "\n");
809+
}

‎examples/common.h

+20-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
#define COMMON_SAMPLE_RATE 16000
1212

1313
//
14-
// CLI argument parsing
14+
// GPT CLI argument parsing
1515
//
1616

1717
struct gpt_params {
@@ -33,6 +33,8 @@ struct gpt_params {
3333

3434
bool interactive = false;
3535
int32_t interactive_port = -1;
36+
37+
int32_t n_gpu_layers = 0;
3638
};
3739

3840
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
@@ -155,3 +157,20 @@ bool vad_simple(
155157

156158
// compute similarity between two strings using Levenshtein distance
157159
float similarity(const std::string & s0, const std::string & s1);
160+
161+
//
162+
// SAM argument parsing
163+
//
164+
165+
struct sam_params {
166+
int32_t seed = -1; // RNG seed
167+
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
168+
169+
std::string model = "models/sam-vit-b/ggml-model-f16.bin"; // model path
170+
std::string fname_inp = "img.jpg";
171+
std::string fname_out = "img.out";
172+
};
173+
174+
bool sam_params_parse(int argc, char ** argv, sam_params & params);
175+
176+
void sam_print_usage(int argc, char ** argv, const sam_params & params);

‎examples/talk.wasm/gpt-2.cpp

+8-9
Original file line numberDiff line numberDiff line change
@@ -191,9 +191,9 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab &
191191
// create the ggml context
192192
{
193193
struct ggml_init_params params = {
194-
.mem_size = ctx_size,
195-
.mem_buffer = NULL,
196-
.no_alloc = false,
194+
/*.mem_size =*/ ctx_size,
195+
/*.mem_buffer =*/ NULL,
196+
/*.no_alloc =*/ false,
197197
};
198198

199199
model.ctx = ggml_init(params);
@@ -420,7 +420,6 @@ bool gpt2_eval(
420420

421421
struct ggml_context * ctx0 = ggml_init(params);
422422
struct ggml_cgraph gf = {};
423-
gf.n_threads = n_threads;
424423

425424
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
426425
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -442,7 +441,7 @@ bool gpt2_eval(
442441
// norm
443442
{
444443
// [ 768, N]
445-
cur = ggml_norm(ctx0, inpL);
444+
cur = ggml_norm(ctx0, inpL, 1e-5f);
446445

447446
// cur = ln_1_g*cur + ln_1_b
448447
// [ 768, N]
@@ -589,7 +588,7 @@ bool gpt2_eval(
589588
{
590589
// norm
591590
{
592-
cur = ggml_norm(ctx0, inpFF);
591+
cur = ggml_norm(ctx0, inpFF, 1e-5f);
593592

594593
// cur = ln_2_g*cur + ln_2_b
595594
// [ 768, N]
@@ -644,7 +643,7 @@ bool gpt2_eval(
644643
// norm
645644
{
646645
// [ 768, N]
647-
inpL = ggml_norm(ctx0, inpL);
646+
inpL = ggml_norm(ctx0, inpL, 1e-5f);
648647

649648
// inpL = ln_f_g*inpL + ln_f_b
650649
// [ 768, N]
@@ -664,8 +663,8 @@ bool gpt2_eval(
664663
//inpL = ggml_soft_max(ctx0, inpL);
665664

666665
// run the computation
667-
ggml_build_forward_expand(&gf, inpL);
668-
ggml_graph_compute (ctx0, &gf);
666+
ggml_build_forward_expand (&gf, inpL);
667+
ggml_graph_compute_with_ctx(ctx0, &gf, n_threads);
669668

670669
//if (n_past%100 == 0) {
671670
// ggml_graph_print (&gf);

‎examples/talk/gpt-2.cpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,7 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab &
379379
// - embd_inp: the embeddings of the tokens in the context
380380
// - embd_w: the predicted logits for the next token
381381
//
382+
// TODO: sync latest version from ggml repo
382383
bool gpt2_eval(
383384
const gpt2_model & model,
384385
const int n_threads,
@@ -420,7 +421,6 @@ bool gpt2_eval(
420421

421422
struct ggml_context * ctx0 = ggml_init(params);
422423
struct ggml_cgraph gf = {};
423-
gf.n_threads = n_threads;
424424

425425
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
426426
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -442,7 +442,7 @@ bool gpt2_eval(
442442
// norm
443443
{
444444
// [ 768, N]
445-
cur = ggml_norm(ctx0, inpL);
445+
cur = ggml_norm(ctx0, inpL, 1e-5f);
446446

447447
// cur = ln_1_g*cur + ln_1_b
448448
// [ 768, N]
@@ -589,7 +589,7 @@ bool gpt2_eval(
589589
{
590590
// norm
591591
{
592-
cur = ggml_norm(ctx0, inpFF);
592+
cur = ggml_norm(ctx0, inpFF, 1e-5f);
593593

594594
// cur = ln_2_g*cur + ln_2_b
595595
// [ 768, N]
@@ -644,7 +644,7 @@ bool gpt2_eval(
644644
// norm
645645
{
646646
// [ 768, N]
647-
inpL = ggml_norm(ctx0, inpL);
647+
inpL = ggml_norm(ctx0, inpL, 1e-5f);
648648

649649
// inpL = ln_f_g*inpL + ln_f_b
650650
// [ 768, N]
@@ -664,8 +664,8 @@ bool gpt2_eval(
664664
//inpL = ggml_soft_max(ctx0, inpL);
665665

666666
// run the computation
667-
ggml_build_forward_expand(&gf, inpL);
668-
ggml_graph_compute (ctx0, &gf);
667+
ggml_build_forward_expand (&gf, inpL);
668+
ggml_graph_compute_with_ctx(ctx0, &gf, n_threads);
669669

670670
//if (n_past%100 == 0) {
671671
// ggml_graph_print (&gf);

0 commit comments

Comments
 (0)