mii/llama.cpp
1
0
Fork 0
mirror of https://github.com/mii443/llama.cpp.git synced 2025-08-22 15:05:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

support API call

Browse Source
This commit is contained in:
mii
2023-03-16 12:50:30 +00:00
parent ac15de7895
commit 8c91c48552
2 changed files with 25170 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24596
json.hpp Normal file
View File

File diff suppressed because it is too large Load Diff

468
main.cpp
View File

@ -11,6 +11,9 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <crow.h>
#include <json.hpp>
#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
#include <signal.h> #include <signal.h>
#include <unistd.h> #include <unistd.h>
@ -86,8 +89,10 @@ struct llama_model {
}; };
// load the model's weights from a file // load the model's weights from a file
bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab & vocab, int n_ctx) { bool llama_model_load(const std::string &fname, llama_model &model,
fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str()); gpt_vocab &vocab, int n_ctx) {
fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", __func__,
fname.c_str());
std::vector<char> f_buf(1024 * 1024); std::vector<char> f_buf(1024 * 1024);
@ -103,7 +108,8 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
uint32_t magic; uint32_t magic;
fin.read((char *)&magic, sizeof(magic)); fin.read((char *)&magic, sizeof(magic));
if (magic != 0x67676d6c) { if (magic != 0x67676d6c) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__,
fname.c_str());
return false; return false;
} }
} }
@ -126,7 +132,9 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
hparams.n_ctx = n_ctx; hparams.n_ctx = n_ctx;
n_ff = ((2*(4*hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult; n_ff =
((2 * (4 * hparams.n_embd) / 3 + hparams.n_mult - 1) / hparams.n_mult) *
hparams.n_mult;
n_parts = LLAMA_N_PARTS.at(hparams.n_embd); n_parts = LLAMA_N_PARTS.at(hparams.n_embd);
fprintf(stderr, "%s: n_vocab = %d\n", __func__, hparams.n_vocab); fprintf(stderr, "%s: n_vocab = %d\n", __func__, hparams.n_vocab);
@ -163,21 +171,29 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
vocab.id_to_token[i] = word; vocab.id_to_token[i] = word;
// if (i < 30000) { // if (i < 30000) {
// fprintf(stderr, "%s: vocab[%d] = '%s'\n", __func__, i, word.c_str()); // fprintf(stderr, "%s: vocab[%d] = '%s'\n", __func__, i,
// word.c_str());
//} //}
} }
} }
// for the big tensors, we have the option to store the data in 16-bit floats or quantized // for the big tensors, we have the option to store the data in 16-bit floats
// in order to save memory and also to speed up the computation // or quantized in order to save memory and also to speed up the computation
ggml_type wtype = GGML_TYPE_COUNT; ggml_type wtype = GGML_TYPE_COUNT;
switch (model.hparams.f16) { switch (model.hparams.f16) {
case 0: wtype = GGML_TYPE_F32; break; case 0:
case 1: wtype = GGML_TYPE_F16; break; wtype = GGML_TYPE_F32;
case 2: wtype = GGML_TYPE_Q4_0; break; break;
case 3: wtype = GGML_TYPE_Q4_1; break; case 1:
default: wtype = GGML_TYPE_F16;
{ break;
case 2:
wtype = GGML_TYPE_Q4_0;
break;
case 3:
wtype = GGML_TYPE_Q4_1;
break;
default: {
fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n", fprintf(stderr, "%s: invalid model file '%s' (bad f16 value %d)\n",
__func__, fname.c_str(), model.hparams.f16); __func__, fname.c_str(), model.hparams.f16);
return false; return false;
@ -204,7 +220,8 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
ctx_size += n_embd * n_vocab * ggml_type_sizef(wtype); // output ctx_size += n_embd * n_vocab * ggml_type_sizef(wtype); // output
ctx_size += n_layer*(n_embd*ggml_type_sizef(GGML_TYPE_F32)); // attention_norm ctx_size +=
n_layer * (n_embd * ggml_type_sizef(GGML_TYPE_F32)); // attention_norm
ctx_size += n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // wq ctx_size += n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // wq
ctx_size += n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // wk ctx_size += n_layer * (n_embd * n_embd * ggml_type_sizef(wtype)); // wk
@ -217,12 +234,15 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
ctx_size += n_layer * (n_ff * n_embd * ggml_type_sizef(wtype)); // w2 ctx_size += n_layer * (n_ff * n_embd * ggml_type_sizef(wtype)); // w2
ctx_size += n_layer * (n_ff * n_embd * ggml_type_sizef(wtype)); // w3 ctx_size += n_layer * (n_ff * n_embd * ggml_type_sizef(wtype)); // w3
ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_k ctx_size +=
ctx_size += n_ctx*n_layer*n_embd*ggml_type_sizef(GGML_TYPE_F32); // memory_v n_ctx * n_layer * n_embd * ggml_type_sizef(GGML_TYPE_F32); // memory_k
ctx_size +=
n_ctx * n_layer * n_embd * ggml_type_sizef(GGML_TYPE_F32); // memory_v
ctx_size += (5 + 10 * n_layer) * 256; // object overhead ctx_size += (5 + 10 * n_layer) * 256; // object overhead
fprintf(stderr, "%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); fprintf(stderr, "%s: ggml ctx size = %6.2f MB\n", __func__,
ctx_size / (1024.0 * 1024.0));
} }
// create the ggml context // create the ggml context
@ -278,18 +298,27 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff); layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
// map by name // map by name
model.tensors["layers." + std::to_string(i) + ".attention_norm.weight"] = layer.attention_norm; model.tensors["layers." + std::to_string(i) + ".attention_norm.weight"] =
layer.attention_norm;
model.tensors["layers." + std::to_string(i) + ".attention.wq.weight"] = layer.wq; model.tensors["layers." + std::to_string(i) + ".attention.wq.weight"] =
model.tensors["layers." + std::to_string(i) + ".attention.wk.weight"] = layer.wk; layer.wq;
model.tensors["layers." + std::to_string(i) + ".attention.wv.weight"] = layer.wv; model.tensors["layers." + std::to_string(i) + ".attention.wk.weight"] =
model.tensors["layers." + std::to_string(i) + ".attention.wo.weight"] = layer.wo; layer.wk;
model.tensors["layers." + std::to_string(i) + ".attention.wv.weight"] =
layer.wv;
model.tensors["layers." + std::to_string(i) + ".attention.wo.weight"] =
layer.wo;
model.tensors["layers." + std::to_string(i) + ".ffn_norm.weight"] = layer.ffn_norm; model.tensors["layers." + std::to_string(i) + ".ffn_norm.weight"] =
layer.ffn_norm;
model.tensors["layers." + std::to_string(i) + ".feed_forward.w1.weight"] = layer.w1; model.tensors["layers." + std::to_string(i) + ".feed_forward.w1.weight"] =
model.tensors["layers." + std::to_string(i) + ".feed_forward.w2.weight"] = layer.w2; layer.w1;
model.tensors["layers." + std::to_string(i) + ".feed_forward.w3.weight"] = layer.w3; model.tensors["layers." + std::to_string(i) + ".feed_forward.w2.weight"] =
layer.w2;
model.tensors["layers." + std::to_string(i) + ".feed_forward.w3.weight"] =
layer.w3;
} }
} }
@ -307,9 +336,11 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); const size_t memory_size =
ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
fprintf(stderr, "%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); fprintf(stderr, "%s: memory_size = %8.2f MB, n_mem = %d\n", __func__,
memory_size / 1024.0 / 1024.0, n_mem);
} }
const size_t file_offset = fin.tellg(); const size_t file_offset = fin.tellg();
@ -327,7 +358,8 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
fname_part += "." + std::to_string(i); fname_part += "." + std::to_string(i);
} }
fprintf(stderr, "%s: loading model part %d/%d from '%s'\n", __func__, i+1, n_parts, fname_part.c_str()); fprintf(stderr, "%s: loading model part %d/%d from '%s'\n", __func__, i + 1,
n_parts, fname_part.c_str());
fin = std::ifstream(fname_part, std::ios::binary); fin = std::ifstream(fname_part, std::ios::binary);
fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size()); fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());
@ -364,7 +396,8 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
fin.read(&name[0], length); fin.read(&name[0], length);
if (model.tensors.find(name.data()) == model.tensors.end()) { if (model.tensors.find(name.data()) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data()); fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__,
name.data());
return false; return false;
} }
@ -404,61 +437,92 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
if (n_dims == 1) { if (n_dims == 1) {
if (ggml_nelements(tensor) != nelements) { if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data()); fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n",
__func__, name.data());
return false; return false;
} }
} else { } else {
if (ggml_nelements(tensor) / n_parts != nelements) { if (ggml_nelements(tensor) / n_parts != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data()); fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n",
__func__, name.data());
return false; return false;
} }
} }
if (n_dims == 1) { if (n_dims == 1) {
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", fprintf(stderr,
__func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]); "%s: tensor '%s' has wrong shape in model file: got [%d, "
"%d], expected [%d, %d]\n",
__func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0],
ne[1]);
return false; return false;
} }
} else { } else {
if (split_type == 0) { if (split_type == 0) {
if (tensor->ne[0] / n_parts != ne[0] || tensor->ne[1] != ne[1]) { if (tensor->ne[0] / n_parts != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", fprintf(stderr,
__func__, name.data(), tensor->ne[0]/n_parts, tensor->ne[1], ne[0], ne[1]); "%s: tensor '%s' has wrong shape in model file: got [%d, "
"%d], expected [%d, %d]\n",
__func__, name.data(), tensor->ne[0] / n_parts,
tensor->ne[1], ne[0], ne[1]);
return false; return false;
} }
} else { } else {
if (tensor->ne[0] != ne[0] || tensor->ne[1] / n_parts != ne[1]) { if (tensor->ne[0] != ne[0] || tensor->ne[1] / n_parts != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", fprintf(stderr,
__func__, name.data(), tensor->ne[0], tensor->ne[1]/n_parts, ne[0], ne[1]); "%s: tensor '%s' has wrong shape in model file: got [%d, "
"%d], expected [%d, %d]\n",
__func__, name.data(), tensor->ne[0],
tensor->ne[1] / n_parts, ne[0], ne[1]);
return false; return false;
} }
} }
} }
if (0) { if (0) {
static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", }; static const char *ftype_str[] = {
fprintf(stderr, "%24s - [%5d, %5d], type = %6s, split = %d\n", name.data(), ne[0], ne[1], ftype_str[ftype], split_type); "f32",
"f16",
"q4_0",
"q4_1",
};
fprintf(stderr, "%24s - [%5d, %5d], type = %6s, split = %d\n",
name.data(), ne[0], ne[1], ftype_str[ftype], split_type);
} }
size_t bpe = 0; size_t bpe = 0;
switch (ftype) { switch (ftype) {
case 0: bpe = ggml_type_size(GGML_TYPE_F32); break; case 0:
case 1: bpe = ggml_type_size(GGML_TYPE_F16); break; bpe = ggml_type_size(GGML_TYPE_F32);
case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break; break;
case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break; case 1:
default: bpe = ggml_type_size(GGML_TYPE_F16);
{ break;
fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype); case 2:
bpe = ggml_type_size(GGML_TYPE_Q4_0);
assert(ne[0] % 64 == 0);
break;
case 3:
bpe = ggml_type_size(GGML_TYPE_Q4_1);
assert(ne[0] % 64 == 0);
break;
default: {
fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__,
ftype);
return false; return false;
} }
}; };
if (n_dims == 1 || n_parts == 1) { if (n_dims == 1 || n_parts == 1) {
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { if ((nelements * bpe) / ggml_blck_size(tensor->type) !=
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", ggml_nbytes(tensor)) {
__func__, name.data(), ggml_nbytes(tensor), nelements*bpe); fprintf(stderr,
"%s: tensor '%s' has wrong size in model file: got %zu, "
"expected %zu\n",
__func__, name.data(), ggml_nbytes(tensor),
nelements * bpe);
return false; return false;
} }
@ -470,27 +534,38 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
total_size += ggml_nbytes(tensor); total_size += ggml_nbytes(tensor);
} else { } else {
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)/n_parts) { if ((nelements * bpe) / ggml_blck_size(tensor->type) !=
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", ggml_nbytes(tensor) / n_parts) {
__func__, name.data(), ggml_nbytes(tensor)/n_parts, nelements*bpe); fprintf(stderr,
"%s: tensor '%s' has wrong size in model file: got %zu, "
"expected %zu\n",
__func__, name.data(), ggml_nbytes(tensor) / n_parts,
nelements * bpe);
return false; return false;
} }
if (split_type == 0) { if (split_type == 0) {
const int np0 = ne[0]; const int np0 = ne[0];
const size_t row_size = (tensor->ne[0]/ggml_blck_size(tensor->type))*ggml_type_size(tensor->type); const size_t row_size =
(tensor->ne[0] / ggml_blck_size(tensor->type)) *
ggml_type_size(tensor->type);
assert(row_size == tensor->nb[1]); assert(row_size == tensor->nb[1]);
for (int i1 = 0; i1 < ne[1]; ++i1) { for (int i1 = 0; i1 < ne[1]; ++i1) {
const size_t offset_row = i1 * row_size; const size_t offset_row = i1 * row_size;
const size_t offset = offset_row + ((part_id*np0)/ggml_blck_size(tensor->type))*ggml_type_size(tensor->type); const size_t offset =
offset_row +
((part_id * np0) / ggml_blck_size(tensor->type)) *
ggml_type_size(tensor->type);
fin.read(reinterpret_cast<char *>(tensor->data) + offset, row_size / n_parts); fin.read(reinterpret_cast<char *>(tensor->data) + offset, row_size / n_parts);
} }
} else { } else {
const int np1 = ne[1]; const int np1 = ne[1];
const size_t row_size = (tensor->ne[0]/ggml_blck_size(tensor->type))*ggml_type_size(tensor->type); const size_t row_size =
(tensor->ne[0] / ggml_blck_size(tensor->type)) *
ggml_type_size(tensor->type);
for (int i1 = 0; i1 < ne[1]; ++i1) { for (int i1 = 0; i1 < ne[1]; ++i1) {
const size_t offset_row = (i1 + part_id * np1) * row_size; const size_t offset_row = (i1 + part_id * np1) * row_size;
@ -501,7 +576,9 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
total_size += ggml_nbytes(tensor) / n_parts; total_size += ggml_nbytes(tensor) / n_parts;
} }
//fprintf(stderr, "%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0); // fprintf(stderr, "%42s - [%5d, %5d], type = %6s, %6.2f MB\n",
// name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16",
// ggml_nbytes(tensor)/1024.0/1024.0);
if (++n_tensors % 8 == 0) { if (++n_tensors % 8 == 0) {
fprintf(stderr, "."); fprintf(stderr, ".");
fflush(stderr); fflush(stderr);
@ -510,7 +587,8 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
fprintf(stderr, " done\n"); fprintf(stderr, " done\n");
fprintf(stderr, "%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors); fprintf(stderr, "%s: model size = %8.2f MB / num tensors = %d\n",
__func__, total_size / 1024.0 / 1024.0, n_tensors);
} }
fin.close(); fin.close();
@ -529,13 +607,9 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
// //
// The GPT-J model requires about 16MB of memory per input token. // The GPT-J model requires about 16MB of memory per input token.
// //
bool llama_eval( bool llama_eval(const llama_model &model, const int n_threads, const int n_past,
const llama_model & model,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> &embd_inp, const std::vector<gpt_vocab::id> &embd_inp,
std::vector<float> & embd_w, std::vector<float> &embd_w, size_t &mem_per_token) {
size_t & mem_per_token) {
const int N = embd_inp.size(); const int N = embd_inp.size();
const auto &hparams = model.hparams; const auto &hparams = model.hparams;
@ -549,14 +623,18 @@ bool llama_eval(
const int d_key = n_embd / n_head; const int d_key = n_embd / n_head;
// TODO: check if this size scales with n_ctx linearly and remove constant. somehow I feel it wasn't the case // TODO: check if this size scales with n_ctx linearly and remove constant.
// somehow I feel it wasn't the case
// static size_t buf_size = hparams.n_ctx*1024*1024; // static size_t buf_size = hparams.n_ctx*1024*1024;
static size_t buf_size = 512u * 1024 * 1024; static size_t buf_size = 512u * 1024 * 1024;
static void *buf = malloc(buf_size); static void *buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token * N > buf_size) { if (mem_per_token > 0 && mem_per_token * N > buf_size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead const size_t buf_size_new =
//fprintf(stderr, "\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); 1.1 *
(mem_per_token * N); // add 10% to account for ggml object overhead
// fprintf(stderr, "\n%s: reallocating buffer from %zu to %zu bytes\n",
// __func__, buf_size, buf_size_new);
// reallocate // reallocate
buf_size = buf_size_new; buf_size = buf_size_new;
@ -591,9 +669,8 @@ bool llama_eval(
cur = ggml_rms_norm(ctx0, inpL); cur = ggml_rms_norm(ctx0, inpL);
// cur = attention_norm*cur // cur = attention_norm*cur
cur = ggml_mul(ctx0, cur = ggml_mul(
ggml_repeat(ctx0, model.layers[il].attention_norm, cur), ctx0, ggml_repeat(ctx0, model.layers[il].attention_norm, cur), cur);
cur);
} }
// self-attention // self-attention
@ -604,42 +681,31 @@ bool llama_eval(
// store key and value to memory // store key and value to memory
if (N >= 1) { if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); struct ggml_tensor *k =
struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past)); ggml_view_1d(ctx0, model.memory_k, N * n_embd,
(ggml_element_size(model.memory_k) * n_embd) *
(il * n_ctx + n_past));
struct ggml_tensor *v =
ggml_view_1d(ctx0, model.memory_v, N * n_embd,
(ggml_element_size(model.memory_v) * n_embd) *
(il * n_ctx + n_past));
ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k)); ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v)); ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
} }
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1,
struct ggml_tensor * Q = // 3)
ggml_permute(ctx0, struct ggml_tensor *Q = ggml_permute(ctx0, ggml_rope(ctx0, ggml_cpy(ctx0, Qcur, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd / n_head, n_head, N)), n_past, n_rot, 0), 0, 2, 1, 3);
ggml_rope(ctx0,
ggml_cpy(ctx0,
Qcur,
ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
n_past, n_rot, 0),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
struct ggml_tensor * K = struct ggml_tensor *K = ggml_permute(ctx0, ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_view_1d(ctx0, model.memory_k, (n_past + N) * n_embd, il * n_ctx * ggml_element_size(model.memory_k) * n_embd), n_embd / n_head, n_head, n_past + N), n_past, n_rot, 1), 0, 2, 1, 3);
ggml_permute(ctx0,
ggml_rope(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
n_past, n_rot, 1),
0, 2, 1, 3);
// K * Q // K * Q
struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q); struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head) // KQ_scaled = KQ / sqrt(n_embd/n_head)
struct ggml_tensor * KQ_scaled = struct ggml_tensor *KQ_scaled = ggml_scale(ctx0, KQ, ggml_new_f32(ctx0, 1.0f / sqrt(float(n_embd) / n_head)));
ggml_scale(ctx0,
KQ,
ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
);
// KQ_masked = mask_past(KQ_scaled) // KQ_masked = mask_past(KQ_scaled)
struct ggml_tensor *KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past); struct ggml_tensor *KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@ -647,10 +713,12 @@ bool llama_eval(
// KQ = soft_max(KQ_masked) // KQ = soft_max(KQ_masked)
struct ggml_tensor *KQ_soft_max = ggml_soft_max(ctx0, KQ_masked); struct ggml_tensor *KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0,
struct ggml_tensor * V_trans = // 3).contiguous()
ggml_permute(ctx0, struct ggml_tensor *V_trans = ggml_permute(
ggml_reshape_3d(ctx0, ctx0,
ggml_reshape_3d(
ctx0,
ggml_view_1d(ctx0, model.memory_v, (n_past + N) * n_embd, il * n_ctx * ggml_element_size(model.memory_v) * n_embd), ggml_view_1d(ctx0, model.memory_v, (n_past + N) * n_embd, il * n_ctx * ggml_element_size(model.memory_v) * n_embd),
n_embd / n_head, n_head, n_past + N), n_embd / n_head, n_head, n_past + N),
1, 2, 0, 3); 1, 2, 0, 3);
@ -662,14 +730,10 @@ bool llama_eval(
struct ggml_tensor *KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); struct ggml_tensor *KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N) // cur = KQV_merged.contiguous().view(n_embd, N)
cur = ggml_cpy(ctx0, cur = ggml_cpy(ctx0, KQV_merged, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
KQV_merged,
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
// projection (no bias) // projection (no bias)
cur = ggml_mul_mat(ctx0, cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur);
model.layers[il].wo,
cur);
} }
struct ggml_tensor *inpFF = ggml_add(ctx0, cur, inpSA); struct ggml_tensor *inpFF = ggml_add(ctx0, cur, inpSA);
@ -681,28 +745,19 @@ bool llama_eval(
cur = ggml_rms_norm(ctx0, inpFF); cur = ggml_rms_norm(ctx0, inpFF);
// cur = ffn_norm*cur // cur = ffn_norm*cur
cur = ggml_mul(ctx0, cur = ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ffn_norm, cur), cur);
ggml_repeat(ctx0, model.layers[il].ffn_norm, cur),
cur);
} }
struct ggml_tensor * tmp = ggml_mul_mat(ctx0, struct ggml_tensor *tmp = ggml_mul_mat(ctx0, model.layers[il].w3, cur);
model.layers[il].w3,
cur);
cur = ggml_mul_mat(ctx0, model.layers[il].w1, cur);
cur = ggml_mul_mat(ctx0,
model.layers[il].w1,
cur);
// SILU activation // SILU activation
cur = ggml_silu(ctx0, cur); cur = ggml_silu(ctx0, cur);
cur = ggml_mul(ctx0, cur, tmp); cur = ggml_mul(ctx0, cur, tmp);
cur = ggml_mul_mat(ctx0, cur = ggml_mul_mat(ctx0, model.layers[il].w2, cur);
model.layers[il].w2,
cur);
} }
cur = ggml_add(ctx0, cur, inpFF); cur = ggml_add(ctx0, cur, inpFF);
@ -716,15 +771,11 @@ bool llama_eval(
inpL = ggml_rms_norm(ctx0, inpL); inpL = ggml_rms_norm(ctx0, inpL);
// inpL = norm*inpL // inpL = norm*inpL
inpL = ggml_mul(ctx0, inpL = ggml_mul(ctx0, ggml_repeat(ctx0, model.norm, inpL), inpL);
ggml_repeat(ctx0, model.norm, inpL),
inpL);
} }
// lm_head // lm_head
{ { inpL = ggml_mul_mat(ctx0, model.output, inpL); }
inpL = ggml_mul_mat(ctx0, model.output, inpL);
}
// logits -> probs // logits -> probs
// inpL = ggml_soft_max(ctx0, inpL); // inpL = ggml_soft_max(ctx0, inpL);
@ -743,7 +794,8 @@ bool llama_eval(
// return result for just the last token // return result for just the last token
embd_w.resize(n_vocab); embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab); memcpy(embd_w.data(), (float *)ggml_get_data(inpL) + (n_vocab * (N - 1)),
sizeof(float) * n_vocab);
if (mem_per_token == 0) { if (mem_per_token == 0) {
mem_per_token = ggml_used_mem(ctx0) / N; mem_per_token = ggml_used_mem(ctx0) / N;
@ -757,7 +809,8 @@ bool llama_eval(
static bool is_interacting = false; static bool is_interacting = false;
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(_WIN32)
void sigint_handler(int signo) { void sigint_handler(int signo) {
printf(ANSI_COLOR_RESET); printf(ANSI_COLOR_RESET);
if (signo == SIGINT) { if (signo == SIGINT) {
@ -795,7 +848,7 @@ int main(int argc, char ** argv) {
const int64_t t_main_start_us = ggml_time_us(); const int64_t t_main_start_us = ggml_time_us();
gpt_params params; gpt_params params;
params.model = "models/llama-7B/ggml-model.bin"; params.model = "models/7B/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) { if (gpt_params_parse(argc, argv, params) == false) {
return 1; return 1;
@ -824,7 +877,8 @@ int main(int argc, char ** argv) {
{ {
const int64_t t_start_us = ggml_time_us(); const int64_t t_start_us = ggml_time_us();
if (!llama_model_load(params.model, model, vocab, params.n_ctx)) { if (!llama_model_load(params.model, model, vocab, params.n_ctx)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); fprintf(stderr, "%s: failed to load model from '%s'\n", __func__,
params.model.c_str());
return 1; return 1;
} }
@ -834,10 +888,35 @@ int main(int argc, char ** argv) {
// print system information // print system information
{ {
fprintf(stderr, "\n"); fprintf(stderr, "\n");
fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", params.n_threads,
params.n_threads, std::thread::hardware_concurrency(), llama_print_system_info()); std::thread::hardware_concurrency(), llama_print_system_info());
} }
printf("Initializing crow...");
crow::SimpleApp app;
CROW_ROUTE(app, "/")([&](const crow::request& req) -> std::string {
nlohmann::json json_data = nlohmann::json::parse(req.body);
std::string input = "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.\n";
auto messages = json_data["messages"];
for (auto& message : messages) {
std::string role = message["role"];
std::string content = message["content"];
if (role == "user") {
input = input + "\nUser: " + content;
} else {
input = input + "\nBob: " + content;
}
}
input = input + "\nBob: ";
std::string result = "";
int n_past = 0; int n_past = 0;
int64_t t_sample_us = 0; int64_t t_sample_us = 0;
@ -846,18 +925,23 @@ int main(int argc, char ** argv) {
std::vector<float> logits; std::vector<float> logits;
// tokenize the prompt // tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(vocab, params.prompt, true); std::vector<gpt_vocab::id> embd_inp =
::llama_tokenize(vocab, input, true);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); params.n_predict =
std::min(params.n_predict, model.hparams.n_ctx - (int)embd_inp.size());
// tokenize the reverse prompt // tokenize the reverse prompt
std::vector<gpt_vocab::id> antiprompt_inp = ::llama_tokenize(vocab, params.antiprompt, false); std::vector<gpt_vocab::id> antiprompt_inp =
::llama_tokenize(vocab, params.antiprompt, false);
fprintf(stderr, "\n"); fprintf(stderr, "\n");
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__,
embd_inp.size());
for (int i = 0; i < (int)embd_inp.size(); i++) { for (int i = 0; i < (int)embd_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).c_str()); fprintf(stderr, "%6d -> '%s'\n", embd_inp[i],
vocab.id_to_token.at(embd_inp[i]).c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
if (params.interactive) { if (params.interactive) {
@ -874,15 +958,22 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: interactive mode on.\n", __func__); fprintf(stderr, "%s: interactive mode on.\n", __func__);
if (antiprompt_inp.size()) { if (antiprompt_inp.size()) {
fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.c_str()); fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__,
fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size()); params.antiprompt.c_str());
fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n",
__func__, antiprompt_inp.size());
for (int i = 0; i < (int)antiprompt_inp.size(); i++) { for (int i = 0; i < (int)antiprompt_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).c_str()); fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i],
vocab.id_to_token.at(antiprompt_inp[i]).c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
} }
} }
fprintf(stderr, "sampling parameters: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty); fprintf(stderr,
"sampling parameters: temp = %f, top_k = %d, top_p = %f, "
"repeat_last_n = %i, repeat_penalty = %f\n",
params.temp, params.top_k, params.top_p, params.repeat_last_n,
params.repeat_penalty);
fprintf(stderr, "\n\n"); fprintf(stderr, "\n\n");
std::vector<gpt_vocab::id> embd; std::vector<gpt_vocab::id> embd;
@ -895,10 +986,11 @@ int main(int argc, char ** argv) {
std::vector<gpt_vocab::id> last_n_tokens(last_n_size); std::vector<gpt_vocab::id> last_n_tokens(last_n_size);
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0); std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
if (params.interactive) { if (params.interactive) {
fprintf(stderr, "== Running in interactive mode. ==\n" fprintf(stderr,
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) "== Running in interactive mode. ==\n"
#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(_WIN32)
" - Press Ctrl+C to interject at any time.\n" " - Press Ctrl+C to interject at any time.\n"
#endif #endif
" - Press Return to return control to LLaMa.\n" " - Press Return to return control to LLaMa.\n"
@ -924,9 +1016,10 @@ int main(int argc, char ** argv) {
if (embd.size() > 0) { if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us(); const int64_t t_start_us = ggml_time_us();
if (!llama_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) { if (!llama_eval(model, params.n_threads, n_past, embd, logits,
mem_per_token)) {
fprintf(stderr, "Failed to predict\n"); fprintf(stderr, "Failed to predict\n");
return 1; return std::string("Failed to predict");
} }
t_predict_us += ggml_time_us() - t_start_us; t_predict_us += ggml_time_us() - t_start_us;
@ -949,7 +1042,9 @@ int main(int argc, char ** argv) {
{ {
const int64_t t_start_sample_us = ggml_time_us(); const int64_t t_start_sample_us = ggml_time_us();
id = llama_sample_top_p_top_k(vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens, repeat_penalty, top_k, top_p, temp, rng); id = llama_sample_top_p_top_k(
vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens,
repeat_penalty, top_k, top_p, temp, rng);
last_n_tokens.erase(last_n_tokens.begin()); last_n_tokens.erase(last_n_tokens.begin());
last_n_tokens.push_back(id); last_n_tokens.push_back(id);
@ -962,11 +1057,9 @@ int main(int argc, char ** argv) {
// echo this to console // echo this to console
input_noecho = false; input_noecho = false;
// decrement remaining sampling budget
--remaining_tokens;
} else { } else {
// some user input remains from prompt or interaction, forward it to processing // some user input remains from prompt or interaction, forward it to
// processing
while (embd_inp.size() > input_consumed) { while (embd_inp.size() > input_consumed) {
embd.push_back(embd_inp[input_consumed]); embd.push_back(embd_inp[input_consumed]);
last_n_tokens.erase(last_n_tokens.begin()); last_n_tokens.erase(last_n_tokens.begin());
@ -978,64 +1071,39 @@ int main(int argc, char ** argv) {
} }
// reset color to default if we there is no pending user input // reset color to default if we there is no pending user input
if (!input_noecho && params.use_color && embd_inp.size() == input_consumed) { if (!input_noecho && params.use_color &&
embd_inp.size() == input_consumed) {
printf(ANSI_COLOR_RESET); printf(ANSI_COLOR_RESET);
} }
} }
// display text
if (!input_noecho) {
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
}
// in interactive mode, and not currently processing queued inputs; // in interactive mode, and not currently processing queued inputs;
// check if we should prompt the user for more // check if we should prompt the user for more
if (params.interactive && embd_inp.size() <= input_consumed) { if (params.interactive && embd_inp.size() <= input_consumed) {
// check for reverse prompt // check for reverse prompt
if (antiprompt_inp.size() && std::equal(antiprompt_inp.rbegin(), antiprompt_inp.rend(), last_n_tokens.rbegin())) { if (antiprompt_inp.size() &&
std::equal(antiprompt_inp.rbegin(), antiprompt_inp.rend(),
last_n_tokens.rbegin())) {
// reverse prompt found // reverse prompt found
printf("Reverse prompt found.\n");
result.erase(result.size() - 4);
is_interacting = true; is_interacting = true;
} }
if (is_interacting) { if (is_interacting) {
// currently being interactive
bool another_line=true;
while (another_line) {
fflush(stdout);
char buf[256] = {0};
int n_read;
if(params.use_color) printf(ANSI_BOLD ANSI_COLOR_GREEN);
if (scanf("%255[^\n]%n%*c", buf, &n_read) <= 0) {
// presumable empty line, consume the newline
scanf("%*c");
n_read=0;
}
if(params.use_color) printf(ANSI_COLOR_RESET);
if (n_read > 0 && buf[n_read-1]=='\\') {
another_line = true;
buf[n_read-1] = '\n';
buf[n_read] = 0;
} else {
another_line = false;
buf[n_read] = '\n';
buf[n_read+1] = 0;
}
std::vector<gpt_vocab::id> line_inp = ::llama_tokenize(vocab, buf, false);
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
remaining_tokens -= line_inp.size();
input_noecho = true; // do not echo this again
}
is_interacting = false; is_interacting = false;
break;
} }
} }
// display text
if (!input_noecho) {
for (auto id : embd) {
result = result + vocab.id_to_token[id].c_str();
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
}
// end of text token // end of text token
if (embd.back() == 2) { if (embd.back() == 2) {
fprintf(stderr, " [end of text]\n"); fprintf(stderr, " [end of text]\n");
@ -1052,13 +1120,23 @@ int main(int argc, char ** argv) {
const int64_t t_main_end_us = ggml_time_us(); const int64_t t_main_end_us = ggml_time_us();
fprintf(stderr, "\n\n"); fprintf(stderr, "\n\n");
fprintf(stderr, "%s: mem per token = %8zu bytes\n", __func__, mem_per_token); fprintf(stderr, "%s: mem per token = %8zu bytes\n", __func__,
fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); mem_per_token);
fprintf(stderr, "%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); fprintf(stderr, "%s: load time = %8.2f ms\n", __func__,
fprintf(stderr, "%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); t_load_us / 1000.0f);
fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); fprintf(stderr, "%s: sample time = %8.2f ms\n", __func__,
t_sample_us / 1000.0f);
fprintf(stderr, "%s: predict time = %8.2f ms / %.2f ms per token\n",
__func__, t_predict_us / 1000.0f, t_predict_us / 1000.0f / n_past);
fprintf(stderr, "%s: total time = %8.2f ms\n", __func__,
(t_main_end_us - t_main_start_us) / 1000.0f);
} }
return result.substr(input.length());
});
app.port(18080).multithreaded().run();
ggml_free(model.ctx); ggml_free(model.ctx);
if (params.use_color) { if (params.use_color) {