diff --git a/examples/common.cpp b/examples/common.cpp index 8eb633e58f4..31cdc0f0e53 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -10,129 +10,6 @@ #include #include -// Function to check if the next argument exists -static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) { - if (i + 1 < argc && argv[i + 1][0] != '-') { - return argv[++i]; - } else { - fprintf(stderr, "error: %s requires one argument.\n", flag.c_str()); - gpt_print_usage(argc, argv, params); - exit(0); - } -} - -bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - - if (arg == "-s" || arg == "--seed") { - params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-t" || arg == "--threads") { - params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-p" || arg == "--prompt") { - params.prompt = get_next_arg(i, argc, argv, arg, params); - } else if (arg == "-n" || arg == "--n_predict") { - params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-np" || arg == "--n_parallel") { - params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--top_k") { - params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--top_p") { - params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--temp") { - params.temp = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--repeat-last-n") { - params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--repeat-penalty") { - params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-b" || arg == "--batch_size") { - params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-c" || arg == "--context") { - params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") { - params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--ignore-eos") { - params.ignore_eos = true; - } else if (arg == "-m" || arg == "--model") { - params.model = get_next_arg(i, argc, argv, arg, params); - } else if (arg == "-i" || arg == "--interactive") { - params.interactive = true; - } else if (arg == "-ip" || arg == "--interactive-port") { - params.interactive = true; - params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-h" || arg == "--help") { - gpt_print_usage(argc, argv, params); - exit(0); - } else if (arg == "-f" || arg == "--file") { - get_next_arg(i, argc, argv, arg, params); - std::ifstream file(argv[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - break; - } - std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); - if (params.prompt.back() == '\n') { - params.prompt.pop_back(); - } - } else if (arg == "-tt" || arg == "--token_test") { - params.token_test = get_next_arg(i, argc, argv, arg, params); - } - else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - gpt_print_usage(argc, argv, params); - exit(0); - } - } - - return true; -} - -void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { - fprintf(stderr, "usage: %s [options]\n", argv[0]); - fprintf(stderr, "\n"); - fprintf(stderr, "options:\n"); - fprintf(stderr, " -h, --help show this help message and exit\n"); - fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); - fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); - fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); - fprintf(stderr, " prompt to start generation with (default: random)\n"); - fprintf(stderr, " -f FNAME, --file FNAME\n"); - fprintf(stderr, " load prompt from a file\n"); - fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n"); - fprintf(stderr, " test tokenization\n"); - fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict); - fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k); - fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p); - fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp); - fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n); - fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty); - fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch); - fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx); - fprintf(stderr, " --ignore-eos ignore EOS token during generation\n"); - fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers); - fprintf(stderr, " -m FNAME, --model FNAME\n"); - fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); - fprintf(stderr, "\n"); -} - -std::string gpt_random_prompt(std::mt19937 & rng) { - const int r = rng() % 10; - switch (r) { - case 0: return "So"; - case 1: return "Once upon a time"; - case 2: return "When"; - case 3: return "The"; - case 4: return "After"; - case 5: return "If"; - case 6: return "import"; - case 7: return "He"; - case 8: return "She"; - case 9: return "They"; - } - - return "The"; -} - std::string trim(const std::string & s) { std::regex e("^\\s+|\\s+$"); return std::regex_replace(s, e, ""); @@ -152,252 +29,6 @@ void gpt_vocab::add_special_token(const std::string & token) { special_tokens.push_back(token); } -std::map json_parse(const std::string & fname) { - std::map result; - - // read file into string - std::string json; - { - std::ifstream ifs(fname); - if (!ifs) { - fprintf(stderr, "Failed to open %s\n", fname.c_str()); - exit(1); - } - - json = std::string((std::istreambuf_iterator(ifs)), - (std::istreambuf_iterator())); - } - - if (json[0] != '{') { - return result; - } - - // parse json - { - bool has_key = false; - bool in_token = false; - - std::string str_key = ""; - std::string str_val = ""; - - int n = json.size(); - for (int i = 1; i < n; ++i) { - if (!in_token) { - if (json[i] == ' ') continue; - if (json[i] == '"') { - in_token = true; - continue; - } - } else { - if (json[i] == '\\' && i+1 < n) { - if (has_key == false) { - str_key += json[i]; - } else { - str_val += json[i]; - } - ++i; - } else if (json[i] == '"') { - if (has_key == false) { - has_key = true; - ++i; - while (json[i] == ' ') ++i; - ++i; // : - while (json[i] == ' ') ++i; - if (json[i] != '\"') { - while (json[i] != ',' && json[i] != '}') { - str_val += json[i++]; - } - has_key = false; - } else { - in_token = true; - continue; - } - } else { - has_key = false; - } - - str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space - str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line - str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> " - - try { - result[str_key] = std::stoi(str_val); - } catch (...) { - //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str()); - - } - str_key = ""; - str_val = ""; - in_token = false; - continue; - } - if (has_key == false) { - str_key += json[i]; - } else { - str_val += json[i]; - } - } - } - } - - return result; -} - -void gpt_split_words(std::string str, std::vector& words) { - const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"; - const std::regex re(pattern); - std::smatch m; - - while (std::regex_search(str, m, re)) { - for (auto x : m) { - words.push_back(x); - } - str = m.suffix(); - } -} - -std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { - std::vector words; - - // first split the text into words - { - std::string str = text; - - // Generate the subpattern from the special_tokens vector if it's not empty - if (!vocab.special_tokens.empty()) { - const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])"); - std::string special_tokens_subpattern; - for (const auto & token : vocab.special_tokens) { - if (!special_tokens_subpattern.empty()) { - special_tokens_subpattern += "|"; - } - special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)"); - } - - std::regex re(special_tokens_subpattern); - std::smatch m; - // Split the text by special tokens. - while (std::regex_search(str, m, re)) { - // Split the substrings in-between special tokens into words. - gpt_split_words(m.prefix(), words); - // Add matched special tokens as words. - for (auto x : m) { - words.push_back(x); - } - str = m.suffix(); - } - // Remaining text without special tokens will be handled below. - } - - gpt_split_words(str, words); - } - - // find the longest token that forms each word in words: - std::vector tokens; - for (const auto & word : words) { - for (int i = 0; i < (int) word.size(); ){ - for (int j = word.size() - 1; j >= i; j--){ - auto cand = word.substr(i, j-i+1); - auto it = vocab.token_to_id.find(cand); - if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab - tokens.push_back(it->second); - i = j + 1; - break; - } - else if (j == i){ // word.substr(i, 1) has no matching - fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data()); - i++; - } - } - } - } - - return tokens; -} - -static std::vector parse_tokens_from_string(const std::string& input, char delimiter) { - std::vector output; - std::stringstream ss(input); - std::string token; - - while (std::getline(ss, token, delimiter)) { - output.push_back(std::stoi(token)); - } - - return output; -} - -static std::map> extract_tests_from_file(const std::string & fpath_test){ - if (fpath_test.empty()){ - fprintf(stderr, "%s : No test file found.\n", __func__); - return std::map>(); - } - - std::map> tests; - - auto fin = std::ifstream(fpath_test, std::ios_base::in); - const char * delimeter = " => "; - const char del_tok = ','; - std::string line; - while (std::getline(fin, line)) { - size_t delimiterPos = line.find(delimeter); - if (delimiterPos != std::string::npos) { - std::string text = line.substr(0, delimiterPos); - std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter)); - tests[text] = parse_tokens_from_string(s_tokens, del_tok); - } - } - return tests; -} - -void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ - std::map> tests = extract_tests_from_file(fpath_test); - - size_t n_fails = 0; - - for (const auto & test : tests) { - std::vector tokens = gpt_tokenize(vocab, test.first); - - if (tokens != test.second){ - n_fails++; - - // print out failure cases - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str()); - fprintf(stderr, "%s : tokens in hf: ", __func__); - for (const auto & t : test.second) { - fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : tokens in ggml: ", __func__); - for (const auto & t : tokens) { - fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); - } - fprintf(stderr, "\n"); - } - } - - fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size()); -} - -bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) { - printf("%s: loading vocab from '%s'\n", __func__, fname.c_str()); - - vocab.token_to_id = ::json_parse(fname); - - for (const auto & kv : vocab.token_to_id) { - vocab.id_to_token[kv.second] = kv.first; - } - - printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size()); - - // print the vocabulary - //for (auto kv : vocab.token_to_id) { - // printf("'%s' -> %d\n", kv.first.data(), kv.second); - //} - - return true; -} - gpt_vocab::id gpt_sample_top_k_top_p( const gpt_vocab & vocab, const float * logits, diff --git a/examples/common.h b/examples/common.h index 8f99df7dd32..aa18b23a359 100644 --- a/examples/common.h +++ b/examples/common.h @@ -11,42 +11,6 @@ #include #include -// -// GPT CLI argument parsing -// - -struct gpt_params { - int32_t seed = -1; // RNG seed - int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); - int32_t n_predict = 200; // new tokens to predict - int32_t n_parallel = 1; // number of parallel streams - int32_t n_batch = 32; // batch size for prompt processing - int32_t n_ctx = 2048; // context size (this is the KV cache max size) - int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU - - bool ignore_eos = false; // ignore EOS token when generating text - - // sampling parameters - int32_t top_k = 40; - float top_p = 0.9f; - float temp = 0.9f; - int32_t repeat_last_n = 64; - float repeat_penalty = 1.00f; - - std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path - std::string prompt = ""; - std::string token_test = ""; - - bool interactive = false; - int32_t interactive_port = -1; -}; - -bool gpt_params_parse(int argc, char ** argv, gpt_params & params); - -void gpt_print_usage(int argc, char ** argv, const gpt_params & params); - -std::string gpt_random_prompt(std::mt19937 & rng); - // // Vocab utils // @@ -69,39 +33,6 @@ struct gpt_vocab { void add_special_token(const std::string & token); }; -// poor-man's JSON parsing -std::map json_parse(const std::string & fname); - -std::string convert_to_utf8(const std::wstring & input); - -std::wstring convert_to_wstring(const std::string & input); - -void gpt_split_words(std::string str, std::vector& words); - -// split text into tokens -// -// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53 -// -// Regex (Python): -// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" -// -// Regex (C++): -// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)" -// -std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text); - -// test outputs of gpt_tokenize -// -// - compare with tokens generated by the huggingface tokenizer -// - test cases are chosen based on the model's main language (under 'prompt' directory) -// - if all sentences are tokenized identically, print 'All tests passed.' -// - otherwise, print sentence, huggingface tokens, ggml tokens -// -void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test); - -// load the tokens from encoder.json -bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab); - // sample next token given probabilities for each embedding // // - consider only the top K tokens diff --git a/scripts/sync-ggml-am.sh b/scripts/sync-ggml-am.sh index 924f67ee5b5..1bf9996e3f0 100755 --- a/scripts/sync-ggml-am.sh +++ b/scripts/sync-ggml-am.sh @@ -78,8 +78,6 @@ while read c; do src/ggml-vulkan/* \ include/ggml*.h \ include/gguf*.h \ - examples/common.h \ - examples/common.cpp \ examples/common-ggml.h \ examples/common-ggml.cpp \ LICENSE \ diff --git a/scripts/sync-ggml.sh b/scripts/sync-ggml.sh index 00162daa05b..715d40f43fc 100755 --- a/scripts/sync-ggml.sh +++ b/scripts/sync-ggml.sh @@ -26,8 +26,6 @@ cp -rpv ../ggml/src/ggml-vulkan/* ./ggml/src/ggml-vulkan/ cp -rpv ../ggml/include/ggml*.h ./ggml/include/ cp -rpv ../ggml/include/gguf*.h ./ggml/include/ -cp -rpv ../ggml/examples/common.h ./examples/common.h -cp -rpv ../ggml/examples/common.cpp ./examples/common.cpp cp -rpv ../ggml/examples/common-ggml.h ./examples/common-ggml.h cp -rpv ../ggml/examples/common-ggml.cpp ./examples/common-ggml.cpp