make llama server configurable

This commit is contained in:
2024-01-26 15:47:02 +01:00
parent 06eca937bf
commit 2aea0bbdac
4 changed files with 20 additions and 5 deletions

View File

@@ -2,6 +2,8 @@
#include "./text_completion_interface.hpp"
#include <solanaceae/util/config_model.hpp>
#include <httplib.h>
#include <nlohmann/json_fwd.hpp>
@@ -9,14 +11,19 @@
#include <atomic>
struct LlamaCppWeb : public TextCompletionI {
// this mutex locks internally
httplib::Client _cli{"http://localhost:8080"};
ConfigModelI& _conf;
// this mutex-locks internally
httplib::Client _cli;
// this is a bad idea
static std::minstd_rand thread_local _rng;
std::atomic<bool> _use_server_cache {true};
LlamaCppWeb(
ConfigModelI& conf
);
~LlamaCppWeb(void);
bool isGood(void) override;