simple llama.cpp server api usage works

This commit is contained in:
2024-01-22 21:14:33 +01:00
commit c497b19b20
11 changed files with 537 additions and 0 deletions

View File

@@ -0,0 +1,23 @@
#pragma once
#include "./llama_cpp_web_interface.hpp"
#include <httplib.h>
#include <nlohmann/json_fwd.hpp>
#include <random>
struct LlamaCppWeb : public LlamaCppWebI {
httplib::Client _cli{"http://localhost:8080"};
std::minstd_rand _rng{std::random_device{}()};
~LlamaCppWeb(void);
bool isHealthy(void) override;
int64_t completeSelect(const std::string_view prompt, const std::vector<std::string_view>& possible) override;
std::string completeLine(const std::string_view prompt) override;
// TODO: expose?
nlohmann::json complete(const nlohmann::json& request_j);
};