File size: 2,148 Bytes
c59fd93 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | #pragma once
#include "common.h"
#include "cli-client.h"
#include "cli-server.h"
#include <atomic>
#include <memory>
#include <optional>
#include <string>
#include <fstream>
struct cli_timings {
double prompt_per_second = 0.0;
double predicted_per_second = 0.0;
};
struct cli_context_impl;
struct cli_context {
common_params params;
cli_client client; // always initialized
std::optional<cli_server> server; // only set when no --server-base is given
// properties of the connected server
// will be populated by fetch_server_props()
std::string model_name;
std::string model_ftype;
std::string build_info;
bool has_vision = false;
bool has_audio = false;
bool has_video = false;
std::optional<std::ofstream> output_file;
cli_context(const common_params & params);
~cli_context();
// connect to --server-base or spawn a local llama-server child;
// argc/argv are needed to forward the server-relevant args to the child
bool init();
// run the interactive chat loop, returns the process exit code
int run();
// stop the local server child (if any)
void shutdown();
// set by the SIGINT handler; cleared once the interrupt has been handled
static std::atomic<bool> & interrupted();
private:
struct generated_content {
std::string reasoning;
std::string content;
};
bool generate_completion(generated_content & content_out, cli_timings & timings);
void fetch_server_props();
void add_system_prompt();
void push_user_message(const std::string & text);
// check if server have multiple models (router mode)
// if yes, list them then ask; do nothing otherwise
bool list_and_ask_models();
// read a file and stage it as a multimodal content part; type is one of
// "image", "audio", "video"; returns false if the file cannot be read
bool stage_media_file(const std::string & fname, const std::string & type);
// no-op if output file is not set
void write_output_file(const std::string & content);
std::unique_ptr<cli_context_impl> impl;
};
|