File size: 4,250 Bytes
88597ff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
'''
-----------------------------------------------------------------------------
Copyright (c) 2023, ai-study-room. All rights reserved.

Authors: freesky-edward

This source code or documentation is under the license under the
LICENSE file in the root directory of this repository
-----------------------------------------------------------------------------
'''

import gradio as gr
import argparse
import os
from inference import *
from inference.config import *
from inference.get_inference import *
from typing import Iterator


def _init_args():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--server_name",
        type=str,
        default="127.0.0.1",
        help="The chatbot bind address")
    parser.add_argument(
        "--server_port",
        type=int,
        default="8089",
        help="The chatbot listen port")
    parser.add_argument(
        "--model_path",
        type=str,
        default="",
        help="The running model path which should contain the model and config.json")

    args = parser.parse_args()
    return args


def _startServer(server, server_address: str, server_port: str):

    server.queue()
    server.launch(server_name=server_address, server_port=server_port)


def _args_check(args):

    if args.model_path == "":
        raise ValueError


def main():

    args = _init_args()
    _args_check(args)

    configfile = './models/llama2/config.yaml'
    cfg = Config(configfile)
    cfg.model_path = args.model_path
    inference = get_inference(cfg)

    server_address = os.getenv("SERVER_ADDRESS", "")
    server_port = os.getenv("SERVER_PORT")
    head = """
      This is a chat-engine
      Source: https://github.com/ai-study-room/chatengine
      Contact: freesky.edward@gmail.com

    """

    if args.server_name != "":
        server_address = args.server_name

    if args.server_port != 8090:
        server_port = args.server_port

    def reply(
            chatbot, chatbot2) -> (Iterator[list[tuple[str, str]]], Iterator[list[tuple[str, str]]]):
        # inp = chatbot[-1][0]
        streamer = inference.generate(chatbot)
        try:
            bot_msg = next(streamer)

            chatbot[-1][1] = "" + bot_msg
            chatbot2[-1][1] = "" + bot_msg
            yield chatbot, chatbot2

            for resp in streamer:
                chatbot[-1][1] = resp
                chatbot2[-1][1] = resp
                yield chatbot, chatbot2

        except StopIteration as si:
            print(si)
            chatbot.append((inp, "error occur1"))
            chatbot2.append((inp, "error occur1"))
            yield chatbot, chatbot1

    def input_msg(inp, history_1, history_2):

        return "", history_1 + [(inp, None)], history_2 + [(inp, None)]

    with gr.Blocks() as bl:

        gr.Markdown(head)

        with gr.Group():
            with gr.Row():
                chatbot = gr.Chatbot(
                    label="Llama-2", show_label=True, container=True)
                chatbot2 = gr.Chatbot(label="Llama-2")
            with gr.Row():
                inp = gr.Textbox(
                    placeholder="welcome to chat engine...",
                    show_label=False)

        with gr.Row():
            submit_btn = gr.Button("Submit")
            clear_btn = gr.ClearButton([chatbot, chatbot2, inp])

        saved_input = gr.State()

        button_event_preprocess = (
            submit_btn.click(
                fn=input_msg,
                inputs=[inp, chatbot, chatbot2],
                outputs=[inp, chatbot, chatbot2],
                api_name=False,
                queue=False,
            ).then(
                fn=reply,
                inputs=[chatbot, chatbot2],
                outputs=[chatbot, chatbot2],
                api_name=False,
            )
        )

        inp.submit(
            fn=input_msg,
            inputs=[inp, chatbot, chatbot2],
            outputs=[inp, chatbot, chatbot2],
            api_name=False,
            queue=False,
        ).then(
            fn=reply,
            inputs=[chatbot, chatbot2],
            outputs=[chatbot, chatbot2],
            api_name=False,
        )

    _startServer(bl, server_address, server_port)


if __name__ == '__main__':
    main()