o ew @sddlZddlZddlZddlZddlZddlZddlmZddlZddl m Z ddl m Z ddl mZmZddlZddlmZmZmZmZddlmZddlmZdd lmZdd lmZdd lmZdd l m!Z!ej"#d GdddeZ$GdddeZ%Gddde%Z&Gddde$e&Z'GdddeZ(Gddde%Z)Gddde$e)Z*GdddeZ+GdddeZ,Gd d!d!eZ-edfd"e.d#dfd$d%Z/eZ0ee/gZ1e0j2ed&gd'd&gd&gd(Gd)d*d*e3Z4e05e3d+d,Z6e0j7d-e1d.d/d0Z8e0j9d1e(e1d2d3ed4e'fd5d6Z:e0j9d7e+e1d2d3ed4e*fd8d9Z;e0jZ=e0je0jeZdZUeeeBed<dZedBed<dZe dBed<dS)CompletionParamspromptN logit_biaslogprobs) rrrr5rrr9dictr:rr r r r!r79s r7c@ eZdZdS) TextRequestNrrrr r r r!r=?r=c@ReZdZUeed<eeed<eeZ eed<eed<dZ eed<eed<dS) TextResponseidchoicescreatedr'text_completionobjectusageN rrrr5rrr;rtimerDrFr r r r!rAC    rAc@sJeZdZUeeed<edddZedBed<edddZ e dBed<dS) ChatParamsmessagesNWIPr#tools tool_choice) rrrrr;rrrNlistrOr5r r r r!rKLs rKc@r<) ChatRequestNr>r r r r!rQRsrQc@r@) ChatResponserBrCrDr'zchat.completionrFrGNrHr r r r!rRWrJrRc@seZdZUeed<dS)ModelInfoResponse model_nameN)rrrr5rr r r r!rS`s rSc@seZdZUeeed<dS)ModelListResponse model_namesN)rrrrr5rr r r r!rUds rU authorizationreturncCsFtdd}|dkr dS|r|dus|d|kr!tddddSdS)NH2OGPT_OPENAI_API_KEYEMPTYzBearer i Unauthorized) status_codedetail)osgetenvr )rWserver_api_keyr r r!verify_api_keyhs  ra*T) allow_originsallow_credentials allow_methods allow_headersc@r<)InvalidRequestErrorNr>r r r r!rgr?rgcs&t|tt|}tt|ddS)Ni)r\)rrgr5r)requestexcexc2r r r!validation_exception_handlers rk/) dependenciescs tddS)NOKcontentr r r r r! options_routes rqz/v1/completions)response_modelrmrh request_datacs@jrfdd}t|Sddlm}|t}t|S)NcsPddlm}|t}|D]}IdH}|rdSdt|iVqdS)Nr)stream_completionsdata)openai_server.backendrtr;is_disconnectedjsondumps)rtresponseresp disconnectedrhrsr r! generators  z%openai_completions..generatorr) completions)r1rrvrr;r)rhrsr~rrzr r}r!openai_completionss  rz/v1/chat/completionscsNjrddlmfdd}t|Sddlm}|t}t|S)Nr)stream_chat_completionscsDt}|D]}IdH}|rdSdt|iVq dS)Nru)r;rwrxry)rzr{r|rhrsrr r!r~s z*openai_chat_completions..generator)chat_completions)r1rvrrrr;r)rhrsr~rrzr rr!openai_chat_completionss   rz /v1/modelsz/v1/models/{model}z/v1/models/{repo}/{model}cs|jj}|tdd}ddlm}t|jdd}dd|D}|s.d|d }t |S||}|dkr?||}t |St d d }t |S) Nz /v1/models/r) gradio_clientz /model_names)api_namecSsg|]}|dqS) base_modelr ).0xr r r! sz!handle_models..rP)rFruINVALID)rT) urlpathlenrvrast literal_evalpredictindexr;r)rhrrTr model_dict base_modelsrz model_indexr r r! handle_modelss"   rz/v1/internal/model/infocddlm}t|dS)Nrget_model_inforo)rvrrrr r r!handle_model_info  rz/v1/internal/model/listcr)Nrget_model_listro)rvrrrr r r!handle_list_modelsrr0.0.0.0c Cs|pdtjd<|p dtjd<|pdtjd<|pdtjd<td tjdp&d }|tjd <ttd |}td |}td |}|rF|rFdnd} td| d|d|td|dtd_t j t ||||ddS)Nhttp GRADIO_PREFIX localhostGRADIO_SERVER_HOST7860GRADIO_SERVER_PORTGRADIO_H2OGPT_H2OGPT_KEYrYrZH2OGPT_OPENAI_PORTH2OGPT_OPENAI_CERT_PATHH2OGPT_OPENAI_KEY_PATHhttpszOpenAI API URL: z://:zOpenAI API key: Fz uvicorn.error)hostport ssl_certfile ssl_keyfile) r^environr_rrinfologging getLogger propagateuvicornrunapp) rrrr gradio_prefix gradio_host gradio_port h2ogpt_keyr`prefixr r r! run_servers     rcKs,|r tdi|dStt|dddS)NT)targetkwargsdaemonr )rrstart)waitrr r r!rsr)rrNNNNNN)T)B contextlibrr^sysrrx threadingrrI tracebackrtypingrpydanticrrrfastapirrr r fastapi.middleware.corsr fastapi.requestsr fastapi.responsesr sse_starletterstarlette.responsesropenai_server.logrrappendrr"r7r=rArKrQrRrSrUr5rar check_keyadd_middleware Exceptionrgexception_handlerrkoptionsrqpostrrgetrrrrrr r r r!s