From a4d2d1c4e621862be5eea545dfa21b1ad080240d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 24 Feb 2024 18:46:20 -0800 Subject: [PATCH] (docs) using openai compatible endpoints --- .../docs/providers/openai_compatible.md | 53 +++++++++---------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/docs/my-website/docs/providers/openai_compatible.md b/docs/my-website/docs/providers/openai_compatible.md index ff68d7a000ec..beaf38cfac69 100644 --- a/docs/my-website/docs/providers/openai_compatible.md +++ b/docs/my-website/docs/providers/openai_compatible.md @@ -6,42 +6,37 @@ To call models hosted behind an openai proxy, make 2 changes: 2. **Do NOT** add anything additional to the base url e.g. `/v1/embedding`. LiteLLM uses the openai-client to make these calls, and that automatically adds the relevant endpoints. -## Usage +## Usage - completion ```python import litellm -from litellm import embedding -litellm.set_verbose = True import os - -litellm_proxy_endpoint = "http://0.0.0.0:8000" -bearer_token = "sk-1234" - -CHOSEN_LITE_LLM_EMBEDDING_MODEL = "openai/GPT-J 6B - Sagemaker Text Embedding (Internal)" - -litellm.set_verbose = False - -print(litellm_proxy_endpoint) - - - -response = embedding( - - model = CHOSEN_LITE_LLM_EMBEDDING_MODEL, # add `openai/` prefix to model so litellm knows to route to OpenAI - - api_key=bearer_token, - - api_base=litellm_proxy_endpoint, # set API Base of your Custom OpenAI Endpoint - - input=["good morning from litellm"], - - api_version='2023-07-01-preview' - +response = litellm.completion( + model="openai/mistral, # add `openai/` prefix to model so litellm knows to route to OpenAI + api_key="sk-1234", # api key to your openai compatible endpoint + api_base="http://0.0.0.0:8000", # set API Base of your Custom OpenAI Endpoint + messages=[ + { + "role": "user", + "content": "Hey, how's it going?", + } + ], ) +print(response) +``` -print('================================================') +## Usage - embedding -print(len(response.data[0]['embedding'])) +```python +import litellm +import os +response = litellm.embedding( + model="openai/GPT-J", # add `openai/` prefix to model so litellm knows to route to OpenAI + api_key="sk-1234", # api key to your openai compatible endpoint + api_base="http://0.0.0.0:8000", # set API Base of your Custom OpenAI Endpoint + input=["good morning from litellm"] +) +print(response) ``` \ No newline at end of file