Skip to content

Commit

Permalink
docs updated
Browse files Browse the repository at this point in the history
  • Loading branch information
ilkersigirci committed Dec 18, 2024
1 parent 043fbe2 commit 8a56dbf
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 18 deletions.
4 changes: 2 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ HF_HOME=DUMMY_PATH/huggingface
HF_HUB_ENABLE_HF_TRANSFER=1
# HF_TOKEN=TO_BE_FILLED
# RUNPOD_API_KEY=TO_BE_FILLED
HF_MODEL_NAME=mistralai/Pixtral-Large-Instruct-2411
SERVED_MODEL_NAME=Pixtral-Large-Instruct-2411
HF_MODEL_NAME=alpindale/c4ai-command-r-plus-GPTQ
SERVED_MODEL_NAME=vLLM-Model
MAX_CONTEXT_LEN=32000
ENABLE_HEALTH_CHECK=1
ENABLE_AUTO_RESTART=0
Expand Down
15 changes: 6 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,12 @@ prepare-env-file: ## Copy .env.example to .env and correct the paths
cp .env.example .env
$(MAKE) change-paths-in-env-file

change-model-env: ## Change the model that is specified in the .env file
# sed -i 's/HF_MODEL_NAME=alpindale\/WizardLM-2-8x22B/HF_MODEL_NAME=CohereForAI\/c4ai-command-r-v01/g' .env
sed -i '/HF_MODEL_NAME=/d' .env
echo "HF_MODEL_NAME=${HF_MODEL_NAME}" >> .env

change-max-context-len-env: ## Change the max context length that is specified in the .env file
# sed -i 's/MAX_CONTEXT_LEN=32000/MAX_CONTEXT_LEN=40000/g' .env
sed -i '/MAX_CONTEXT_LEN=/d' .env
echo "MAX_CONTEXT_LEN=${MAX_CONTEXT_LEN}" >> .env
replace-value-in-env-file: ## Replace a variable in the .env file
@if [ -z "${variable_name}" ] || [ -z "${new_value}" ]; then \
echo "Usage: make replace-value-in-env-file variable_name=<variable> new_value=<value>"; \
exit 1; \
fi
sed -i "s|${variable_name}=.*|${variable_name}=${new_value}|g" .env

install-uv:
! command -v uv &> /dev/null && curl -LsSf https://astral.sh/uv/install.sh | sh
Expand Down
13 changes: 6 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,20 @@ make gui
- Example: Change default model and its context length to CohereForAI/c4ai-command-r-plus-GPTQ

```bash
make change-model-env HF_MODEL_NAME=CohereForAI/c4ai-command-r-plus-GPTQ
make change-max-context-len-env MAX_CONTEXT_LEN=40000

make replace-value-in-env-file variable_name=HF_MODEL_NAME new_value=CohereForAI/c4ai-command-r-plus-GPTQ
make replace-value-in-env-file variable_name=MAX_CONTEXT_LEN new_value=40000
```

## cURL Examples

- Request with system message
- Request with system message assuming `SERVED_MODEL_NAME=vLLM-Model`

```bash
curl --request POST \
--url http://0.0.0.0:8000/v1/chat/completions \
--header "Content-Type: application/json" \
--data '{
"model": "c4ai-command-r-plus-GPTQ",
"model": "vLLM-Model",
"messages": [
{
"role": "system",
Expand All @@ -69,14 +68,14 @@ curl --request POST \
}'
```

- Request without system message
- Request without system message assuming `SERVED_MODEL_NAME=vLLM-Model`

```bash
curl --request POST \
--url http://0.0.0.0:8000/v1/chat/completions \
--header "Content-Type: application/json" \
--data '{
"model": "c4ai-command-r-plus-GPTQ",
"model": "vLLM-Model",
"messages": [
{
"role": "user",
Expand Down

0 comments on commit 8a56dbf

Please sign in to comment.