From fa5f688c70d955da3d758c1b8b5ed72e74ce525b Mon Sep 17 00:00:00 2001 From: Spike Lu Date: Mon, 31 Jul 2023 21:28:45 -0700 Subject: [PATCH] add logger and release pipeline --- .github/workflows/release.yml | 36 ++ .goreleaser.yml | 29 ++ CHANGELOG.md | 10 + LICENSE | 7 + README.md | 501 ++++++++++++++++++++++++++- assets/bricks-logo.png | Bin 0 -> 7282 bytes cmd/atlas/main.go | 75 ---- cmd/bricksllm/main.go | 69 ++++ config/config.go | 175 +++++++++- atlas.yaml => example/bricksllm.yaml | 13 +- go.mod | 7 +- go.sum | 8 + internal/client/openai/cost.go | 23 ++ internal/client/openai/error.go | 23 ++ internal/client/openai/openai.go | 88 ++++- internal/logger/api_messag.go | 27 -- internal/logger/api_message.go | 160 +++++++++ internal/logger/error_message.go | 30 ++ internal/logger/llm_message.go | 155 ++++++++- internal/logger/logger.go | 11 +- internal/logger/zap/zap.go | 115 ++++++ internal/prompt/prompt.go | 2 +- internal/server/web/web.go | 228 ++++++++++-- internal/util/util.go | 23 ++ scripts/release_notes.sh | 10 + 25 files changed, 1650 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yml create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 assets/bricks-logo.png delete mode 100644 cmd/atlas/main.go create mode 100644 cmd/bricksllm/main.go rename atlas.yaml => example/bricksllm.yaml (73%) create mode 100644 internal/client/openai/cost.go create mode 100644 internal/client/openai/error.go delete mode 100644 internal/logger/api_messag.go create mode 100644 internal/logger/api_message.go create mode 100644 internal/logger/error_message.go create mode 100644 internal/logger/zap/zap.go create mode 100644 scripts/release_notes.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..2a8ea1d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,36 @@ +name: goreleaser + +on: + push: + tags: + - '*' + +permissions: + contents: write + packages: write + issues: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: 1.19.x + check-latest: true + + - name: Check Out Repo + uses: actions/checkout@v4 + + - name: Release Notes + run: ./resources/scripts/release_notes.sh > ./release_notes.md + + - name: GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + version: latest + args: release --release-notes=./release_notes.md --timeout 60m + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..6041ace --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,29 @@ +builds: + - id: bricksllm + main: cmd/bricksllm/main.go + binary: bricksllm + goos: [ windows, darwin, linux ] + goarch: [ amd64, arm, arm64 ] + goarm: ["6", "7"] + ignore: + - goos: windows + goarch: arm + - goos: darwin + goarch: arm + env: + - CGO_ENABLED=0 +archives: + - id: bricksllm + builds: [ bricksllm ] + format: tar.gz + files: + - README.md + - CHANGELOG.md + - LICENSE +dist: target/dist +release: + github: + owner: bricks-cloud + name: bricks + prerelease: auto + disable: false \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9b27090 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +## 0.0.1 - 2023-07-31 +### Added +- Added a http web server for hosting `openai` prompts +- Added a `yaml` parser for reading bricksllm configurations +- Added support for [`cors`](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) configuration in the web server +- Added support for specifying `input` json schema +- Added support for `{input.field}` like syntax in the prompt template +- Added comprehensive logging in production and developer mode +- Added logger configuration for hiding sensitive fields using the `logger` field +- Added support for API key authenticaiton with `key_auth` field diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4dde2e9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2023 Bricks Cloud Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 1d240d0..18b7281 100644 --- a/README.md +++ b/README.md @@ -1 +1,500 @@ -# atlas \ No newline at end of file +

+ +

+ +# **BricksLLM: A Declarative Approach To Building LLM Applications** + +

+ PRs Welcome + Join BricksLLM on Discord + License +

+ +**BricksLLM** is a declarative Go framework that gives you building blocks to create reliable LLM workflows. Productionizing LLM applications is difficult due to the technology's probalistic nature. **BricksLLM** solves this issue by accelerating the developer development cycle and allowing developers to relentlessly test and improve their LLM applications continuously. + +* **Build**: Use BricksLLM building blocks to quickly build out your LLM backend. +* **Test**: Unit test prompts and BricksLLM APIs in CI/CD. +* **Version Control**: Version control your prompt strategies through declaractive configuration. +* **Deploy**: Containerize and deploy BricksLLM anywhere. +* **Monitor**: Out of the box detailed logging and monitoring. +* **A/B Testing**: Fine tune your prompt strategies through A/B testing. + +## Overview +Here is an example of BricksLLM config yaml. + +```yaml +openai: + api_credential: ${OPENAI_KEY} + +routes: + - path: /travel + provider: openai + key_auth: + key: ${API_KEY} + cors: + allowed_origins: ["*"] + allowed_credentials: true + input: + plan: + type: object + properties: + place: + type: string + openai_config: + model: gpt-3.5-turbo + prompts: + - role: assistant + content: say hi to {{ plan.place }} + + - path: /test + provider: openai + input: + name: + type: string + openai_config: + model: gpt-3.5-turbo + prompts: + - role: assistant + content: say hi to {{ name }} + +``` +### Core Components +Each BricksLLM application has to have at least one route configuration + +```yaml +routes: + - path: /travel + provider: openai + key_auth: + key: ${API_KEY} + cors: + allowed_origins: ["*"] + allowed_credentials: true + input: + plan: + type: object + properties: + place: + type: string + openai_config: + model: gpt-3.5-turbo + prompts: + - role: assistant + content: say hi to {{ plan.place }} +``` + +### Observability Components +There are also observability components such as `logger` that let you control how BricksLLM exposes log data. + +```yaml +logger: + api: + hide_ip: true + hide_headers: true + llm: + hide_headers: true + hide_response_content: true + hide_prompt_content: true +``` + +### Resource Components +Resource components let you specify available resources that could be used in the core components + +```yaml +openai: + api_credential: ${OPENAI_KEY} +``` + +# Documentation +## routes +### Example +```yaml +routes: + - path: /travel + provider: openai + key_auth: + key: ${API_KEY} + cors: + allowed_origins: ["*"] + allowed_credentials: true + input: + plan: + type: object + properties: + place: + type: string + openai_config: + model: gpt-3.5-turbo + prompts: + - role: assistant + content: say hi to {{ plan.place }} +``` +### Fields +#### `routes` +##### Required: ```true``` +##### Type: ```array``` +A list of route configurations connected with LLM APIs. + +#### `routes[].path` +##### Required: ```true``` +##### Type: ```string``` +A path specifies the resource that gets exposed to the client. + +```yaml +routes: + - path: /weathers +``` + +#### `routes[].provider` +##### Required: ```true``` +##### Type: ```enum``` +##### Options: [`openai`] +A provider is the name of the service that provides the LLM API. Right now, Bricks only supports OpenAI. + +#### `routes[].key_auth` +##### Required: ```false``` +##### Type: ```object``` +Contains configurations for using API key authentication. If set up, BricksLLM would start checking API header `X-Api-Key` of incoming request for authentication and return ```401``` if the API call is unauthorized. + +#### `routes[].key_auth.key` +##### Required: `true` +##### Type: `string` +A unique key to authenticate the API. + +#### `routes[].cors` +##### Required: `false` +##### Type: `object` +The `cors` field is used to specify Cross-Origin Resource Sharing settings. It includes subfields for setting up CORS policies. + +#### `routes[].cors.allowed_origins` +##### Required: `true` +##### Type: `array` +An array of strings specifying allowed origins for CORS. + +#### `routes[].cors.allowed_credentials` +##### Required: `true` +##### Type: `boolean` +A boolean value specifying if CORS response can include credentials. It sets `Access-Control-Allow-Credentials` to `true` in server responses. You can read more about it [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials). + +#### `routes[].input` +##### Required: `false` +##### Type: `object` +The `input` field is used to specify the input JSON schema of the route. + +```yaml + input: + plan: + type: object + properties: + place: + type: string + +``` +This translates to the following JSON schema. + +```json +{ + "plan": { + "type": "", + } +} +``` +BricksLLM would use this schema to validate incoming HTTP requests. It would return `400` if expected fields in the request body are empty or the data type does not match the schema. + +#### `routes[].input[field_name].type` +##### Required: `false` +##### Type: `enum` +##### Options: [`string`, `boolean`, `object`, `number`] +The `type` field within input specifies the data type of the expected + +#### `routes[].input[field_name].properties` +##### Required: `true` (Only if the field data type is `object`) +##### Type: `object` +The `properties` field specifies the schema of the `object` field. + +#### `routes[].openai_config` +##### Required: `false` +##### Type: `object` +The `openai_config` field is used to specify the configuration details for OpenAI. + +#### `routes[].openai_config.api_credential` +##### Required: `true` (Only required `openai.api_credential` is not specified) +##### Type: `string` +OpenAI API credential. If `openai.api_credential` is specified, the credential here will overwrite it in the API call to OpenAI. + +```yaml + openai_config: + model: gpt-3.5-turbo + api_credential: ${OPENAI_KEY} +``` + +:warning: **Store OpenAI key securely, and only use it as an environment variable.** + +#### `routes[].openai_config.model` +##### Required: `true` +##### Type: `enum` +##### Options: [`gpt-3.5-turbo`, `gpt-3.5-turbo-16k`, `gpt-3.5-turbo-0613`, `gpt-3.5-turbo-16k-0613`, `gpt-4`, `gpt-4-0613`, `gpt-4-32k`, `gpt-4-32k-0613`] + +The `model` field specifies the version of the model to use. Right now, BricksLLM supports all `gpt-3.5` and `gpt-4` OpenAI models. + +#### `routes[].openai_config.prompts` +##### Required: `true` +##### Type: `array` +An array of objects that define how the model should respond to input. + +#### `routes[].openai_config.prompts[].role` +##### Required: `true` +##### Type: `enum` +##### Options: [`assisstant, system, user`] +The `role` field specifies the role the model should take when responding to input. + +#### `routes[].openai_config.prompts[].content` +##### Required: `true` +##### Type: `string` +The `content` field specifies content of the prompt. + +## openai +### Example +```yaml +openai: + api_credential: ${OPENAI_KEY} +``` + +### Fields +#### `openai` +##### Required: `true` +##### Type: `string` +It contains the configuration of OpenAI API. + +#### `openai.api_credential` +##### Required: `true` +##### Type: `string` +OpenAI API credential. If `routes[].openai_config.api_credential` is specified, it will overwrite this credential in the OpenAI API call. + +:warning: **Store OpenAI key securely, and only use it as an environment variable.** + +## logger +### Example +```yaml +logger: + api: + hide_ip: true + hide_headers: true + llm: + hide_headers: true + hide_response_content: true + hide_prompt_content: true +``` + +### Fields +#### `logger` +##### Required: `false` +##### Type: `object` +It contains the configuration of the logger. + +#### `logger.api` +##### Required: `false` +##### Type: `object` +It contains the configuration for API log. Here is an example of the API log: +```json +{ + "clientIp": "", + "instanceId": "fcfc0aa8-0f57-4d24-82f9-247b68e4dcaf", + "latency": { + "proxy": 886, + "bricksllm": 3, + "total": 889 + }, + "created_at": 1690773969, + "route": { + "path": "/travel", + "protocol": "http" + }, + "response": { + "headers": { + "Access-Control-Allow-Credentials": [ + "true" + ], + "Access-Control-Allow-Origin": [ + "https://figma.com" + ] + }, + "createdAt": 1690773970, + "status": 200, + "size": 295 + }, + "request": { + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Encoding": [ + "gzip, deflate, br" + ], + "Connection": [ + "keep-alive" + ], + "Content-Length": [ + "50" + ], + "Content-Type": [ + "application/json" + ], + "Origin": [ + "https://figma.com" + ], + "Postman-Token": [ + "d05b5651-542e-4a4d-9fa1-f00d005488ad" + ], + "User-Agent": [ + "PostmanRuntime/7.32.2" + ] + }, + "size": 50 + }, + "type": "api" +} +``` + + +#### `logger.llm` +##### Required: `false` +##### Type: `object` +It contains the configuration for LLM API log. Here is an example of the LLM API log. +```json +{ + "instanceId": "fcfc0aa8-0f57-4d24-82f9-247b68e4dcaf", + "type": "llm", + "token": { + "prompt_tokens": 12, + "completion_tokens": 10, + "total": 22 + }, + "response": { + "id": "chatcmpl-7iDpqAjqdAp1AeBT6ZVRpYrgrMQ3z", + "headers": { + "Access-Control-Allow-Origin": [ + "*" + ], + "Alt-Svc": [ + "h3=\":443\"; ma=86400" + ], + "Cache-Control": [ + "no-cache, must-revalidate" + ], + "Cf-Cache-Status": [ + "DYNAMIC" + ], + "Cf-Ray": [ + "7ef2bcffd9dd173a-SJC" + ], + "Content-Type": [ + "application/json" + ], + "Date": [ + "Mon, 31 Jul 2023 03:26:10 GMT" + ], + "Openai-Model": [ + "gpt-3.5-turbo-0613" + ], + "Openai-Organization": [ + "acme" + ], + "Openai-Processing-Ms": [ + "560" + ], + "Openai-Version": [ + "2020-10-01" + ], + "Server": [ + "cloudflare" + ], + "Strict-Transport-Security": [ + "max-age=15724800; includeSubDomains" + ], + "X-Ratelimit-Limit-Requests": [ + "3500" + ], + "X-Ratelimit-Limit-Tokens": [ + "90000" + ], + "X-Ratelimit-Remaining-Requests": [ + "3499" + ], + "X-Ratelimit-Remaining-Tokens": [ + "89977" + ], + "X-Ratelimit-Reset-Requests": [ + "17ms" + ], + "X-Ratelimit-Reset-Tokens": [ + "14ms" + ], + "X-Request-Id": [ + "200f64b5d1ad324c5293dba96132b31c" + ] + }, + "created_at": 1690773970, + "size": 435, + "status": 200, + "choices": [ + { + "role": "assistant", + "content": "Hi Beijing! How can I assist you today?", + "finish_reason": "stop" + } + ] + }, + "request": { + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "assistant", + "content": "say hi to beijing" + } + ], + "size": 89, + "created_at": 1690773969 + }, + "provider": "openai", + "estimated_cost": 0.038000000000000006, + "created_at": 1690773969, + "latency": 886 +} +``` + +#### `logger.api.hide_ip` +##### Required: `false` +##### Default: `false` +##### Type: `boolean` +This field prevents logger from logging the http request ip. + +#### `logger.api.hide_headers` +##### Required: `false` +##### Default: `false` +##### Type: `boolean` +This field prevents logger from logging the http request and response headers. + + +#### `logger.llm.hide_headers` +##### Required: `false` +##### Default: `false` +##### Type: `boolean` +This field prevents logger from logging the upstream llm http request and llm response headers. + +#### `logger.llm.hide_response_content` +##### Required: `false` +##### Default: `false` +##### Type: `boolean` +This field prevents logger from logging the upstream llm http response content. + +#### `logger.llm.hide_prompt_content` +##### Required: `false` +##### Default: `false` +##### Type: `boolean` +This field prevents logger from logging the upstream llm http request prompt content. \ No newline at end of file diff --git a/assets/bricks-logo.png b/assets/bricks-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..288db1fee84ee1af828e824a0d940104c49800e6 GIT binary patch literal 7282 zcmX|GcQ{*b*bb_#Rjt~yx=5_lEJ~EB6+39{RE^lHv_{Ypzfz+mBt~P#s#T*lm9|Q$ z5wo@`rABQb_>O+x_5E?KoOeCX{oeO|p7Wk7FO3YeSQvR3K_C!|j`lqh5QxSUxD3vp z1$va}j&A`!4Bpxg{Xih5%cmC&C@q5<=%n#8(NYJMedk{S8fTpE>fZ%{DifHfNO};6 z|D4XfyNIVWtCO$<{tkS{MnO<2CkyY4c>IrR*Ls{()xtZ!W`kQnZUV{`$+z$5)q7l6 zrDJ~+F!wb}QpwRWlg{q3jIx+CJ$GW2fF`^7n?TgF9&A1NR~coTOqL<5Oq!Vv9UXxN zzjwz(X3ZWi6crZ~CxarCR#aAFR{ZCsmt)?pTW+R4 z0L3PQ-gYwS7%yY5Us28thquJ6rR-ljg#4)Y8yr`@-1f$XhX3}8fC$Y~IZDkqHexyE zQ|1HBNgocB`etp>)ugbR+@Bj>F*jS4DcvJU9hUNaLqUHl6;@Q%gH^_-iO}@bo zG`~mB`xi=(Tzwmp*J|An)h+l!f{?=|Ze$E*Z^qP6kYlps&2l?abGmTXM3CsCT zX&fwyqZ=>ZmPYxyWTcFb&0wjC7Rvpp)9JeP+T6Oac>c!-b|o+X(^_VO-LTlQlnteh z!)(6>hwW?OVGVqL_H8$JJZN0+IWAW^YItm^s90Apl*dd=$G*AvYT>yLcLqLDwl(JI zVzbmQQo;$?vFJfd*tVTQ{Fy=qWCJ}pk^kO@Xen8HRbRF@=M=PFWEw;7|4huFcb-IV z{`T5jJZ2gLZ50%9kf>?w0Xnh4=%ME1zW)#Ezb#PtqbiMpiJ!_+sHpF zg)PMHLa(Q?)TCQ#{_v(QsksgeDWMZXLQTV2JZdsf%SOsJn)0EA&HCxJni3=`xVVCs z7~R~<>Uc3wW7?5qG;YZ8BPS)>hLNvIsoM}Uneb-psJ|~gTi(??Tf!CpxcIl*x`^HS zd80QqCd(CpzRLK~lcd|Vo`sp75UsjpmsrdDVmp7s)=&7x^^IeiyZoa|D9mJWR&!~ zs3NwV(rxPR53LkA%5GSQhteQ1<{pQ<82Of;QY#+j*a&82y2RNlRLE_wIVgU-chT5;oGT>N6{0_I+KYL3AomqEz@SfDev{~24` zRX*&}yNcxPSoS=il$>MUFqRVWF#gsj8Fj;zm26yf-2G9X zlVBue0d%@>sN7x&4X_ALwj}xAAq?ubewn+xY(BzpXa;~Ckc=t{5;OH=)G(6C1# zN+&jV-G;b`NGF36ShFllf%Yg*M1B?- z?2-L**jLfYHm+|~@)q!k;#Phnc=?x3)v;y9_i2Ng$fEj}vPU7lNI@f!^^yS|gGH}w z%hPzB^YOu127}cTH2TDus-YneE0_RgsB#yk8DI$nZ1KQ`QCYeAQsx$1M=WjiQNQw zL_V}<>Yi?us{6*)?`9Sk#m9{(88b#x&#O^lA_Z}U@aWr_t$6th%a;g z3e`SEU-7@8WWFvF_Jf%(3LKr@mjC75{Gvl6O2hpD-nWin0`(Nda6JVT=MAfX;qm+2#oYHrA z=18?$i{fg(fDkYa##3FEq@o-VM7_$%L5-6Q+%&nIC$9QY9zq^^WE=|gVV2tU^FEw68P&8J z40`RtNw`xb4fpf9>e#}bQYq33Dm|nve^&;tf5~puw@{+XIF7`vr(;hw7<^izhijLF zkgswt96CAT05{Y|Kqa0}8n!&-B|hspcsECi`dRR9zTD@_n#hTG`M)5(o39kfSoT;$ z)fn@TMbF(EoP-(m@$x`hLt|`^%QKvl=T*6$gTqYUR6MO?rAU~{=avvD3H^?!eDyg*$a+pX~g=_-(ljwd&c(^$zkl758>kQy|Ced?wPmr;1lEBlAU!I{N4)4=A{6ZOBw znPpUX!}%FHZ9#_iG@iP=t~~^0Y;7@D^6=l&0HBI0uI4Ti+Zd6BJLGew?J$_usVAUl zbYYjCy73Zqb1Vj?p(XnxWe^%`dz{^w5$wJ@H}L)~m~^i?J{D|zJM*35hoStbIPCPw z{nXUd9s;h7>GD?zfKeoPPqsXF`69vA*%HM#?T*lh5BHI2^)Cmzu>+j+7KOS@a}8|e zEF}Mpnn-Z^LCNVWR1$O;$f$qn?loWg6TYaR%2T{&mWxh<;ONQ)#>HsPph zQ(Oh?J18d8Mgt594;=hf#DZ(o8nj`j7a$7@KO0%$XDB$o((e8`;BGjdRw8j;O!hwplS<_R(rS66q9~ zaf1w$`;>~$FaT0a4VcB{PJK&&)rI|3)7(gpgj6^Jk~P=pebUK4kj!Ru=L2LY+ct5R zE;zsp_p9^|aDWU7U7Qn4caLg)?3l6{>$X1jdR5r;`SArlPoHfyk+PL3dBuD}Qvx{B z?1#C-E@OH?oN-}S7*qIx@w+qGDJn6!M$H?T7+d~}#cIuUW!IT&Joj%RA`X0PbhzJT zyuDjUyEEJH2KqoH#zhi=y~7Fm1&mzG$oM1AJhZ`Srp_T6`$t!OW;+pHrTK)tc!fwC z^Rwa2MfaLZxG!P$D!jX^yn--@a_035%F;%iKkKgP;_b?D>ux%mIwfTTFYE-59-Yj- zhdGywB&K~ing7-Z?3!JJ%61{RLr2thm6^05y?pc6BN2gk9cTa?8bg6JIJ7KpkIj=3 zN;kqo*9KT6NH7O_sWpE9^$kn3O{X}2V52XJ_7=n4{6-%W&HK7#&h3@TT(685KFabd zG+BmN7fJrq#2LUfcAN=2)h4j(hzKVe3u#^xKAqbaq6e?yy;tH%#wpRcyqt**uD6G~ zOQ4xhD@4-C(t6BPs77pqh?*X_ckdThrkAy9o}kL#Tl`!u%^#M>Z81f+TCRn#7f&Pw~^4v&MY_(7% zDO&%n7xQ!5<}@Mq6J86`+#5HsD6I;QBUUu}r~R113$L@Pz^O`s@~N(4Xb>c>tk>?p zbQR$cV9vj*3WL+`(5xar?hkADdX}a_(&K{^;_HAiQ!5+LE7e#88tma0QL<&WqrR2p z&t??*zfTkXi6-fgrsJEeW8362c?=zy^;!H! zyv~_{|9oD{HV}0>%n4wwLhy=G|xBehL%_ zzHNdBUHa(Z*LI6jpCo1nqBC+dY;_(DH6roSd`{5 zMxFn?@qN2esyi$eudE3=iy+pYANtEo=i}v!L`&NkW6W3aGa_)($TBqZ3gW236D|EI zOMLE!o|BR==b4DE>i_DEhAadtHEUE=yA2Owo{_^vF_e6Of4Qo4yFb4pTtcl9-K1fP z3|%YK$Kx;FpO^6vKH_>_afv8~D=A8n3jFb7ztvS(WRRkh^@d|l?Glk*cB8%3KbL~m ziWA~;lhtZ2oJa@WQo_kbHnoW)Y93*~kMV0vaMpRU9FYsdk~2<3DwrE9F!cY)=_!5f zIsWkdR#H+H+#>0Wi0@A5)*N-UVNf-2moK&o)b)K^pacpECAZ7e5Tj?(*kNPBpebOk|$T z*H*wMt~L~l;tBUm@vY+`1^fYQ7$(!qvI}Gq;cZimv}kTW%;_#_@Nx=&B+Zq_e#qjX z?+I9jUfvgPUO>ZlcFWg|@Mj%F0;gn^yJ1OeF0$vAk?8n~j5VroEVJqWn|?yNa$SwG z%K2Q0b33%-_QSLnW9qaWd3(%=C(rid^4jyVxhvw@uV2wyi%2`U{b~+>tHbw4Lf?ZE838(hVXX>8Kl))h8wKO%`b?4?uQSY|Rcm=dqM{FYfSI3Bhr(lE z+tdBlbu^(w5qK!a?jl`$B@D*y^Ab84ceqr5AcCOsqNeyREc##!@P<84$KNQFBTh1- zt{2-I{)Bmp?sfgS;yrtetL zISqsSp!^IW4iB6Y`CkgkXFf5XZqOafqG|2<9iTF7^A*7SOGUC;-dY4?V?)7h0;YHa zPLqC~{5P3LFRx50TaK~fC6wjfvS495viR}e{`pD%MzPethkXmE@`oA30bz#1`I9mDDb>mD- z-rq*7#{}#o21?%-5R^m=br`M-qbC2+1EuelSLLj$l*+T5GG=l6ijsK#KUxLKLN2HW zZf4ClY6i2 zdAAUqzum}@kD;6eoJVRxfxhgidHn4!sBbyr4OD_zQ6XYbi}bGpwzju!Qnv>8CV^tI zxy}x(m0{jGQm8-Y`XWK)bT_+uE(wgE&mx1Kpu<$-4ndag0O}>E%JM26nt2w21>3W( z8qbdY%}kR}&$PQp1X;qp9^m8A>+gIB^rCHo5VE)Xwt?n$8f{xf7b)Zglzt^knnitt z>QEJU>L3QmA*K!t5@Kyfds~f+=I&N8N{?Dvi~Z+>aDhmasHA*mVyEmHj0?nhkvKal z3a@G)%j>KP7fi`@CkMT%3K~bVJt5tAT3{4yqAM61w6S2kZa#gd+z5Z+ zr0zO=2YQ-+9hg3Bg!l4gs600uc-t)@Mp`Xs=1UdcC`v!=A|t}#;f`PLs+hlP{j*}{dW)*U{aBE zkNg_pPOdb)`wX|Lvb?~+Vo_ZhjM|F_x--~_;Vk?li>ep^26j4o9##l@)_q3N{#*&A zrpQs7P!Qx;{L}Cz-o1U0Iv00t`ExJg$&(99TZ7Q<1kN%k#k$VNibf*5dtwz(AlH#V z4Df~ouNs5yAU*yq4&vD%zwT}V8^IuCUw2jjA}cxSOK8AdnkNo&tyi>wv~u0}3*QH- z=!MYKA@BFpnT0j)wd9w6SxV^mVHu4ezquph^%;0=R!ZRS9YSb}zdMX!cy`1pH(YK} z=DjvOyECyD{r9^aeQfP50D^|2D~_D&uduGXAoY=OD1}LYQNO_!TNGG^t&$+}IcfC1 ze`Q`f+|Cnf3LQM$ySAYwfh^vhX}Q<`r!%)~_#mKvI*__#zc0r3o{V}9ED5k4GwI3s z+}#UeYPvQqIIvi`=@U4rtV1ietebJB#AI66cG*SZ=3`}(2+Fs|=5pjAanrLdu|g_$ zuu0?qslNw+Qa}AyyhoJ{+PC?13U3PYi6Bent)4`_yOnuU+pG%Clch6tyi>J1MY}SV zsG7i~Bf~NlAx#rJ9_hYG7zWEYR&+n^vPbl+bwjL*AD&qdNjR3b33ExBO;p`<(y)Pendu}p?^-)ns8>ad(+;s=m?u?s5W z%Wm~eI34%EBFdT}1twL}@Xhi=SC%v9`1aJ##my<9V_|*Hu0&Nsw=Y#yiTcG4^-Rx- zhkqnaA8)9mw;KIWIenPzXZt!_!KKz`RjLZVtrJ3Sv<9854~Rwp_P9tCN3ajC(80&KEmcI#g4Cf(pIs(Uxkx1cCL9~*aQx!Ir zz9K-T!|M3=$D%VOcL9q;7G0CKs(+=1QZ%rWC6|l98u4%sLN8K@cc4Q@uHodK<$q9*h^A*_N>=sEK~9r@x91CGI9N2c zZBVz4i61h0%q3sYR|GY0O{H$mcAT)=RX+hBK{dU>W{vX649y&{nrhy z1;TK)r5yI09BH5rPkJ3=xd)^^P8$riYl?Awxaq#ft%}($xkRk+>}+zka4mBhuK?A? zCFODtaB1Kkxh*U>O6c>*t`48+Kz1dzoM~sD^6Bs5aP56p;;_#>kgA2dDtelNz||HH z6ZksbTAS*{yriIo%eh1p7V{aoFP1v1(fjW6X+Vp6xSW3LYG0l*&NlZ*u*N~qMWRbS z#ayyJoqE0LkwaJ8mr{l&nf#vZ6 literal 0 HcmV?d00001 diff --git a/cmd/atlas/main.go b/cmd/atlas/main.go deleted file mode 100644 index f31dc83..0000000 --- a/cmd/atlas/main.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - "os/signal" - "syscall" - "time" - - "github.com/bricks-cloud/atlas/config" - "github.com/bricks-cloud/atlas/internal/server/web" - "github.com/gin-gonic/gin" - "go.uber.org/zap" -) - -func main() { - gin.SetMode(gin.ReleaseMode) - - rawJSON := []byte(`{ - "level": "debug", - "encoding": "json", - "outputPaths": ["stdout", "/tmp/logs"], - "errorOutputPaths": ["stderr"], - "encoderConfig": { - "messageKey": "message", - "levelKey": "level", - "levelEncoder": "lowercase" - } - }`) - - var cfg zap.Config - - if err := json.Unmarshal(rawJSON, &cfg); err != nil { - panic(err) - } - - logger := zap.Must(cfg.Build()) - defer logger.Sync() - - filePath := "atlas.yaml" - - c, err := config.NewConfig(filePath) - if err != nil { - logger.Sugar().Fatalf("error parsing yaml config %s : %w", filePath, err) - } - - logger.Sugar().Infof("successfuly parsed atlas yaml config file from path: %s", filePath) - - ws, err := web.NewWebServer(c, logger.Sugar()) - if err != nil { - logger.Sugar().Fatalf("error creating http server: %w", err) - } - - ws.Run() - - quit := make(chan os.Signal) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - <-quit - - logger.Sugar().Info("shutting down server...") - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := ws.Shutdown(ctx); err != nil { - logger.Sugar().Fatalf("server shutdown: %w", err) - } - - select { - case <-ctx.Done(): - logger.Sugar().Info("timeout of 5 seconds") - } - - logger.Sugar().Info("server exited") -} diff --git a/cmd/bricksllm/main.go b/cmd/bricksllm/main.go new file mode 100644 index 0000000..37f2feb --- /dev/null +++ b/cmd/bricksllm/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "flag" + "os" + "os/signal" + "syscall" + "time" + + "github.com/bricks-cloud/bricksllm/config" + "github.com/bricks-cloud/bricksllm/internal/logger/zap" + "github.com/bricks-cloud/bricksllm/internal/server/web" + "github.com/gin-gonic/gin" +) + +func main() { + modePtr := flag.String("mode", "dev", "select the mode that bricksllm runs in") + + filePathPtr := flag.String("path", "", "enter the file path to the config file") + + flag.Parse() + + gin.SetMode(gin.ReleaseMode) + + logger := zap.NewLogger(*modePtr) + + logger.Infof("running bricksllm in %s mode", *modePtr) + + defer logger.Sync() + + if filePathPtr == nil || len(*filePathPtr) == 0 { + logger.Fatal("path is not specified") + } + + filePath := *filePathPtr + + c, err := config.NewConfig(filePath) + if err != nil { + logger.Fatalf("error parsing yaml config: %v", err) + } + + logger.Infof("successfuly parsed bricksllm yaml config file from path: %s", filePath) + + ws, err := web.NewWebServer(c, logger, *modePtr) + if err != nil { + logger.Fatalf("error creating http server: %v", err) + } + + ws.Run() + + quit := make(chan os.Signal) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + logger.Infof("shutting down server...") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := ws.Shutdown(ctx); err != nil { + logger.Fatalf("server shutdown: %v", err) + } + + select { + case <-ctx.Done(): + logger.Infof("timeout of 5 seconds") + } + + logger.Infof("server exited") +} diff --git a/config/config.go b/config/config.go index cf746a5..2550ed4 100644 --- a/config/config.go +++ b/config/config.go @@ -4,12 +4,11 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "strings" "time" - "github.com/bricks-cloud/atlas/internal/util" + "github.com/bricks-cloud/bricksllm/internal/util" "gopkg.in/yaml.v3" ) @@ -25,12 +24,84 @@ const ( Https Protocol = "https" ) +func (p Protocol) Valid() bool { + if p != Http && p != Https { + return false + } + + return true +} + type Provider string const ( - openaiProvider Provider = "openai" + OpenaiProvider Provider = "openai" ) +func (p Provider) Valid() bool { + if p != OpenaiProvider { + return false + } + + return true +} + +type ApiLoggerConfig struct { + HideIp bool `yaml:"hide_ip"` + HideHeaders bool `yaml:"hide_headers"` +} + +func (alc *ApiLoggerConfig) GetHideIp() bool { + if alc == nil { + return false + } + + return alc.HideIp +} + +func (alc *ApiLoggerConfig) GetHideHeaders() bool { + if alc == nil { + return false + } + + return alc.HideHeaders +} + +type LlmLoggerConfig struct { + HideHeaders bool `yaml:"hide_headers"` + HideResponseContent bool `yaml:"hide_response_content"` + HidePromptContent bool `yaml:"hide_prompt_content"` +} + +func (llc *LlmLoggerConfig) GetHideResponseContent() bool { + if llc == nil { + return false + } + + return llc.HideResponseContent +} + +func (llc *LlmLoggerConfig) GetHidePromptContent() bool { + if llc == nil { + return false + } + + return llc.HidePromptContent +} + +func (llc *LlmLoggerConfig) GetHideHeaders() bool { + if llc == nil { + return false + } + + return llc.HideHeaders +} + +type LoggerConfig struct { + Api *ApiLoggerConfig `yaml:"api"` + Llm *LlmLoggerConfig `yaml:"llm"` +} + type ServerConfig struct { Port int `yaml:"port"` } @@ -45,6 +116,14 @@ const ( BooleanDataType DataType = "boolean" ) +func (d DataType) Valid() bool { + if d != StringDataType && d != NumberDataType && d != ArrayDataType && d != ObjectDataType && d != BooleanDataType { + return false + } + + return true +} + type InputValue struct { DataType DataType `yaml:"type"` Properties map[string]interface{} `yaml:"properties"` @@ -97,23 +176,46 @@ type RouteConfig struct { type OpenAiMessageRole string const ( - system OpenAiMessageRole = "system" - user OpenAiMessageRole = "user" - assitant OpenAiMessageRole = "assitant" - function OpenAiMessageRole = "function" + SystemMessageRole OpenAiMessageRole = "system" + UserMessageRole OpenAiMessageRole = "user" + AssitantMessageRole OpenAiMessageRole = "assistant" + FunctionMessageRole OpenAiMessageRole = "function" ) +func (r OpenAiMessageRole) Valid() bool { + if r != SystemMessageRole && r != UserMessageRole && r != AssitantMessageRole && r != FunctionMessageRole { + return false + } + + return true +} + type OpenAiPrompt struct { - Role string `yaml:"role"` - Content string `yaml:"content"` + Role OpenAiMessageRole `yaml:"role"` + Content string `yaml:"content"` } type OpenAiModel string const ( - gpt35Turbo OpenAiModel = "gpt-3.5-turbo" + Gpt35Turbo OpenAiModel = "gpt-3.5-turbo" + Gpt35Turbo16k OpenAiModel = "gpt-3.5-turbo-16k" + Gpt35Turbo0613 OpenAiModel = "gpt-3.5-turbo-0613" + Gpt35Turbo16k0613 OpenAiModel = "gpt-3.5-turbo-16k-0613" + Gpt4 OpenAiModel = "gpt-4" + Gpt40613 OpenAiModel = "gpt-4-0613" + Gpt432k OpenAiModel = "gpt-4-32k" + Gpt432k0613 OpenAiModel = "gpt-4-32k-0613" ) +func (m OpenAiModel) Valid() bool { + if m != Gpt35Turbo && m != Gpt35Turbo16k && m != Gpt35Turbo0613 && m != Gpt35Turbo16k0613 && m != Gpt4 && m != Gpt40613 && m != Gpt432k && m != Gpt432k0613 { + return false + } + + return true +} + type OpenAiRouteConfig struct { ApiCredential string `yaml:"api_credential"` Model OpenAiModel `yaml:"model"` @@ -125,19 +227,19 @@ type OpenAiConfig struct { } type Config struct { + LoggerConfig *LoggerConfig `yaml:"logger"` Routes []*RouteConfig `yaml:"routes"` Server *ServerConfig `yaml:"server"` OpenAiConfig *OpenAiConfig `yaml:"openai"` } func NewConfig(filePath string) (*Config, error) { - yamlFile, err := ioutil.ReadFile(filePath) + yamlFile, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("unable to read config file with path %s: %w", filePath, err) } yamlFile = []byte(os.ExpandEnv(string(yamlFile))) - c := &Config{} err = yaml.Unmarshal(yamlFile, c) if err != nil { @@ -151,10 +253,27 @@ func NewConfig(filePath string) (*Config, error) { } } + // routes have to be configured if len(c.Routes) == 0 { return nil, fmt.Errorf("routes are not configured in config file %s", filePath) } + // default server port to 8080 + if c.LoggerConfig == nil { + c.LoggerConfig = &LoggerConfig{ + Api: &ApiLoggerConfig{ + HideIp: false, + HideHeaders: false, + }, + + Llm: &LlmLoggerConfig{ + HideHeaders: false, + HideResponseContent: false, + HidePromptContent: false, + }, + } + } + apiCredentialConfigured := false if c.OpenAiConfig != nil && len(c.OpenAiConfig.ApiCredential) != 0 { apiCredentialConfigured = true @@ -178,6 +297,10 @@ func parseRouteConfig(rc *RouteConfig, isOpenAiConfigured bool) error { return errors.New("provider is empty") } + if !rc.Provider.Valid() { + return errors.New("provider must be openai") + } + if rc.CorsConfig != nil { if len(rc.CorsConfig.AllowedOrgins) == 0 { return fmt.Errorf("cors config is present but allowed_origins is not specified for route: %s", rc.Path) @@ -190,14 +313,26 @@ func parseRouteConfig(rc *RouteConfig, isOpenAiConfigured bool) error { } } - if rc.Provider == openaiProvider { + if rc.Provider == OpenaiProvider { if rc.OpenAiConfig == nil { return errors.New("openai config is not provided") } + if len(rc.OpenAiConfig.Model) == 0 { + return errors.New("openai model cannot be empty") + } + + if !rc.OpenAiConfig.Model.Valid() { + return errors.New("open ai model must be of gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-3.5-turbo-0613, gpt-3.5-turbo-16k-0613, gpt-4, gpt-4-0613, gpt-4-32k and gpt-4-32k-0613") + } + for _, prompt := range rc.OpenAiConfig.Prompts { if len(prompt.Role) == 0 { - return errors.New("role is not provided in openai prompt") + return errors.New("role cannot be empty in openai prompt") + } + + if !prompt.Role.Valid() { + return errors.New("role must be of user, system, assistant or function") } if !isOpenAiConfigured && len(rc.OpenAiConfig.ApiCredential) == 0 { @@ -231,6 +366,10 @@ func parseRouteConfig(rc *RouteConfig, isOpenAiConfigured bool) error { rc.Protocol = Http } + if !rc.Protocol.Valid() { + return errors.New("protocol must be of http or https") + } + return nil } @@ -258,11 +397,15 @@ func validateInput(input map[string]InputValue, variableMap map[string]string) e for index, part := range parts { value, found := innerInput[part] if !found { - return errors.New("referenced value in prompt does not exist") + return fmt.Errorf("referenced var: %s in prompt does not exist", value) } if index != len(parts)-1 && value.DataType != ObjectDataType { - return errors.New("input value is not represented as object") + return fmt.Errorf("input value is not represented as object for referenced var: %s", reference) + } + + if value.DataType == ObjectDataType && len(value.Properties) == 0 { + return fmt.Errorf("object properties is empty for referenced var: %s", reference) } js, err := json.Marshal(value.Properties) diff --git a/atlas.yaml b/example/bricksllm.yaml similarity index 73% rename from atlas.yaml rename to example/bricksllm.yaml index 3612594..f525eef 100644 --- a/atlas.yaml +++ b/example/bricksllm.yaml @@ -1,6 +1,15 @@ openai: api_credential: ${OPENAI_KEY} +# logger: +# api: +# hide_ip: true +# hide_headers: true +# llm: +# hide_headers: true +# hide_response_content: true +# hide_prompt_content: true + routes: - path: /travel provider: openai @@ -18,7 +27,7 @@ routes: openai_config: model: gpt-3.5-turbo prompts: - - role: assitant + - role: assistant content: say hi to {{ plan.place }} - path: /test @@ -29,5 +38,5 @@ routes: openai_config: model: gpt-3.5-turbo prompts: - - role: assitant + - role: assistant content: say hi to {{ name }} diff --git a/go.mod b/go.mod index 05e811c..79ea8f2 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ -module github.com/bricks-cloud/atlas +module github.com/bricks-cloud/bricksllm go 1.19 require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/fatih/color v1.7.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect @@ -14,13 +14,14 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/go-hclog v0.14.1 // indirect github.com/hashicorp/go-plugin v1.4.10 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect - github.com/mattn/go-colorable v0.1.4 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 343c0e0..8428e9c 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -37,6 +39,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= @@ -52,8 +56,11 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= @@ -115,6 +122,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/client/openai/cost.go b/internal/client/openai/cost.go new file mode 100644 index 0000000..02eadcb --- /dev/null +++ b/internal/client/openai/cost.go @@ -0,0 +1,23 @@ +package openai + +import "strings" + +func EstimateCost(model string, promptTokens int, completionTokens int) float64 { + if strings.HasPrefix(model, "gpt-4") { + if strings.HasPrefix(model, "gpt-4-32k") { + return float64(promptTokens)*0.06 + float64(completionTokens)*0.12 + } + + return float64(promptTokens)*0.03 + float64(completionTokens)*0.06 + } + + if strings.HasPrefix(model, "gpt-3.5-turbo") { + if strings.HasPrefix(model, "gpt-3.5-turbo-16k") { + return float64(promptTokens)*0.003 + float64(completionTokens)*0.004 + } + + return float64(promptTokens)*0.0015 + float64(completionTokens)*0.002 + } + + return 0 +} diff --git a/internal/client/openai/error.go b/internal/client/openai/error.go new file mode 100644 index 0000000..1c08484 --- /dev/null +++ b/internal/client/openai/error.go @@ -0,0 +1,23 @@ +package openai + +type OpenAiError struct { + message string + errorType string + code int +} + +func NewOpenAiError(message string, errorType string, code int) *OpenAiError { + return &OpenAiError{ + message: message, + errorType: errorType, + code: code, + } +} + +func (e *OpenAiError) Error() string { + return e.message +} + +func (e *OpenAiError) StatusCode() int { + return e.code +} diff --git a/internal/client/openai/openai.go b/internal/client/openai/openai.go index 2b1e2e6..fe5fa24 100644 --- a/internal/client/openai/openai.go +++ b/internal/client/openai/openai.go @@ -5,10 +5,13 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" + "time" - "github.com/bricks-cloud/atlas/config" + "github.com/bricks-cloud/bricksllm/config" + "github.com/bricks-cloud/bricksllm/internal/logger" + "github.com/bricks-cloud/bricksllm/internal/util" ) type OpenAiClient struct { @@ -38,7 +41,7 @@ type choice struct { FinishReason string `json:"finish_reason"` } -type openAiResponse struct { +type OpenAiResponse struct { Id string `json:"id"` Object string `json:"object"` Created int64 `json:"created"` @@ -46,18 +49,39 @@ type openAiResponse struct { Usage usage `json:"usage"` } -func (c OpenAiClient) Send(rc *config.OpenAiRouteConfig, prompts []*config.OpenAiPrompt) (*openAiResponse, error) { +type OpenAiErrorResponse struct { + Error *OpenAiErrorContent `json:"error"` +} + +type OpenAiErrorContent struct { + Message string `json:"message"` + Type string `json:"type"` +} + +const ( + AuthorizationHeader string = "Authorization" + ContentTypeHeader string = "Content-Type" +) + +func (c OpenAiClient) Send(rc *config.OpenAiRouteConfig, prompts []*config.OpenAiPrompt, lm *logger.LlmMessage) (*OpenAiResponse, error) { if len(c.apiCredential) == 0 && len(rc.ApiCredential) == 0 { return nil, errors.New("openai api credentials not found") } messages := []message{} + loggerMessages := []logger.Message{} for _, prompt := range prompts { messages = append(messages, message{ Role: string(prompt.Role), Content: prompt.Content, }) + + loggerMessages = append(loggerMessages, logger.Message{ + Role: string(prompt.Role), + Content: prompt.Content, + }) } + lm.SetRequestMessages(loggerMessages) p := openAiPayload{ Model: string(rc.Model), @@ -81,19 +105,65 @@ func (c OpenAiClient) Send(rc *config.OpenAiRouteConfig, prompts []*config.OpenA selected = rc.ApiCredential } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", selected)) + req.Header.Set(ContentTypeHeader, "application/json") + req.Header.Set(AuthorizationHeader, fmt.Sprintf("Bearer %s", selected)) + + lm.SetRequestHeaders(util.FilterHeaders(req.Header, []string{ + AuthorizationHeader, + })) + lm.SetRequestSize(req.ContentLength) + lm.SetRequestCreatedAt(time.Now().Unix()) res, err := c.httpClient.Do(req) - b, err = ioutil.ReadAll(res.Body) + lm.SetResponseCreatedAt(time.Now().Unix()) + if res != nil { + lm.SetResponseStatus(res.StatusCode) + lm.SetResponseHeaders(res.Header) + } + if err != nil { - return nil, fmt.Errorf("error reading response body: %v", err) + return nil, fmt.Errorf("error sending http requests: %w", err) + } + + b, err = io.ReadAll(res.Body) + lm.SetResponseBodySize(int64(len(b))) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + + if res.StatusCode != http.StatusOK { + openAiErr := &OpenAiErrorResponse{} + err = json.Unmarshal(b, openAiErr) + if err != nil { + return nil, fmt.Errorf("error unmarshaling open ai error response : %w", err) + } + + if openAiErr.Error == nil { + return nil, fmt.Errorf("cannot parse open ai error response : %w", err) + } + + return nil, NewOpenAiError(openAiErr.Error.Message, openAiErr.Error.Type, res.StatusCode) } defer res.Body.Close() - openaiRes := &openAiResponse{} + openaiRes := &OpenAiResponse{} err = json.Unmarshal(b, openaiRes) + if err != nil { + return nil, fmt.Errorf("error unmarshaling open ai response : %w", err) + } + + lm.SetEstimatedCost(EstimateCost(string(rc.Model), openaiRes.Usage.PromptTokens, openaiRes.Usage.CompletionTokens)) + + choices := []logger.Choice{} + for _, choice := range openaiRes.Choices { + choices = append(choices, logger.Choice{ + Role: choice.Message.Role, + Content: choice.Message.Content, + FinishReason: choice.FinishReason, + }) + } + lm.SetResponseChoices(choices) return openaiRes, err } diff --git a/internal/logger/api_messag.go b/internal/logger/api_messag.go deleted file mode 100644 index 5762d7d..0000000 --- a/internal/logger/api_messag.go +++ /dev/null @@ -1,27 +0,0 @@ -package logger - -type ApiMessage struct { - ClientIp string `json:"clientIp"` - InstanceId string `json:"instanceId"` - Latency *Latency `json:"latency"` - CreatedAt int64 `json:"created_at"` -} - -type Latency struct { - Proxy int `json:"proxy"` - Atlas int `json:"atlas"` - Total int `json:"total"` -} - -type Route struct { - Path string `json:"path"` - Protocol string `json:"protocol"` -} - -type Response struct { - Headers map[string]string `json:"headers"` - Status int `json:"status"` - Type MessageType `json:"type"` - Size int `json:"size"` - Route *Route `json:"route"` -} diff --git a/internal/logger/api_message.go b/internal/logger/api_message.go new file mode 100644 index 0000000..c6e64ad --- /dev/null +++ b/internal/logger/api_message.go @@ -0,0 +1,160 @@ +package logger + +import ( + "fmt" + + "github.com/fatih/color" +) + +type ApiMessage struct { + ClientIp string `json:"clientIp"` + InstanceId string `json:"instanceId"` + Latency *Latency `json:"latency"` + CreatedAt int64 `json:"created_at"` + Route *Route `json:"route"` + Response *Response `json:"response"` + Request *Request `json:"request"` + Type MessageType `json:"type"` +} + +type Latency struct { + Proxy int64 `json:"proxy"` + BricksLlm int64 `json:"bricksllm"` + Total int64 `json:"total"` +} + +type Route struct { + Path string `json:"path"` + Protocol string `json:"protocol"` +} + +type Request struct { + Headers map[string][]string `json:"headers"` + Size int64 `json:"size"` +} + +type Response struct { + Headers map[string][]string `json:"headers"` + CreatedAt int64 `json:"createdAt"` + Status int `json:"status"` + Size int64 `json:"size"` +} + +func NewApiMessage() *ApiMessage { + return &ApiMessage{ + Latency: &Latency{}, + Route: &Route{}, + Request: &Request{}, + Response: &Response{}, + Type: ApiMessageType, + } +} + +func colorStatusCode(status int) string { + green := color.New(color.BgGreen) + red := color.New(color.BgRed) + yellow := color.New(color.BgYellow) + if status >= 500 { + return red.Sprintf(" %d ", status) + } + + if status >= 400 { + return yellow.Sprintf(" %d ", status) + } + + return green.Sprintf(" %d ", status) +} + +func (am *ApiMessage) DevLogContext() string { + result := "API | " + + if am.Response.Status != 0 { + result += (colorStatusCode(am.Response.Status) + " |") + } + + if am.Latency.Total != 0 { + result += fmt.Sprintf(" %dms |", am.Latency.Total) + } + + if len(am.Route.Path) != 0 { + result += fmt.Sprintf(" %s |", am.Route.Path) + } + + return result +} + +func (am *ApiMessage) SetBricksLlmLatency(latency int64) { + am.Latency.BricksLlm = latency +} + +func (am *ApiMessage) SetTotalLatency(latency int64) { + am.Latency.Total = latency +} + +func (am *ApiMessage) SetClientIp(ip string) { + am.ClientIp = ip +} + +func (am *ApiMessage) SetPath(path string) { + am.Route.Path = path +} + +func (am *ApiMessage) SetProtocol(protocol string) { + am.Route.Protocol = protocol +} + +func (am *ApiMessage) SetInstanceId(id string) { + am.InstanceId = id +} + +func (am *ApiMessage) SetRequestHeaders(headers map[string][]string) { + am.Request.Headers = headers +} + +func (am *ApiMessage) SetResponseHeaders(headers map[string][]string) { + am.Response.Headers = headers +} + +func (am *ApiMessage) SetCreatedAt(createdAt int64) { + am.CreatedAt = createdAt +} + +func (am *ApiMessage) SetProxyLatency(latency int64) { + am.Latency.Proxy = latency +} + +func (am *ApiMessage) SetRequestBodySize(size int64) { + am.Request.Size = size +} + +func (am *ApiMessage) SetResponseBodySize(size int64) { + am.Response.Size = size +} + +func (am *ApiMessage) SetResponseStatus(status int) { + am.Response.Status = status +} + +func (am *ApiMessage) SetResponseCreatedAt(createdAt int64) { + am.Response.CreatedAt = createdAt +} + +func (am *ApiMessage) GetProxyLatency() int64 { + return am.Latency.Proxy +} + +type apiLoggerConfig interface { + GetHideIp() bool + GetHideHeaders() bool +} + +func (am *ApiMessage) ModifyFileds(c apiLoggerConfig) { + if c.GetHideIp() { + am.ClientIp = "" + } + + if c.GetHideHeaders() { + am.Request.Headers = map[string][]string{} + am.Response.Headers = map[string][]string{} + } +} diff --git a/internal/logger/error_message.go b/internal/logger/error_message.go new file mode 100644 index 0000000..0d09bb1 --- /dev/null +++ b/internal/logger/error_message.go @@ -0,0 +1,30 @@ +package logger + +type ErrorMessage struct { + Type MessageType `json:"type"` + InstanceId string `json:"instanceId"` + Message string `json:"message"` + CreatedAt int64 `json:"createdAt"` +} + +func (em *ErrorMessage) DevLogContext() string { + return "ERROR | " +} + +func (em *ErrorMessage) SetInstanceId(instanceId string) { + em.InstanceId = instanceId +} + +func (em *ErrorMessage) SetMessage(message string) { + em.Message = message +} + +func (em *ErrorMessage) SetCreatedAt(createdAt int64) { + em.CreatedAt = createdAt +} + +func NewErrorMessage() *ErrorMessage { + return &ErrorMessage{ + Type: ErrorMessageType, + } +} diff --git a/internal/logger/llm_message.go b/internal/logger/llm_message.go index 6976753..66c405c 100644 --- a/internal/logger/llm_message.go +++ b/internal/logger/llm_message.go @@ -1,13 +1,14 @@ package logger +import "fmt" + type LlmResponse struct { - Id string `json:"id"` - Model string `json:"model"` - CreatedAt int64 `json:"created_at"` - Token Token `json:"token"` - Size int `json:"size"` - Status int `json:"status"` - Choices []*Choice `json:"choices"` + Id string `json:"id"` + Headers map[string][]string `json:"headers"` + CreatedAt int64 `json:"created_at"` + Size int64 `json:"size"` + Status int `json:"status"` + Choices []Choice `json:"choices"` } type Choice struct { @@ -16,10 +17,17 @@ type Choice struct { FinishReason string `json:"finish_reason"` } +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + type LlmRequest struct { - Headers map[string]string `json:"headers"` - Model string `json:"model"` - CreatedAt int64 `json:"created_at"` + Headers map[string][]string `json:"headers"` + Model string `json:"model"` + Messages []Message `json:"messages"` + Size int64 `json:"size"` + CreatedAt int64 `json:"created_at"` } type Token struct { @@ -31,10 +39,135 @@ type Token struct { type LlmMessage struct { InstanceId string `json:"instanceId"` Type MessageType `json:"type"` + Token *Token `json:"token"` Response *LlmResponse `json:"response"` Request *LlmRequest `json:"request"` Provider string `json:"provider"` EstimatedCost float64 `json:"estimated_cost"` CreatedAt int64 `json:"created_at"` - Latency int `json:"latency"` + Latency int64 `json:"latency"` +} + +func NewLlmMessage() *LlmMessage { + return &LlmMessage{ + Type: LlmMessageType, + Token: &Token{}, + Response: &LlmResponse{}, + Request: &LlmRequest{}, + } +} + +func (lm *LlmMessage) DevLogContext() string { + result := "LLM | " + + if lm.Response.Status != 0 { + result += (colorStatusCode(lm.Response.Status) + " |") + } + + if lm.Latency != 0 { + result += fmt.Sprintf(" %dms |", lm.Latency) + } + + if lm.Token.Total != 0 { + result += fmt.Sprintf(" %d tokens |", lm.Token.Total) + } + + return result +} + +func (lm *LlmMessage) SetResponseId(id string) { + lm.Response.Id = id +} + +func (lm *LlmMessage) SetResponseCreatedAt(createdAt int64) { + lm.Response.CreatedAt = createdAt +} + +func (lm *LlmMessage) SetResponseHeaders(headers map[string][]string) { + lm.Response.Headers = headers +} + +func (lm *LlmMessage) SetResponseBodySize(size int64) { + lm.Response.Size = size +} + +func (lm *LlmMessage) SetResponseStatus(status int) { + lm.Response.Status = status +} + +func (lm *LlmMessage) SetResponseChoices(choices []Choice) { + lm.Response.Choices = choices +} + +func (lm *LlmMessage) SetRequestHeaders(headers map[string][]string) { + lm.Request.Headers = headers +} + +func (lm *LlmMessage) SetRequestModel(model string) { + lm.Request.Model = model +} + +func (lm *LlmMessage) SetRequestSize(size int64) { + lm.Request.Size = size +} + +func (lm *LlmMessage) SetRequestCreatedAt(createdAt int64) { + lm.Request.CreatedAt = createdAt +} + +func (lm *LlmMessage) SetPromptTokens(tokens int) { + lm.Token.PromptTokens = tokens +} + +func (lm *LlmMessage) SetCompletionTokens(tokens int) { + lm.Token.CompletionTokens = tokens +} + +func (lm *LlmMessage) SetTotalTokens(tokens int) { + lm.Token.Total = tokens +} + +func (lm *LlmMessage) SetInstanceId(id string) { + lm.InstanceId = id +} + +func (lm *LlmMessage) SetProvider(provider string) { + lm.Provider = provider +} + +func (lm *LlmMessage) SetEstimatedCost(cost float64) { + lm.EstimatedCost = cost +} + +func (lm *LlmMessage) SetCreatedAt(createdAt int64) { + lm.CreatedAt = createdAt +} + +func (lm *LlmMessage) SetLatency(latency int64) { + lm.Latency = latency +} + +func (lm *LlmMessage) SetRequestMessages(messages []Message) { + lm.Request.Messages = messages +} + +type llmLoggerConfig interface { + GetHideHeaders() bool + GetHideResponseContent() bool + GetHidePromptContent() bool +} + +func (lm *LlmMessage) ModifyFileds(c llmLoggerConfig) { + if c.GetHideResponseContent() { + lm.Response.Choices = []Choice{} + } + + if c.GetHidePromptContent() { + lm.Request.Messages = []Message{} + } + + if c.GetHideHeaders() { + lm.Request.Headers = map[string][]string{} + lm.Response.Headers = map[string][]string{} + } } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 1a6e3e8..a634fb2 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -4,11 +4,18 @@ type Logger interface { Infow(msg string, keysAndValues ...interface{}) Info(args ...interface{}) Infof(template string, args ...interface{}) + Sync() error + Debug(args ...interface{}) + Debugf(template string, args ...interface{}) + Debugw(template string, args ...interface{}) + Fatalf(template string, args ...interface{}) + Fatal(args ...interface{}) } type MessageType string const ( - LlmMessageType string = "llm" - ApiMessageType string = "api" + LlmMessageType MessageType = "llm" + ApiMessageType MessageType = "api" + ErrorMessageType MessageType = "error" ) diff --git a/internal/logger/zap/zap.go b/internal/logger/zap/zap.go new file mode 100644 index 0000000..cc88c1e --- /dev/null +++ b/internal/logger/zap/zap.go @@ -0,0 +1,115 @@ +package zap + +import ( + "encoding/json" + "time" + + "github.com/bricks-cloud/bricksllm/internal/logger" + "github.com/fatih/color" + "github.com/mattn/go-colorable" + "go.uber.org/zap" + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +type prependEncoder struct { + // embed a zapcore encoder + // this makes prependEncoder implement the interface without extra work + zapcore.Encoder + cfg zapcore.EncoderConfig + // zap buffer pool + pool buffer.Pool +} + +func (e *prependEncoder) Clone() zapcore.Encoder { + return &prependEncoder{ + // cloning the encoder with the base config + Encoder: zapcore.NewConsoleEncoder(e.cfg), + pool: buffer.NewPool(), + cfg: e.cfg, + } +} + +// implementing only EncodeEntry +func (e *prependEncoder) EncodeEntry(entry zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + // new log buffer + buf := e.pool.Get() + blue := color.New(color.BgBlue) + red := color.New(color.BgRed) + + coloredPrefix := blue.Sprint("[BRICKSLLM]") + if entry.Level != zap.InfoLevel { + coloredPrefix = red.Sprint("[BRICKSLLM]") + } + + buf.AppendString(coloredPrefix) + buf.AppendString(" ") + buf.AppendString(e.toAtalasPrefix(entry.Level)) + buf.AppendString(" | ") + buf.AppendString(time.Now().Format(time.RFC3339)) + buf.AppendString(" | ") + + // calling the embedded encoder's EncodeEntry to keep the original encoding format + consolebuf, err := e.Encoder.EncodeEntry(entry, fields) + if err != nil { + return nil, err + } + + // just write the output into your own buffer + _, err = buf.Write(consolebuf.Bytes()) + if err != nil { + return nil, err + } + return buf, nil +} + +func (e *prependEncoder) toAtalasPrefix(lvl zapcore.Level) string { + switch lvl { + case zapcore.DebugLevel: + return "DEBUG" + case zapcore.InfoLevel: + return "INFO " + } + return "" +} + +func NewLogger(mode string) logger.Logger { + rawJSON := []byte(`{ + "level": "debug", + "encoding": "json", + "outputPaths": ["stdout"], + "errorOutputPaths": ["stderr"], + "encoderConfig": { + "messageKey": "message", + "levelKey": "level", + "levelEncoder": "lowercase" + } + }`) + + var cfg zap.Config + + if err := json.Unmarshal(rawJSON, &cfg); err != nil { + panic(err) + } + + if mode == "production" { + cfg.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel) + return zap.Must(cfg.Build()).Sugar() + } + + cfg.EncoderConfig.LevelKey = zapcore.OmitKey + + enc := &prependEncoder{ + Encoder: zapcore.NewConsoleEncoder(cfg.EncoderConfig), + pool: buffer.NewPool(), + cfg: cfg.EncoderConfig, + } + + zapLogger := zap.New(zapcore.NewCore( + enc, + zapcore.AddSync(colorable.NewColorableStdout()), + zapcore.DebugLevel, + )) + + return zapLogger.Sugar() +} diff --git a/internal/prompt/prompt.go b/internal/prompt/prompt.go index 0107b56..4f6ee24 100644 --- a/internal/prompt/prompt.go +++ b/internal/prompt/prompt.go @@ -5,7 +5,7 @@ import ( "regexp" "strings" - "github.com/bricks-cloud/atlas/config" + "github.com/bricks-cloud/bricksllm/config" ) type Prompt struct { diff --git a/internal/server/web/web.go b/internal/server/web/web.go index 6c9df85..8d8af98 100644 --- a/internal/server/web/web.go +++ b/internal/server/web/web.go @@ -5,17 +5,20 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "log" + "net" "net/http" "reflect" "strings" + "time" - "github.com/bricks-cloud/atlas/config" - "github.com/bricks-cloud/atlas/internal/client/openai" - "github.com/bricks-cloud/atlas/internal/logger" - "github.com/bricks-cloud/atlas/internal/util" + "github.com/bricks-cloud/bricksllm/config" + "github.com/bricks-cloud/bricksllm/internal/client/openai" + "github.com/bricks-cloud/bricksllm/internal/logger" + "github.com/bricks-cloud/bricksllm/internal/util" "github.com/gin-gonic/gin" + "github.com/google/uuid" ) type WebServer struct { @@ -23,14 +26,14 @@ type WebServer struct { openAiClinet *openai.OpenAiClient } -func NewWebServer(c *config.Config, lg logger.Logger) (*WebServer, error) { +func NewWebServer(c *config.Config, lg logger.Logger, mode string) (*WebServer, error) { port := c.Server.Port - router := gin.Default() + router := gin.New() openAiClient := openai.NewOpenAiClient(c.OpenAiConfig.ApiCredential) for _, rc := range c.Routes { - r, err := NewRoute(rc, openAiClient, lg) + r, err := NewRoute(rc, openAiClient, lg, c.LoggerConfig, mode) if err != nil { return nil, errors.New("errors with setting up the web server") } @@ -75,9 +78,25 @@ type keyAuthConfig interface { GetKey() string } +type apiLoggerConfig interface { + GetHideIp() bool + GetHideHeaders() bool +} + +type llmLoggerConfig interface { + GetHideHeaders() bool + GetHideResponseContent() bool + GetHidePromptContent() bool +} + type Route struct { + mode string logger logger.Logger + apiLoggerConfig apiLoggerConfig + llmLoggerConfig llmLoggerConfig + provider string path string + protocol string openAiRouteConfig *config.OpenAiRouteConfig openAiClient openai.OpenAiClient corsConfig corsConfig @@ -85,7 +104,7 @@ type Route struct { dataSchema reflect.Type } -func NewRoute(rc *config.RouteConfig, openAiClient openai.OpenAiClient, lg logger.Logger) (*Route, error) { +func NewRoute(rc *config.RouteConfig, openAiClient openai.OpenAiClient, lg logger.Logger, lc *config.LoggerConfig, mode string) (*Route, error) { structSchema, err := newInputStruct(rc.Input) if err != nil { return nil, err @@ -93,6 +112,11 @@ func NewRoute(rc *config.RouteConfig, openAiClient openai.OpenAiClient, lg logge return &Route{ logger: lg, + apiLoggerConfig: lc.Api, + llmLoggerConfig: lc.Llm, + mode: mode, + provider: string(rc.Provider), + protocol: string(rc.Protocol), path: rc.Path, openAiClient: openAiClient, openAiRouteConfig: rc.OpenAiConfig, @@ -102,9 +126,134 @@ func NewRoute(rc *config.RouteConfig, openAiClient openai.OpenAiClient, lg logge }, nil } +func (r *Route) newApiMessage() *logger.ApiMessage { + am := logger.NewApiMessage() + am.SetCreatedAt(time.Now().Unix()) + am.SetPath(r.path) + am.SetProtocol(r.protocol) + return am +} + +func (r *Route) newLlmMessage() *logger.LlmMessage { + lm := logger.NewLlmMessage() + lm.SetProvider(r.provider) + lm.SetCreatedAt(time.Now().Unix()) + lm.SetRequestModel(string(r.openAiRouteConfig.Model)) + return lm +} + +func newErrMessage() *logger.ErrorMessage { + return logger.NewErrorMessage() +} + +func newUuid() string { + return uuid.New().String() +} + +const ( + apiKeyHeader string = "X-Api-Key" + forwardedFor string = "X-Forwarded-For" +) + +func readUserIP(r *http.Request) string { + address := r.Header.Get(forwardedFor) + parts := strings.Split(address, ",") + ip := "" + + if len(parts) > 0 { + ip = parts[0] + } + + if len(ip) == 0 { + ip = r.RemoteAddr + } + + return ip +} + +type openAiError interface { + Error() string + StatusCode() int +} + func (r *Route) newRequestHandler() gin.HandlerFunc { return func(c *gin.Context) { - apiKey := c.Request.Header.Get("x-api-key") + am := r.newApiMessage() + lm := r.newLlmMessage() + instanceId := newUuid() + am.SetInstanceId(instanceId) + lm.SetInstanceId(instanceId) + + em := newErrMessage() + em.SetInstanceId(instanceId) + var err error + + var proxyStart time.Time + + if c.Request != nil { + ip := net.ParseIP(readUserIP(c.Request)) + if ip != nil { + am.SetClientIp(ip.String()) + } + + am.SetRequestBodySize(c.Request.ContentLength) + am.SetRequestHeaders(util.FilterHeaders(c.Request.Header, []string{ + apiKeyHeader, + forwardedFor, + })) + + } + + start := time.Now() + defer func() { + now := time.Now() + total := now.Sub(start).Milliseconds() + latency := now.Sub(proxyStart).Milliseconds() + + errExists := false + if err != nil { + em.SetCreatedAt(now.Unix()) + errExists = true + } + + am.SetTotalLatency(total) + am.SetProxyLatency(latency) + lm.SetLatency(latency) + am.SetBricksLlmLatency(total - am.GetProxyLatency()) + + am.SetResponseHeaders(c.Writer.Header()) + am.SetResponseStatus(c.Writer.Status()) + am.ModifyFileds(r.apiLoggerConfig) + lm.ModifyFileds(r.llmLoggerConfig) + + if err != nil { + em.SetMessage(err.Error()) + } + + if r.mode == "production" { + r.logger.Infow("api message", "context", am) + r.logger.Infow("llm message", "context", lm) + r.logger.Debugw("error message", "context", em) + return + } + + data, err := json.MarshalIndent(em, "", " ") + if errExists && err == nil { + r.logger.Debug(em.DevLogContext(), "\n", string(data)) + } + + data, err = json.MarshalIndent(am, "", " ") + if err == nil { + r.logger.Info(am.DevLogContext(), "\n", string(data)) + } + + data, err = json.MarshalIndent(lm, "", " ") + if err == nil { + r.logger.Info(lm.DevLogContext(), "\n", string(data)) + } + }() + + apiKey := c.Request.Header.Get(apiKeyHeader) if r.keyAuthConfig.Enabled() { if r.keyAuthConfig.GetKey() != apiKey { c.Status(http.StatusUnauthorized) @@ -112,9 +261,9 @@ func (r *Route) newRequestHandler() gin.HandlerFunc { } } - jsonData, err := ioutil.ReadAll(c.Request.Body) + jsonData, err := io.ReadAll(c.Request.Body) if err != nil { - c.IndentedJSON(http.StatusInternalServerError, err.Error()) + c.JSON(http.StatusInternalServerError, err.Error()) return } @@ -134,25 +283,48 @@ func (r *Route) newRequestHandler() gin.HandlerFunc { data := reflect.New(r.dataSchema) err = json.Unmarshal(jsonData, data.Interface()) if err != nil { - c.IndentedJSON(http.StatusInternalServerError, err.Error()) + c.JSON(http.StatusInternalServerError, err.Error()) return } prompts, err := r.populatePrompts(data) if err != nil { - c.IndentedJSON(http.StatusInternalServerError, err.Error()) + c.JSON(http.StatusInternalServerError, err.Error()) + return + } + + proxyStart = time.Now() + res, err := r.openAiClient.Send(r.openAiRouteConfig, prompts, lm) + am.SetResponseCreatedAt(time.Now().Unix()) + if err != nil { + if oae, ok := err.(openAiError); ok { + c.JSON(oae.StatusCode(), err.Error()) + return + } + + c.JSON(http.StatusInternalServerError, err.Error()) return } - res, err := r.openAiClient.Send(r.openAiRouteConfig, prompts) + lm.SetCompletionTokens(res.Usage.CompletionTokens) + lm.SetPromptTokens(res.Usage.PromptTokens) + lm.SetTotalTokens(res.Usage.TotalTokens) + lm.SetResponseId(res.Id) + + resData, err := json.Marshal(res) if err != nil { - c.IndentedJSON(http.StatusInternalServerError, err.Error()) + c.JSON(http.StatusInternalServerError, err.Error()) return } - r.logger.Infow("successful request") + size, err := c.Writer.Write(resData) + if err != nil { + c.JSON(http.StatusInternalServerError, err.Error()) + return + } - c.IndentedJSON(http.StatusOK, res) + am.SetResponseBodySize(int64(size)) + c.Status(http.StatusOK) } } @@ -269,10 +441,12 @@ func populateVariablesInPromptTemplate(propmtContent string, val reflect.Value) return "", err } - bs, err := json.Marshal(val) - - populated = strings.ReplaceAll(populated, old, string(bs)) + stringified := fmt.Sprint(val) + if len(stringified) == 0 { + return "", fmt.Errorf("input value is empty: %v", err) + } + populated = strings.ReplaceAll(populated, old, stringified) } return populated, nil @@ -304,14 +478,12 @@ func accessValueFromDataStruct(val reflect.Value, reference string) (interface{} continue } - if index != len(parts)-1 { - inner = inner.FieldByName(strings.Title(part)) - if inner.IsZero() { - return nil, fmt.Errorf("referenced data struct is empty: %s", part) - } - - continue + inner = inner.FieldByName(strings.Title(part)) + if inner.IsZero() { + return nil, fmt.Errorf("referenced data struct is empty: %s", part) } + + continue } return inner.Interface(), nil diff --git a/internal/util/util.go b/internal/util/util.go index 851b6ec..4a76fed 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -1,6 +1,7 @@ package util import ( + "net/http" "regexp" "strings" ) @@ -16,3 +17,25 @@ func GetVariableMap(str string) map[string]string { } return variables } + +func FilterHeaders(headers http.Header, filters []string) map[string][]string { + result := map[string][]string{} + for header, val := range headers { + + filtered := false + for _, filter := range filters { + if strings.ToLower(filter) == strings.ToLower(header) { + filtered = true + break + } + } + + if filtered { + continue + } + + result[header] = val + } + + return result +} diff --git a/scripts/release_notes.sh b/scripts/release_notes.sh new file mode 100644 index 0000000..02fe119 --- /dev/null +++ b/scripts/release_notes.sh @@ -0,0 +1,10 @@ +#!/bin/sh +cat CHANGELOG.md | awk ' + /^## [0-9]/ { + release++; + } + !/^## [0-9]/ { + if ( release == 1 ) print; + if ( release > 1 ) exit; + }' +echo "The full change log can be [found here](https://github.com/bricks-cloud/bricks/blob/main/CHANGELOG.md)." \ No newline at end of file