Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Image ingestion improvements #2

Merged
Merged
Prev Previous commit
Next Next commit
Fixed test script
Signed-off-by: Melanie Buehler <[email protected]>
mhbuehler committed Oct 22, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 3320b4823344726a9b7ecff9835b02ef36dd4424
20 changes: 10 additions & 10 deletions tests/dataprep/test_dataprep_multimodal_redis_langchain.sh
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ image_fn="apple.png"
function build_docker_images() {
cd $WORKPATH
echo $(pwd)
docker build --no-cache -t opea/dataprep-multimodal-redis:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/multimodal/redis/langchain/Dockerfile .
docker build --no-cache -t opea/dataprep-multimodal-redis:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg no_proxy=$no_proxy -f comps/dataprep/multimodal/redis/langchain/Dockerfile .

if [ $? -ne 0 ]; then
echo "opea/dataprep-multimodal-redis built fail"
@@ -32,14 +32,14 @@ function build_docker_images() {
function build_lvm_docker_images() {
cd $WORKPATH
echo $(pwd)
docker build --no-cache -t opea/lvm-llava:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/llava/dependency/Dockerfile .
docker build --no-cache -t opea/lvm-llava:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg no_proxy=$no_proxy -f comps/lvms/llava/dependency/Dockerfile .
if [ $? -ne 0 ]; then
echo "opea/lvm-llava built fail"
exit 1
else
echo "opea/lvm-llava built successful"
fi
docker build --no-cache -t opea/lvm-llava-svc:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/llava/Dockerfile .
docker build --no-cache -t opea/lvm-llava-svc:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg no_proxy=$no_proxy -f comps/lvms/llava/Dockerfile .
if [ $? -ne 0 ]; then
echo "opea/lvm-llava-svc built fail"
exit 1
@@ -49,9 +49,9 @@ function build_lvm_docker_images() {
}

function start_lvm_service() {
unset http_proxy
docker run -d --name="test-comps-lvm-llava" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 5029:8399 --ipc=host opea/lvm-llava:comps
docker run -d --name="test-comps-lvm-llava-svc" -e LVM_ENDPOINT=http://$ip_address:5029 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p ${LVM_PORT}:9399 --ipc=host opea/lvm-llava-svc:comps
# unset http_proxy
docker run -d --name="test-comps-lvm-llava" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -p 5029:8399 --ipc=host opea/lvm-llava:comps
docker run -d --name="test-comps-lvm-llava-svc" -e LVM_ENDPOINT=http://$ip_address:5029 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -p ${LVM_PORT}:9399 --ipc=host opea/lvm-llava-svc:comps
sleep 5m
}

@@ -69,13 +69,13 @@ function start_service() {
# start redis
echo "Starting Redis server"
REDIS_PORT=6380
docker run -d --name="test-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p $REDIS_PORT:6379 -p 8002:8001 --ipc=host redis/redis-stack:7.2.0-v9
docker run -d --name="test-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -p $REDIS_PORT:6379 -p 8002:8001 --ipc=host redis/redis-stack:7.2.0-v9

# start dataprep microservice
echo "Starting dataprep microservice"
dataprep_service_port=5013
REDIS_URL="redis://${ip_address}:${REDIS_PORT}"
docker run -d --name="test-comps-dataprep-multimodal-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME -e LVM_ENDPOINT=$LVM_ENDPOINT -p ${dataprep_service_port}:6007 --runtime=runc --ipc=host opea/dataprep-multimodal-redis:comps
docker run -d --name="test-comps-dataprep-multimodal-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e REDIS_URL=$REDIS_URL -e REDIS_HOST=$ip_address -e INDEX_NAME=$INDEX_NAME -e LVM_ENDPOINT=$LVM_ENDPOINT -p ${dataprep_service_port}:6007 --runtime=runc --ipc=host opea/dataprep-multimodal-redis:comps

sleep 1m
}
@@ -116,10 +116,10 @@ place to watch it is on BlackmagicShine.com. We're right here on the smoking
00:00:45.240 --> 00:00:47.440
tire.""" > ${transcript_fn}

#echo "Downloading Image"
echo "Downloading Image"
wget https://github.com/docarray/docarray/blob/main/tests/toydata/image-data/apple.png?raw=true -O ${image_fn}

#echo "Downloading Video"
echo "Downloading Video"
wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/WeAreGoingOnBullrun.mp4 -O ${video_fn}

}