From 5dbe7d993a4dbc07f7771142cdfac6a60283b9be Mon Sep 17 00:00:00 2001 From: Alex Aizman Date: Mon, 11 Mar 2024 17:52:24 -0400 Subject: [PATCH] project layout: move and consolidate scripts Signed-off-by: Alex Aizman --- .github/workflows/test-python.yml | 2 +- .github/workflows/test-short.yml | 2 +- .gitlab-ci.yml | 55 ++++++++++--------- Makefile | 2 +- README.md | 2 +- action.yml | 2 +- cmd/README.md | 2 +- cmd/cli/Makefile | 2 +- cmd/cli/README.md | 6 +- cmd/cli/cli/app.go | 2 +- deploy/README.md | 2 +- deploy/dev/local/README.md | 2 +- deploy/dev/local/deploy.sh | 5 +- docs/cli.md | 6 +- docs/cli/help.md | 4 +- docs/development.md | 12 ++-- docs/environment-vars.md | 2 +- docs/getting_started.md | 2 +- docs/index.md | 2 +- docs/s3inventory.md | 6 +- docs/tools.md | 2 +- {deploy/scripts => scripts}/README.md | 0 {deploy/scripts => scripts}/bench.sh | 0 {deploy/scripts => scripts}/bootstrap.sh | 13 +++-- {deploy/scripts => scripts}/clean_deploy.sh | 2 +- {deploy/scripts => scripts}/config/.pylintrc | 0 .../scripts => scripts}/config/.style.yapf | 0 {deploy/scripts => scripts}/github_release.sh | 0 .../install_from_binaries.sh | 0 scripts/s3/README.md | 3 + .../s3/delete-bucket-inventory.sh | 0 .../s3/get-bucket-inventory.sh | 0 .../s3/list-bucket-inventory.sh | 0 .../s3/put-bucket-inventory.sh | 0 .../s3/put-bucket-policy.sh | 0 {deploy/scripts => scripts}/utils.sh | 0 36 files changed, 75 insertions(+), 65 deletions(-) rename {deploy/scripts => scripts}/README.md (100%) rename {deploy/scripts => scripts}/bench.sh (100%) rename {deploy/scripts => scripts}/bootstrap.sh (91%) rename {deploy/scripts => scripts}/clean_deploy.sh (97%) rename {deploy/scripts => scripts}/config/.pylintrc (100%) rename {deploy/scripts => scripts}/config/.style.yapf (100%) rename {deploy/scripts => scripts}/github_release.sh (100%) rename {deploy/scripts => scripts}/install_from_binaries.sh (100%) create mode 100644 scripts/s3/README.md rename {deploy/scripts => scripts}/s3/delete-bucket-inventory.sh (100%) rename {deploy/scripts => scripts}/s3/get-bucket-inventory.sh (100%) rename {deploy/scripts => scripts}/s3/list-bucket-inventory.sh (100%) rename {deploy/scripts => scripts}/s3/put-bucket-inventory.sh (100%) rename {deploy/scripts => scripts}/s3/put-bucket-policy.sh (100%) rename {deploy/scripts => scripts}/utils.sh (100%) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 28cf17bb566..81f69c62f96 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -32,7 +32,7 @@ jobs: - name: Run AIStore run: | export GOPATH="$(go env GOPATH)" - deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 1 + scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 1 - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/test-short.yml b/.github/workflows/test-short.yml index 38507323d42..0cbb4399fa7 100644 --- a/.github/workflows/test-short.yml +++ b/.github/workflows/test-short.yml @@ -33,5 +33,5 @@ jobs: - name: Run and test AIStore run: | export GOPATH="$(go env GOPATH)" - deploy/scripts/clean_deploy.sh --target-cnt 3 --proxy-cnt 1 --mountpath-cnt 2 + scripts/clean_deploy.sh --target-cnt 3 --proxy-cnt 1 --mountpath-cnt 2 NUM_PROXY=1 NUM_TARGET=3 BUCKET="ais://test" make test-short diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3afa6f137c1..d3ae9f38c1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,8 @@ include: file: /templates/static-analysis/Checkmarx-main-csv.gitlab-ci.yml variables: - MODE: debug # run tests with debug asserts + MODE: debug # run aistore and tests with debug asserts enabled + SCRIPTS_DIR: "./scripts" NUM_TARGET: value: "5" description: "Number of targets deployed." @@ -101,7 +102,7 @@ variables: stage: test-long tags: - ais - timeout: 4h ## NOTE: when changing, make sure to update deploy/scripts/bootstrap.sh and GitLab /settings/ci_cd + timeout: 4h ## NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "web"' @@ -113,7 +114,7 @@ variables: stage: test-long tags: - ais - timeout: 4h ## NOTE: when changing, make sure to update deploy/scripts/bootstrap.sh and GitLab /settings/ci_cd + timeout: 4h ## NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"' when: manual @@ -157,7 +158,7 @@ test:short: variables: BUCKET: "ais://test" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all - make test-short - FLAGS="--duration=10s" make test-aisloader @@ -168,7 +169,7 @@ test:short:python: AIS_ENDPOINT: "http://localhost:8080" BUCKET: "aws://ais-ci-python" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws - cd python - make python_sdk_tests - make python_botocore_tests @@ -183,7 +184,7 @@ test:short:python-etl: tags: - ais-k8s script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all - cd python - make python_etl_tests except: @@ -205,7 +206,7 @@ test:short:minimal: variables: BUCKET: "ais://test" script: - - deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all - NUM_PROXY=1 NUM_TARGET=1 make test-short test:short:s3-compat: @@ -213,7 +214,7 @@ test:short:s3-compat: tags: - ais script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all - ais config cluster features Provide-S3-API-via-Root - ais config cluster checksum.type=md5 - cd python/tests/s3compat @@ -234,7 +235,7 @@ test:short:authn: BUCKET: "ais://test" RE: "TestAuth" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT - ais auth login $AIS_AUTHN_SU_NAME -p $AIS_AUTHN_SU_PASS - make test-run - ais auth logout @@ -250,7 +251,7 @@ test:short:https: BUCKET: "ais://ais-ci" script: - openssl req -x509 -out $AIS_SERVER_CRT -keyout $AIS_SERVER_KEY -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <( printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https - make test-short test:short:s3rproxy: @@ -260,7 +261,7 @@ test:short:s3rproxy: TAGS: "s3rproxy" RE: "S3" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT - make test-short test:short:aws: @@ -268,7 +269,7 @@ test:short:aws: variables: BUCKET: "aws://ais-cicd" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws - make test-short test:short:gcp: @@ -278,7 +279,7 @@ test:short:gcp: BUCKET: "gs://ais-ci" script: - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - make test-short test:short:azure: @@ -286,7 +287,7 @@ test:short:azure: variables: BUCKET: "az://ais-ci" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure - make test-short test:long: @@ -295,7 +296,7 @@ test:long: NUM_PROXY: 6 BUCKET: "ais://ais-ci" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT - make test-long test:long:aws: @@ -304,7 +305,7 @@ test:long:aws: NUM_PROXY: 6 BUCKET: "aws://ais-cicd" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws - make test-long test:long:gcp: @@ -315,7 +316,7 @@ test:long:gcp: BUCKET: "gs://ais-ci" script: - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - make test-long test:long:azure: @@ -324,7 +325,7 @@ test:long:azure: NUM_PROXY: 6 BUCKET: "az://ais-ci" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure - make test-long # test:long:hdfs: @@ -335,7 +336,7 @@ test:long:azure: # BUCKET: "hdfs://ais-ci" # script: # - bash deploy/ci/setup_hdfs.sh -# - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --hdfs +# - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --hdfs # - ais bucket create "${BUCKET}" --props="extra.hdfs.ref_directory=/" # - make test-long @@ -347,7 +348,7 @@ test:long:aisloader: variables: AIS_ENDPOINT: "http://localhost:8080" script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT - sleep 10 # make sure that cluster properly starts - FLAGS="--duration=5m" make test-aisloader - cd ./python; make PYAISLOADER_TEST_TYPE=long test-pyaisloader @@ -396,7 +397,7 @@ test:short:k8s: - export NUM_TARGET=3 - export NUM_PROXY=3 - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - make test-short - status=$? make kill && exit $status @@ -419,7 +420,7 @@ test:short:assorted:k8s: - export NUM_TARGET=3 - export NUM_PROXY=3 - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - make test-run - status=$? make kill && exit $status <<: *gather_logs_def @@ -427,7 +428,7 @@ test:short:assorted:k8s: test:long:k8s: stage: test-long <<: *test_k8s_long_def - timeout: 4h ## NOTE: when changing, make sure to update deploy/scripts/bootstrap.sh and GitLab /settings/ci_cd + timeout: 4h ## NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/' @@ -437,7 +438,7 @@ test:long:k8s: script: - export NUM_TARGET=5 - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - make test-run - status=$? make kill && exit $status <<: *k8s_long_exit_def @@ -453,7 +454,7 @@ test:long:k8s:single-target: allow_failure: true script: - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp - status=0 - make test-run - status=$? make kill && exit $status @@ -469,7 +470,7 @@ test:long:k8s:aisloader: when: manual allow_failure: true script: - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT - sleep 10 # Give some time for the cluster to stabilize. - BUCKET="ais://test" FLAGS="--duration=2m --etl" make test-aisloader - status=$? make kill && exit $status @@ -493,7 +494,7 @@ test:long:k8s:all: - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus script: - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}" - - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --aws + - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --aws - | echo "----- RUNNING K8S TESTS -----" BUCKET="aws://ais-blossom" RE="TestETL|TestConfig|TestMountpath" make test-run diff --git a/Makefile b/Makefile index 36fc1f66e7c..33cd6ffb5e0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # SHELL := /bin/bash DEPLOY_DIR = ./deploy/dev/local -SCRIPTS_DIR = ./deploy/scripts +SCRIPTS_DIR = ./scripts BUILD_DIR = ./cmd BUILD_SRC = $(BUILD_DIR)/aisnode/main.go diff --git a/README.md b/README.md index 34355096ed7..f3f8a3dbf64 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ But there's more. In v3.22, we introduce [blob downloader](docs/blob_downloader. Generally, AIStore (cluster) requires at least some sort of [deployment](/deploy#contents) procedure. There are standalone binaries, though, that can be [built](Makefile) from source or, alternatively, installed directly from GitHub: ```console -$ ./deploy/scripts/install_from_binaries.sh --help +$ ./scripts/install_from_binaries.sh --help ``` The script installs [aisloader](/docs/aisloader.md) and [CLI](/docs/cli.md) from the most recent, or the previous, GitHub [release](https://github.com/NVIDIA/aistore/releases). For CLI, it'll also enable auto-completions (which is strongly recommended). diff --git a/action.yml b/action.yml index 1806969ee15..0f44b34ab8b 100644 --- a/action.yml +++ b/action.yml @@ -37,5 +37,5 @@ runs: export GOPATH="$(go env GOPATH)" git clone https://github.com/NVIDIA/aistore.git cd aistore - ./deploy/scripts/clean_deploy.sh --target-cnt ${{ inputs.targets }} --proxy-cnt ${{ inputs.proxies }} --mountpath-cnt ${{ inputs.mountpaths }} + ./scripts/clean_deploy.sh --target-cnt ${{ inputs.targets }} --proxy-cnt ${{ inputs.proxies }} --mountpath-cnt ${{ inputs.mountpaths }} make cli diff --git a/cmd/README.md b/cmd/README.md index 2b16f1cb5d2..d3d59b4e9d3 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -27,7 +27,7 @@ Standalone binaries, on the other hand, can be [built](Makefile) from source or In particular: ```console -$ ./deploy/scripts/install_from_binaries.sh --help +$ ./scripts/install_from_binaries.sh --help NAME: install_from_binaries.sh - install 'ais' (CLI) and 'aisloader' from release binaries diff --git a/cmd/cli/Makefile b/cmd/cli/Makefile index a79d5dbb844..5dc28f87589 100644 --- a/cmd/cli/Makefile +++ b/cmd/cli/Makefile @@ -1,5 +1,5 @@ AISTORE_PATH = $(shell git rev-parse --show-toplevel) -SCRIPTS_DIR = $(AISTORE_PATH)/deploy/scripts +SCRIPTS_DIR = $(AISTORE_PATH)/scripts # Do not print enter/leave directory when doing 'make -C DIR ' MAKEFLAGS += --no-print-directory diff --git a/cmd/cli/README.md b/cmd/cli/README.md index 90d112c7abf..3f8fd8d2ae8 100644 --- a/cmd/cli/README.md +++ b/cmd/cli/README.md @@ -15,15 +15,15 @@ $ make cli-autocompletions # 2. install CLI autocompletions (Bash and/or Zsh) Alternatively, install directly from GitHub: -* [Install CLI from release binaries](https://github.com/NVIDIA/aistore/blob/main/deploy/scripts/install_from_binaries.sh) +* [Install CLI from release binaries](https://github.com/NVIDIA/aistore/blob/main/scripts/install_from_binaries.sh) For example, the following command extracts CLI binary to the specified destination and, secondly, installs Bash autocompletions: ```console -$ ./deploy/scripts/install_from_binaries.sh --dstdir /tmp/www --completions +$ ./scripts/install_from_binaries.sh --dstdir /tmp/www --completions ``` -For more usage options, run: `./deploy/scripts/install_from_binaries.sh --help` +For more usage options, run: `./scripts/install_from_binaries.sh --help` You can also install Bash and/or Zsh autocompletions separately at any (later) time: diff --git a/cmd/cli/cli/app.go b/cmd/cli/cli/app.go index 9166306a5bd..e583f2dc222 100644 --- a/cmd/cli/cli/app.go +++ b/cmd/cli/cli/app.go @@ -32,7 +32,7 @@ const ( cliDescr = `If completion doesn't work: * download ` + cmn.GitHubHome + `/tree/main/cmd/cli/autocomplete * run 'cmd/cli/autocomplete/install.sh' - To install CLI directly from GitHub: ` + cmn.GitHubHome + `/blob/main/deploy/scripts/install_from_binaries.sh` + To install CLI directly from GitHub: ` + cmn.GitHubHome + `/blob/main/scripts/install_from_binaries.sh` // custom cli.AppHelpTemplate // "You can render custom help text by setting this variable." (from github.com/urfave/cli) diff --git a/deploy/README.md b/deploy/README.md index 0620295c424..a15596cc8c1 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -2,7 +2,7 @@ |Folder | Intended for | When and how to use | Documentation | | --- | --- | --- | --- | -| [deploy/dev/local](/deploy/dev/local) | developers | use it for non-containerized development on your workstation, laptop, VM, etc. | run `make help` to see usage examples and supported options; visit [deploy/scripts](/deploy/scripts) for alternative scripted ways to run cluster locally | +| [deploy/dev/local](/deploy/dev/local) | developers | use it for non-containerized development on your workstation, laptop, VM, etc. | run `make help` to see usage examples and supported options; visit [scripts](/scripts) for alternative scripted ways to run cluster locally | | [deploy/dev/ansible](/deploy/dev/ansible) | developers | use it for non-containerized development and testing on multiple nodes | [readme](/deploy/dev/ansible/README.md) | | **Docker** | --- | --- | --- | | [deploy/dev/docker](/deploy/dev/docker) | developers | run AIS cluster consisting of one or more `aisnode` containers; use `AIS_ENDPOINT` to test and/or develop with it | [readme](/deploy/dev/docker/README.md) | diff --git a/deploy/dev/local/README.md b/deploy/dev/local/README.md index 457cb7e7656..922f2aa2206 100644 --- a/deploy/dev/local/README.md +++ b/deploy/dev/local/README.md @@ -52,7 +52,7 @@ Note that we are still running everything locally. Therefore: remove the previously generated configuration (if any) and redeploy from scratch a minimal cluster consisting of a single gateway and a single target: ```console -$ deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 0 --deployment local --remote-alias remais --gcp --aws +$ scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 0 --deployment local --remote-alias remais --gcp --aws # or, same: $ make kill clean cli deploy <<< $'1\n1\n0\ny\ny\nn\nn\n0\n' diff --git a/deploy/dev/local/deploy.sh b/deploy/dev/local/deploy.sh index c21acba2a48..20d8ea09d8a 100755 --- a/deploy/dev/local/deploy.sh +++ b/deploy/dev/local/deploy.sh @@ -51,8 +51,11 @@ else fi fi -AISTORE_PATH=$(cd "$(dirname "$0")/../../../"; pwd -P) # absolute path to aistore directory +## NOTE: absolute path to aistore root +## (should we use `git rev-parse --show-toplevel` instead?) +AISTORE_PATH=$(cd "$(dirname "$0")/../../../"; pwd -P) source $AISTORE_PATH/deploy/dev/utils.sh + AIS_USE_HTTPS=${AIS_USE_HTTPS:-false} AIS_HTTP_CHUNKED_TRANSFER=true HTTP_WRITE_BUFFER_SIZE=65536 diff --git a/docs/cli.md b/docs/cli.md index 03d666c3236..f75b63767d0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -76,15 +76,15 @@ $ MODE=debug make cli Alternatively, install directly from GitHub: -* [Install CLI from release binaries](https://github.com/NVIDIA/aistore/blob/main/deploy/scripts/install_from_binaries.sh) +* [Install CLI from release binaries](https://github.com/NVIDIA/aistore/blob/main/scripts/install_from_binaries.sh) For example, the following command extracts CLI binary to the specified destination and, secondly, installs `bash` autocompletions: ```console -$ ./deploy/scripts/install_from_binaries.sh --dstdir /tmp/www --completions +$ ./scripts/install_from_binaries.sh --dstdir /tmp/www --completions ``` -For more usage options, run: `./deploy/scripts/install_from_binaries.sh --help` +For more usage options, run: `./scripts/install_from_binaries.sh --help` You can also install `bash` and/or `zsh` autocompletions separately at any (later) time: diff --git a/docs/cli/help.md b/docs/cli/help.md index a5828ed805f..ee0fc749984 100644 --- a/docs/cli/help.md +++ b/docs/cli/help.md @@ -14,7 +14,7 @@ This readme is a loose assortment of quick tips. The default destination is /usr/local/bin but here we install into /tmp/www ```console -$ deploy/scripts/install_from_binaries.sh --dstdir /tmp/www +$ scripts/install_from_binaries.sh --dstdir /tmp/www Installing aisloader => /tmp/www/aisloader % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed @@ -74,7 +74,7 @@ DESCRIPTION: If completion doesn't work: * download https://github.com/NVIDIA/aistore/tree/main/cmd/cli/autocomplete * run 'cmd/cli/autocomplete/install.sh' - To install CLI directly from GitHub: https://github.com/NVIDIA/aistore/blob/main/deploy/scripts/install_from_binaries.sh + To install CLI directly from GitHub: https://github.com/NVIDIA/aistore/blob/main/scripts/install_from_binaries.sh COMMANDS: bucket create/destroy buckets, list bucket's contents, show existing buckets and their properties diff --git a/docs/development.md b/docs/development.md index 19b758ce814..1e4c9b8e6f6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -91,7 +91,7 @@ Performs cleanup and then deploys a new instance of an AIS cluster. To make it even more convenient, consider setting up an alias: ```bash -alias cais="bash ${GOPATH}/src/github.com/NVIDIA/aistore/deploy/scripts/clean-deploy --aws --gcp" +alias cais="bash ${GOPATH}/src/github.com/NVIDIA/aistore/scripts/clean-deploy --aws --gcp" ``` #### Example: minimal remote cluster @@ -107,13 +107,13 @@ The script will not only deploy the two clusters - it will also assign the remot and attach one cluster to another, thus forming a [global namespace](providers.md#remote-ais-cluster). ```console -$ deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 4 --deployment all --remote-alias remais +$ scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt 4 --deployment all --remote-alias remais ``` Here's another example that illustrates multi-node (6 + 6) cluster with storage targets utilizing loopback devices to simulate actual non-shared storage disks (one disk per target mountpath): ```console -$ deploy/scripts/clean_deploy.sh --target-cnt 6 --proxy-cnt 6 --mountpath-cnt 4 --deployment all --loopback 123M --remote-alias remais --gcp --aws +$ scripts/clean_deploy.sh --target-cnt 6 --proxy-cnt 6 --mountpath-cnt 4 --deployment all --loopback 123M --remote-alias remais --gcp --aws ``` > Overall, this line above will create 4 loopbacks of total size 123M * 4 = 0.5GiB. It'll take maybe a few extra seconds but only at the very first run - subsequent cluster restarts will utilize already provisioned devices and other persistent configuration. @@ -145,7 +145,7 @@ The command below starts a cluster with 5 proxies and 5 targets with GCP cloud e Remember to set `GOOGLE_APPLICATION_CREDENTIALS` env when using GCP cloud! ```console -$ bash ./deploy/scripts/clean-deploy --gcp +$ bash ./scripts/clean-deploy --gcp ``` The example below deploys: @@ -155,7 +155,7 @@ The example below deploys: - with AWS support ```console -$ bash ./deploy/scripts/clean-deploy --deployment all --remote-alias remoteAIS --target-cnt 3 --proxy-cnt 3 --aws +$ bash ./scripts/clean-deploy --deployment all --remote-alias remoteAIS --target-cnt 3 --proxy-cnt 3 --aws ``` #### Options @@ -196,7 +196,7 @@ It also can quickly provide an answer if the change that was made actually impro The command below will compare the benchmark(s) `BenchmarkRandom*` between the current commit and `f9a1536f...`. ```console -$ bash ./deploy/scripts/bootstrap.sh bench cmp f9a1536f4c9af0d1ac84c200e68f2ba73676c487 --dir bench/tools/aisloader --bench BenchmarkRandom +$ bash ./scripts/bootstrap.sh bench cmp f9a1536f4c9af0d1ac84c200e68f2ba73676c487 --dir bench/tools/aisloader --bench BenchmarkRandom ``` #### Options diff --git a/docs/environment-vars.md b/docs/environment-vars.md index c211d7fd031..5f6d409f87e 100644 --- a/docs/environment-vars.md +++ b/docs/environment-vars.md @@ -107,7 +107,7 @@ See also: | `NUM_PROXY` | (ditto) | See also: -* [deploy/scripts/clean_deploy.sh](https://github.com/NVIDIA/aistore/blob/main/deploy/scripts/clean_deploy.sh) +* [scripts/clean_deploy.sh](https://github.com/NVIDIA/aistore/blob/main/scripts/clean_deploy.sh) * [wait-for-cluster](https://github.com/NVIDIA/aistore/blob/main/ais/test/main_test.go#L47-L56) ## Kubernetes diff --git a/docs/getting_started.md b/docs/getting_started.md index c3e5745542d..ee20f5e3883 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -137,7 +137,7 @@ $ cd $GOPATH/src/github.com/NVIDIA $ git clone https://github.com/NVIDIA/aistore.git $ cd aistore # Optionally, run `make mod-tidy` to preload dependencies -$ ./deploy/scripts/clean_deploy.sh +$ ./scripts/clean_deploy.sh $ ais show cluster ``` diff --git a/docs/index.md b/docs/index.md index 31d04d42668..7bfe268c609 100644 --- a/docs/index.md +++ b/docs/index.md @@ -91,7 +91,7 @@ But there's more. In v3.22, we introduce [blob downloader](docs/blob_downloader. Generally, AIStore (cluster) requires at least some sort of [deployment](/deploy#contents) procedure. There are standalone binaries, though, that can be [built](Makefile) from source or, alternatively, installed directly from GitHub: ```console -$ ./deploy/scripts/install_from_binaries.sh --help +$ ./scripts/install_from_binaries.sh --help ``` The script installs [aisloader](/docs/aisloader.md) and [CLI](/docs/cli.md) from the most recent, or the previous, GitHub [release](https://github.com/NVIDIA/aistore/releases). For CLI, it'll also enable auto-completions (which is strongly recommended). diff --git a/docs/s3inventory.md b/docs/s3inventory.md index 122758f74e4..689014e6507 100644 --- a/docs/s3inventory.md +++ b/docs/s3inventory.md @@ -31,7 +31,9 @@ Notwithstanding CLI examples above, the feature (as always) is provided via AISt ## Managing inventories -As of Q1 2024, the operations to enable, list, disable inventories are _scripted_. The scripts themselves can be located in [`deploy/dev/scripts/s3`](https://github.com/NVIDIA/aistore/tree/main/deploy/dev/s3), and include: +As of Q1 2024, the operations to enable, list, disable inventories are _scripted_. The scripts themselves can be found in directory [`scripts/s3`](https://github.com/NVIDIA/aistore/tree/main/scripts/s3). + +They include: | script | description | | --- | --- | @@ -58,7 +60,7 @@ ID PREFIX FREQUENCY Example of `get` (more detailed): ``` -$ ./deploy/scripts/aws/get-bucket-inventory.sh -b ais-vm -n 1234 +$ ./scripts/aws/get-bucket-inventory.sh -b ais-vm -n 1234 ID 1234 Prefix inv-all Frequency Daily diff --git a/docs/tools.md b/docs/tools.md index 46dcb7ad3be..710e5ca5c09 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -37,7 +37,7 @@ Standalone binaries, on the other hand, can be [built](Makefile) from source or In particular: ```console -$ ./deploy/scripts/install_from_binaries.sh --help +$ ./scripts/install_from_binaries.sh --help NAME: install_from_binaries.sh - install 'ais' (CLI) and 'aisloader' from release binaries diff --git a/deploy/scripts/README.md b/scripts/README.md similarity index 100% rename from deploy/scripts/README.md rename to scripts/README.md diff --git a/deploy/scripts/bench.sh b/scripts/bench.sh similarity index 100% rename from deploy/scripts/bench.sh rename to scripts/bench.sh diff --git a/deploy/scripts/bootstrap.sh b/scripts/bootstrap.sh similarity index 91% rename from deploy/scripts/bootstrap.sh rename to scripts/bootstrap.sh index d46a5298d8e..92ae10effa1 100644 --- a/deploy/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -37,11 +37,12 @@ run_tests() { fi } -AISTORE_PATH="$(cd "$(dirname "$0")/../../"; pwd -P)" +AISTORE_PATH="$(cd "$(dirname "$0")/../"; pwd -P)" ## NOTE: this assumes `bootstrap.sh` itself is one level below +SCRIPTS_DIR=${AISTORE_PATH}/scripts PYLINT_STYLE="$(dirname ${0})/config/.pylintrc" EXTERNAL_SRC_REGEX=".*\(venv\|build\|3rdparty\|dist\|.idea\|.vscode\)/.*" # This script is used by Makefile to run commands. -source ${AISTORE_PATH}/deploy/scripts/utils.sh +source ${SCRIPTS_DIR}/utils.sh case $1 in lint) @@ -153,18 +154,18 @@ test-docker) test-bench) echo "Running benchmark tests..." >&2 - SCRIPT_ROOT=${AISTORE_PATH}/deploy/scripts - . ${SCRIPT_ROOT}/bench.sh cmp --dir "${AISTORE_PATH}" --verbose --post-checkout "${SCRIPT_ROOT}/clean_deploy.sh --dir ${AISTORE_PATH}" + SCRIPTS_DIR=${AISTORE_PATH}/scripts + . ${SCRIPTS_DIR}/bench.sh cmp --dir "${AISTORE_PATH}" --verbose --post-checkout "${SCRIPTS_DIR}/clean_deploy.sh --dir ${AISTORE_PATH}" ;; bench) shift - . ${AISTORE_PATH}/deploy/scripts/bench.sh + . ${SCRIPTS_DIR}/bench.sh ;; clean-deploy) shift - . ${AISTORE_PATH}/deploy/scripts/clean_deploy.sh + . ${SCRIPTS_DIR}/clean_deploy.sh ;; dev-init) diff --git a/deploy/scripts/clean_deploy.sh b/scripts/clean_deploy.sh similarity index 97% rename from deploy/scripts/clean_deploy.sh rename to scripts/clean_deploy.sh index b99b66dc734..c589edc23a0 100755 --- a/deploy/scripts/clean_deploy.sh +++ b/scripts/clean_deploy.sh @@ -20,7 +20,7 @@ function retry { done } -root_dir="$(cd "$(dirname "$0")/../../"; pwd -P)" +root_dir="$(cd "$(dirname "$0")/../"; pwd -P)" ## NOTE: this assumes `clean_deploy.sh` itself is one level below # Default values aws_provider="n" diff --git a/deploy/scripts/config/.pylintrc b/scripts/config/.pylintrc similarity index 100% rename from deploy/scripts/config/.pylintrc rename to scripts/config/.pylintrc diff --git a/deploy/scripts/config/.style.yapf b/scripts/config/.style.yapf similarity index 100% rename from deploy/scripts/config/.style.yapf rename to scripts/config/.style.yapf diff --git a/deploy/scripts/github_release.sh b/scripts/github_release.sh similarity index 100% rename from deploy/scripts/github_release.sh rename to scripts/github_release.sh diff --git a/deploy/scripts/install_from_binaries.sh b/scripts/install_from_binaries.sh similarity index 100% rename from deploy/scripts/install_from_binaries.sh rename to scripts/install_from_binaries.sh diff --git a/scripts/s3/README.md b/scripts/s3/README.md new file mode 100644 index 00000000000..7e465e8b051 --- /dev/null +++ b/scripts/s3/README.md @@ -0,0 +1,3 @@ +This folder contains assorted scripts to manage S3 bucket inventory. + +For details and background, please see [S3 bucket inventory](/docs/s3inventory.md) document. diff --git a/deploy/scripts/s3/delete-bucket-inventory.sh b/scripts/s3/delete-bucket-inventory.sh similarity index 100% rename from deploy/scripts/s3/delete-bucket-inventory.sh rename to scripts/s3/delete-bucket-inventory.sh diff --git a/deploy/scripts/s3/get-bucket-inventory.sh b/scripts/s3/get-bucket-inventory.sh similarity index 100% rename from deploy/scripts/s3/get-bucket-inventory.sh rename to scripts/s3/get-bucket-inventory.sh diff --git a/deploy/scripts/s3/list-bucket-inventory.sh b/scripts/s3/list-bucket-inventory.sh similarity index 100% rename from deploy/scripts/s3/list-bucket-inventory.sh rename to scripts/s3/list-bucket-inventory.sh diff --git a/deploy/scripts/s3/put-bucket-inventory.sh b/scripts/s3/put-bucket-inventory.sh similarity index 100% rename from deploy/scripts/s3/put-bucket-inventory.sh rename to scripts/s3/put-bucket-inventory.sh diff --git a/deploy/scripts/s3/put-bucket-policy.sh b/scripts/s3/put-bucket-policy.sh similarity index 100% rename from deploy/scripts/s3/put-bucket-policy.sh rename to scripts/s3/put-bucket-policy.sh diff --git a/deploy/scripts/utils.sh b/scripts/utils.sh similarity index 100% rename from deploy/scripts/utils.sh rename to scripts/utils.sh