Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bump the npm_and_yarn group across 1 directory with 3 updates #6

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+43 −32
Open

Bump the npm_and_yarn group across 1 directory with 3 updates #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fb00fee
optionally disable batch job dependency install
zendesk-klross Jul 21, 2023
8ee3245
Merge branch 'Netflix:master' into master
zendesk-klross Jul 21, 2023
ced41f1
lint
zendesk-klross Jul 21, 2023
280db55
Bump the npm_and_yarn group across 1 directory with 3 updates
dependabot[bot] Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions metaflow/metaflow_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,12 @@
# default to True.
BATCH_EMIT_TAGS = from_conf("BATCH_EMIT_TAGS", False)

# Set to False to disable default package installation in docker images during job launch.
# This allows better dependency control for your flow, but please ensure that your
# environment has all the required packages for your datastore type (any necessary
# dependencies should be explicitly installed in your custom container).
AUTO_INSTALL_DEPENDENCIES = from_conf("AUTO_INSTALL_DEPENDENCIES", True)

###
# AWS Step Functions configuration
###
Expand Down
47 changes: 26 additions & 21 deletions metaflow/metaflow_environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from metaflow.extension_support import dump_module_info
from metaflow.mflog import BASH_MFLOG
from . import R
from metaflow.metaflow_config import AUTO_INSTALL_DEPENDENCIES

version_cache = None

Expand Down Expand Up @@ -118,32 +119,36 @@ def _get_download_code_package_cmd(self, code_package_url, datastore_type):
% datastore_type
)

def _get_install_dependencies_cmd(self, datastore_type):
cmds = ["%s -m pip install requests -qqq" % self._python()]
if datastore_type == "s3":
cmds.append("%s -m pip install awscli boto3 -qqq" % self._python())
elif datastore_type == "azure":
cmds.append(
"%s -m pip install azure-identity azure-storage-blob simple-azure-blob-downloader -qqq"
% self._python()
)
elif datastore_type == "gs":
cmds.append(
"%s -m pip install google-cloud-storage google-auth simple-gcp-object-downloader -qqq"
% self._python()
)
else:
raise NotImplementedError(
"We don't know how to generate an install dependencies cmd for datastore %s"
% datastore_type
)
return " && ".join(cmds)
def _get_install_dependencies_cmd(
self, datastore_type, install_dependencies=AUTO_INSTALL_DEPENDENCIES
):
cmds = []
if install_dependencies:
cmds.append("%s -m pip install requests -qqq" % self._python())
if datastore_type == "s3":
cmds.append("%s -m pip install awscli boto3 -qqq" % self._python())
elif datastore_type == "azure":
cmds.append(
"%s -m pip install azure-identity azure-storage-blob simple-azure-blob-downloader -qqq"
% self._python()
)
elif datastore_type == "gs":
cmds.append(
"%s -m pip install google-cloud-storage google-auth simple-gcp-object-downloader -qqq"
% self._python()
)
else:
raise NotImplementedError(
"We don't know how to generate an install dependencies cmd for datastore %s"
% datastore_type
)
return cmds

def get_package_commands(self, code_package_url, datastore_type):
cmds = [
BASH_MFLOG,
"mflog 'Setting up task environment.'",
self._get_install_dependencies_cmd(datastore_type),
*self._get_install_dependencies_cmd(datastore_type),
"mkdir metaflow",
"cd metaflow",
"mkdir .metaflow", # mute local datastore creation log
Expand Down
2 changes: 1 addition & 1 deletion metaflow/plugins/cards/ui/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"cypress-svelte-unit-test": "^3.3.4",
"eslint": "^8.44.0",
"eslint-plugin-svelte3": "^3.4.0",
"postcss": "^8.4.24",
"postcss": "^8.4.31",
"rollup": "^2.63.0",
"rollup-plugin-css-only": "^3.1.0",
"rollup-plugin-livereload": "^2.0.5",
Expand Down
20 changes: 10 additions & 10 deletions metaflow/plugins/cards/ui/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1444,9 +1444,9 @@ flatted@^3.1.0:
integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==

follow-redirects@^1.14.7:
version "1.15.2"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
version "1.15.6"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==

forever-agent@~0.6.1:
version "0.6.1"
Expand Down Expand Up @@ -2541,10 +2541,10 @@ postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0:
resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514"
integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==

postcss@^8.4.24:
version "8.4.24"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.24.tgz#f714dba9b2284be3cc07dbd2fc57ee4dc972d2df"
integrity sha512-M0RzbcI0sO/XJNucsGjvWU9ERWxb/ytp1w6dKtxTKgixdtQDq4rmx/g8W1hnaheq9jgwL/oyEdH5Bc4WwJKMqg==
postcss@^8.4.31:
version "8.4.31"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d"
integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==
dependencies:
nanoid "^3.3.6"
picocolors "^1.0.0"
Expand Down Expand Up @@ -3297,9 +3297,9 @@ wrappy@1:
integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==

ws@^7.4.3:
version "7.5.9"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
version "7.5.10"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==

yallist@^4.0.0:
version "4.0.0"
Expand Down
Loading