From 3e4b0d2112d457cc8aae323d56eea5c28dd9f5b4 Mon Sep 17 00:00:00 2001 From: Luke Baumann Date: Tue, 4 Feb 2025 11:06:36 -0800 Subject: [PATCH 1/8] Update the pod_name query for pathways workloads (#362) --- src/xpk/core/pathways.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/xpk/core/pathways.py b/src/xpk/core/pathways.py index 71ed27ea1..6d7b9735c 100644 --- a/src/xpk/core/pathways.py +++ b/src/xpk/core/pathways.py @@ -174,15 +174,12 @@ def ensure_pathways_workload_prerequisites(args, system) -> bool: def get_pathways_unified_query_link(args) -> str: """Get the unified query link for the pathways workload.""" - pw_suffixes = ['main', 'rm', 'proxy'] - pw_pod_names = [f'"{args.workload}-{suffix}-0"' for suffix in pw_suffixes] - pw_pod_names_query = '%20OR%20'.join(pw_pod_names + ['worker-0-0']) query_params = ( 'resource.type%3D"k8s_container"%0A' f'resource.labels.project_id%3D"{args.project}"%0A' f'resource.labels.location%3D"{zone_to_region(args.zone)}"%0A' f'resource.labels.cluster_name%3D"{args.cluster}"%0A' - f'resource.labels.pod_name:{pw_pod_names_query}%0A' + f'resource.labels.pod_name:"{args.workload}-"%0A' 'severity>%3DDEFAULT' ) From 666b4c534e48f9479d311f4f11a9e59e4adbcde0 Mon Sep 17 00:00:00 2001 From: Roshani Narasimhan Date: Tue, 4 Feb 2025 13:47:30 -0800 Subject: [PATCH 2/8] Support custom Pathways args. (#364) --- src/xpk/core/pathways.py | 30 ++++++++++++++++++++++++++++++ src/xpk/parser/workload.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/xpk/core/pathways.py b/src/xpk/core/pathways.py index 6d7b9735c..f3b1976bc 100644 --- a/src/xpk/core/pathways.py +++ b/src/xpk/core/pathways.py @@ -44,6 +44,8 @@ def get_pathways_worker_args(args) -> str: - --resource_manager_address={rm_address} - --gcs_scratch_location={args.pathways_gcs_location}""" if args.use_pathways: + if args.custom_pathways_worker_args: + yaml = append_custom_pathways_args(yaml, args.custom_pathways_worker_args) return yaml.format(args=args, rm_address=get_rm_address(args)) else: return '' @@ -62,6 +64,10 @@ def get_pathways_proxy_args(args) -> str: - --gcs_scratch_location={args.pathways_gcs_location}""" if args.use_pathways: + if args.custom_pathways_proxy_server_args: + yaml = append_custom_pathways_args( + yaml, args.custom_pathways_proxy_server_args + ) return yaml.format(args=args, rm_address=get_rm_address(args)) else: return '' @@ -200,6 +206,8 @@ def get_pathways_rm_args(args, system: SystemCharacteristics) -> str: - --instance_count={instance_count} - --instance_type={instance_type}""" if args.use_pathways: + if args.custom_pathways_server_args: + yaml = append_custom_pathways_args(yaml, args.custom_pathways_server_args) return yaml.format( args=args, instance_count=args.num_slices, @@ -209,6 +217,28 @@ def get_pathways_rm_args(args, system: SystemCharacteristics) -> str: return '' +def append_custom_pathways_args(yaml, custom_args) -> str: + """Append custom Pathways args to the YAML with proper indentation. + + Args: + yaml (string): existing yaml containing args + + Returns: + yaml (string): yaml with additional args appended. + """ + second_line = yaml.split('\n')[1] + if ( + not second_line + ): # to cover edge case if only one arg remains, we would have to look at the entire YAML in this case. + return yaml + # Calculate the indentation based on the second line of existing YAML. + indentation = ' ' * (len(second_line) - len(second_line.lstrip())) + custom_args = custom_args.split(' ') + for arg in custom_args: + yaml += '\n' + indentation + '- ' + arg + return yaml + + def get_user_workload_for_pathways(args, system: SystemCharacteristics) -> str: """ Create a user workload container for Pathways. diff --git a/src/xpk/parser/workload.py b/src/xpk/parser/workload.py index 023c2edf4..e598b8a96 100644 --- a/src/xpk/parser/workload.py +++ b/src/xpk/parser/workload.py @@ -244,6 +244,39 @@ def set_workload_parsers(workload_parser): required=False, ) + workload_create_pathways_parser_optional_arguments.add_argument( + '--custom-pathways-server-args', + type=str, + default=None, + help=( + 'Provide custom Pathways server args as follows -' + " --custom-pathways-server-args='--arg_1=xxx --arg2=yyy'" + ), + required=False, + ) + + workload_create_pathways_parser_optional_arguments.add_argument( + '--custom-pathways-proxy-server-args', + type=str, + default=None, + help=( + 'Provide custom Pathways proxy server args as follows -' + " --custom-pathways-proxy-server-args='--arg_1=xxx --arg2=yyy'" + ), + required=False, + ) + + workload_create_pathways_parser_optional_arguments.add_argument( + '--custom-pathways-worker-args', + type=str, + default=None, + help=( + 'Provide custom Pathways worker args as follows -' + " --custom-pathways-worker-args='--arg_1=xxx --arg2=yyy'" + ), + required=False, + ) + add_shared_workload_create_required_arguments([ workload_create_parser_required_arguments, workload_create_pathways_parser_required_arguments, From c8f20956107c8a2b57064415d548555b11ec1413 Mon Sep 17 00:00:00 2001 From: Roshani Narasimhan Date: Tue, 4 Feb 2025 15:59:18 -0800 Subject: [PATCH 3/8] Remove Pathways specific args from workload create flow. (#365) * Remove Pathways specific optional args from workload create. * Deprecate --use-pathways from workload create, set it in create-pathways. * Remove --enable-pathways from cluster create flow. * Adding --use-pathways and --enable-pathways args - they are used to determine Pathways flows. --- src/xpk/commands/workload.py | 15 +++-- src/xpk/parser/cluster.py | 5 +- src/xpk/parser/workload.py | 109 ++++++++++++++++------------------- 3 files changed, 58 insertions(+), 71 deletions(-) diff --git a/src/xpk/commands/workload.py b/src/xpk/commands/workload.py index 81648d1eb..a390e82e7 100644 --- a/src/xpk/commands/workload.py +++ b/src/xpk/commands/workload.py @@ -352,6 +352,13 @@ def workload_create_pathways(args) -> None: 0 if successful and 1 otherwise. """ args.use_pathways = True + if args.headless: + xpk_print( + 'Please use kubectl port forwarding to connect to the Pathways proxy.' + ' kubectl get pods kubectl port-forward 29000:29000' + ' JAX_PLATFORMS=proxy JAX_BACKEND_TARGET=grpc://127.0.0.1:29000 python' + " -c 'import pathwaysutils; import jax; print(jax.devices())'" + ) workload_create(args) @@ -366,14 +373,6 @@ def workload_create(args) -> None: """ add_zone_and_project(args) - if args.headless: - xpk_print( - 'Please use kubectl port forwarding to connect to the Pathways proxy.' - ' kubectl get pods kubectl port-forward 29000:29000' - ' JAX_PLATFORMS=proxy JAX_BACKEND_TARGET=grpc://127.0.0.1:29000 python' - " -c 'import pathwaysutils; import jax; print(jax.devices())'" - ) - set_cluster_command_code = set_cluster_command(args) if set_cluster_command_code != 0: xpk_exit(set_cluster_command_code) diff --git a/src/xpk/parser/cluster.py b/src/xpk/parser/cluster.py index bc2a944d2..b4ce61c36 100644 --- a/src/xpk/parser/cluster.py +++ b/src/xpk/parser/cluster.py @@ -93,11 +93,10 @@ def set_cluster_parser(cluster_parser): '--enable-pathways', action='store_true', help=( - 'DEPRECATING SOON!!! Please use `xpk cluster create-pathways`.' - ' Enable cluster to accept Pathways workloads.' + 'Please use `xpk cluster create-pathways` instead to' + ' enable cluster to accept Pathways workloads.' ), ) - ### Autoprovisioning arguments specific to "cluster create" cluster_create_autoprovisioning_arguments = ( cluster_create_parser.add_argument_group( diff --git a/src/xpk/parser/workload.py b/src/xpk/parser/workload.py index e598b8a96..638af6d36 100644 --- a/src/xpk/parser/workload.py +++ b/src/xpk/parser/workload.py @@ -67,11 +67,7 @@ def set_workload_parsers(workload_parser): 'Arguments for configuring autoprovisioning.', ) ) - workload_pathways_workload_arguments = workload_create_parser.add_argument_group( - 'Pathways Image Arguments', - 'If --use-pathways is provided, user wants to set up a' - 'Pathways workload on xpk.', - ) + workload_vertex_tensorboard_arguments = ( workload_create_parser.add_argument_group( 'Vertex Tensorboard Arguments', @@ -151,6 +147,15 @@ def set_workload_parsers(workload_parser): ), ) + workload_create_parser_optional_arguments.add_argument( + '--use-pathways', + action='store_true', + help=( + 'Please use `xpk workload create-pathways` instead to' + ' create Pathways workloads.' + ), + ) + # Autoprovisioning workload arguments workload_create_autoprovisioning_arguments.add_argument( '--on-demand', @@ -178,16 +183,6 @@ def set_workload_parsers(workload_parser): ), ) - # Pathways workload arguments - workload_pathways_workload_arguments.add_argument( - '--use-pathways', - action='store_true', - help=( - 'DECRATING SOON!!! Please use `xpk workload create-pathways` instead.' - ' Provide this argument to create Pathways workloads.' - ), - ) - # "workload create-pathways" command parser. workload_create_pathways_parser = workload_subcommands.add_parser( 'create-pathways', help='Create a new job.' @@ -230,6 +225,45 @@ def set_workload_parsers(workload_parser): help='The tpu type to use, v5litepod-16, etc.', ) + ### "workload create-pathways" Optional arguments, specific to Pathways + workload_create_pathways_parser_optional_arguments.add_argument( + '--headless', + action='store_true', + help=( + 'Please provide this argument to create Pathways workloads in' + ' headless mode. This arg can only be used in `xpk workload' + ' create-pathways`.' + ), + ) + workload_create_pathways_parser_optional_arguments.add_argument( + '--proxy-server-image', + type=str, + default=( + 'us-docker.pkg.dev/cloud-tpu-v2-images/pathways/proxy_server:latest' + ), + help=( + 'Please provide the proxy server image for Pathways. This arg can' + ' only be used in `xpk workload create-pathways`.' + ), + ) + workload_create_pathways_parser_optional_arguments.add_argument( + '--server-image', + type=str, + default='us-docker.pkg.dev/cloud-tpu-v2-images/pathways/server:latest', + help=( + 'Please provide the server image for Pathways. This arg can only be' + ' used in `xpk workload create-pathways`.' + ), + ) + workload_create_pathways_parser_optional_arguments.add_argument( + '--pathways-gcs-location', + type=str, + default='gs://cloud-pathways-staging/tmp', + help=( + 'Please provide the GCS location to store Pathways artifacts. This' + ' arg can only be used in `xpk workload create-pathways`.' + ), + ) workload_create_pathways_parser_optional_arguments.add_argument( '--command', type=str, @@ -555,51 +589,6 @@ def add_shared_workload_create_optional_arguments(args_parsers): ' default on Pathways workloads.' ), ) - custom_parser.add_argument( - '--headless', - action='store_true', - help=( - 'Please provide this argument to create Pathways workloads in' - ' headless mode. This arg can only be used in `xpk workload' - ' create-pathways`(preferred) or `xpk workload create' - ' --use-pathways.` (--use-pathways will be deprecated soon).' - ), - ) - custom_parser.add_argument( - '--proxy-server-image', - type=str, - default=( - 'us-docker.pkg.dev/cloud-tpu-v2-images/pathways/proxy_server:latest' - ), - help=( - 'Please provide the proxy server image for Pathways. This arg can' - ' only be used in `xpk workload create-pathways`(preferred) or `xpk' - ' workload create --use-pathways.` (--use-pathways will be' - ' deprecated soon).' - ), - ) - custom_parser.add_argument( - '--server-image', - type=str, - default='us-docker.pkg.dev/cloud-tpu-v2-images/pathways/server:latest', - help=( - 'Please provide the server image for Pathways. This arg can only be' - ' used in `xpk workload create-pathways`(preferred) or `xpk' - ' workload create --use-pathways.` (--use-pathways will be' - ' deprecated soon).' - ), - ) - custom_parser.add_argument( - '--pathways-gcs-location', - type=str, - default='gs://cloud-pathways-staging/tmp', - help=( - 'Please provide the GCS location to store Pathways artifacts. This' - ' arg can only be used in `xpk workload create-pathways`(preferred)' - ' or `xpk workload create --use-pathways.` (--use-pathways will be' - ' deprecated soon).' - ), - ) custom_parser.add_argument( '--ramdisk-directory', type=str, From 0ee68502aa2b745375f535330dc463f5414ff356 Mon Sep 17 00:00:00 2001 From: Akanksha Gupta Date: Fri, 7 Feb 2025 19:32:08 +0000 Subject: [PATCH 4/8] Add env variables for collecting Streamz metrics --- src/xpk/commands/workload.py | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/xpk/commands/workload.py b/src/xpk/commands/workload.py index a390e82e7..a880f6ac4 100644 --- a/src/xpk/commands/workload.py +++ b/src/xpk/commands/workload.py @@ -240,6 +240,21 @@ containers: - args: {pathways_worker_args} + env: + - name: PROJECT_ID + value: "tpu-prod-env-one-vm" + - name: LOCATION + value: "asia-northeast1" + - name: CLUSTER_NAME + value: "bodaborg-v6e-256-rxc" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONTAINER_NAME + value: "pathways-worker" + - name: NAMESPACE + value: "cloud_prod" image: {args.server_image} imagePullPolicy: Always name: pathways-worker @@ -295,6 +310,20 @@ value: $(JOBSET_NAME)-$(REPLICATED_JOB_NAME)-0-0.$(JOBSET_NAME) - name: TPU_SKIP_MDS_QUERY value: "true" + - name: PROJECT_ID + value: "tpu-prod-env-one-vm" + - name: LOCATION + value: "asia-northeast1" + - name: CLUSTER_NAME + value: "bodaborg-v6e-256-rxc" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONTAINER_NAME + value: "pathways-rm" + - name: NAMESPACE + value: "cloud_prod" image: {args.server_image} imagePullPolicy: Always name: pathways-rm @@ -329,6 +358,23 @@ containers: - args: {pathways_proxy_args} + env: + - name: PROJECT_ID + value: "tpu-prod-env-one-vm" + - name: LOCATION + value: "asia-northeast1" + - name: CLUSTER_NAME + value: "bodaborg-v6e-256-rxc" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONTAINER_NAME + value: "pathways-proxy" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: {args.proxy_server_image} imagePullPolicy: Always name: pathways-proxy From 1bd3114b0bf06f1d8941037407f44365cfa35bd0 Mon Sep 17 00:00:00 2001 From: Akanksha Gupta Date: Sat, 1 Mar 2025 00:37:20 +0000 Subject: [PATCH 5/8] Streamz env --- src/xpk/commands/workload.py | 38 +++++++++++++++++++++++++++--------- src/xpk/core/pathways.py | 2 ++ src/xpk/parser/cluster.py | 2 +- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/xpk/commands/workload.py b/src/xpk/commands/workload.py index a880f6ac4..22a457656 100644 --- a/src/xpk/commands/workload.py +++ b/src/xpk/commands/workload.py @@ -242,11 +242,11 @@ {pathways_worker_args} env: - name: PROJECT_ID - value: "tpu-prod-env-one-vm" + value: {args.project} - name: LOCATION - value: "asia-northeast1" + value: {args.zone} - name: CLUSTER_NAME - value: "bodaborg-v6e-256-rxc" + value: {args.cluster} - name: POD_NAME valueFrom: fieldRef: @@ -255,6 +255,26 @@ value: "pathways-worker" - name: NAMESPACE value: "cloud_prod" + - name: MEGASCALE_GRPC_ENABLE_XOR_TRACER + value: "false" + - name: MEGASCALE_NUM_SLICES + valueFrom: + fieldRef: + fieldPath: "metadata.labels['jobset.sigs.k8s.io/replicatedjob-replicas']" + - name: JOBSET_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['jobset.sigs.k8s.io/jobset-name'] + - name: REPLICATED_JOB_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['jobset.sigs.k8s.io/replicatedjob-name'] + - name: MEGASCALE_SLICE_ID + valueFrom: + fieldRef: + fieldPath: "metadata.labels['jobset.sigs.k8s.io/job-index']" + - name: MEGASCALE_COORDINATOR_ADDRESS + value: "$(JOBSET_NAME)-$(REPLICATED_JOB_NAME)-$(MEGASCALE_SLICE_ID)-0.$(JOBSET_NAME)" image: {args.server_image} imagePullPolicy: Always name: pathways-worker @@ -311,11 +331,11 @@ - name: TPU_SKIP_MDS_QUERY value: "true" - name: PROJECT_ID - value: "tpu-prod-env-one-vm" + value: {args.project} - name: LOCATION - value: "asia-northeast1" + value: {args.zone} - name: CLUSTER_NAME - value: "bodaborg-v6e-256-rxc" + value: {args.cluster} - name: POD_NAME valueFrom: fieldRef: @@ -360,11 +380,11 @@ {pathways_proxy_args} env: - name: PROJECT_ID - value: "tpu-prod-env-one-vm" + value: {args.project} - name: LOCATION - value: "asia-northeast1" + value: {args.zone} - name: CLUSTER_NAME - value: "bodaborg-v6e-256-rxc" + value: {args.cluster} - name: POD_NAME valueFrom: fieldRef: diff --git a/src/xpk/core/pathways.py b/src/xpk/core/pathways.py index f3b1976bc..b7b2baf6e 100644 --- a/src/xpk/core/pathways.py +++ b/src/xpk/core/pathways.py @@ -269,6 +269,8 @@ def get_user_workload_for_pathways(args, system: SystemCharacteristics) -> str: {container} nodeSelector: cloud.google.com/gke-nodepool: cpu-user-np + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet restartPolicy: OnFailure volumes: - hostPath: diff --git a/src/xpk/parser/cluster.py b/src/xpk/parser/cluster.py index b4ce61c36..0b9a7644a 100644 --- a/src/xpk/parser/cluster.py +++ b/src/xpk/parser/cluster.py @@ -445,7 +445,7 @@ def add_shared_cluster_create_optional_arguments(args_parsers): custom_parser.add_argument( '--pathways-gce-machine-type', type=str, - default='n1-standard-32', + default='e2-standard-32', help='The CPU type for Pathways CPU nodepools', ) custom_parser.add_argument( From 59a35f30d9b5970452f711cef17f66fecbbd7894 Mon Sep 17 00:00:00 2001 From: Akanksha Gupta Date: Sat, 1 Mar 2025 05:38:56 +0000 Subject: [PATCH 6/8] Remove old env --- %3DDEFAULT | 0 kueue | 1 + maxtext-prev | 1 + src/xpk/commands/workload.py | 20 -------------------- 4 files changed, 2 insertions(+), 20 deletions(-) create mode 100644 %3DDEFAULT create mode 160000 kueue create mode 160000 maxtext-prev diff --git a/%3DDEFAULT b/%3DDEFAULT new file mode 100644 index 000000000..e69de29bb diff --git a/kueue b/kueue new file mode 160000 index 000000000..70b32a41e --- /dev/null +++ b/kueue @@ -0,0 +1 @@ +Subproject commit 70b32a41e689866631b8a5a58f3dd869345eb29f diff --git a/maxtext-prev b/maxtext-prev new file mode 160000 index 000000000..20c480f96 --- /dev/null +++ b/maxtext-prev @@ -0,0 +1 @@ +Subproject commit 20c480f96752d6978a427279dc6e3cb541fac251 diff --git a/src/xpk/commands/workload.py b/src/xpk/commands/workload.py index 22a457656..bc4f7e879 100644 --- a/src/xpk/commands/workload.py +++ b/src/xpk/commands/workload.py @@ -255,26 +255,6 @@ value: "pathways-worker" - name: NAMESPACE value: "cloud_prod" - - name: MEGASCALE_GRPC_ENABLE_XOR_TRACER - value: "false" - - name: MEGASCALE_NUM_SLICES - valueFrom: - fieldRef: - fieldPath: "metadata.labels['jobset.sigs.k8s.io/replicatedjob-replicas']" - - name: JOBSET_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['jobset.sigs.k8s.io/jobset-name'] - - name: REPLICATED_JOB_NAME - valueFrom: - fieldRef: - fieldPath: metadata.annotations['jobset.sigs.k8s.io/replicatedjob-name'] - - name: MEGASCALE_SLICE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.labels['jobset.sigs.k8s.io/job-index']" - - name: MEGASCALE_COORDINATOR_ADDRESS - value: "$(JOBSET_NAME)-$(REPLICATED_JOB_NAME)-$(MEGASCALE_SLICE_ID)-0.$(JOBSET_NAME)" image: {args.server_image} imagePullPolicy: Always name: pathways-worker From 2b35e68b43ef708a8574526653c2b4330d472717 Mon Sep 17 00:00:00 2001 From: Akanksha Gupta Date: Sat, 1 Mar 2025 06:01:50 +0000 Subject: [PATCH 7/8] remove unncessary files --- %3DDEFAULT | 0 kueue | 1 - src/xpk/core/pathways.py | 2 -- src/xpk/parser/cluster.py | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 %3DDEFAULT delete mode 160000 kueue diff --git a/%3DDEFAULT b/%3DDEFAULT deleted file mode 100644 index e69de29bb..000000000 diff --git a/kueue b/kueue deleted file mode 160000 index 70b32a41e..000000000 --- a/kueue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 70b32a41e689866631b8a5a58f3dd869345eb29f diff --git a/src/xpk/core/pathways.py b/src/xpk/core/pathways.py index b7b2baf6e..f3b1976bc 100644 --- a/src/xpk/core/pathways.py +++ b/src/xpk/core/pathways.py @@ -269,8 +269,6 @@ def get_user_workload_for_pathways(args, system: SystemCharacteristics) -> str: {container} nodeSelector: cloud.google.com/gke-nodepool: cpu-user-np - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet restartPolicy: OnFailure volumes: - hostPath: diff --git a/src/xpk/parser/cluster.py b/src/xpk/parser/cluster.py index 0b9a7644a..b4ce61c36 100644 --- a/src/xpk/parser/cluster.py +++ b/src/xpk/parser/cluster.py @@ -445,7 +445,7 @@ def add_shared_cluster_create_optional_arguments(args_parsers): custom_parser.add_argument( '--pathways-gce-machine-type', type=str, - default='e2-standard-32', + default='n1-standard-32', help='The CPU type for Pathways CPU nodepools', ) custom_parser.add_argument( From 2c5db3f33d6cadc7ce4cd0b6340dda6debf73caf Mon Sep 17 00:00:00 2001 From: Akanksha Gupta Date: Sat, 1 Mar 2025 06:02:45 +0000 Subject: [PATCH 8/8] remove unncessary files --- maxtext-prev | 1 - 1 file changed, 1 deletion(-) delete mode 160000 maxtext-prev diff --git a/maxtext-prev b/maxtext-prev deleted file mode 160000 index 20c480f96..000000000 --- a/maxtext-prev +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 20c480f96752d6978a427279dc6e3cb541fac251