From b5b00efe86900a79236408528e06697859d542d0 Mon Sep 17 00:00:00 2001 From: Sneha Prabhu <122787562+prabhusneha@users.noreply.github.com> Date: Mon, 28 Oct 2024 20:38:54 +0530 Subject: [PATCH] remove redundant scheduler_lock column from dag table (#43023) * remove scheduler_lock column from dag table * add migration script * add migration script * add migration script * add migration script * delete duplicate file * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * resolve conflicts --------- Co-authored-by: Sneha Prabhu --- airflow/api_connexion/openapi/v1.yaml | 8 - airflow/api_connexion/schemas/dag_schema.py | 1 - .../core_api/openapi/v1-generated.yaml | 21 - .../api_fastapi/core_api/serializers/dags.py | 1 - airflow/cli/commands/dag_command.py | 1 - ...0043_3_0_0_remove_scheduler_lock_column.py | 49 + airflow/models/dag.py | 2 - airflow/serialization/pydantic/dag.py | 1 - .../ui/openapi-gen/requests/schemas.gen.ts | 39 - airflow/ui/openapi-gen/requests/types.gen.ts | 3 - .../ui/src/pages/DagsList/DagCard.test.tsx | 1 - airflow/utils/db.py | 2 +- airflow/www/static/js/types/api-generated.ts | 6 - docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 998 +++++++++--------- docs/apache-airflow/migrations-ref.rst | 4 +- .../endpoints/test_dag_endpoint.py | 32 - .../api_connexion/schemas/test_dag_schema.py | 3 - .../core_api/routes/public/test_dags.py | 2 - 19 files changed, 551 insertions(+), 625 deletions(-) create mode 100644 airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 252d4cd6ed3eb..38911a57538cb 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3069,14 +3069,6 @@ components: Time when the DAG last received a refresh signal (e.g. the DAG's "refresh" button was clicked in the web UI) - *New in version 2.3.0* - scheduler_lock: - type: boolean - readOnly: true - nullable: true - description: | - Whether (one of) the scheduler is scheduling this DAG at the moment - *New in version 2.3.0* pickle_id: type: string diff --git a/airflow/api_connexion/schemas/dag_schema.py b/airflow/api_connexion/schemas/dag_schema.py index 3ca650ac9a5e9..4fb1dd6ae4b39 100644 --- a/airflow/api_connexion/schemas/dag_schema.py +++ b/airflow/api_connexion/schemas/dag_schema.py @@ -56,7 +56,6 @@ class Meta: last_parsed_time = auto_field(dump_only=True) last_pickled = auto_field(dump_only=True) last_expired = auto_field(dump_only=True) - scheduler_lock = auto_field(dump_only=True) pickle_id = auto_field(dump_only=True) default_view = auto_field(dump_only=True) fileloc = auto_field(dump_only=True) diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index a6ac7ac79d28d..136d6a4ed0271 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -1681,12 +1681,6 @@ components: format: date-time - type: 'null' title: Last Expired - scheduler_lock: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Scheduler Lock pickle_id: anyOf: - type: string @@ -1851,7 +1845,6 @@ components: - last_parsed_time - last_pickled - last_expired - - scheduler_lock - pickle_id - default_view - fileloc @@ -1928,12 +1921,6 @@ components: format: date-time - type: 'null' title: Last Expired - scheduler_lock: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Scheduler Lock pickle_id: anyOf: - type: string @@ -2028,7 +2015,6 @@ components: - last_parsed_time - last_pickled - last_expired - - scheduler_lock - pickle_id - default_view - fileloc @@ -2238,12 +2224,6 @@ components: format: date-time - type: 'null' title: Last Expired - scheduler_lock: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Scheduler Lock pickle_id: anyOf: - type: string @@ -2343,7 +2323,6 @@ components: - last_parsed_time - last_pickled - last_expired - - scheduler_lock - pickle_id - default_view - fileloc diff --git a/airflow/api_fastapi/core_api/serializers/dags.py b/airflow/api_fastapi/core_api/serializers/dags.py index a3e11de36039b..c6294324c7e67 100644 --- a/airflow/api_fastapi/core_api/serializers/dags.py +++ b/airflow/api_fastapi/core_api/serializers/dags.py @@ -45,7 +45,6 @@ class DAGResponse(BaseModel): last_parsed_time: datetime | None last_pickled: datetime | None last_expired: datetime | None - scheduler_lock: datetime | None pickle_id: datetime | None default_view: str | None fileloc: str diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index 83d0430a717bd..89c5fd477f056 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -227,7 +227,6 @@ def _get_dagbag_dag_details(dag: DAG) -> dict: "last_parsed_time": None, "last_pickled": None, "last_expired": None, - "scheduler_lock": None, "pickle_id": dag.pickle_id, "default_view": dag.default_view, "fileloc": dag.fileloc, diff --git a/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py b/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py new file mode 100644 index 0000000000000..12d1b16d9729d --- /dev/null +++ b/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +remove scheduler_lock column. + +Revision ID: 486ac7936b78 +Revises: d59cbbef95eb +Create Date: 2024-10-23 07:48:52.494396 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "486ac7936b78" +down_revision = "d59cbbef95eb" +branch_labels = None +depends_on = None +airflow_version = "3.0.0" + + +def upgrade(): + """Apply remove scheduler_lock column.""" + with op.batch_alter_table("dag", schema=None) as batch_op: + batch_op.drop_column("scheduler_lock") + + +def downgrade(): + """Unapply remove scheduler_lock column.""" + with op.batch_alter_table("dag", schema=None) as batch_op: + batch_op.add_column(sa.Column("scheduler_lock", sa.BOOLEAN(), autoincrement=False, nullable=True)) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index fd1c67debe248..d72eb0928cb46 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2818,8 +2818,6 @@ class DagModel(Base): # Time when the DAG last received a refresh signal # (e.g. the DAG's "refresh" button was clicked in the web UI) last_expired = Column(UtcDateTime) - # Whether (one of) the scheduler is scheduling this DAG at the moment - scheduler_lock = Column(Boolean) # Foreign key to the latest pickle_id pickle_id = Column(Integer) # The location of the file containing the DAG object diff --git a/airflow/serialization/pydantic/dag.py b/airflow/serialization/pydantic/dag.py index f7cb90797d5c8..4e37a633da058 100644 --- a/airflow/serialization/pydantic/dag.py +++ b/airflow/serialization/pydantic/dag.py @@ -82,7 +82,6 @@ class DagModelPydantic(BaseModelPydantic): last_parsed_time: Optional[datetime] last_pickled: Optional[datetime] last_expired: Optional[datetime] - scheduler_lock: Optional[bool] pickle_id: Optional[int] fileloc: str processor_subdir: Optional[str] diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index a30712d02acea..4bbf448ca8eec 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -295,18 +295,6 @@ export const $DAGDetailsResponse = { ], title: "Last Expired", }, - scheduler_lock: { - anyOf: [ - { - type: "string", - format: "date-time", - }, - { - type: "null", - }, - ], - title: "Scheduler Lock", - }, pickle_id: { anyOf: [ { @@ -607,7 +595,6 @@ export const $DAGDetailsResponse = { "last_parsed_time", "last_pickled", "last_expired", - "scheduler_lock", "pickle_id", "default_view", "fileloc", @@ -712,18 +699,6 @@ export const $DAGResponse = { ], title: "Last Expired", }, - scheduler_lock: { - anyOf: [ - { - type: "string", - format: "date-time", - }, - { - type: "null", - }, - ], - title: "Scheduler Lock", - }, pickle_id: { anyOf: [ { @@ -889,7 +864,6 @@ export const $DAGResponse = { "last_parsed_time", "last_pickled", "last_expired", - "scheduler_lock", "pickle_id", "default_view", "fileloc", @@ -1196,18 +1170,6 @@ export const $DAGWithLatestDagRunsResponse = { ], title: "Last Expired", }, - scheduler_lock: { - anyOf: [ - { - type: "string", - format: "date-time", - }, - { - type: "null", - }, - ], - title: "Scheduler Lock", - }, pickle_id: { anyOf: [ { @@ -1380,7 +1342,6 @@ export const $DAGWithLatestDagRunsResponse = { "last_parsed_time", "last_pickled", "last_expired", - "scheduler_lock", "pickle_id", "default_view", "fileloc", diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 174190f9f493a..e3df16ea82980 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -69,7 +69,6 @@ export type DAGDetailsResponse = { last_parsed_time: string | null; last_pickled: string | null; last_expired: string | null; - scheduler_lock: string | null; pickle_id: string | null; default_view: string | null; fileloc: string; @@ -132,7 +131,6 @@ export type DAGResponse = { last_parsed_time: string | null; last_pickled: string | null; last_expired: string | null; - scheduler_lock: string | null; pickle_id: string | null; default_view: string | null; fileloc: string; @@ -225,7 +223,6 @@ export type DAGWithLatestDagRunsResponse = { last_parsed_time: string | null; last_pickled: string | null; last_expired: string | null; - scheduler_lock: string | null; pickle_id: string | null; default_view: string | null; fileloc: string; diff --git a/airflow/ui/src/pages/DagsList/DagCard.test.tsx b/airflow/ui/src/pages/DagsList/DagCard.test.tsx index b039d7c90b467..c01c627e23f73 100644 --- a/airflow/ui/src/pages/DagsList/DagCard.test.tsx +++ b/airflow/ui/src/pages/DagsList/DagCard.test.tsx @@ -53,7 +53,6 @@ const mockDag = { next_dagrun_data_interval_start: "2024-08-22T00:00:00+00:00", owners: ["airflow"], pickle_id: null, - scheduler_lock: null, tags: [], timetable_description: "", timetable_summary: "", diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 2cae52db8e4c1..6961302156adb 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -97,7 +97,7 @@ class MappedClassProtocol(Protocol): "2.9.2": "686269002441", "2.10.0": "22ed7efa9da2", "2.10.3": "5f2621c13b39", - "3.0.0": "d59cbbef95eb", + "3.0.0": "486ac7936b78", } diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index e5c3434ae9003..18ea142c5b401 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1058,12 +1058,6 @@ export interface components { * *New in version 2.3.0* */ last_expired?: string | null; - /** - * @description Whether (one of) the scheduler is scheduling this DAG at the moment - * - * *New in version 2.3.0* - */ - scheduler_lock?: boolean | null; /** * @description Foreign key to the latest pickle_id * diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index aad8dfbe06ef3..0278ab85b0b04 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -621811bf2476a947cf52d8b75149e2be7e0ec9d383eda551b1c6496c44f2a22d \ No newline at end of file +247d6705552b669793ce92a4e15c9b225b1f852d5cbc066f204ec7eb21e77ca7 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index c9232c64f2d82..c0f395e89ab1a 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -4,11 +4,11 @@ - - + + %3 - + log @@ -396,23 +396,23 @@ asset_alias - -asset_alias - -id - - [INTEGER] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +asset_alias + +id + + [INTEGER] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL @@ -433,9 +433,9 @@ asset_alias--asset_alias_asset - -0..N -1 + +0..N +1 @@ -456,9 +456,9 @@ asset_alias--asset_alias_asset_event - -0..N -1 + +0..N +1 @@ -489,57 +489,57 @@ asset_alias--dag_schedule_asset_alias_reference - -0..N -1 + +0..N +1 asset - -asset - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -extra - - [JSON] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +extra + + [JSON] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL asset--asset_alias_asset - -0..N -1 + +0..N +1 @@ -560,16 +560,16 @@ asset--asset_active - -1 -1 + +1 +1 asset--asset_active - -1 -1 + +1 +1 @@ -600,9 +600,9 @@ asset--dag_schedule_asset_reference - -0..N -1 + +0..N +1 @@ -638,9 +638,9 @@ asset--task_outlet_asset_reference - -0..N -1 + +0..N +1 @@ -666,58 +666,58 @@ asset--asset_dag_run_queue - -0..N -1 + +0..N +1 asset_event - -asset_event - -id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -extra - - [JSON] - NOT NULL - -source_dag_id - - [VARCHAR(250)] - -source_map_index - - [INTEGER] - -source_run_id - - [VARCHAR(250)] - -source_task_id - - [VARCHAR(250)] - -timestamp - - [TIMESTAMP] - NOT NULL + +asset_event + +id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +extra + + [JSON] + NOT NULL + +source_dag_id + + [VARCHAR(250)] + +source_map_index + + [INTEGER] + +source_run_id + + [VARCHAR(250)] + +source_task_id + + [VARCHAR(250)] + +timestamp + + [TIMESTAMP] + NOT NULL asset_event--asset_alias_asset_event - -0..N -1 + +0..N +1 @@ -738,151 +738,147 @@ asset_event--dagrun_asset_event - -0..N -1 + +0..N +1 dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -asset_expression - - [JSON] - -dag_display_name - - [VARCHAR(2000)] - -default_view - - [VARCHAR(25)] - -description - - [TEXT] - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_active - - [BOOLEAN] - -is_paused - - [BOOLEAN] - -last_expired - - [TIMESTAMP] - -last_parsed_time - - [TIMESTAMP] - -last_pickled - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -pickle_id - - [INTEGER] - -processor_subdir - - [VARCHAR(2000)] - -scheduler_lock - - [BOOLEAN] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +asset_expression + + [JSON] + +dag_display_name + + [VARCHAR(2000)] + +default_view + + [VARCHAR(25)] + +description + + [TEXT] + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_active + + [BOOLEAN] + +is_paused + + [BOOLEAN] + +last_expired + + [TIMESTAMP] + +last_parsed_time + + [TIMESTAMP] + +last_pickled + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +pickle_id + + [INTEGER] + +processor_subdir + + [VARCHAR(2000)] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] dag--dag_schedule_asset_alias_reference - -0..N -1 + +0..N +1 dag--dag_schedule_asset_reference - -0..N -1 + +0..N +1 dag--task_outlet_asset_reference - -0..N -1 + +0..N +1 dag--asset_dag_run_queue - -0..N -1 + +0..N +1 @@ -903,9 +899,9 @@ dag--dag_tag - -0..N -1 + +0..N +1 @@ -931,9 +927,9 @@ dag--dag_owner_attributes - -0..N -1 + +0..N +1 @@ -964,9 +960,9 @@ dag--dag_warning - -0..N -1 + +0..N +1 @@ -1421,7 +1417,7 @@ task_instance--task_reschedule -0..N +0..N 1 @@ -2061,318 +2057,318 @@ session - -session - -id - - [INTEGER] - NOT NULL - -data - - [BYTEA] - -expiry - - [TIMESTAMP] - -session_id - - [VARCHAR(255)] + +session + +id + + [INTEGER] + NOT NULL + +data + + [BYTEA] + +expiry + + [TIMESTAMP] + +session_id + + [VARCHAR(255)] alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL ab_user - -ab_user - -id - - [INTEGER] - NOT NULL - -active - - [BOOLEAN] - -changed_by_fk - - [INTEGER] - -changed_on - - [TIMESTAMP] - -created_by_fk - - [INTEGER] - -created_on - - [TIMESTAMP] - -email - - [VARCHAR(512)] - NOT NULL - -fail_login_count - - [INTEGER] - -first_name - - [VARCHAR(256)] - NOT NULL - -last_login - - [TIMESTAMP] - -last_name - - [VARCHAR(256)] - NOT NULL - -login_count - - [INTEGER] - -password - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_user + +id + + [INTEGER] + NOT NULL + +active + + [BOOLEAN] + +changed_by_fk + + [INTEGER] + +changed_on + + [TIMESTAMP] + +created_by_fk + + [INTEGER] + +created_on + + [TIMESTAMP] + +email + + [VARCHAR(512)] + NOT NULL + +fail_login_count + + [INTEGER] + +first_name + + [VARCHAR(256)] + NOT NULL + +last_login + + [TIMESTAMP] + +last_name + + [VARCHAR(256)] + NOT NULL + +login_count + + [INTEGER] + +password + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} ab_user--ab_user - -0..N -{0,1} + +0..N +{0,1} ab_user_role - -ab_user_role - -id - - [INTEGER] - NOT NULL - -role_id - - [INTEGER] - -user_id - - [INTEGER] + +ab_user_role + +id + + [INTEGER] + NOT NULL + +role_id + + [INTEGER] + +user_id + + [INTEGER] ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_register_user - -ab_register_user - -id - - [INTEGER] - NOT NULL - -email - - [VARCHAR(512)] - NOT NULL - -first_name - - [VARCHAR(256)] - NOT NULL - -last_name - - [VARCHAR(256)] - NOT NULL - -password - - [VARCHAR(256)] - -registration_date - - [TIMESTAMP] - -registration_hash - - [VARCHAR(256)] - -username - - [VARCHAR(512)] - NOT NULL + +ab_register_user + +id + + [INTEGER] + NOT NULL + +email + + [VARCHAR(512)] + NOT NULL + +first_name + + [VARCHAR(256)] + NOT NULL + +last_name + + [VARCHAR(256)] + NOT NULL + +password + + [VARCHAR(256)] + +registration_date + + [TIMESTAMP] + +registration_hash + + [VARCHAR(256)] + +username + + [VARCHAR(512)] + NOT NULL ab_permission - -ab_permission - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +ab_permission + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL ab_permission_view - -ab_permission_view - -id - - [INTEGER] - NOT NULL - -permission_id - - [INTEGER] - -view_menu_id - - [INTEGER] + +ab_permission_view + +id + + [INTEGER] + NOT NULL + +permission_id + + [INTEGER] + +view_menu_id + + [INTEGER] ab_permission--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} ab_permission_view_role - -ab_permission_view_role - -id - - [INTEGER] - NOT NULL - -permission_view_id - - [INTEGER] - -role_id - - [INTEGER] + +ab_permission_view_role + +id + + [INTEGER] + NOT NULL + +permission_view_id + + [INTEGER] + +role_id + + [INTEGER] ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} ab_view_menu - -ab_view_menu - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(250)] - NOT NULL + +ab_view_menu + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(250)] + NOT NULL ab_view_menu--ab_permission_view - -0..N -{0,1} + +0..N +{0,1} ab_role - -ab_role - -id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(64)] - NOT NULL + +ab_role + +id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(64)] + NOT NULL ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} alembic_version_fab - -alembic_version_fab - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version_fab + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 3719ece1e95e9..4d3cc11e12516 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``d59cbbef95eb`` (head) | ``05234396c6fc`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | +| ``486ac7936b78`` (head) | ``d59cbbef95eb`` | ``3.0.0`` | remove scheduler_lock column. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``d59cbbef95eb`` | ``05234396c6fc`` | ``3.0.0`` | Add UUID primary key to ``task_instance`` table. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``05234396c6fc`` | ``3a8972ecb8f9`` | ``3.0.0`` | Rename dataset as asset. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 83785fdbc1de8..ecb0052a8a03b 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -188,7 +188,6 @@ def test_should_respond_200(self): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -229,7 +228,6 @@ def test_should_respond_200_with_schedule_none(self, session): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -336,7 +334,6 @@ def test_should_respond_200(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "template_searchpath": None, @@ -399,7 +396,6 @@ def test_should_respond_200_with_asset_expression(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "template_searchpath": None, @@ -450,7 +446,6 @@ def test_should_response_200_with_doc_md_none(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "template_searchpath": None, @@ -501,7 +496,6 @@ def test_should_response_200_for_null_start_date(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": None, "tags": [], "template_searchpath": None, @@ -561,7 +555,6 @@ def test_should_respond_200_serialized(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "template_searchpath": None, @@ -622,7 +615,6 @@ def test_should_respond_200_serialized(self, url_safe_serializer): "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": "2 2 * * *", - "scheduler_lock": None, "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "template_searchpath": None, @@ -718,7 +710,6 @@ def test_should_respond_200(self, session, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -746,7 +737,6 @@ def test_should_respond_200(self, session, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -786,7 +776,6 @@ def test_only_active_true_returns_active_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -827,7 +816,6 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -855,7 +843,6 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1012,7 +999,6 @@ def test_paused_true_returns_paused_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1052,7 +1038,6 @@ def test_paused_false_returns_unpaused_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1092,7 +1077,6 @@ def test_paused_none_returns_all_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1120,7 +1104,6 @@ def test_paused_none_returns_all_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1210,7 +1193,6 @@ def test_should_respond_200_on_patch_is_paused(self, url_safe_serializer, sessio "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1309,7 +1291,6 @@ def test_should_respond_200_with_update_mask(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1404,7 +1385,6 @@ def test_should_respond_200_on_patch_is_paused(self, session, url_safe_serialize "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1432,7 +1412,6 @@ def test_should_respond_200_on_patch_is_paused(self, session, url_safe_serialize "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1485,7 +1464,6 @@ def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1513,7 +1491,6 @@ def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1606,7 +1583,6 @@ def test_only_active_true_returns_active_dags(self, url_safe_serializer, session "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1655,7 +1631,6 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1683,7 +1658,6 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1882,7 +1856,6 @@ def test_should_respond_200_and_pause_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1910,7 +1883,6 @@ def test_should_respond_200_and_pause_dags(self, url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1959,7 +1931,6 @@ def test_should_respond_200_and_pause_dag_pattern(self, session, url_safe_serial "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -1987,7 +1958,6 @@ def test_should_respond_200_and_pause_dag_pattern(self, session, url_safe_serial "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -2038,7 +2008,6 @@ def test_should_respond_200_and_reverse_ordering(self, session, url_safe_seriali "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, @@ -2066,7 +2035,6 @@ def test_should_respond_200_and_reverse_ordering(self, session, url_safe_seriali "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": False, "pickle_id": None, diff --git a/tests/api_connexion/schemas/test_dag_schema.py b/tests/api_connexion/schemas/test_dag_schema.py index 1b83b798422b5..1576f9cf3d6ee 100644 --- a/tests/api_connexion/schemas/test_dag_schema.py +++ b/tests/api_connexion/schemas/test_dag_schema.py @@ -70,7 +70,6 @@ def test_serialize_test_dag_schema(url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": None, "pickle_id": None, @@ -107,7 +106,6 @@ def test_serialize_test_dag_collection_schema(url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": None, "pickle_id": None, @@ -135,7 +133,6 @@ def test_serialize_test_dag_collection_schema(url_safe_serializer): "last_pickled": None, "default_view": None, "last_parsed_time": None, - "scheduler_lock": None, "timetable_description": None, "has_import_errors": None, "pickle_id": None, diff --git a/tests/api_fastapi/core_api/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py index 03253dbfa7a7a..72f4f70179f35 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -333,7 +333,6 @@ def test_dag_details( "pickle_id": None, "render_template_as_native_obj": False, "timetable_summary": None, - "scheduler_lock": None, "start_date": start_date.replace(tzinfo=None).isoformat() + "Z", # pydantic datetime format "tags": [], "template_search_path": None, @@ -386,7 +385,6 @@ def test_get_dag(self, test_client, query_params, dag_id, expected_status_code, "last_pickled": None, "default_view": "grid", "last_parsed_time": last_parsed_time, - "scheduler_lock": None, "timetable_description": "Never, external triggers only", "has_import_errors": False, "pickle_id": None,