diff --git a/data-archive/archive-jobs/ida_archive_table_info.json b/data-archive/archive-jobs/ida_archive_table_info.json index 464f280d..c335ad48 100644 --- a/data-archive/archive-jobs/ida_archive_table_info.json +++ b/data-archive/archive-jobs/ida_archive_table_info.json @@ -10,7 +10,6 @@ }, { "source_table": "otp_transaction", - "archive_table": "mosip_ida_otp_transaction", "id_column": "id", "date_column": "cr_dtimes", "retention_days": 30, diff --git a/data-archive/archive-jobs/kernel_archive_table_info.json b/data-archive/archive-jobs/kernel_archive_table_info.json index 2782cb8d..97494cf0 100644 --- a/data-archive/archive-jobs/kernel_archive_table_info.json +++ b/data-archive/archive-jobs/kernel_archive_table_info.json @@ -2,7 +2,6 @@ "tables_info": [ { "source_table": "otp_transaction", - "archive_table": "mosip_kernel_otp_transaction", "id_column": "id", "date_column": "generated_dtimes", "retention_days": 7, diff --git a/data-archive/archive-jobs/mosip_archive_main.py b/data-archive/archive-jobs/mosip_archive_main.py index 2c3089ec..b22b65a3 100644 --- a/data-archive/archive-jobs/mosip_archive_main.py +++ b/data-archive/archive-jobs/mosip_archive_main.py @@ -178,6 +178,7 @@ def data_archive(db_name, db_param, tables_info, batch_size): total_archived = 0 total_deleted = 0 total_batches_processed = 0 # Counter for total batches processed + try: print(f'Connecting to the PostgreSQL source and archive databases for {db_name}...') @@ -215,15 +216,29 @@ def data_archive(db_name, db_param, tables_info, batch_size): for table_info in tables_info: source_table_name = table_info['source_table'] - archive_table_name = table_info['archive_table'] + archive_table_name = table_info.get('archive_table') id_column = table_info['id_column'] operation_type = table_info.get('operation_type', 'none').lower() batch_count = 0 # Counter for batches for the current table - - # Skip the table if operation_type is 'none' - if operation_type == 'none': + # Validate operation_type-specific fields + if operation_type == 'archive_delete': + if 'archive_table' not in table_info: + print(f"Error: 'archive_table' is required for operation_type 'archive_delete' in table {source_table_name}.") + continue + elif operation_type == 'delete': + if 'archive_table' in table_info: + print(f"Warning: 'archive_table' will be ignored for operation_type 'delete' in table {source_table_name}.") + elif operation_type == 'none': print(f"Skipping archival for table {source_table_name} as operation type is 'none'.") continue + else: + print(f"Error: Unsupported operation_type '{operation_type}' for table {source_table_name}.") + continue + + # # Skip the table if operation_type is 'none' + # if operation_type == 'none': + # print(f"Skipping archival for table {source_table_name} as operation type is 'none'.") + # continue # Prepare the select query based on operation type and retention settings if 'retention_days' in table_info and 'date_column' in table_info: @@ -249,6 +264,7 @@ def data_archive(db_name, db_param, tables_info, batch_size): # Increment the batch count and print batch_count += 1 print(f"Starting batch {batch_count} for table {source_table_name}...") + # Accumulate records for bulk insert or delete bulk_insert_values = [get_tablevalues(row) for row in rows] @@ -320,11 +336,12 @@ def data_archive(db_name, db_param, tables_info, batch_size): if archive_conn is not None: archive_conn.close() print('Archive database connection closed.') + if total_batches_processed > 0: + print(f"Data archival completed for {db_name}.") + print(f"Total records archived: {total_archived}") + print(f"Total records deleted: {total_deleted}") + print(f"Total batches processed for {db_name}: {total_batches_processed}") - print(f"Data archival completed for {db_name}.") - print(f"Total records archived: {total_archived}") - print(f"Total records deleted: {total_deleted}") - print(f"Total batches processed for {db_name}: {total_batches_processed}") def main(): diff --git a/data-archive/archive-jobs/pms_archive_table_info.json b/data-archive/archive-jobs/pms_archive_table_info.json index 3b0ea10b..d39fcb2f 100644 --- a/data-archive/archive-jobs/pms_archive_table_info.json +++ b/data-archive/archive-jobs/pms_archive_table_info.json @@ -26,7 +26,6 @@ }, { "source_table": "notifications", - "archive_table": "", "id_column": "id", "date_column": "cr_dtimes", "retention_days": 60, diff --git a/data-archive/archive-jobs/prereg_archive_table_info.json b/data-archive/archive-jobs/prereg_archive_table_info.json index a1a08a0b..9fa719a7 100644 --- a/data-archive/archive-jobs/prereg_archive_table_info.json +++ b/data-archive/archive-jobs/prereg_archive_table_info.json @@ -2,7 +2,6 @@ "tables_info": [ { "source_table": "otp_transaction", - "archive_table": "mosip_prereg_otp_transaction", "id_column": "id", "date_column": "cr_dtimes", "retention_days": 30, diff --git a/data-archive/archive-jobs/resident_archive_table_info.json b/data-archive/archive-jobs/resident_archive_table_info.json index 5c0a9fd3..b19b1572 100644 --- a/data-archive/archive-jobs/resident_archive_table_info.json +++ b/data-archive/archive-jobs/resident_archive_table_info.json @@ -2,7 +2,6 @@ "tables_info": [ { "source_table": "otp_transaction", - "archive_table": "mosip_resident_otp_transaction", "id_column": "id", "date_column": "cr_dtimes", "retention_days": 30, diff --git a/data-archive/db_scripts/mosip_archive/ddl.sql b/data-archive/db_scripts/mosip_archive/ddl.sql index 89635ba6..1c7f2184 100644 --- a/data-archive/db_scripts/mosip_archive/ddl.sql +++ b/data-archive/db_scripts/mosip_archive/ddl.sql @@ -6,8 +6,8 @@ \ir ddl/archive-credential-batch_job_instance.sql \ir ddl/archive-credential-batch_step_execution.sql \ir ddl/archive-credential-batch_step_execution_context.sql -\ir ddl/archive-ida-credential_event_store.sql \ir ddl/archive-credential-credential_transaction.sql +\ir ddl/archive-ida-credential_event_store.sql \ir ddl/archive-esignet-consent_history.sql \ir ddl/archive-ida-anonymous_profile.sql \ir ddl/archive-ida-auth_transaction.sql @@ -17,11 +17,9 @@ \ir ddl/archive-ida-batch_job_instance.sql \ir ddl/archive-ida-batch_step_execution_context.sql \ir ddl/archive-ida-batch_step_execution.sql -\ir ddl/archive-ida-otp_transaction.sql \ir ddl/archive-idrepo-anonymous_profile.sql \ir ddl/archive-idrepo-credential_request_status.sql \ir ddl/archive-idrepo-uin_draft.sql -\ir ddl/archive-kernel-otp_transaction.sql \ir ddl/archive-master-bulkupload_transaction.sql \ir ddl/archive-master-device_master_h.sql \ir ddl/archive-master-machine_master_h.sql @@ -31,9 +29,7 @@ \ir ddl/archive-pms-auth_policy_h.sql \ir ddl/archive-pms-partner_h.sql \ir ddl/archive-pms-secure_biometric_interface_h.sql -\ir ddl/archive-prereg-otp_transaction.sql \ir ddl/archive-resident_grievance_ticket.sql -\ir ddl/archive-resident-otp_transaction.sql \ir ddl/archive-resident_session.sql \ir ddl/archive-resident_transaction.sql \ir ddl/archive-resident_user_actions.sql diff --git a/data-archive/db_scripts/mosip_archive/ddl/archive-ida-otp_transaction.sql b/data-archive/db_scripts/mosip_archive/ddl/archive-ida-otp_transaction.sql deleted file mode 100644 index ffca685b..00000000 --- a/data-archive/db_scripts/mosip_archive/ddl/archive-ida-otp_transaction.sql +++ /dev/null @@ -1,66 +0,0 @@ --- ------------------------------------------------------------------------------------------------- --- Database Name: mosip_ida --- Table Name : mosip_ida_otp_transaction --- Purpose : OTP Transaction: All OTP related data and validation details are maintained here for ID Authentication. --- --- Create By : Sadanandegowda DM --- Created Date : Sep-2020 --- --- Modified Date Modified By Comments / Remarks --- ------------------------------------------------------------------------------------------ - --- ------------------------------------------------------------------------------------------ --- object: archive.mosip_ida_otp_transaction | type: TABLE -- --- DROP TABLE IF EXISTS archive.mosip_ida_otp_transaction CASCADE; -CREATE TABLE archive.mosip_ida_otp_transaction( - id character varying(36) NOT NULL, - ref_id character varying(64) NOT NULL, - otp_hash character varying(512) NOT NULL, - generated_dtimes timestamp, - expiry_dtimes timestamp, - validation_retry_count smallint, - status_code character varying(36), - lang_code character varying(3), - cr_by character varying(256) NOT NULL, - cr_dtimes timestamp NOT NULL, - upd_by character varying(256), - upd_dtimes timestamp, - is_deleted boolean DEFAULT FALSE, - del_dtimes timestamp, - CONSTRAINT pk_otpt_id PRIMARY KEY (id) - -); --- ddl-end -- ---index section starts---- -CREATE INDEX ind_otphsh ON archive.mosip_ida_otp_transaction (ref_id,status_code); ---index section ends------ -COMMENT ON TABLE archive.mosip_ida_otp_transaction IS 'OTP Transaction: All OTP related data and validation details are maintained here for ID Authentication module.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.id IS 'ID: Key alias id is a unique identifier (UUID) used as an alias of the encryption key stored in keystore like HSM (hardware security module).'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.ref_id IS 'Reference ID: Reference ID is a reference information received from OTP requester which can be used while validating the OTP. AM: please give examples of ref_id'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.otp_hash IS 'OTP Hash: Hash of id, ref_id and otp which is generated based on the configuration setup and sent to the requester application / module.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.generated_dtimes IS 'Generated Date Time: Date and Time when the OTP was generated'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.expiry_dtimes IS 'Expiry Date Time: Date Time when the OTP will be expired'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.validation_retry_count IS 'Validation Retry Count: Validation retry counts of this OTP request. If the validation retry crosses the threshold limit, then the OTP will be de-activated.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.status_code IS 'Status Code: Status of the OTP whether it is active or expired. AM: please enumerate the status types. They are only a few, not infinite'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.lang_code IS 'Language Code : For multilanguage implementation this attribute Refers master.language.code. The value of some of the attributes in current record is stored in this respective language.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.cr_by IS 'Created By : ID or name of the user who create / insert record'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.cr_dtimes IS 'Created DateTimestamp : Date and Timestamp when the record is created/inserted'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.upd_by IS 'Updated By : ID or name of the user who update the record with new values'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.upd_dtimes IS 'Updated DateTimestamp : Date and Timestamp when any of the fields in the record is updated with new values.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.is_deleted IS 'IS_Deleted : Flag to mark whether the record is Soft deleted.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_ida_otp_transaction.del_dtimes IS 'Deleted DateTimestamp : Date and Timestamp when the record is soft deleted with is_deleted=TRUE'; --- ddl-end -- diff --git a/data-archive/db_scripts/mosip_archive/ddl/archive-kernel-otp_transaction.sql b/data-archive/db_scripts/mosip_archive/ddl/archive-kernel-otp_transaction.sql deleted file mode 100644 index fe2a34e1..00000000 --- a/data-archive/db_scripts/mosip_archive/ddl/archive-kernel-otp_transaction.sql +++ /dev/null @@ -1,56 +0,0 @@ - - --- object: archive.mosip_kernel_otp_transaction | type: TABLE -- --- DROP TABLE IF EXISTS archive.mosip_kernel_otp_transaction CASCADE; -CREATE TABLE archive.mosip_kernel_otp_transaction( - id character varying(64) NOT NULL, - ref_id character varying(64), - ref_id_type character varying(64), - otp character varying(8), - generated_dtimes timestamp, - expiry_dtimes timestamp, - validation_retry_count smallint, - status_code character varying(64), - lang_code character varying(3), - cr_by character varying(256), - cr_dtimes timestamp, - upd_by character varying(256), - upd_dtimes timestamp, - is_deleted boolean DEFAULT FALSE, - del_dtimes timestamp, - CONSTRAINT pk_otptrn_id PRIMARY KEY (id) - -); --- ddl-end -- -COMMENT ON TABLE archive.mosip_kernel_otp_transaction IS 'OTP Transaction: All OTP related data and validation details are maintained here. '; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.id IS 'ID: Unique transaction id for each otp transaction request'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.ref_id IS 'Reference ID: Reference ID is a reference information received from OTP requester which can be used while validating the OTP. AM: please give examples of ref_id'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.ref_id_type IS 'Reference ID Type: Type of information in Reference ID field, used to reference this OTP request. AM: i guess only email & ph are the types. Please specify'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.otp IS 'OTP: One Time Pin which is generated based on the configuration setup and sent to the requester application / module.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.generated_dtimes IS 'Generated Date Time: Date and Time when the OTP was generated'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.expiry_dtimes IS 'Expiry Date Time: Date Time when the OTP will be expired'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.validation_retry_count IS 'Validation Retry Count: Validation retry counts of this OTP request. If the validation retry crosses the threshold limit, then the OTP will be de-activated.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.status_code IS 'Status Code: Status of the OTP whether it is active or expired. AM: please enumerate the status types. They are only a few, not infinite'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.lang_code IS 'Language Code : For multilanguage implementation this attribute Refers master.language.code. The value of some of the attributes in current record is stored in this respective language. '; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.cr_by IS 'Created By : ID or name of the user who create / insert record'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.cr_dtimes IS 'Created DateTimestamp : Date and Timestamp when the record is created/inserted'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.upd_by IS 'Updated By : ID or name of the user who update the record with new values'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.upd_dtimes IS 'Updated DateTimestamp : Date and Timestamp when any of the fields in the record is updated with new values.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.is_deleted IS 'IS_Deleted : Flag to mark whether the record is Soft deleted.'; --- ddl-end -- -COMMENT ON COLUMN archive.mosip_kernel_otp_transaction.del_dtimes IS 'Deleted DateTimestamp : Date and Timestamp when the record is soft deleted with is_deleted=TRUE'; --- ddl-end -- diff --git a/data-archive/db_scripts/mosip_archive/ddl/archive-prereg-otp_transaction.sql b/data-archive/db_scripts/mosip_archive/ddl/archive-prereg-otp_transaction.sql deleted file mode 100644 index 18d16edc..00000000 --- a/data-archive/db_scripts/mosip_archive/ddl/archive-prereg-otp_transaction.sql +++ /dev/null @@ -1,35 +0,0 @@ --- This is used to save the OTP for the user whenever user requests for one using the email id / phone number to log into the application. - -CREATE TABLE archive.mosip_prereg_otp_transaction( - id character varying(36) NOT NULL, - ref_id character varying(64) NOT NULL, - otp_hash character varying(512) NOT NULL, - generated_dtimes timestamp, - expiry_dtimes timestamp, - validation_retry_count smallint, - status_code character varying(36), - lang_code character varying(3), - cr_by character varying(256) NOT NULL, - cr_dtimes timestamp NOT NULL, - upd_by character varying(256), - upd_dtimes timestamp, - is_deleted boolean, - del_dtimes timestamp, - CONSTRAINT pk_otptrans_id PRIMARY KEY (id) -); - -COMMENT ON TABLE archive.mosip_prereg_otp_transaction IS 'All OTP related data and validation details are maintained here for Pre Registration module.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.id IS 'OTP id is a unique identifier (UUID) used as an unique key to identify the OTP transaction'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.ref_id IS 'Reference ID is a reference information received from OTP requester which can be used while validating the OTP. AM: please give examples of ref_id'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.otp_hash IS 'Hash of id, ref_id and otp which is generated based on the configuration setup and sent to the requester application / module.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.generated_dtimes IS 'Date and Time when the OTP was generated'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.expiry_dtimes IS 'Date Time when the OTP will be expired'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.validation_retry_count IS 'Validation retry counts of this OTP request. If the validation retry crosses the threshold limit, then the OTP will be de-activated.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.status_code IS 'Current status of the transaction. Refers to code field of master.status_list table.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.lang_code IS 'For multilanguage implementation this attribute Refers master.language.code. The value of some of the attributes in current record is stored in this respective language.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.cr_by IS 'ID or name of the user who create / insert record.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.cr_dtimes IS 'Date and Timestamp when the record is created/inserted'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.upd_by IS 'ID or name of the user who update the record with new values'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.upd_dtimes IS 'Date and Timestamp when any of the fields in the record is updated with new values.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.is_deleted IS 'Flag to mark whether the record is Soft deleted.'; -COMMENT ON COLUMN archive.mosip_prereg_otp_transaction.del_dtimes IS 'Date and Timestamp when the record is soft deleted with is_deleted=TRUE'; diff --git a/data-archive/db_scripts/mosip_archive/ddl/archive-resident-otp_transaction.sql b/data-archive/db_scripts/mosip_archive/ddl/archive-resident-otp_transaction.sql deleted file mode 100644 index 7f3d8572..00000000 --- a/data-archive/db_scripts/mosip_archive/ddl/archive-resident-otp_transaction.sql +++ /dev/null @@ -1,47 +0,0 @@ --- ------------------------------------------------------------------------------------------------- --- Database Name: mosip_resident --- Release Version : 1.2.1 --- Purpose : Database scripts for Resident Service DB. --- Create By : Manoj SP --- Created Date : April-2022 --- --- Modified Date Modified By Comments / Remarks --- -------------------------------------------------------------------------------------------------- --- April-2022 Manoj SP Added otp_transaction table creation scripts with comments. ------------------------------------------------------------------------------------------------------ - --- This Table is used to save the OTP for the user whenever user requests for one using the email id / phone number to log into the application. -CREATE TABLE archive.mosip_resident_otp_transaction( - id character varying(36) NOT NULL, - ref_id character varying(1024) NOT NULL, - otp_hash character varying(512) NOT NULL, - generated_dtimes timestamp, - expiry_dtimes timestamp, - validation_retry_count smallint, - status_code character varying(36), - lang_code character varying(3), - cr_by character varying(256) NOT NULL, - cr_dtimes timestamp NOT NULL, - upd_by character varying(256), - upd_dtimes timestamp, - is_deleted boolean, - del_dtimes timestamp, - CONSTRAINT pk_otpt_id_resident PRIMARY KEY (id) -); - -COMMENT ON TABLE archive.mosip_resident_otp_transaction IS 'All OTP related data and validation details are maintained here for Pre Registration module.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.id IS 'OTP id is a unique identifier (UUID) used as an unique key to identify the OTP transaction'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.ref_id IS 'Reference ID is a reference information received from OTP requester which can be used while validating the OTP. AM: please give examples of ref_id'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.otp_hash IS 'Hash of id, ref_id and otp which is generated based on the configuration setup and sent to the requester application / module.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.generated_dtimes IS 'Date and Time when the OTP was generated'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.expiry_dtimes IS 'Date Time when the OTP will be expired'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.validation_retry_count IS 'Validation retry counts of this OTP request. If the validation retry crosses the threshold limit, then the OTP will be de-activated.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.status_code IS 'Current status of the transaction. Refers to code field of master.status_list table.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.lang_code IS 'For multilanguage implementation this attribute Refers master.language.code. The value of some of the attributes in current record is stored in this respective language.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.cr_by IS 'ID or name of the user who create / insert record.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.cr_dtimes IS 'Date and Timestamp when the record is created/inserted'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.upd_by IS 'ID or name of the user who update the record with new values'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.upd_dtimes IS 'Date and Timestamp when any of the fields in the record is updated with new values.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.is_deleted IS 'Flag to mark whether the record is Soft deleted.'; -COMMENT ON COLUMN archive.mosip_resident_otp_transaction.del_dtimes IS 'Date and Timestamp when the record is soft deleted with is_deleted=TRUE'; ------------------------------------------------------------------------------------------------------