diff --git a/base_import_async/README.rst b/base_import_async/README.rst deleted file mode 100644 index 26629481f2..0000000000 --- a/base_import_async/README.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License: AGPL-3 - -Odoo Asynchronous import module -=============================== - -This module extends the standard CSV import functionality -to import files in the background using the OCA/connector -framework. - -Usage -===== - -The user is presented with a new checkbox in the import -screen. When selected, the import is delayed in a background -job. - -This job in turn splits the CSV file in chunks of minimum -100 lines (or more to align with record boundaries). Each -chunk is then imported in a separate background job. - -When an import fails, the job is marked as such and the -user can read the error in the job status. The CSV chunk -being imported is stored as an attachment to the job, making -it easy to download it, fix it and run a new import, possibly -in synchronous mode since the chunks are small. - -Any file that can be imported by the standard import mechanism -can also be imported in the background. - -This module's scope is limited to making standard imports -asynchronous. It does not attempt to transform the data nor -automate ETL flows. - -Other modules may benefit from this infrastructure in the following way -(as illustrated in the test suite): - -1. create an instance of `base_import.import` and populate its fields - (`res_model`, `file`, `file_name`), -2. invoke the `do` method with appropriate options - (`header`, `encoding`, `separator`, `quoting`, - `use_connector`, `chunk_size`). - -Known issues / Roadmap -====================== - -* There is currently no user interface to control the chunk size, - which is currently 100 by default. Should this proves to be an issue, - it is easy to add an option to extend the import screen. -* Validation cannot be run in the background. - - -Bug Tracker -=========== - -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. - - -Credits -======= - -Contributors ------------- - -Sébastien Beau (Akretion) authored the initial prototype. - -Stéphane Bidoul (ACSONE) extended it to version 1.0 to support -multi-line records, store data to import as attachments -and let the user control the asynchronous behaviour. - -Other contributors include: - -* Anthony Muschang (ACSONE) -* David Béal (Akretion) -* Jonathan Nemry (ACSONE) -* Laurent Mignon (ACSONE) - -Maintainer ----------- - -.. image:: http://odoo-community.org/logo.png - :alt: Odoo Community Association - :target: http://odoo-community.org - -This module is maintained by the OCA. - -OCA, or the Odoo Community Association, is a nonprofit organization whose -mission is to support the collaborative development of Odoo features and -promote its widespread use. - -To contribute to this module, please visit http://odoo-community.org. diff --git a/base_import_async/__init__.py b/base_import_async/__init__.py index 184a3e03cb..31660d6a96 100644 --- a/base_import_async/__init__.py +++ b/base_import_async/__init__.py @@ -1,25 +1,3 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import models diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 758696214d..ea0a3ad5fb 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -1,31 +1,17 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + { 'name': 'Asynchronous Import', - 'version': '10.0.1.0.0', + 'summary': 'This module extends the standard CSV import ' + 'functionality to import files in the background', + 'version': '11.0.1.0.0', 'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', + 'website': 'https://github.com/OCA/queue', 'category': 'Generic Modules', 'depends': [ 'base_import', diff --git a/base_import_async/models/__init__.py b/base_import_async/models/__init__.py index 29ab48105d..2af21f4102 100644 --- a/base_import_async/models/__init__.py +++ b/base_import_async/models/__init__.py @@ -1,26 +1,4 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import base_import_import from . import queue_job diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 063d5e49bd..997641c3fb 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -1,33 +1,15 @@ -# -*- coding: utf-8 -*- -############################################################################### -# -# Module for Odoo -# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). -# Copyright (C) 2013 Akretion (http://www.akretion.com). -# @author Stéphane Bidoul -# @author Sébastien BEAU -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################### +# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu). +# Copyright (C) 2013 Akretion (http://www.akretion.com). +# @author Stéphane Bidoul +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). import csv -import os -from cStringIO import StringIO +from os.path import splitext +from base64 import encodebytes, decodebytes +from io import StringIO -from odoo import api, _ -from odoo.models import TransientModel +from odoo import api, models, _ from odoo.models import fix_import_export_id_paths from odoo.addons.queue_job.job import job, related_action @@ -48,15 +30,7 @@ DEFAULT_CHUNK_SIZE = 100 -def _encode(row, encoding): - return [cell.encode(encoding) for cell in row] - - -def _decode(row, encoding): - return [cell.decode(encoding) for cell in row] - - -class BaseImportImport(TransientModel): +class BaseImportImport(models.TransientModel): _inherit = 'base_import.import' @api.multi @@ -74,7 +48,7 @@ def do(self, fields, options, dryrun=False): except ValueError as e: return [{ 'type': 'error', - 'message': unicode(e), + 'message': str(e), 'record': False, }] @@ -117,13 +91,14 @@ def _create_csv_attachment(self, fields, data, options, file_name): delimiter=str(options.get(OPT_SEPARATOR)), quotechar=str(options.get(OPT_QUOTING))) encoding = options.get(OPT_ENCODING, 'utf-8') - writer.writerow(_encode(fields, encoding)) + writer.writerow(fields) for row in data: - writer.writerow(_encode(row, encoding)) + writer.writerow(row) # create attachment + datas = encodebytes(f.getvalue().encode(encoding)) attachment = self.env['ir.attachment'].create({ 'name': file_name, - 'datas': f.getvalue().encode('base64'), + 'datas': datas, 'datas_fname': file_name }) return attachment.id @@ -131,20 +106,22 @@ def _create_csv_attachment(self, fields, data, options, file_name): @api.model def _read_csv_attachment(self, att_id, options): att = self.env['ir.attachment'].browse(att_id) - f = StringIO(att.datas.decode('base64')) + decoded_datas = decodebytes(att.datas) + encoding = options.get(OPT_ENCODING, 'utf-8') + f = StringIO(decoded_datas.decode(encoding)) reader = csv.reader(f, delimiter=str(options.get(OPT_SEPARATOR)), quotechar=str(options.get(OPT_QUOTING))) - encoding = options.get(OPT_ENCODING, 'utf-8') - fields = _decode(reader.next(), encoding) - data = [_decode(row, encoding) for row in reader] + + fields = next(reader) + data = [row for row in reader] return fields, data @api.model def _extract_records(self, model_obj, fields, data, chunk_size): """ Split the data on record boundaries, in chunks of minimum chunk_size """ - fields = map(fix_import_export_id_paths, fields) + fields = list(map(fix_import_export_id_paths, fields)) row_from = 0 for rows in model_obj._extract_records(fields, data): @@ -180,7 +157,7 @@ def _split_file(self, model_name, translated_model_name, row_from + 1 + header_offset, row_to + 1 + header_offset) # create a CSV attachment and enqueue the job - root, ext = os.path.splitext(file_name) + root, ext = splitext(file_name) att_id = self._create_csv_attachment( fields, data[row_from:row_to + 1], options, file_name=root + '-' + chunk + ext) diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py index b9bcb79cc6..f1476c575f 100644 --- a/base_import_async/models/queue_job.py +++ b/base_import_async/models/queue_job.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright 2017 ACSONE SA/NV -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from odoo import api, models, _ diff --git a/base_import_async/readme/CONTRIBUTORS.rst b/base_import_async/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..48e3feaae6 --- /dev/null +++ b/base_import_async/readme/CONTRIBUTORS.rst @@ -0,0 +1,12 @@ +Sébastien Beau (Akretion) authored the initial prototype. + +Stéphane Bidoul (ACSONE) extended it to version 1.0 to support +multi-line records, store data to import as attachments +and let the user control the asynchronous behaviour. + +Other contributors include: + +* Anthony Muschang (ACSONE) +* David Béal (Akretion) +* Jonathan Nemry (ACSONE) +* Laurent Mignon (ACSONE) diff --git a/base_import_async/readme/DESCRIPTION.rst b/base_import_async/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..b5996832ec --- /dev/null +++ b/base_import_async/readme/DESCRIPTION.rst @@ -0,0 +1,3 @@ +This module extends the standard CSV import functionality +to import files in the background using the OCA/connector +framework. diff --git a/base_import_async/readme/ROADMAP.rst b/base_import_async/readme/ROADMAP.rst new file mode 100644 index 0000000000..eccbbb6e4b --- /dev/null +++ b/base_import_async/readme/ROADMAP.rst @@ -0,0 +1,4 @@ +* There is currently no user interface to control the chunk size, + which is currently 100 by default. Should this proves to be an issue, + it is easy to add an option to extend the import screen. +* Validation cannot be run in the background. diff --git a/base_import_async/readme/USAGE.rst b/base_import_async/readme/USAGE.rst new file mode 100644 index 0000000000..3618b71b4c --- /dev/null +++ b/base_import_async/readme/USAGE.rst @@ -0,0 +1,29 @@ +The user is presented with a new checkbox in the import +screen. When selected, the import is delayed in a background +job. + +This job in turn splits the CSV file in chunks of minimum +100 lines (or more to align with record boundaries). Each +chunk is then imported in a separate background job. + +When an import fails, the job is marked as such and the +user can read the error in the job status. The CSV chunk +being imported is stored as an attachment to the job, making +it easy to download it, fix it and run a new import, possibly +in synchronous mode since the chunks are small. + +Any file that can be imported by the standard import mechanism +can also be imported in the background. + +This module's scope is limited to making standard imports +asynchronous. It does not attempt to transform the data nor +automate ETL flows. + +Other modules may benefit from this infrastructure in the following way +(as illustrated in the test suite): + +1. create an instance of `base_import.import` and populate its fields + (`res_model`, `file`, `file_name`), +2. invoke the `do` method with appropriate options + (`header`, `encoding`, `separator`, `quoting`, + `use_connector`, `chunk_size`). diff --git a/base_import_async/static/src/js/import.js b/base_import_async/static/src/js/import.js index 7ab7d08f7a..9cff3b58af 100644 --- a/base_import_async/static/src/js/import.js +++ b/base_import_async/static/src/js/import.js @@ -3,11 +3,9 @@ odoo.define('base_import_async.import', function (require) { var core = require('web.core'); var _t = core._t; - require('base_import.import'); + var DataImport = require('base_import.import').DataImport; - var DataImport = core.action_registry.get('import'); - - DataImport = DataImport.include({ + DataImport.include({ import_options: function () { var options = this._super.apply(this, arguments); @@ -16,9 +14,11 @@ odoo.define('base_import_async.import', function (require) { }, onimported: function () { - var self = this; if (this.$('input.oe_import_queue').prop('checked')) { - this.do_notify(_t("Your request is being processed"), _t("You can check the status of this job in menu 'Queue / Jobs'.")); + this.do_notify( + _t("Your request is being processed"), + _t("You can check the status of this job in menu 'Queue / Jobs'.") + ); } this._super.apply(this, arguments); }, diff --git a/base_import_async/static/src/xml/import.xml b/base_import_async/static/src/xml/import.xml index 22300db512..7819613d51 100644 --- a/base_import_async/static/src/xml/import.xml +++ b/base_import_async/static/src/xml/import.xml @@ -1,3 +1,4 @@ + diff --git a/base_import_async/views/base_import_async.xml b/base_import_async/views/base_import_async.xml index da49cf8c90..56e5645a7f 100644 --- a/base_import_async/views/base_import_async.xml +++ b/base_import_async/views/base_import_async.xml @@ -1,10 +1,10 @@ - - - - - + + + + + diff --git a/test_base_import_async/__init__.py b/test_base_import_async/__init__.py index 8939873f07..b08b489909 100644 --- a/test_base_import_async/__init__.py +++ b/test_base_import_async/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ############################################################################### # # Module for OpenERP diff --git a/test_base_import_async/__manifest__.py b/test_base_import_async/__manifest__.py index ba9ca1eda6..b1ef4c0db3 100644 --- a/test_base_import_async/__manifest__.py +++ b/test_base_import_async/__manifest__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ############################################################################### # # Module for OpenERP @@ -21,11 +20,12 @@ ############################################################################### { 'name': 'Test suite for base_import_async', - 'version': '10.0.1.0.0', + 'version': '11.0.1.0.0', 'author': 'ACSONE SA/NV, Odoo Community Association (OCA)', 'license': 'AGPL-3', + 'website': 'https://github.com/OCA/queue', 'category': 'Generic Modules', - 'description': """Test suite for base_import_async. + 'summary': """Test suite for base_import_async. Normally you don't need to install this. """, diff --git a/test_base_import_async/tests/__init__.py b/test_base_import_async/tests/__init__.py index c315b360fd..5ed374a195 100644 --- a/test_base_import_async/tests/__init__.py +++ b/test_base_import_async/tests/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ############################################################################### # # Module for OpenERP diff --git a/test_base_import_async/tests/test_base_import_async.py b/test_base_import_async/tests/test_base_import_async.py index 9f612f0f3f..36552b4487 100644 --- a/test_base_import_async/tests/test_base_import_async.py +++ b/test_base_import_async/tests/test_base_import_async.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ############################################################################### # # Module for OpenERP