[MIG] base_import_async and test_base_import_async to 11.0

base_import_async/__init__.py

@@ -1,25 +1,3 @@
-# -*- coding: utf-8 -*-
-###############################################################################
-#
-#   Module for Odoo
-#   Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
-#   Copyright (C) 2013 Akretion (http://www.akretion.com).
-#   @author Stéphane Bidoul <[email protected]>
-#   @author Sébastien BEAU <[email protected]>
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as
-#   published by the Free Software Foundation, either version 3 of the
-#   License, or (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-###############################################################################
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from . import models

base_import_async/__manifest__.py

@@ -1,31 +1,17 @@
-# -*- coding: utf-8 -*-
-###############################################################################
-#
-#   Module for Odoo
-#   Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
-#   Copyright (C) 2013 Akretion (http://www.akretion.com).
-#   @author Stéphane Bidoul <[email protected]>
-#   @author Sébastien BEAU <[email protected]>
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as
-#   published by the Free Software Foundation, either version 3 of the
-#   License, or (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-###############################################################################
+# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
+# Copyright (C) 2013 Akretion (http://www.akretion.com).
+# @author Stéphane Bidoul <[email protected]>
+# @author Sébastien BEAU <[email protected]>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
 {
     'name': 'Asynchronous Import',
-    'version': '10.0.1.0.0',
+    'summary': 'This module extends the standard CSV import '
+               'functionality to import files in the background',
+    'version': '11.0.1.0.0',
     'author': 'Akretion, ACSONE SA/NV, Odoo Community Association (OCA)',
     'license': 'AGPL-3',
+    'website': 'https://github.com/OCA/queue',
     'category': 'Generic Modules',
     'depends': [
         'base_import',

base_import_async/models/__init__.py

@@ -1,26 +1,4 @@
-# -*- coding: utf-8 -*-
-###############################################################################
-#
-#   Module for Odoo
-#   Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
-#   Copyright (C) 2013 Akretion (http://www.akretion.com).
-#   @author Stéphane Bidoul <[email protected]>
-#   @author Sébastien BEAU <[email protected]>
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as
-#   published by the Free Software Foundation, either version 3 of the
-#   License, or (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-###############################################################################
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from . import base_import_import
 from . import queue_job

base_import_async/models/base_import_import.py

@@ -1,33 +1,15 @@
-# -*- coding: utf-8 -*-
-###############################################################################
-#
-#   Module for Odoo
-#   Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
-#   Copyright (C) 2013 Akretion (http://www.akretion.com).
-#   @author Stéphane Bidoul <[email protected]>
-#   @author Sébastien BEAU <[email protected]>
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as
-#   published by the Free Software Foundation, either version 3 of the
-#   License, or (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-###############################################################################
+# Copyright (C) 2014 ACSONE SA/NV (http://acsone.eu).
+# Copyright (C) 2013 Akretion (http://www.akretion.com).
+# @author Stéphane Bidoul <[email protected]>
+# @author Sébastien BEAU <[email protected]>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 import csv
-import os
-from cStringIO import StringIO
+from os.path import splitext
+from base64 import encodebytes, decodebytes
+from io import StringIO
 
-from odoo import api, _
-from odoo.models import TransientModel
+from odoo import api, models, _
 from odoo.models import fix_import_export_id_paths
 
 from odoo.addons.queue_job.job import job, related_action
@@ -48,15 +30,7 @@
 DEFAULT_CHUNK_SIZE = 100
 
 
-def _encode(row, encoding):
-    return [cell.encode(encoding) for cell in row]
-
-
-def _decode(row, encoding):
-    return [cell.decode(encoding) for cell in row]
-
-
-class BaseImportImport(TransientModel):
+class BaseImportImport(models.TransientModel):
     _inherit = 'base_import.import'
 
     @api.multi
@@ -74,7 +48,7 @@ def do(self, fields, options, dryrun=False):
         except ValueError as e:
             return [{
                 'type': 'error',
-                'message': unicode(e),
+                'message': str(e),
                 'record': False,
             }]
 
@@ -117,34 +91,37 @@ def _create_csv_attachment(self, fields, data, options, file_name):
                             delimiter=str(options.get(OPT_SEPARATOR)),
                             quotechar=str(options.get(OPT_QUOTING)))
         encoding = options.get(OPT_ENCODING, 'utf-8')
-        writer.writerow(_encode(fields, encoding))
+        writer.writerow(fields)
         for row in data:
-            writer.writerow(_encode(row, encoding))
+            writer.writerow(row)
         # create attachment
+        datas = encodebytes(f.getvalue().encode(encoding))
         attachment = self.env['ir.attachment'].create({
             'name': file_name,
-            'datas': f.getvalue().encode('base64'),
+            'datas': datas,
             'datas_fname': file_name
         })
         return attachment.id
 
     @api.model
     def _read_csv_attachment(self, att_id, options):
         att = self.env['ir.attachment'].browse(att_id)
-        f = StringIO(att.datas.decode('base64'))
+        decoded_datas = decodebytes(att.datas)
+        encoding = options.get(OPT_ENCODING, 'utf-8')
+        f = StringIO(decoded_datas.decode(encoding))
         reader = csv.reader(f,
                             delimiter=str(options.get(OPT_SEPARATOR)),
                             quotechar=str(options.get(OPT_QUOTING)))
-        encoding = options.get(OPT_ENCODING, 'utf-8')
-        fields = _decode(reader.next(), encoding)
-        data = [_decode(row, encoding) for row in reader]
+
+        fields = next(reader)
+        data = [row for row in reader]
         return fields, data
 
     @api.model
     def _extract_records(self, model_obj, fields, data, chunk_size):
         """ Split the data on record boundaries,
         in chunks of minimum chunk_size """
-        fields = map(fix_import_export_id_paths, fields)
+        fields = list(map(fix_import_export_id_paths, fields))
         row_from = 0
         for rows in model_obj._extract_records(fields,
                                                data):
@@ -180,7 +157,7 @@ def _split_file(self, model_name, translated_model_name,
                                          row_from + 1 + header_offset,
                                          row_to + 1 + header_offset)
             # create a CSV attachment and enqueue the job
-            root, ext = os.path.splitext(file_name)
+            root, ext = splitext(file_name)
             att_id = self._create_csv_attachment(
                 fields, data[row_from:row_to + 1], options,
                 file_name=root + '-' + chunk + ext)

base_import_async/models/queue_job.py

@@ -1,6 +1,5 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 ACSONE SA/NV
-# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from odoo import api, models, _
 

base_import_async/readme/CONTRIBUTORS.rst

@@ -0,0 +1,12 @@
+Sébastien Beau (Akretion) authored the initial prototype.
+
+Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
+multi-line records, store data to import as attachments
+and let the user control the asynchronous behaviour.
+
+Other contributors include:
+
+* Anthony Muschang (ACSONE)
+* David Béal (Akretion)
+* Jonathan Nemry (ACSONE)
+* Laurent Mignon (ACSONE)

base_import_async/readme/DESCRIPTION.rst

@@ -0,0 +1,3 @@
+This module extends the standard CSV import functionality
+to import files in the background using the OCA/connector
+framework.

base_import_async/readme/ROADMAP.rst

@@ -0,0 +1,4 @@
+* There is currently no user interface to control the chunk size,
+  which is currently 100 by default. Should this proves to be an issue,
+  it is easy to add an option to extend the import screen.
+* Validation cannot be run in the background.

base_import_async/readme/USAGE.rst

@@ -0,0 +1,29 @@
+The user is presented with a new checkbox in the import
+screen. When selected, the import is delayed in a background
+job.
+
+This job in turn splits the CSV file in chunks of minimum
+100 lines (or more to align with record boundaries). Each
+chunk is then imported in a separate background job.
+
+When an import fails, the job is marked as such and the
+user can read the error in the job status. The CSV chunk
+being imported is stored as an attachment to the job, making
+it easy to download it, fix it and run a new import, possibly
+in synchronous mode since the chunks are small.
+
+Any file that can be imported by the standard import mechanism
+can also be imported in the background.
+
+This module's scope is limited to making standard imports
+asynchronous. It does not attempt to transform the data nor
+automate ETL flows.
+
+Other modules may benefit from this infrastructure in the following way
+(as illustrated in the test suite):
+
+1. create an instance of `base_import.import` and populate its fields
+   (`res_model`, `file`, `file_name`),
+2. invoke the `do` method with appropriate options
+   (`header`, `encoding`, `separator`, `quoting`,
+   `use_connector`, `chunk_size`).