[14.0][ADD] queue_job_chunk : backport from v16

queue_job_chunk/README.rst

@@ -0,0 +1,90 @@
+===============
+Job Queue Chunk
+===============
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:2ec876f81add7cb79bdb91363bbf23f50aeea267b12cc9e76f6da9cca47d2931
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Beta
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github
+    :target: https://github.com/OCA/queue/tree/14.0/queue_job_chunk
+    :alt: OCA/queue
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/queue-14-0/queue-14-0-queue_job_chunk
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=14.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+Adds the notion of queue job chunks, essentially a queue job with some metadata.
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Usage
+=====
+
+1. Create a queue job chunk using relevant fields to specify which service to use
+2. Use menu to check its status and if you want to, modify its contents to re-run it
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/queue/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/queue/issues/new?body=module:%20queue_job_chunk%0Aversion:%2014.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+~~~~~~~
+
+* Akretion
+
+Contributors
+~~~~~~~~~~~~
+
+* Kevin Khao <kevin.khao@akretion.com>
+* Sébastien Beau <sebastien.beau@akretion.com>
+
+Other credits
+~~~~~~~~~~~~~
+
+The development of this module has been financially supported by:
+
+* Akretion <www.akretion.com>
+
+Maintainers
+~~~~~~~~~~~
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/14.0/queue_job_chunk>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

queue_job_chunk/__init__.py

@@ -0,0 +1 @@
+from . import models

queue_job_chunk/__manifest__.py

@@ -0,0 +1,18 @@
+#  Copyright (c) Akretion 2020
+#  License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
+
+{
+    "name": "Job Queue Chunk",
+    "version": "14.0.2.0.0",
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "website": "https://github.com/OCA/queue",
+    "license": "AGPL-3",
+    "category": "Generic Modules",
+    "depends": ["queue_job"],
+    "data": [
+        "views/queue_job_chunk.xml",
+        "security/security.xml",
+        "security/ir.model.access.csv",
+    ],
+    "installable": True,
+}

queue_job_chunk/models/__init__.py

@@ -0,0 +1 @@
+from . import queue_job_chunk

queue_job_chunk/models/queue_job_chunk.py

@@ -0,0 +1,115 @@
+#  Copyright (c) Akretion 2020
+#  License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
+
+import json
+import traceback
+
+from psycopg2 import OperationalError
+
+from odoo import api, fields, models
+from odoo.service.model import PG_CONCURRENCY_ERRORS_TO_RETRY
+
+from odoo.addons.queue_job.exception import RetryableJobError
+
+# Use to bypass chunks entirely for easier debugging
+DEBUG_MODE = False
+
+
+class QueueJobChunk(models.Model):
+    _name = "queue.job.chunk"
+    _description = "Queue Job Chunk"
+    _order = "id desc"
+
+    processor = fields.Selection([])
+    data_str = fields.Text(string="Editable data")
+    state = fields.Selection(
+        [("pending", "Pending"), ("done", "Done"), ("fail", "Failed")],
+        default="pending",
+        readonly=True,
+    )
+    state_info = fields.Text("Additional state information", readonly=True)
+    model_name = fields.Char(readonly=True)
+    record_id = fields.Integer(readonly=True)
+    reference = fields.Reference(
+        selection="_selection_target_model",
+        compute="_compute_reference",
+        store=True,
+        readonly=True,
+    )
+    company_id = fields.Many2one(
+        "res.company",
+        compute="_compute_reference",
+        store=True,
+        readonly=True,
+    )
+    stack_trace = fields.Text(readonly=True)
+
+    @api.model
+    def _selection_target_model(self):
+        models = self.env["ir.model"].search([])
+        return [(model.model, model.name) for model in models]
+
+    @api.depends("model_name", "record_id")
+    def _compute_reference(self):
+        for rec in self:
+            rec.company_id = self.env.user.company_id
+            if rec.model_name and rec.record_id:
+                rec.reference = "{},{}".format(rec.model_name, rec.record_id or 0)
+                record = self.env[rec.model_name].browse(rec.record_id)
+                if "company_id" in record._fields:
+                    rec.company_id = record.company_id
+            else:
+                rec.reference = False
+
+    @api.model_create_multi
+    def create(self, vals):
+        result = super().create(vals)
+        for rec in result:
+            rec.enqueue_job()
+        return result
+
+    def button_retry(self):
+        self.enqueue_job()
+
+    def enqueue_job(self):
+        if DEBUG_MODE:
+            return self.process_chunk()
+        else:
+            return self.with_delay().process_chunk()
+
+    def _get_processor(self):
+        # return here whatever class you want
+        # it can be a pure python class, an odoo TransientModel ...
+        raise NotImplementedError
+
+    def _get_data(self):
+        return json.loads(self.data_str)
+
+    def process_chunk(self):
+        self.ensure_one()
+        try:
+            with self.env.cr.savepoint():
+                processor = self._get_processor()
+                result = processor.run()
+        except RetryableJobError:
+            raise
+        except Exception as e:
+            if DEBUG_MODE:
+                raise
+            # TODO maybe it will be simplier to have a kind of inherits
+            #  on queue.job to avoid a double error management
+            # so a failling chunk will have a failling job
+            if (
+                isinstance(e, OperationalError)
+                and e.pgcode in PG_CONCURRENCY_ERRORS_TO_RETRY
+            ):
+                # In that case we raise an error so queue_job
+                # will do a RetryableJobError
+                raise
+            self.state = "fail"
+            self.state_info = type(e).__name__ + str(e.args)
+            self.stack_trace = traceback.format_exc()
+            return False
+        self.state_info = ""
+        self.state = "done"
+        return result

queue_job_chunk/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"

queue_job_chunk/readme/CONTRIBUTORS.rst

@@ -0,0 +1,2 @@
+* Kevin Khao <kevin.khao@akretion.com>
+* Sébastien Beau <sebastien.beau@akretion.com>

queue_job_chunk/readme/CREDITS.rst

@@ -0,0 +1,3 @@
+The development of this module has been financially supported by:
+
+* Akretion <www.akretion.com>

queue_job_chunk/readme/DESCRIPTION.rst

@@ -0,0 +1 @@
+Adds the notion of queue job chunks, essentially a queue job with some metadata.

queue_job_chunk/readme/USAGE.rst

@@ -0,0 +1,2 @@
+1. Create a queue job chunk using relevant fields to specify which service to use
+2. Use menu to check its status and if you want to, modify its contents to re-run it

queue_job_chunk/security/ir.model.access.csv

@@ -0,0 +1,3 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_queue_job_chunk_manager,queue job manager,model_queue_job_chunk,queue_job.group_queue_job_manager,1,1,1,1
+access_queue_job_queue_job_chunk_user,queue job manager,queue_job.model_queue_job,group_queue_job_chunk_user,1,0,0,0

queue_job_chunk/security/security.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo>
+
+    <record id="group_queue_job_chunk_user" model="res.groups">
+        <field name="name">Job Queue chunk User</field>
+        <field name="category_id" ref="queue_job.module_category_queue_job" />
+    </record>
+
+    <record id="queue_job.group_queue_job_manager" model="res.groups">
+        <field
+            name="implied_ids"
+            eval="[(4, ref('queue_job_chunk.group_queue_job_chunk_user'))]"
+        />
+    </record>
+
+    <record id="queue_job_chunk_comp_rule" model="ir.rule">
+        <field name="name">Job Queue chunk multi-company</field>
+        <field name="model_id" ref="model_queue_job_chunk" />
+        <field
+            name="domain_force"
+        >['|', ('company_id','=',False), ('company_id','in',company_ids)]</field>
+    </record>
+
+</odoo>

queue_job_chunk/tests/__init__.py

@@ -0,0 +1 @@
+from . import test_queue_job_chunk

queue_job_chunk/tests/models.py

@@ -0,0 +1,44 @@
+# Copyright 2022 Akretion (https://www.akretion.com).
+# @author Sébastien BEAU <sebastien.beau@akretion.com>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+from odoo import fields, models
+
+
+# Exemple of pure python processor (inherit is not possible)
+class TestPythonPartnerProcessor:
+    def __init__(self, chunk):
+        super().__init__()
+        self.env = chunk.env
+        self.chunk = chunk
+
+    def run(self):
+        return self.env["res.partner"].create(self.chunk._get_data())
+
+
+# Exemple of odoo processor (inherit is possible)
+class TestOdooStateProcessor(models.TransientModel):
+    _name = "test.odoo.state.processor"
+    _description = "Chunk Processor State Create"
+
+    chunk_id = fields.Many2one("queue.job.chunk", "Chunk")
+
+    def run(self):
+        return self.env["res.country.state"].create(self.chunk_id._get_data())
+
+
+class QueueJobChunk(models.Model):
+    _inherit = "queue.job.chunk"
+
+    processor = fields.Selection(
+        selection_add=[
+            ("test_partner", "Test create Partner"),
+            ("test_state", "Test create Country State"),
+        ],
+    )
+
+    def _get_processor(self):
+        if self.processor == "test_partner":
+            return TestPythonPartnerProcessor(self)
+        elif self.processor == "test_state":
+            return self.env["test.odoo.state.processor"].new({"chunk_id": self.id})

queue_job_chunk/tests/test_queue_job_chunk.py

@@ -0,0 +1,103 @@
+#  Copyright (c) Akretion 2020
+#  License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
+
+
+from odoo_test_helper import FakeModelLoader
+
+from odoo.tests import SavepointCase
+
+
+class TestQueueJobChunk(SavepointCase, FakeModelLoader):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        cls.loader = FakeModelLoader(cls.env, cls.__module__)
+        cls.loader.backup_registry()
+        from .models import QueueJobChunk, TestOdooStateProcessor
+
+        cls.loader.update_registry((TestOdooStateProcessor, QueueJobChunk))
+
+        cls.env = cls.env(context=dict(cls.env.context, test_queue_job_no_delay=True))
+        cls.main_company = cls.env.ref("base.main_company")
+        cls.another_company_partner = cls.env["res.partner"].create(
+            {"name": "Company2"}
+        )
+        cls.another_company = cls.env["res.company"].create(
+            {
+                "name": cls.another_company_partner.name,
+                "partner_id": cls.another_company_partner.id,
+                "currency_id": cls.env.ref("base.main_company").currency_id.id,
+            }
+        )
+        cls.partner = cls.env.ref("base.res_partner_3")
+        cls.chunk_data_contact = [
+            {
+                "data_str": '{"name": "Steve Queue Job"}',
+                "processor": "test_partner",
+                "model_name": "res.partner",
+                "record_id": cls.partner.id,
+            },
+            {
+                "data_str": '{"name": "Other"}',
+                "processor": "test_partner",
+                "model_name": "res.partner",
+                "record_id": cls.partner.id,
+            },
+        ]
+        cls.chunk_data_bad = {
+            "data_str": "{''(;,),x*}",
+            "processor": "test_partner",
+            "model_name": "res.partner",
+            "record_id": cls.partner.id,
+        }
+        USA = cls.env.ref("base.us")
+        cls.chunk_data_state = {
+            "processor": "test_state",
+            "data_str": '{"name": "New Stateshire", "code": "NS", "country_id": %d}'
+            % USA.id,
+            "model_name": "res.country",
+            "record_id": USA.id,
+        }
+
+    def test_create_chunk(self):
+        partner_count = self.env["res.partner"].search_count([])
+        chunk = self.env["queue.job.chunk"].create(self.chunk_data_contact)
+        new_partner_count = self.env["res.partner"].search_count([])
+        self.assertEqual(chunk[0].state, "done")
+        self.assertEqual(chunk[1].state, "done")
+        self.assertEqual(partner_count + 2, new_partner_count)
+
+    def test_create_chunk_fail_retry(self):
+        partner_count = self.env["res.partner"].search_count([])
+        # fail with bad data
+        chunk = self.env["queue.job.chunk"].create(self.chunk_data_bad)
+        self.assertEqual(chunk.state, "fail")
+
+        # retry with correct data
+        chunk.data_str = '{"name": "Steve Queue Job"}'
+        chunk.button_retry()
+        new_partner_count = self.env["res.partner"].search_count([])
+        self.assertEqual(partner_count + 1, new_partner_count)
+
+    def test_create_chunk_without_company_id(self):
+        chunk = self.env["queue.job.chunk"].create(self.chunk_data_state)
+        self.assertEqual(chunk.company_id, self.env.user.company_id)
+
+    def test_create_chunk_with_company_id(self):
+        company = self.partner.company_id
+        chunk = self.env["queue.job.chunk"].create(self.chunk_data_contact[0])
+        self.assertEqual(company, chunk.company_id)
+
+    def test_reference(self):
+        chunk = self.env["queue.job.chunk"].create(self.chunk_data_contact[0])
+        self.assertEqual(chunk.reference, self.partner)
+
+    def test_create_chunk_job(self):
+        job_count = self.env["queue.job"].search_count([])
+        self.env = self.env(
+            context=dict(self.env.context, test_queue_job_no_delay=False)
+        )
+        self.env["queue.job.chunk"].create(self.chunk_data_contact[0])
+        new_job_count = self.env["queue.job"].search_count([])
+        self.assertEqual(job_count + 1, new_job_count)

queue_job_chunk/views/queue_job_chunk.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo>
+
+    <record id="view_queue_job_form" model="ir.ui.view">
+        <field name="model">queue.job.chunk</field>
+        <field name="arch" type="xml">
+            <form>
+                <header>
+                    <button
+                        name="button_retry"
+                        type="object"
+                        string="Retry job"
+                        attrs="{'invisible': [('state', '=', 'done')]}"
+                        class="btn-primary"
+                    />
+                    <field name="state" widget="statusbar" />
+                </header>
+                <sheet>
+                    <group>
+                        <group>
+                            <field name="processor" />
+                            <field name="reference" />
+                            <field name="state_info" />
+                            <field name="stack_trace" groups="base.group_system" />
+                            <field name="data_str" />
+                        </group>
+                    </group>
+                </sheet>
+            </form>
+        </field>
+    </record>
+
+    <record id="view_queue_job_tree" model="ir.ui.view">
+        <field name="model">queue.job.chunk</field>
+        <field name="arch" type="xml">
+            <tree>
+                <field name="create_date" />
+                <field name="processor" />
+                <field name="reference" />
+                <field name="state" />
+                <field name="data_str" optional="hide" />
+            </tree>
+        </field>
+    </record>
+
+    <record id="view_queue_job_search" model="ir.ui.view">
+        <field name="model">queue.job.chunk</field>
+        <field name="arch" type="xml">
+            <search>
+                <field name="data_str" />
+                <field name="stack_trace" />
+                <filter
+                    string="Pending"
+                    name="pending"
+                    domain="[('state', '=', 'pending')]"
+                />
+                <filter
+                    string="Failed"
+                    name="failed"
+                    domain="[('state', '=', 'fail')]"
+                />
+            </search>
+        </field>
+    </record>
+
+    <record id="action_queue_job_chunk" model="ir.actions.act_window">
+        <field name="name">Queue Job Chunks</field>
+        <field name="type">ir.actions.act_window</field>
+        <field name="res_model">queue.job.chunk</field>
+        <field name="view_mode">tree,form</field>
+    </record>
+
+    <menuitem
+        id="menu_queue_job_chunk"
+        name="Queue Chunks"
+        parent="queue_job.menu_queue_job_root"
+        action="action_queue_job_chunk"
+    />
+
+</odoo>

setup/queue_job_chunk/odoo/addons/queue_job_chunk

@@ -0,0 +1 @@
+../../../../queue_job_chunk

setup/queue_job_chunk/setup.py

@@ -0,0 +1,6 @@
+import setuptools
+
+setuptools.setup(
+    setup_requires=['setuptools-odoo'],
+    odoo_addon=True,
+)