From 790d68d5b9d70fd006c4bcc0f84adb182ef80d20 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 10:34:24 +0100 Subject: [PATCH 01/69] Add edi_storage --- edi_storage_oca/README.rst | 105 ++++ edi_storage_oca/__init__.py | 2 + edi_storage_oca/__manifest__.py | 17 + edi_storage_oca/components/__init__.py | 3 + edi_storage_oca/components/base.py | 54 +++ edi_storage_oca/components/check.py | 83 ++++ edi_storage_oca/components/send.py | 39 ++ edi_storage_oca/demo/edi_backend_demo.xml | 14 + edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 67 +++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/readme/DESCRIPTION.rst | 23 + edi_storage_oca/readme/USAGE.rst | 1 + edi_storage_oca/security/ir_model_access.xml | 12 + edi_storage_oca/static/description/icon.png | Bin 0 -> 9455 bytes edi_storage_oca/static/description/index.html | 450 ++++++++++++++++++ .../tests/_TODO_test_components_base.py | 43 ++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 116 +++++ .../tests/test_edi_backend_storage.py | 204 ++++++++ edi_storage_oca/views/edi_backend_views.xml | 21 + 21 files changed, 1257 insertions(+) create mode 100644 edi_storage_oca/README.rst create mode 100644 edi_storage_oca/__init__.py create mode 100644 edi_storage_oca/__manifest__.py create mode 100644 edi_storage_oca/components/__init__.py create mode 100644 edi_storage_oca/components/base.py create mode 100644 edi_storage_oca/components/check.py create mode 100644 edi_storage_oca/components/send.py create mode 100644 edi_storage_oca/demo/edi_backend_demo.xml create mode 100644 edi_storage_oca/models/__init__.py create mode 100644 edi_storage_oca/models/edi_backend.py create mode 100644 edi_storage_oca/readme/CONTRIBUTORS.rst create mode 100644 edi_storage_oca/readme/DESCRIPTION.rst create mode 100644 edi_storage_oca/readme/USAGE.rst create mode 100644 edi_storage_oca/security/ir_model_access.xml create mode 100644 edi_storage_oca/static/description/icon.png create mode 100644 edi_storage_oca/static/description/index.html create mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/__init__.py create mode 100644 edi_storage_oca/tests/common.py create mode 100644 edi_storage_oca/tests/test_edi_backend_storage.py create mode 100644 edi_storage_oca/views/edi_backend_views.xml diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst new file mode 100644 index 0000000000..fffe958379 --- /dev/null +++ b/edi_storage_oca/README.rst @@ -0,0 +1,105 @@ +=========================== +EDI Storage backend support +=========================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github + :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :alt: OCA/edi +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/226/13.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* ACSONE + +Contributors +~~~~~~~~~~~~ + +* Simone Orsi + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/edi `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__init__.py b/edi_storage_oca/__init__.py new file mode 100644 index 0000000000..f24d3e2426 --- /dev/null +++ b/edi_storage_oca/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py new file mode 100644 index 0000000000..9d4ff373fd --- /dev/null +++ b/edi_storage_oca/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "EDI Storage backend support", + "summary": """ + Base module to allow exchanging files via storage backend (eg: SFTP). + """, + "version": "13.0.1.0.0", + "development_status": "Alpha", + "license": "AGPL-3", + "author": "ACSONE,Odoo Community Association (OCA)", + "depends": ["edi", "storage_backend", "component"], + "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "demo": ["demo/edi_backend_demo.xml"], +} diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py new file mode 100644 index 0000000000..9f64b792bb --- /dev/null +++ b/edi_storage_oca/components/__init__.py @@ -0,0 +1,3 @@ +from . import base +from . import check +from . import send diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py new file mode 100644 index 0000000000..2a337fed17 --- /dev/null +++ b/edi_storage_oca/components/base.py @@ -0,0 +1,54 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from pathlib import PurePath + +from odoo.addons.component.core import AbstractComponent + + +class EDIStorageSendComponentMixin(AbstractComponent): + + _name = "edi.storage.component.mixin" + + @property + def storage(self): + return self.backend.storage_id + + def _dir_by_state(self, direction, state): + """Return remote directory path by direction and state. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :return: PurePath object + """ + assert direction in ("input", "output") + assert state in ("pending", "done", "error") + return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + + def _remote_file_path(self, direction, state, filename): + """Return remote file path by direction and state for give filename. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :param filename: string for file name + :return: PurePath object + """ + return self._dir_by_state(direction, state) / filename.strip("/ ") + + def _get_remote_file(self, state, filename=None, binary=False): + """Get file for current exchange_record in the given destination state. + + :param state: string ("pending", "done", "error") + :param filename: custom file name, exchange_record filename used by default + :return: remote file content as string + """ + filename = filename or self.exchange_record.exchange_filename + path = self._remote_file_path(self.exchange_record.direction, state, filename) + try: + # TODO: support match via pattern (eg: filename-prefix-*) + # otherwise is impossible to retrieve input files and acks + # (the date will never match) + return self.storage.get(path.as_posix(), binary=binary) + except FileNotFoundError: + return None diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py new file mode 100644 index 0000000000..e76ec75aac --- /dev/null +++ b/edi_storage_oca/components/check.py @@ -0,0 +1,83 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo.tools import pycompat + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + + +class EDIStorageCheckComponentMixin(Component): + + _name = "edi.storage.component.check" + _inherit = [ + "edi.component.check.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.check" + + def check(self): + return self._exchange_output_check() + + def _exchange_output_check(self): + """Check status output exchange and update record. + + 1. check if the file has been processed already (done) + 2. if yes, post message and exit + 3. if not, check for errors + 4. if no errors, return + + :return: boolean + * False if there's nothing else to be done + * True if file still need action + """ + if self._get_remote_file("done"): + _logger.info( + "%s done for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + if ( + not self.exchange_record.edi_exchange_state + == "output_sent_and_processed" + ): + self.exchange_record.edi_exchange_state = "output_sent_and_processed" + self.backend._notify_done(self.exchange_record) + if self.exchange_record.type_id.ack_needed: + self._exchange_output_handle_ack() + return False + + error = self._get_remote_file("error") + if error: + _logger.info( + "%s error for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + # Assume a text file will be placed there w/ the same name and error suffix + err_filename = self.exchange_record.exchange_filename + ".error" + error_report = self._get_remote_file("error", filename=err_filename) + if self.exchange_record.edi_exchange_state == "output_sent": + self.exchange_record.update( + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": pycompat.to_text(error_report), + } + ) + self.backend._notify_error(self.exchange_record, "process_ko") + return False + return True + + def _exchange_output_handle_ack(self): + ack_file = self._get_remote_file( + "done", filename=self.exchange_record.ack_filename + ) + if ack_file: + self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend._notify_ack_received(self.exchange_record) + else: + self.backend._notify_ack_missing(self.exchange_record) diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py new file mode 100644 index 0000000000..4d3f376758 --- /dev/null +++ b/edi_storage_oca/components/send.py @@ -0,0 +1,39 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class EDIStorageSendComponent(Component): + + _name = "edi.storage.component.send" + _inherit = [ + "edi.component.send.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.send" + + def send(self): + # If the file has been sent already, refresh its state + # TODO: double check if this is useless + # since the backend checks the state already + checker = self.component(usage="edi.storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + filedata = self.exchange_record.exchange_file + path = self._remote_file_path(direction, "pending", filename) + self.storage.add(path.as_posix(), filedata, binary=False) + # TODO: delegate this to generic storage backend + # except paramiko.ssh_exception.AuthenticationException: + # # TODO this exc handling should be moved to sftp backend IMO + # error = _("Authentication error") + # state = "error_on_send" + # TODO: catch other specific exceptions + # this will swallow all the exceptions! + return True diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml new file mode 100644 index 0000000000..d8ff5d972c --- /dev/null +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -0,0 +1,14 @@ + + + + Storage Demo EDI backend + + + demo_in/pending + demo_in/done + demo_in/error + demo_out/pending + demo_out/done + demo_out/error + + diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py new file mode 100644 index 0000000000..4b2c54ac39 --- /dev/null +++ b/edi_storage_oca/models/__init__.py @@ -0,0 +1 @@ +from . import edi_backend diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py new file mode 100644 index 0000000000..5ba9336391 --- /dev/null +++ b/edi_storage_oca/models/edi_backend.py @@ -0,0 +1,67 @@ +# Copyright 2020 ACSONE SA +# @author Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class EDIBackend(models.Model): + + _inherit = "edi.backend" + + storage_id = fields.Many2one( + string="Storage backend", + comodel_name="storage.backend", + help="Storage for in-out files", + ondelete="restrict", + ) + """ + We assume the exchanges happen it 2 ways (input, output) + and we have a hierarchy of directory like: + + from_A_to_B + |- pending + |- done + |- error + from_B_to_A + |- pending + |- done + |- error + + where A and B are the partners exchanging data and they are in turn + sender and receiver and vice versa. + """ + # TODO: these paths should probably be by type instead + # Here we can maybe set a common root folder for this exchange. + input_dir_pending = fields.Char( + "Input pending directory", help="Path to folder for pending operations" + ) + input_dir_done = fields.Char( + "Input done directory", help="Path to folder for doneful operations" + ) + input_dir_error = fields.Char( + "Input error directory", help="Path to folder for error operations" + ) + output_dir_pending = fields.Char( + "Output pending directory", help="Path to folder for pending operations" + ) + output_dir_done = fields.Char( + "Output done directory", help="Path to folder for doneful operations" + ) + output_dir_error = fields.Char( + "Output error directory", help="Path to folder for error operations" + ) + + def _get_component_usage_candidates(self, exchange_record, key): + candidates = super()._get_component_usage_candidates(exchange_record, key) + if not self.storage_id: + return candidates + storage_generic = "edi.storage.{}".format(key) + storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type + type_code = exchange_record.type_id.code + return [ + storage_by_backend_type + "." + type_code, + storage_by_backend_type, + storage_generic + "." + exchange_record.direction, + storage_generic, + ] + candidates diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..f583948be8 --- /dev/null +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -0,0 +1 @@ +* Simone Orsi diff --git a/edi_storage_oca/readme/DESCRIPTION.rst b/edi_storage_oca/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..bdef14a8ad --- /dev/null +++ b/edi_storage_oca/readme/DESCRIPTION.rst @@ -0,0 +1,23 @@ +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. diff --git a/edi_storage_oca/readme/USAGE.rst b/edi_storage_oca/readme/USAGE.rst new file mode 100644 index 0000000000..a47c64c841 --- /dev/null +++ b/edi_storage_oca/readme/USAGE.rst @@ -0,0 +1 @@ +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. diff --git a/edi_storage_oca/security/ir_model_access.xml b/edi_storage_oca/security/ir_model_access.xml new file mode 100644 index 0000000000..b7d7b84987 --- /dev/null +++ b/edi_storage_oca/security/ir_model_access.xml @@ -0,0 +1,12 @@ + + + + access_storage_backend EDI manager + + + + + + + + diff --git a/edi_storage_oca/static/description/icon.png b/edi_storage_oca/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html new file mode 100644 index 0000000000..70dda24beb --- /dev/null +++ b/edi_storage_oca/static/description/index.html @@ -0,0 +1,450 @@ + + + + + + +EDI Storage backend support + + + +
+

EDI Storage backend support

+ + +

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Allow exchange files using storage backends from OCA/storage.

+

This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files.

+

Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files).

+

Inside this folder you have this hierarchy:

+
+input/output folder
+    |- pending
+    |- done
+    |- error
+
+
    +
  • pending folder contains files that have been just sent
    • +
  • done folder contains files that have been processes successfully
    • +
  • error folder contains files with errors and cannot be processed
    • +
+

The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly.

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Usage

+

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/edi project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py new file mode 100644 index 0000000000..ede7d946e2 --- /dev/null +++ b/edi_storage_oca/tests/_TODO_test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from freezegun import freeze_time + +from .common import EDIBackendCommonTestCase + + +class EDIBackendTestCase(EDIBackendCommonTestCase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.backend._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("input", "WHATEVER", "foo.csv") + + @freeze_time("2020-10-21 10:00:00") + def test_create_record(self): + vals = { + "model": self.partner._name, + "res_id": self.partner.id, + } + record = self.backend.create_record("test_csv_input", vals) + expected = { + "type_id": self.exchange_type_in.id, + "record_id": self.partner, + "edi_exchange_state": "new", + "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", + } + self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py new file mode 100644 index 0000000000..d81ae344d1 --- /dev/null +++ b/edi_storage_oca/tests/__init__.py @@ -0,0 +1 @@ +from . import test_edi_backend_storage diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py new file mode 100644 index 0000000000..2d02206ab4 --- /dev/null +++ b/edi_storage_oca/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +import functools + +import mock + +from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase + + +class TestEDIStorageBase(EDIBackendCommonComponentTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + @classmethod + def _setup_records(cls): + super()._setup_records() + cls.filedata = base64.b64encode(b"This is a simple file") + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": cls.filedata, + } + cls.record = cls.backend.create_record("test_csv_output", vals) + + cls.fakepath = "/tmp/{}".format(cls._filename(cls)) + with open(cls.fakepath, "w+b") as fakefile: + fakefile.write(b"filecontent") + + cls.fakepath_ack = "/tmp/{}.ack".format(cls._filename(cls)) + with open(cls.fakepath_ack, "w+b") as fakefile: + fakefile.write(b"ACK filecontent") + + cls.fakepath_error = "/tmp/{}.error".format(cls._filename(cls)) + with open(cls.fakepath_error, "w+b") as fakefile: + fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + + cls.checker = cls.backend._get_component( + ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + ) + cls.sender = cls.backend._get_component( + ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + ) + + def setUp(self): + super().setUp() + self._storage_backend_calls = [] + + def _filename(self, record=None, ack=False): + record = record or self.record + return record.exchange_filename if not ack else record.ack_filename + + def _file_fullpath(self, state, record=None, ack=False): + record = record or self.record + fname = self._filename(record, ack=ack) + if state == "error-report": + # Exception as we read from the same path but w/ error suffix + state = "error" + fname += ".error" + return ( + self.checker._remote_file_path(record.direction, state, fname) + ).as_posix() + + def _mocked_backend_get(self, mocked_paths, path, **kwargs): + self._storage_backend_calls.append(path) + if mocked_paths.get(path): + with open(mocked_paths.get(path), "rb") as remote_file: + return remote_file.read() + raise FileNotFoundError() + + def _mocked_backend_add(self, path, data, **kwargs): + self._storage_backend_calls.append(path) + + def _mock_storage_backend_get(self, mocked_paths): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + mocked = functools.partial(self._mocked_backend_get, mocked_paths) + return mock.patch(mock_path + ".get", mocked) + + def _mock_storage_backend_add(self): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + return mock.patch(mock_path + ".add", self._mocked_backend_add) + + def _test_result( + self, record, expected_values, expected_messages=None, state_paths=None, + ): + state_paths = state_paths or ("done", "pending", "error") + # Paths will be something like: + # [ + # 'demo_out/pending/$filename.csv', + # 'demo_out/pending/$filename.csv', + # 'demo_out/error/$filename.csv', + # ] + for state in state_paths: + path = self._file_fullpath(state, record=record) + self.assertIn(path, self._storage_backend_calls) + self.assertRecordValues(record, [expected_values]) + if expected_messages: + self.assertEqual(len(record.record.message_ids), len(expected_messages)) + for msg_rec, expected in zip(record.record.message_ids, expected_messages): + self.assertIn(expected["message"], msg_rec.body) + self.assertIn("level-" + expected["level"], msg_rec.body) + # TODO: test content of file sent + + def _test_send(self, record, mocked_paths=None): + with self._mock_storage_backend_add(): + if mocked_paths: + with self._mock_storage_backend_get(mocked_paths): + self.backend.exchange_send(record) + else: + self.backend.exchange_send(record) + + def _test_run_cron(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_get(mocked_paths): + self.backend._cron_check_output_exchange_sync() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py new file mode 100644 index 0000000000..de240c9552 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -0,0 +1,204 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 + +from freezegun import freeze_time + +from odoo.tools import mute_logger + +from .common import TestEDIStorageBase + +LOGGERS = ( + "odoo.addons.edi_storage.components.check", + "odoo.addons.edi.models.edi_backend", +) + + +@freeze_time("2020-10-21 10:30:00") +class TestEDIBackendOutput(TestEDIStorageBase): + @mute_logger(*LOGGERS) + def test_export_file_sent(self): + """Send, no errors.""" + self.record.edi_exchange_state = "output_pending" + mocked_paths = {self._file_fullpath("pending"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + # self._test_send(self.record, mocked_paths=mocked_paths) + self._test_run_cron(mocked_paths) + self._test_result( + self.record, + {"edi_exchange_state": "output_sent"}, + expected_messages=[ + { + "message": self.record._exchange_status_message("send_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done(self): + """Already sent, successfully.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = {self._file_fullpath("done"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `done` folder, + # we should get the final good state + # and only one call to ftp + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_not_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + } + self._test_run_cron(mocked_paths) + # No ack file found, warning message is posted + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_missing"), + "level": "warning", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + self._file_fullpath("done", ack=True): self.fakepath_ack, + } + self._test_run_cron(mocked_paths) + # Found ack file, set on record + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_processed", + "ack_file": base64.b64encode(b"ACK filecontent"), + }, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_received"), + "level": "info", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_already_sent_process_error(self): + """Already sent, error process.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = { + self._file_fullpath("error"): self.fakepath, + self._file_fullpath("error-report"): self.fakepath_error, + } + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `error` folder, + # we should get a call for: done, error and then the read of the report. + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_cron_full_flow(self): + """Already sent, update the state via cron.""" + self.record.edi_exchange_state = "output_sent" + rec1 = self.record + partner2 = self.env.ref("base.res_partner_2") + partner3 = self.env.ref("base.res_partner_3") + rec2 = self.record.copy( + { + "model": partner2._name, + "res_id": partner2.id, + "exchange_filename": "rec2.csv", + } + ) + rec3 = self.record.copy( + { + "model": partner3._name, + "res_id": partner3.id, + "exchange_filename": "rec3.csv", + "edi_exchange_state": "output_sent_and_error", + } + ) + mocked_paths = { + self._file_fullpath("done", record=rec1): self.fakepath, + self._file_fullpath("error", record=rec2): self.fakepath, + self._file_fullpath("error-report", record=rec2): self.fakepath_error, + self._file_fullpath("done", record=rec3): self.fakepath, + } + self._test_run_cron(mocked_paths) + self._test_result( + rec1, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec1._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + self._test_result( + rec2, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": rec2._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + self._test_result( + rec3, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec3._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml new file mode 100644 index 0000000000..5c7916eb1e --- /dev/null +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -0,0 +1,21 @@ + + + + edi.backend + + + + + + + + + + + + + + + + + From 59e050cc97d6ce27f5f2c06ff523f5195885b806 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 13:50:23 +0100 Subject: [PATCH 02/69] edi_storage: add base tests for component --- .../tests/_TODO_test_components_base.py | 43 ------------------- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 10 +++-- edi_storage_oca/tests/test_components_base.py | 43 +++++++++++++++++++ 4 files changed, 50 insertions(+), 47 deletions(-) delete mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/test_components_base.py diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py deleted file mode 100644 index ede7d946e2..0000000000 --- a/edi_storage_oca/tests/_TODO_test_components_base.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 ACSONE -# @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). - -from freezegun import freeze_time - -from .common import EDIBackendCommonTestCase - - -class EDIBackendTestCase(EDIBackendCommonTestCase): - def test_remote_file_path(self): - to_test = ( - (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), - (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), - (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), - (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), - (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), - (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), - ) - for _args, expected in to_test: - path_obj = self.backend._remote_file_path(*_args) - self.assertEqual(path_obj.as_posix(), expected) - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("input", "WHATEVER", "foo.csv") - - @freeze_time("2020-10-21 10:00:00") - def test_create_record(self): - vals = { - "model": self.partner._name, - "res_id": self.partner.id, - } - record = self.backend.create_record("test_csv_input", vals) - expected = { - "type_id": self.exchange_type_in.id, - "record_id": self.partner, - "edi_exchange_state": "new", - "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", - } - self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index d81ae344d1..b788a00714 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1 +1,2 @@ from . import test_edi_backend_storage +from . import test_components_base diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 2d02206ab4..a6af8cb003 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -7,6 +7,10 @@ from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +STORAGE_BACKEND_MOCK_PATH = ( + "odoo.addons.storage_backend.models.storage_backend.StorageBackend" +) + class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod @@ -73,13 +77,11 @@ def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) def _mock_storage_backend_get(self, mocked_paths): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" mocked = functools.partial(self._mocked_backend_get, mocked_paths) - return mock.patch(mock_path + ".get", mocked) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) def _mock_storage_backend_add(self): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" - return mock.patch(mock_path + ".add", self._mocked_backend_add) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( self, record, expected_values, expected_messages=None, state_paths=None, diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py new file mode 100644 index 0000000000..c2ba7275f6 --- /dev/null +++ b/edi_storage_oca/tests/test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import mock + +from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase + + +class EDIBackendTestCase(TestEDIStorageBase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.checker._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + + def test_get_remote_file(self): + with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: + self.checker._get_remote_file("pending") + mocked.assert_called_with( + "demo_out/pending/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("done") + mocked.assert_called_with( + "demo_out/done/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("error") + mocked.assert_called_with( + "demo_out/error/{}".format(self._filename(self.record)), binary=False + ) From 776bde0ac58dc92ff0bc7961dd255c1a3ba9d1bb Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sun, 29 Nov 2020 18:58:26 +0100 Subject: [PATCH 03/69] edi_storage: fix messages test isolation --- edi_storage_oca/tests/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index a6af8cb003..d49b443a56 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -98,8 +98,12 @@ def _test_result( self.assertIn(path, self._storage_backend_calls) self.assertRecordValues(record, [expected_values]) if expected_messages: - self.assertEqual(len(record.record.message_ids), len(expected_messages)) - for msg_rec, expected in zip(record.record.message_ids, expected_messages): + # consider only edi related messages + messages = record.record.message_ids.filtered( + lambda x: "edi-exchange" in x.body + ) + self.assertEqual(len(messages), len(expected_messages)) + for msg_rec, expected in zip(messages, expected_messages): self.assertIn(expected["message"], msg_rec.body) self.assertIn("level-" + expected["level"], msg_rec.body) # TODO: test content of file sent From 8b87e33375f9b0f5d425993be67fd12798ca9c0a Mon Sep 17 00:00:00 2001 From: oca-travis Date: Mon, 30 Nov 2020 11:50:12 +0000 Subject: [PATCH 04/69] [UPD] Update edi_storage.pot --- edi_storage_oca/i18n/edi_storage.pot | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage.pot diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot new file mode 100644 index 0000000000..d6944e83bd --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage.pot @@ -0,0 +1,77 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 13.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage +#: model:ir.model,name:edi_storage.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From 8fd2948f75728aabbdf5dedd8249cf7e09476308 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 30 Nov 2020 12:00:54 +0000 Subject: [PATCH 05/69] edi_storage 13.0.1.1.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 9d4ff373fd..0d0500a2f3 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.0.0", + "version": "13.0.1.1.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 9b617494d6fd9a8837b2cf166c10fbccc4cc5409 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:04:25 +0100 Subject: [PATCH 06/69] edi_storage: adapt to edi api changes --- edi_storage_oca/components/check.py | 8 ++++---- edi_storage_oca/tests/common.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index e76ec75aac..835f979b83 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -46,7 +46,7 @@ def _exchange_output_check(self): == "output_sent_and_processed" ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" - self.backend._notify_done(self.exchange_record) + self.exchange_record._notify_done() if self.exchange_record.type_id.ack_needed: self._exchange_output_handle_ack() return False @@ -68,7 +68,7 @@ def _exchange_output_check(self): "exchange_error": pycompat.to_text(error_report), } ) - self.backend._notify_error(self.exchange_record, "process_ko") + self.exchange_record._notify_error("process_ko") return False return True @@ -78,6 +78,6 @@ def _exchange_output_handle_ack(self): ) if ack_file: self.exchange_record._set_file_content(ack_file, field_name="ack_file") - self.backend._notify_ack_received(self.exchange_record) + self.exchange_record._notify_ack_received() else: - self.backend._notify_ack_missing(self.exchange_record) + self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d49b443a56..d55fe67741 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,10 +40,10 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") - cls.checker = cls.backend._get_component( + cls.checker = cls.backend._find_component( ["edi.storage.check"], work_ctx={"exchange_record": cls.record} ) - cls.sender = cls.backend._get_component( + cls.sender = cls.backend._find_component( ["edi.storage.send"], work_ctx={"exchange_record": cls.record} ) From 4010b36e5c646d0898ed0c218a5dec46bfa29c38 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 2 Dec 2020 14:00:21 +0000 Subject: [PATCH 07/69] edi_storage 13.0.1.2.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0d0500a2f3..494055c6c4 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.1.0", + "version": "13.0.1.2.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 9eb8589ea80b530fb6100779afbaeadec70bab21 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:38:55 +0100 Subject: [PATCH 08/69] edi_storage: put ack handling on hold Ack files should be treated as incoming records. Receive files is not handled yet by the framework and will be added soon. --- edi_storage_oca/components/check.py | 21 +++- edi_storage_oca/tests/common.py | 4 +- .../tests/test_edi_backend_storage.py | 109 +++++++++--------- 3 files changed, 72 insertions(+), 62 deletions(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 835f979b83..965b01dfe4 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -47,8 +47,6 @@ def _exchange_output_check(self): ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" self.exchange_record._notify_done() - if self.exchange_record.type_id.ack_needed: - self._exchange_output_handle_ack() return False error = self._get_remote_file("error") @@ -72,12 +70,23 @@ def _exchange_output_check(self): return False return True + # FIXME: this is not used ATM -> should be refactored + # into an incoming exchange. + # The backend will look for records needing an ack + # and generate and ack record. def _exchange_output_handle_ack(self): - ack_file = self._get_remote_file( - "done", filename=self.exchange_record.ack_filename - ) + ack_type = self.exchange_record.type_id.ack_type_id + filename = ack_type._make_exchange_filename(self.exchange_record) + ack_file = self._get_remote_file("done", filename=filename) if ack_file: - self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend.create_record( + ack_type.code, + { + "parent_id": self.exchange_record.id, + "exchange_file": ack_file, + "edi_exchange_state": "input_received", + }, + ) self.exchange_record._notify_ack_received() else: self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d55fe67741..992c4b44f8 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -53,7 +53,9 @@ def setUp(self): def _filename(self, record=None, ack=False): record = record or self.record - return record.exchange_filename if not ack else record.ack_filename + if ack: + record.type_id.ack_type_id._make_exchange_filename(record) + return record.exchange_filename def _file_fullpath(self, state, record=None, ack=False): record = record or self.record diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index de240c9552..1eda66121f 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,7 +1,5 @@ # Copyright 2020 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -import base64 - from freezegun import freeze_time from odoo.tools import mute_logger @@ -47,7 +45,7 @@ def test_export_file_already_done(self): # and only one call to ftp self._test_result( self.record, - {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + {"edi_exchange_state": "output_sent_and_processed"}, state_paths=("done",), expected_messages=[ { @@ -57,59 +55,60 @@ def test_export_file_already_done(self): ], ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_not_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - } - self._test_run_cron(mocked_paths) - # No ack file found, warning message is posted - self._test_result( - self.record, - {"edi_exchange_state": "output_sent_and_processed"}, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_missing"), - "level": "warning", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # FIXME: ack should be handle as an incoming record (new machinery to be added) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_not_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # } + # self._test_run_cron(mocked_paths) + # # No ack file found, warning message is posted + # self._test_result( + # self.record, + # {"edi_exchange_state": "output_sent_and_processed"}, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_missing"), + # "level": "warning", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - self._file_fullpath("done", ack=True): self.fakepath_ack, - } - self._test_run_cron(mocked_paths) - # Found ack file, set on record - self._test_result( - self.record, - { - "edi_exchange_state": "output_sent_and_processed", - "ack_file": base64.b64encode(b"ACK filecontent"), - }, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_received"), - "level": "info", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # self._file_fullpath("done", ack=True): self.fakepath_ack, + # } + # self._test_run_cron(mocked_paths) + # # Found ack file, set on record + # self._test_result( + # self.record, + # { + # "edi_exchange_state": "output_sent_and_processed", + # "ack_file": base64.b64encode(b"ACK filecontent"), + # }, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_received"), + # "level": "info", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) @mute_logger(*LOGGERS) def test_already_sent_process_error(self): From d1349ff9f3e8304e87b2553a313fa64551186a2e Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 2 Dec 2020 16:30:21 +0000 Subject: [PATCH 09/69] edi_storage 13.0.1.3.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 494055c6c4..838e6579a4 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.2.0", + "version": "13.0.1.3.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 123bb07954dbca69fc7eace9f4cd0bf01d69b9fa Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 8 Feb 2021 09:47:58 +0100 Subject: [PATCH 10/69] edi_storage: adapt to new components lookup --- edi_storage_oca/components/base.py | 14 ++- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 4 +- edi_storage_oca/models/edi_backend.py | 29 ++++--- edi_storage_oca/tests/common.py | 8 +- edi_storage_oca/tests/test_component_match.py | 86 +++++++++++++++++++ edi_storage_oca/tests/test_components_base.py | 2 +- 7 files changed, 128 insertions(+), 17 deletions(-) create mode 100644 edi_storage_oca/tests/test_component_match.py diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 2a337fed17..c6e67c1487 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -7,9 +7,21 @@ from odoo.addons.component.core import AbstractComponent -class EDIStorageSendComponentMixin(AbstractComponent): +class EDIStorageComponentMixin(AbstractComponent): _name = "edi.storage.component.mixin" + _inherit = "edi.component.mixin" + # Components having `_storage_backend_type` will have precedence. + # If the value is not set, generic components will be used. + _storage_backend_type = None + + @classmethod + def _component_match(cls, work, usage=None, model_name=None, **kw): + res = super()._component_match(work, usage=usage, model_name=model_name, **kw) + storage_type = kw.get("storage_backend_type") + if storage_type and cls._storage_backend_type: + return cls._storage_backend_type == storage_type + return res @property def storage(self): diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 965b01dfe4..6e3cfa99a8 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -18,7 +18,7 @@ class EDIStorageCheckComponentMixin(Component): "edi.component.check.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.check" + _usage = "storage.check" def check(self): return self._exchange_output_check() diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index 4d3f376758..b5ec45b5cd 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -12,13 +12,13 @@ class EDIStorageSendComponent(Component): "edi.component.send.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.send" + _usage = "storage.send" def send(self): # If the file has been sent already, refresh its state # TODO: double check if this is useless # since the backend checks the state already - checker = self.component(usage="edi.storage.check") + checker = self.component(usage="storage.check") result = checker.check() if not result: # all good here diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5ba9336391..63aa102490 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -52,16 +52,25 @@ class EDIBackend(models.Model): "Output error directory", help="Path to folder for error operations" ) + _storage_actions = ("check", "send", "receive") + def _get_component_usage_candidates(self, exchange_record, key): candidates = super()._get_component_usage_candidates(exchange_record, key) - if not self.storage_id: + if not self.storage_id or key not in self._storage_actions: return candidates - storage_generic = "edi.storage.{}".format(key) - storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type - type_code = exchange_record.type_id.code - return [ - storage_by_backend_type + "." + type_code, - storage_by_backend_type, - storage_generic + "." + exchange_record.direction, - storage_generic, - ] + candidates + return ["storage.{}".format(key)] + candidates + + def _component_match_attrs(self, exchange_record, key): + # Override to inject storage_backend_type + res = super()._component_match_attrs(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return res + res["storage_backend_type"] = self.storage_id.backend_type + return res + + def _component_sort_key(self, component_class): + res = super()._component_sort_key(component_class) + # Override to give precedence by storage_backend_type when needed. + if not self.storage_id: + return res + return (1 if component_class._storage_backend_type else 0,) + res diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 992c4b44f8..9cd239422e 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -41,10 +41,14 @@ def _setup_records(cls): fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") cls.checker = cls.backend._find_component( - ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record}, ) cls.sender = cls.backend._find_component( - ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.send"], + work_ctx={"exchange_record": cls.record}, ) def setUp(self): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py new file mode 100644 index 0000000000..4cd1bad876 --- /dev/null +++ b/edi_storage_oca/tests/test_component_match.py @@ -0,0 +1,86 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component +from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage") + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + def test_component_match(self): + """Lookup with special match method.""" + + class SFTPCheck(Component): + _name = "sftp.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class SFTPSend(Component): + _name = "sftp.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class S3Check(Component): + _name = "s3.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + class S3Send(Component): + _name = "s3.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Check._name) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPCheck._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPSend._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Send._name) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index c2ba7275f6..27969a6054 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -7,7 +7,7 @@ from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase -class EDIBackendTestCase(TestEDIStorageBase): +class EDIStorageComponentTestCase(TestEDIStorageBase): def test_remote_file_path(self): to_test = ( (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), From 3a1eec3c36e3eeee1436f6d0ba1ad184c11a1650 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 10 Mar 2021 06:49:58 +0000 Subject: [PATCH 11/69] edi_storage 13.0.1.4.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 838e6579a4..20db765f15 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.3.0", + "version": "13.0.1.4.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 4a2b22e4fd51598f2f00635caf57a7b35a7c14bc Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 10 Mar 2021 13:56:20 +0100 Subject: [PATCH 12/69] [CHG] edi: Use more permissive licence: AGPL-> LGPL --- edi_storage_oca/README.rst | 8 ++++---- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/components/base.py | 2 +- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 2 +- edi_storage_oca/models/edi_backend.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_component_match.py | 2 +- edi_storage_oca/tests/test_components_base.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index fffe958379..cc387fb2d0 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -10,9 +10,9 @@ EDI Storage backend support .. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png :target: https://odoo-community.org/page/development-status :alt: Alpha -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png - :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html - :alt: License: AGPL-3 +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github :target: https://github.com/OCA/edi/tree/13.0/edi_storage :alt: OCA/edi @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 20db765f15..503ae5cfed 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). { "name": "EDI Storage backend support", @@ -9,7 +9,7 @@ """, "version": "13.0.1.4.0", "development_status": "Alpha", - "license": "AGPL-3", + "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index c6e67c1487..01d0c8b2c7 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from pathlib import PurePath diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 6e3cfa99a8..878cfffcb3 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index b5ec45b5cd..cbb6ed309a 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 63aa102490..2437a6bc10 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo import fields, models diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 70dda24beb..b034d89fe3 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 9cd239422e..134c807a9b 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). import base64 import functools diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index 4cd1bad876..e422c9698e 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 27969a6054..57abd13856 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import mock diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 1eda66121f..f49138fc0a 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). from freezegun import freeze_time from odoo.tools import mute_logger From 91c753f99e248c66e945015a2d9ce235a9f368d9 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 15 Mar 2021 21:46:44 +0000 Subject: [PATCH 13/69] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index cc387fb2d0..0cceccfbfe 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. From f6545439294f8f7976ea1a93521739253305e779 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 15 Mar 2021 21:46:54 +0000 Subject: [PATCH 14/69] edi_storage 13.0.1.4.1 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 503ae5cfed..0df82391d4 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.0", + "version": "13.0.1.4.1", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 79fafc6ab576b8520e939fd78f2c0f36539effaf Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 7 Apr 2021 10:35:07 +0200 Subject: [PATCH 15/69] [FIX] edi_storage: sort key was not working properly --- edi_storage_oca/models/edi_backend.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2437a6bc10..0ba578fa09 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -73,4 +73,6 @@ def _component_sort_key(self, component_class): # Override to give precedence by storage_backend_type when needed. if not self.storage_id: return res - return (1 if component_class._storage_backend_type else 0,) + res + return ( + 1 if getattr(component_class, "_storage_backend_type", False) else 0, + ) + res From 176ad6571fdca7d9dbfb4a5b5177534ce8191b39 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Fri, 9 Apr 2021 09:39:31 +0000 Subject: [PATCH 16/69] edi_storage 13.0.1.5.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0df82391d4..0243ab8cf2 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.1", + "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 2a69f8ec181b73080e6e6d4481387eb2e5195f2b Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 14:43:32 +0530 Subject: [PATCH 17/69] [IMP] edi_storage_oca: isort, black, prettier. --- edi_storage_oca/__manifest__.py | 1 + edi_storage_oca/tests/common.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0243ab8cf2..488f606134 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -10,6 +10,7 @@ "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", + "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 134c807a9b..c5ed77c459 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -90,7 +90,11 @@ def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( - self, record, expected_values, expected_messages=None, state_paths=None, + self, + record, + expected_values, + expected_messages=None, + state_paths=None, ): state_paths = state_paths or ("done", "pending", "error") # Paths will be something like: From 8faf61aa1376f494b3d8e9f3f0c64cf685b4d014 Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 15:07:19 +0530 Subject: [PATCH 18/69] [MIG] Migrate module edi_storage_oca to v14. --- edi_storage_oca/README.rst | 11 ++++++----- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/demo/edi_backend_demo.xml | 2 +- edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/static/description/index.html | 6 +++--- edi_storage_oca/tests/common.py | 4 ++-- edi_storage_oca/tests/test_component_match.py | 4 ++-- edi_storage_oca/tests/test_edi_backend_storage.py | 4 ++-- edi_storage_oca/views/edi_backend_views.xml | 2 +- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 0cceccfbfe..14f8cd6920 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -14,13 +14,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github - :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :target: https://github.com/OCA/edi/tree/14.0/edi_storage_oca :alt: OCA/edi .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/13.0 + :target: https://runbot.odoo-community.org/runbot/226/14.0 :alt: Try me on Runbot |badge1| |badge2| |badge3| |badge4| |badge5| @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -86,6 +86,7 @@ Contributors ~~~~~~~~~~~~ * Simone Orsi +* Foram Shah Maintainers ~~~~~~~~~~~ @@ -100,6 +101,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi `_ project on GitHub. +This module is part of the `OCA/edi `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 488f606134..b8606ae4c2 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,12 +7,12 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.5.0", + "version": "14.0.1.0.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", - "depends": ["edi", "storage_backend", "component"], + "depends": ["edi_oca", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml index d8ff5d972c..5b003460f7 100644 --- a/edi_storage_oca/demo/edi_backend_demo.xml +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -2,7 +2,7 @@ Storage Demo EDI backend - + demo_in/pending demo_in/done diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index f583948be8..e3fa6d8e96 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1 +1,2 @@ * Simone Orsi +* Foram Shah diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index b034d89fe3..da7347ce60 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -441,7 +441,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi project on GitHub.

+

This module is part of the OCA/edi project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index c5ed77c459..4be4c1daf8 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -5,7 +5,7 @@ import mock -from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentTestCase STORAGE_BACKEND_MOCK_PATH = ( "odoo.addons.storage_backend.models.storage_backend.StorageBackend" @@ -15,7 +15,7 @@ class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") @classmethod def _setup_records(cls): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e422c9698e..e4fbfbe827 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -10,11 +10,11 @@ class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls._load_module_components(cls, "edi_storage") + cls._load_module_components(cls, "edi_storage_oca") @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") def test_component_match(self): """Lookup with special match method.""" diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index f49138fc0a..319d6e1cd3 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -7,8 +7,8 @@ from .common import TestEDIStorageBase LOGGERS = ( - "odoo.addons.edi_storage.components.check", - "odoo.addons.edi.models.edi_backend", + "odoo.addons.edi_storage_oca.components.check", + "odoo.addons.edi_oca.models.edi_backend", ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 5c7916eb1e..6d2e4d5d35 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -2,7 +2,7 @@ edi.backend - + From fe8e17370f8958e355cf1f74c9748a0aa9a9034e Mon Sep 17 00:00:00 2001 From: oca-travis Date: Thu, 27 May 2021 13:11:31 +0000 Subject: [PATCH 19/69] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 0000000000..fdd8638b17 --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,92 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 14.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +msgid "Display Name" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +msgid "ID" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +msgid "Last Modified on" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From d2dc6846e3485adfc116d693f8dd2ee51e5c7f45 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Thu, 27 May 2021 13:16:44 +0000 Subject: [PATCH 20/69] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/static/description/index.html | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 14f8cd6920..5160047107 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index da7347ce60..a52f95fdab 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -432,6 +432,7 @@

Authors

Contributors

From 5d912f488c1ef0c1e868cdf3486ee5c80e76090b Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:32:31 +0200 Subject: [PATCH 21/69] [IMP] edi_storage_oca: implement basis of the input process. * A cron job check related storage on edi backends to create the needed exchange files. * Add basic receive component. --- edi_storage_oca/__manifest__.py | 6 ++- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/base.py | 8 ++-- edi_storage_oca/components/receive.py | 27 ++++++++++++++ edi_storage_oca/data/cron.xml | 15 ++++++++ edi_storage_oca/models/edi_backend.py | 49 +++++++++++++++++++++++++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 edi_storage_oca/components/receive.py create mode 100644 edi_storage_oca/data/cron.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index b8606ae4c2..c15efdd671 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -13,6 +13,10 @@ "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi_oca", "storage_backend", "component"], - "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "data": [ + "data/cron.xml", + "security/ir_model_access.xml", + "views/edi_backend_views.xml", + ], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 9f64b792bb..7333ccfb34 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -1,3 +1,4 @@ from . import base from . import check from . import send +from . import receive diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 01d0c8b2c7..ca0fa5cef9 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state): + def _dir_by_state(self, direction, state, absolute=False): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,9 +36,11 @@ def _dir_by_state(self, direction, state): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") + if absolute: + return PurePath(self.backend[direction + "_dir_" + state] or "") return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) - def _remote_file_path(self, direction, state, filename): + def _remote_file_path(self, direction, state, filename, absolute=False): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -46,7 +48,7 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state) / filename.strip("/ ") + return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py new file mode 100644 index 0000000000..e421df38a6 --- /dev/null +++ b/edi_storage_oca/components/receive.py @@ -0,0 +1,27 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageReceiveComponent(Component): + + _name = "edi.storage.component.receive" + _inherit = [ + "edi.component.receive.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.receive" + + def receive(self): + checker = self.component(usage="storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + path = self._remote_file_path(direction, "pending", filename, absolute=True) + filedata = self.storage.get(path.as_posix(), binary=True) + return filedata diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml new file mode 100644 index 0000000000..0dad93786b --- /dev/null +++ b/edi_storage_oca/data/cron.xml @@ -0,0 +1,15 @@ + + + + EDI backend storage check pending input + + + 1 + hours + -1 + + + code + model.search([])._cron_check_storage_pending_input() + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0ba578fa09..315b63b16d 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,9 +1,14 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging + from odoo import fields, models +_logger = logging.getLogger(__name__) + class EDIBackend(models.Model): @@ -76,3 +81,47 @@ def _component_sort_key(self, component_class): return ( 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res + + def _cron_check_storage_pending_input(self, **kw): + for backend in self.filtered(lambda b: b.storage_id): + backend._check_storage_pending_input(**kw) + + def _check_storage_pending_input(self, **kw): + self.ensure_one() + if not self.storage_id or not self.input_dir_pending: + _logger.info( + "%s ignored. No storage and/or input directory specified.", self.name + ) + return False + + pending_files = self.storage_id.list_files(self.input_dir_pending) + exchange_type = self.env["edi.exchange.type"].search( + self._domain_exchange_type_storage_pending_input() + ) + if not len(exchange_type) == 1: + _logger.info("%s ignored. More than one exchange type found.", self.name) + return False + for file_name in pending_files: + existing = self.env["edi.exchange.record"].search( + [ + ("backend_id", "=", self.id), + ("type_id", "=", exchange_type.id), + ("exchange_filename", "=", file_name), + ] + ) + if existing: + continue + self.create_record( + exchange_type.code, self._get_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return True + + def _domain_exchange_type_storage_pending_input(self): + return [ + ("backend_type_id", "=", self.backend_type_id.id), + ("direction", "=", "input"), + ] + + def _get_exchange_record_vals(self, file_name): + return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index e3fa6d8e96..55ca1c7e05 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1,2 +1,3 @@ * Simone Orsi * Foram Shah +* Lois Rilo From 89cfa929385a96e41a726840c408330a92dd1438 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:34:51 +0200 Subject: [PATCH 22/69] [FIX] edi_oca: show the exchange file name while there is no file yet. --- edi_storage_oca/components/base.py | 12 +++--- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/models/edi_backend.py | 4 +- edi_storage_oca/tests/common.py | 38 ++++++++++++++++--- .../tests/test_edi_backend_storage.py | 29 ++++++++++++++ 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index ca0fa5cef9..5010e68449 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state, absolute=False): + def _dir_by_state(self, direction, state): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,11 +36,11 @@ def _dir_by_state(self, direction, state, absolute=False): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") - if absolute: - return PurePath(self.backend[direction + "_dir_" + state] or "") - return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + return PurePath( + (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/") + ) - def _remote_file_path(self, direction, state, filename, absolute=False): + def _remote_file_path(self, direction, state, filename): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -48,7 +48,7 @@ def _remote_file_path(self, direction, state, filename, absolute=False): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") + return self._dir_by_state(direction, state) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index e421df38a6..695c407784 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -22,6 +22,6 @@ def receive(self): direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename, absolute=True) - filedata = self.storage.get(path.as_posix(), binary=True) + path = self._remote_file_path(direction, "pending", filename) + filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 315b63b16d..4514958761 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -83,7 +83,7 @@ def _component_sort_key(self, component_class): ) + res def _cron_check_storage_pending_input(self, **kw): - for backend in self.filtered(lambda b: b.storage_id): + for backend in self: backend._check_storage_pending_input(**kw) def _check_storage_pending_input(self, **kw): @@ -102,7 +102,7 @@ def _check_storage_pending_input(self, **kw): _logger.info("%s ignored. More than one exchange type found.", self.name) return False for file_name in pending_files: - existing = self.env["edi.exchange.record"].search( + existing = self.env["edi.exchange.record"].search_count( [ ("backend_id", "=", self.id), ("type_id", "=", exchange_type.id), diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 4be4c1daf8..7519ccfbab 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,6 +40,14 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + cls.fakepath_input_pending_1 = "/tmp/test-input-001.csv" + with open(cls.fakepath_input_pending_1, "w+b") as fakefile: + fakefile.write(b"I received this in my storage.") + + cls.fakepath_input_pending_2 = "/tmp/test-input-002.csv" + with open(cls.fakepath_input_pending_2, "w+b") as fakefile: + fakefile.write(b"I received that in my storage.") + cls.checker = cls.backend._find_component( cls.partner._name, ["storage.check"], @@ -61,16 +69,19 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath(self, state, record=None, ack=False): + def _file_fullpath( + self, state, record=None, ack=False, direction=False, fname=None + ): record = record or self.record - fname = self._filename(record, ack=ack) + if not fname: + fname = self._filename(record, ack=ack) + if not direction: + direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return ( - self.checker._remote_file_path(record.direction, state, fname) - ).as_posix() + return (self.checker._remote_file_path(direction, state, fname)).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) @@ -82,6 +93,14 @@ def _mocked_backend_get(self, mocked_paths, path, **kwargs): def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) + def _mocked_backend_list_files(self, mocked_paths, path, **kwargs): + files = [] + path_length = len(path) + for p in mocked_paths.keys(): + if path in p and path != p: + files.append(p[path_length:]) + return files + def _mock_storage_backend_get(self, mocked_paths): mocked = functools.partial(self._mocked_backend_get, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) @@ -89,6 +108,10 @@ def _mock_storage_backend_get(self, mocked_paths): def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) + def _mock_storage_backend_list_files(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _test_result( self, record, @@ -130,3 +153,8 @@ def _test_run_cron(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_get(mocked_paths): self.backend._cron_check_output_exchange_sync() + + def _test_run_cron_pending_input(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_list_files(mocked_paths): + self.backend._cron_check_storage_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 319d6e1cd3..d4779a704d 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -201,3 +201,32 @@ def test_cron_full_flow(self): } ], ) + + @mute_logger(*LOGGERS) + def test_create_input_exchange_file_from_file_received(self): + exch_type = self.exchange_type_in + input_dir = "/test_input/pending/" + file_names = ["some-file.csv", "another-file.csv"] + self.backend.input_dir_pending = input_dir + mocked_paths = { + input_dir: "/tmp/", + self._file_fullpath( + "pending", direction="input", fname=file_names[0] + ): self.fakepath_input_pending_1, + self._file_fullpath( + "pending", direction="input", fname=file_names[1] + ): self.fakepath_input_pending_2, + } + existing = self.env["edi.exchange.record"].search_count( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(existing, 0) + # Run cron action: + self._test_run_cron_pending_input(mocked_paths) + new_records = self.env["edi.exchange.record"].search( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(len(new_records), 2) + for rec in new_records: + self.assertIn(rec.exchange_filename, file_names) + self.assertEqual(rec.edi_exchange_state, "input_pending") From 165d24d862707df7692baf1e846577fd552f118e Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 23 Jun 2021 15:46:00 +0200 Subject: [PATCH 23/69] edi_storage: improve input process --- edi_storage_oca/data/cron.xml | 2 +- edi_storage_oca/models/edi_backend.py | 78 ++++++++++++++++++--------- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index 0dad93786b..d57316cd49 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,6 @@ code - model.search([])._cron_check_storage_pending_input() + model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 4514958761..4f14f99359 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -4,6 +4,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging +import os from odoo import fields, models @@ -82,46 +83,73 @@ def _component_sort_key(self, component_class): 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res - def _cron_check_storage_pending_input(self, **kw): + def _storage_cron_check_pending_input(self, **kw): for backend in self: - backend._check_storage_pending_input(**kw) + backend._storage_check_pending_input(**kw) - def _check_storage_pending_input(self, **kw): + def _storage_check_pending_input(self, **kw): + """Create new exchange records if new files found. + + Collect input exchange types and for each of them, + check by pattern if the a new exchange record is required. + """ self.ensure_one() if not self.storage_id or not self.input_dir_pending: _logger.info( - "%s ignored. No storage and/or input directory specified.", self.name + "%s ignored: no storage and/or input directory specified.", self.name ) return False - pending_files = self.storage_id.list_files(self.input_dir_pending) - exchange_type = self.env["edi.exchange.type"].search( - self._domain_exchange_type_storage_pending_input() + exchange_types = self.env["edi.exchange.type"].search( + self._storage_exchange_type_pending_input_domain() ) - if not len(exchange_type) == 1: - _logger.info("%s ignored. More than one exchange type found.", self.name) - return False - for file_name in pending_files: - existing = self.env["edi.exchange.record"].search_count( - [ - ("backend_id", "=", self.id), - ("type_id", "=", exchange_type.id), - ("exchange_filename", "=", file_name), - ] - ) - if existing: - continue - self.create_record( - exchange_type.code, self._get_exchange_record_vals(file_name) + for exchange_type in exchange_types: + # NOTE: this call might keep hanging the cron + # if the remote storage is slow (eg: too many files) + # We should probably run this code in a separate job per exchange type. + file_names = self._storage_get_input_filenames(exchange_type) + _logger.info( + "Processing exchange type '%s': found %s files to process", + exchange_type.display_name, + len(file_names), ) - _logger.debug("%s: new exchange record generated.", self.name) + for file_name in file_names: + # TODO: add config for job + self.with_delay()._storage_create_record_if_missing( + exchange_type, file_name + ) return True - def _domain_exchange_type_storage_pending_input(self): + def _storage_exchange_type_pending_input_domain(self): + """Domain for retrieving input exchange types. + """ return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), ] - def _get_exchange_record_vals(self, file_name): + def _storage_create_record_if_missing(self, exchange_type, remote_file_name): + """Create a new exchange record for given type and file name if missing. + """ + file_name = os.path.basename(remote_file_name) + extra_domain = [("exchange_filename", "=", file_name)] + existing = self._find_existing_exchange_records( + exchange_type, extra_domain=extra_domain, count_only=True + ) + if existing: + return + record = self.create_record( + exchange_type.code, self._storage_new_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return record.identifier + + def _storage_get_input_filenames(self, exchange_type): + bits = [exchange_type.exchange_filename_pattern] + if exchange_type.exchange_file_ext: + bits.append("*." + exchange_type.exchange_file_ext) + pattern = "".join(bits) + return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + + def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} From 58b8b495b990cc7fb896cd04880f2323af71a849 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Wed, 23 Jun 2021 17:10:11 +0200 Subject: [PATCH 24/69] [FIX] edi_storage_oca: support the use or not of filename patern + small runtime fixes. --- edi_storage_oca/data/cron.xml | 4 +++- edi_storage_oca/models/edi_backend.py | 7 ++++++- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index d57316cd49..df21892a08 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,8 @@ code - model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() + model.search([('storage_id', '!=', False)])._storage_cron_check_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 4f14f99359..2b9674a531 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,11 +145,16 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + if not exchange_type.exchange_filename_pattern: + # If there is not patter, return everything + return self.storage_id.list_files(self.input_dir_pending) + + # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append("*." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + return self.storage_id.find_files(pattern, self.input_dir_pending) def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 7519ccfbab..5aaa9951e8 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -157,4 +157,4 @@ def _test_run_cron(self, mocked_paths): def _test_run_cron_pending_input(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_list_files(mocked_paths): - self.backend._cron_check_storage_pending_input() + self.backend._storage_cron_check_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index d4779a704d..06448474b2 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -203,8 +203,9 @@ def test_cron_full_flow(self): ) @mute_logger(*LOGGERS) - def test_create_input_exchange_file_from_file_received(self): + def test_create_input_exchange_file_from_file_received_no_pattern(self): exch_type = self.exchange_type_in + exch_type.exchange_filename_pattern = "" input_dir = "/test_input/pending/" file_names = ["some-file.csv", "another-file.csv"] self.backend.input_dir_pending = input_dir From 0d877ca4a3e87e062e42bf7e0be047ae6d77b0c3 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 09:29:10 +0200 Subject: [PATCH 25/69] [IMP] edi_storage_oca: add specific channel and job function definitions --- edi_storage_oca/__manifest__.py | 2 ++ edi_storage_oca/data/job_channel_data.xml | 6 ++++++ edi_storage_oca/data/queue_job_function_data.xml | 7 +++++++ edi_storage_oca/models/edi_backend.py | 1 - 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 edi_storage_oca/data/job_channel_data.xml create mode 100644 edi_storage_oca/data/queue_job_function_data.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c15efdd671..c950bd37fc 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -15,6 +15,8 @@ "depends": ["edi_oca", "storage_backend", "component"], "data": [ "data/cron.xml", + "data/job_channel_data.xml", + "data/queue_job_function_data.xml", "security/ir_model_access.xml", "views/edi_backend_views.xml", ], diff --git a/edi_storage_oca/data/job_channel_data.xml b/edi_storage_oca/data/job_channel_data.xml new file mode 100644 index 0000000000..b3b3b770d7 --- /dev/null +++ b/edi_storage_oca/data/job_channel_data.xml @@ -0,0 +1,6 @@ + + + edi_storage + + + diff --git a/edi_storage_oca/data/queue_job_function_data.xml b/edi_storage_oca/data/queue_job_function_data.xml new file mode 100644 index 0000000000..1965284193 --- /dev/null +++ b/edi_storage_oca/data/queue_job_function_data.xml @@ -0,0 +1,7 @@ + + + + _storage_create_record_if_missing + + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2b9674a531..5e229a8a44 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -114,7 +114,6 @@ def _storage_check_pending_input(self, **kw): len(file_names), ) for file_name in file_names: - # TODO: add config for job self.with_delay()._storage_create_record_if_missing( exchange_type, file_name ) From fc4ef99816ee315ecbab164ba2f639d49a50dced Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 12:57:49 +0200 Subject: [PATCH 26/69] [FIX] edi_storage_oca: support regex pattern searching when fetching filenames from storage. Also extend help in `exchange_filename_pattern` field to clarify usage. --- edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 7 ++++--- edi_storage_oca/models/edi_exchange_type.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 edi_storage_oca/models/edi_exchange_type.py diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py index 4b2c54ac39..f34a72163f 100644 --- a/edi_storage_oca/models/__init__.py +++ b/edi_storage_oca/models/__init__.py @@ -1 +1,2 @@ from . import edi_backend +from . import edi_exchange_type diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5e229a8a44..21d34095eb 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -148,12 +148,13 @@ def _storage_get_input_filenames(self, exchange_type): # If there is not patter, return everything return self.storage_id.list_files(self.input_dir_pending) - # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: - bits.append("*." + exchange_type.exchange_file_ext) + bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(pattern, self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) + pending_path_len = len(self.input_dir_pending) + return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py new file mode 100644 index 0000000000..42a7f70088 --- /dev/null +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -0,0 +1,21 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import fields, models + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + # Extend help to explain new usage. + exchange_filename_pattern = fields.Char( + help="For output exchange types this should be a formatting string " + "with the following variables available (to be used between " + "brackets, `{}`): `exchange_record`, `record_name`, `type` and " + "`dt`. For instance, a valid string would be " + "{record_name}-{type.code}-{dt}\n" + "For input exchange types related to storage backends " + "it should be a regex expression to filter " + "the files to be fetched from the pending directory in the related " + "storage. E.g: `.*my-type-[0-9]*.\\.csv`" + ) From cb7e36977a2282df4e13459753bdfef36106f698 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 2 Jul 2021 09:23:52 +0200 Subject: [PATCH 27/69] [FIX] edi_storage: consider if the exchange type has a specific backend assigned. --- edi_storage_oca/models/edi_backend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 21d34095eb..0f8196be7b 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -125,6 +125,9 @@ def _storage_exchange_type_pending_input_domain(self): return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), + "|", + ("backend_id", "=", False), + ("backend_id", "=", self.id), ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): From 0ae27850f72d764e4f474f2037e96e21a1d971ba Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 15:56:58 +0200 Subject: [PATCH 28/69] [IMP] : black, isort, prettier --- edi_storage_oca/models/edi_backend.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0f8196be7b..5f95e33827 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -120,8 +120,7 @@ def _storage_check_pending_input(self, **kw): return True def _storage_exchange_type_pending_input_domain(self): - """Domain for retrieving input exchange types. - """ + """Domain for retrieving input exchange types.""" return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), @@ -131,8 +130,7 @@ def _storage_exchange_type_pending_input_domain(self): ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): - """Create a new exchange record for given type and file name if missing. - """ + """Create a new exchange record for given type and file name if missing.""" file_name = os.path.basename(remote_file_name) extra_domain = [("exchange_filename", "=", file_name)] existing = self._find_existing_exchange_records( From 410d2a7d869437646e7d54c9cf99d4117c0de489 Mon Sep 17 00:00:00 2001 From: oca-travis Date: Tue, 6 Jul 2021 09:43:30 +0000 Subject: [PATCH 29/69] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index fdd8638b17..784ca4c680 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -15,6 +15,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name msgid "Display Name" msgstr "" @@ -23,8 +24,33 @@ msgstr "" msgid "EDI Backend" msgstr "" +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id msgid "ID" msgstr "" @@ -45,6 +71,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update msgid "Last Modified on" msgstr "" From 3c7a47cb3e4122b667791c863734ed2f96e53ab3 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 6 Jul 2021 09:49:11 +0000 Subject: [PATCH 30/69] [UPD] README.rst --- edi_storage_oca/README.rst | 1 + edi_storage_oca/static/description/index.html | 1 + 2 files changed, 2 insertions(+) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 5160047107..dc3a04a126 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -87,6 +87,7 @@ Contributors * Simone Orsi * Foram Shah +* Lois Rilo Maintainers ~~~~~~~~~~~ diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index a52f95fdab..257c2d221a 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -433,6 +433,7 @@

Contributors

From 83d1961f5eeb682e3ab2ed4e9b170b992722d92f Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 6 Jul 2021 09:49:11 +0000 Subject: [PATCH 31/69] edi_storage_oca 14.0.1.1.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c950bd37fc..e25ee456dd 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.0.0", + "version": "14.0.1.1.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 4f89da5a61ec118c8ea99e3b68358729ccd7bed4 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 18:36:03 +0200 Subject: [PATCH 32/69] [IMP] edi_storage: add listener to move files to done/error dirs when a file is processed the listener will move it to the done directory or the error directory based on the result of processing the file. --- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/listener.py | 59 +++++++++++++++ edi_storage_oca/tests/__init__.py | 1 + .../tests/test_edi_storage_listener.py | 71 +++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 edi_storage_oca/components/listener.py create mode 100644 edi_storage_oca/tests/test_edi_storage_listener.py diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 7333ccfb34..7bff9ddfcc 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -2,3 +2,4 @@ from . import check from . import send from . import receive +from . import listener diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py new file mode 100644 index 0000000000..a1f248b7ad --- /dev/null +++ b/edi_storage_oca/components/listener.py @@ -0,0 +1,59 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import functools +from pathlib import PurePath + +from odoo.addons.component.core import Component + + +class EdiStorageListener(Component): + _name = "edi.storage.component.listener" + _inherit = "base.event.listener" + + def _move_file(self, storage, from_dir_str, to_dir_str, filename): + from_dir = PurePath(from_dir_str) + to_dir = PurePath(to_dir_str) + if filename not in storage.list_files(from_dir): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook( + storage._move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + ) + return True + + def _add_after_commit_hook(self, move_func, sftp_filepath, sftp_destination_path): + """Add hook after commit to move the file when transaction is over.""" + self.env.cr.after( + "commit", + functools.partial(move_func, sftp_filepath, sftp_destination_path), + ) + + def on_edi_exchange_done(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + done_dir = record.backend_id.input_dir_done + error_dir = record.backend_id.input_dir_error + if not done_dir: + return res + res = self._move_file(storage, pending_dir, done_dir, file) + if not res: + # If a file previously failed it should have been previously + # moved to the error dir, therefore it is not present in the + # pending dir and we need to retry from error dir. + res = self._move_file(storage, error_dir, done_dir, file) + return res + + def on_edi_exchange_error(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + error_dir = record.backend_id.input_dir_error + if error_dir: + res = self._move_file(storage, pending_dir, error_dir, file) + return res diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index b788a00714..a91d5f238c 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,2 +1,3 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py new file mode 100644 index 0000000000..4f12f3311f --- /dev/null +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -0,0 +1,71 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 + +import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.fake_components import FakeInputProcess + +LISTENER_MOCK_PATH = ( + "odoo.addons.edi_storage_oca.components.listener.EdiStorageListener" +) + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + cls._build_components( + cls, + FakeInputProcess, + ) + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + } + cls.record = cls.backend.create_record("test_csv_input", vals) + cls.fake_move_args = None + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def setUp(self): + super().setUp() + FakeInputProcess.reset_faked() + + def _move_file_mocked(self, *args): + self.fake_move_args = [*args] + if not all([*args]): + return False + return True + + def _mock_listener_move_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + + def test_01_process_record_success(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_done) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_error) + self.assertEqual(filename, self.record.exchange_filename) From 6c7597a416781b10a184c94a30911bacdca2c0fd Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 9 Jul 2021 11:25:05 +0200 Subject: [PATCH 33/69] [FIX] edi_storage: `list_files` expects a string. --- edi_storage_oca/components/listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index a1f248b7ad..7b551c81e8 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -14,7 +14,7 @@ class EdiStorageListener(Component): def _move_file(self, storage, from_dir_str, to_dir_str, filename): from_dir = PurePath(from_dir_str) to_dir = PurePath(to_dir_str) - if filename not in storage.list_files(from_dir): + if filename not in storage.list_files(from_dir.as_posix()): # The file might have been moved after a previous error. return False self._add_after_commit_hook( From 8596037aa77377d82bb9f293193cbdf5e8bf7994 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 14 Jul 2021 11:43:32 +0000 Subject: [PATCH 34/69] edi_storage_oca 14.0.1.2.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index e25ee456dd..fb84ffcbb8 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.1.0", + "version": "14.0.1.2.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From f60d5da23077d4b32ea9af97526885b9994fee64 Mon Sep 17 00:00:00 2001 From: JordiMForgeFlow Date: Wed, 4 Aug 2021 09:23:35 +0200 Subject: [PATCH 35/69] [14.0][FIX] edi_storage_oca: remove duplicate backend_type_id field in edi backend form --- edi_storage_oca/views/edi_backend_views.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 6d2e4d5d35..520965fb3e 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -6,7 +6,6 @@ - From beb54a1486d3d318463c0c8038be0b07beef7223 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 4 Aug 2021 09:00:35 +0000 Subject: [PATCH 36/69] edi_storage_oca 14.0.1.2.1 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index fb84ffcbb8..7fd2405b11 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.0", + "version": "14.0.1.2.1", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From e7464c8bec5315645b0e19f3954dbce1502e9cab Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 6 Sep 2021 12:04:31 +0200 Subject: [PATCH 37/69] [FIX] edi_storage: incorrect call to output checker in receive component. checker is only relevant for output direction. --- edi_storage_oca/components/receive.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 695c407784..0061ac1d94 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,12 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - checker = self.component(usage="storage.check") - result = checker.check() - if not result: - # all good here - return True - direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename path = self._remote_file_path(direction, "pending", filename) From 3eeee9fe78134382abfed7bf544784c192a19474 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Fri, 1 Oct 2021 09:07:16 +0000 Subject: [PATCH 38/69] edi_storage_oca 14.0.1.2.2 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 7fd2405b11..efe19ecca8 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.1", + "version": "14.0.1.2.2", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From d01b27cd3822f7e19429adc10901f3139aaf932e Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 17 Sep 2021 17:14:19 +0200 Subject: [PATCH 39/69] edi: improve backend view extension --- edi_storage_oca/views/edi_backend_views.xml | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 520965fb3e..ead51ef513 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -4,17 +4,19 @@ edi.backend - - - - - - - - - - - + + + + + + + + + + + + + From 56be5e45365bcfcaf85cd661cac6f8946e29010b Mon Sep 17 00:00:00 2001 From: oca-travis Date: Wed, 13 Oct 2021 14:22:09 +0000 Subject: [PATCH 40/69] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index 784ca4c680..eb9e806888 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -108,6 +108,11 @@ msgstr "" msgid "Path to folder for pending operations" msgstr "" +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id msgid "Storage backend" From 8bc49905a033e56f9f2e1b45d63c0619454022d0 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 13 Oct 2021 14:53:40 +0000 Subject: [PATCH 41/69] edi_storage_oca 14.0.1.3.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index efe19ecca8..3481bef650 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.2", + "version": "14.0.1.3.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 9e77f673aa3674a173e50f441ea4d7d74e89cf44 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 22 Sep 2021 20:07:37 +0200 Subject: [PATCH 42/69] edi: load main vars at init --- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/test_component_match.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index a91d5f238c..678979f8c6 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,3 +1,4 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_component_match from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e4fbfbe827..f0a776ae90 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -3,7 +3,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component -from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @@ -49,9 +49,13 @@ class S3Send(Component): self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + # Record not relevant for these tests + work_ctx = {"exchange_record": self.env["edi.exchange.record"].browse()} + component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", @@ -61,6 +65,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -70,6 +75,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -79,6 +85,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", From e2d6e869abf8e279534ed896855cbad2442014de Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sat, 30 Oct 2021 08:52:28 +0200 Subject: [PATCH 43/69] edi_storage: fix find files test --- edi_storage_oca/tests/common.py | 6 ++++++ edi_storage_oca/tests/test_edi_backend_storage.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 5aaa9951e8..31e4077c05 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -112,6 +112,12 @@ def _mock_storage_backend_list_files(self, mocked_paths): mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _mock_storage_backend_find_files(self, result): + def _result(self, pattern, relative_path=None, **kw): + return result + + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".find_files", _result) + def _test_result( self, record, diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 06448474b2..2f396d39d6 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -223,7 +223,9 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): ) self.assertEqual(existing, 0) # Run cron action: - self._test_run_cron_pending_input(mocked_paths) + found_files = [input_dir + fname for fname in file_names] + with self._mock_storage_backend_find_files(found_files): + self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) From 4959f62c9b0f23d0d495c8634da06ed5b374ccc3 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 2 Nov 2021 12:01:22 +0000 Subject: [PATCH 44/69] edi_storage_oca 14.0.1.4.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 3481bef650..d0cb44158f 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.3.0", + "version": "14.0.1.4.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From c1935b59084320d4c14fbb0467178ce680fae4c5 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 12:41:35 +0100 Subject: [PATCH 45/69] edi: get rid of name, use identifier Name contained redundant information like type name and related record. On for and tree view is totally useless. To still provide information on linked resource name the 'related_name' field has been added. --- edi_storage_oca/components/check.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 878cfffcb3..90f13c9517 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -37,9 +37,8 @@ def _exchange_output_check(self): """ if self._get_remote_file("done"): _logger.info( - "%s done for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s done", + self.exchange_record.identifier, ) if ( not self.exchange_record.edi_exchange_state @@ -52,9 +51,8 @@ def _exchange_output_check(self): error = self._get_remote_file("error") if error: _logger.info( - "%s error for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s error", + self.exchange_record.identifier, ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" From e1e7dfdbcc466f955aa8f98ac4a85fc39cd097f8 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 16:28:30 +0100 Subject: [PATCH 46/69] edi_storage: fail gracefully w/o error report --- edi_storage_oca/components/check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 90f13c9517..2cd97fb4a0 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -56,7 +56,9 @@ def _exchange_output_check(self): ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" - error_report = self._get_remote_file("error", filename=err_filename) + error_report = ( + self._get_remote_file("error", filename=err_filename) or "no-report" + ) if self.exchange_record.edi_exchange_state == "output_sent": self.exchange_record.update( { From 75024b0655c8905cfb1afe704c15b3a048e53598 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Sat, 19 Mar 2022 15:05:52 +0000 Subject: [PATCH 47/69] edi_storage_oca 14.0.1.5.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index d0cb44158f..7742522b13 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.4.0", + "version": "14.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 73870d5118c2fb65300b80aac5e6c9e496d04a1c Mon Sep 17 00:00:00 2001 From: jcoux Date: Tue, 17 May 2022 16:50:06 +0200 Subject: [PATCH 48/69] edi_storage_oca: improve error handling When trying to get done/failed directories, if we don't found it on SFTP server, we ignore the error. Current implementation only ignores FileNotFoundError, but depending on the SFTP server, we can have OSError. --- edi_storage_oca/components/base.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 5010e68449..40e1dcb2f8 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,11 +1,13 @@ # Copyright 2020 ACSONE # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). - +import logging from pathlib import PurePath from odoo.addons.component.core import AbstractComponent +_logger = logging.getLogger(__file__) + class EDIStorageComponentMixin(AbstractComponent): @@ -65,4 +67,19 @@ def _get_remote_file(self, state, filename=None, binary=False): # (the date will never match) return self.storage.get(path.as_posix(), binary=binary) except FileNotFoundError: + _logger.info( + "Ignored FileNotFoundError when trying " + "to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None + except OSError: + _logger.info( + "Ignored OSError when trying to get file %s into path %s for state %s", + filename, + path, + state, + ) return None From 4badf8828bfa68ce4f6ffd87a7e1dbefb2f55888 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Thu, 19 May 2022 07:16:15 +0000 Subject: [PATCH 49/69] edi_storage_oca 14.0.1.5.1 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 7742522b13..b0b2c16e81 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.5.0", + "version": "14.0.1.5.1", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From ed65d404bdb27061893d7f09813adb7be766d1ac Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 6 May 2022 12:20:57 +0200 Subject: [PATCH 50/69] edi_storage: path configurable by type and param By setting specific storage path on exchange type advanced settings, we can configure specific path for each exchange type (input or output). Path can also be configured by config parameters to allow to easily set different values depending of the environment. --- edi_storage_oca/components/base.py | 18 ++++- edi_storage_oca/components/listener.py | 20 ++++-- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/components/send.py | 5 +- edi_storage_oca/models/edi_backend.py | 16 +++-- edi_storage_oca/models/edi_exchange_type.py | 41 ++++++++++++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 15 +++-- edi_storage_oca/tests/test_components_base.py | 12 ++-- .../tests/test_edi_backend_storage.py | 13 ++-- edi_storage_oca/tests/test_exchange_type.py | 67 +++++++++++++++++++ 11 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 edi_storage_oca/tests/test_exchange_type.py diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 40e1dcb2f8..db1a2c8a32 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,4 +1,5 @@ # Copyright 2020 ACSONE +# Copyright 2022 Camptocamp # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging @@ -50,8 +51,22 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ + _logger.warning( + "Call of deprecated function `_remote_file_path`. " + "Please use `_get_remote_file_path` instead.", + ) return self._dir_by_state(direction, state) / filename.strip("/ ") + def _get_remote_file_path(self, state, filename=None): + """Retrieve remote path for current exchange record.""" + filename = filename or self.exchange_record.exchange_filename + direction = self.exchange_record.direction + directory = self._dir_by_state(direction, state).as_posix() + path = self.exchange_record.type_id._storage_fullpath( + directory=directory, filename=filename + ) + return path + def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. @@ -59,8 +74,7 @@ def _get_remote_file(self, state, filename=None, binary=False): :param filename: custom file name, exchange_record filename used by default :return: remote file content as string """ - filename = filename or self.exchange_record.exchange_filename - path = self._remote_file_path(self.exchange_record.direction, state, filename) + path = self._get_remote_file_path(state, filename=filename) try: # TODO: support match via pattern (eg: filename-prefix-*) # otherwise is impossible to retrieve input files and acks diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index 7b551c81e8..54ad1ec01c 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -34,9 +34,15 @@ def on_edi_exchange_done(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - done_dir = record.backend_id.input_dir_done - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + done_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_done + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if not done_dir: return res res = self._move_file(storage, pending_dir, done_dir, file) @@ -52,8 +58,12 @@ def on_edi_exchange_error(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if error_dir: res = self._move_file(storage, pending_dir, error_dir, file) return res diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 0061ac1d94..b0ab8257ec 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,8 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index cbb6ed309a..95c6ecdc6e 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -23,11 +23,8 @@ def send(self): if not result: # all good here return True - - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename filedata = self.exchange_record.exchange_file - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") self.storage.add(path.as_posix(), filedata, binary=False) # TODO: delegate this to generic storage backend # except paramiko.ssh_exception.AuthenticationException: diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5f95e33827..3669eabf92 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,16 +145,24 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + full_input_dir_pending = exchange_type._storage_fullpath( + self.input_dir_pending + ).as_posix() if not exchange_type.exchange_filename_pattern: - # If there is not patter, return everything - return self.storage_id.list_files(self.input_dir_pending) + # If there is not pattern, return everything + filenames = [ + x + for x in self.storage_id.list_files(full_input_dir_pending) + if x.strip("/") + ] + return filenames bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) - pending_path_len = len(self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, full_input_dir_pending) + pending_path_len = len(full_input_dir_pending) return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py index 42a7f70088..258576edc3 100644 --- a/edi_storage_oca/models/edi_exchange_type.py +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -1,6 +1,8 @@ # Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from pathlib import PurePath + from odoo import fields, models @@ -19,3 +21,42 @@ class EDIExchangeType(models.Model): "the files to be fetched from the pending directory in the related " "storage. E.g: `.*my-type-[0-9]*.\\.csv`" ) + + def _storage_path(self): + """Retrieve specific path for current exchange type. + + In your exchange type you can pass this config: + + storage: + # simple string + path: path/to/file + + Or + + storage: + # name of the param containing the path + path_config_param: path/to/file + + Thanks to the param you could even configure it by env. + """ + self.ensure_one() + storage_settings = self.advanced_settings.get("storage", {}) + path = storage_settings.get("path") + if path: + return PurePath(path) + path_config_param = storage_settings.get("path_config_param") + if path_config_param: + icp = self.env["ir.config_parameter"].sudo() + path = icp.get_param(path_config_param) + if path: + return PurePath(path) + + def _storage_fullpath(self, directory=None, filename=None): + self.ensure_one() + path_prefix = self._storage_path() + path = PurePath((directory or "").strip().rstrip("/")) + if path_prefix: + path = path_prefix / path + if filename: + path = path / filename.strip("/") + return path diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index 678979f8c6..18bc953002 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -2,3 +2,4 @@ from . import test_components_base from . import test_component_match from . import test_edi_storage_listener +from . import test_exchange_type diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 31e4077c05..65cf70608f 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -27,6 +27,7 @@ def _setup_records(cls): "exchange_file": cls.filedata, } cls.record = cls.backend.create_record("test_csv_output", vals) + cls.record_input = cls.backend.create_record("test_csv_input", vals) cls.fakepath = "/tmp/{}".format(cls._filename(cls)) with open(cls.fakepath, "w+b") as fakefile: @@ -53,6 +54,11 @@ def _setup_records(cls): ["storage.check"], work_ctx={"exchange_record": cls.record}, ) + cls.checker_input = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record_input}, + ) cls.sender = cls.backend._find_component( cls.partner._name, ["storage.send"], @@ -69,19 +75,16 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath( - self, state, record=None, ack=False, direction=False, fname=None - ): + def _file_fullpath(self, state, record=None, ack=False, fname=None, checker=None): record = record or self.record + checker = checker or self.checker if not fname: fname = self._filename(record, ack=ack) - if not direction: - direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return (self.checker._remote_file_path(direction, state, fname)).as_posix() + return checker._get_remote_file_path(state, filename=fname).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 57abd13856..9388993b0b 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -18,14 +18,16 @@ def test_remote_file_path(self): (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), ) for _args, expected in to_test: - path_obj = self.checker._remote_file_path(*_args) + direction, state, filename = _args + if direction == "input": + checker = self.checker_input + else: + checker = self.checker + path_obj = checker._get_remote_file_path(state, filename) self.assertEqual(path_obj.as_posix(), expected) with self.assertRaises(AssertionError): - self.checker._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + self.checker_input._get_remote_file_path("WHATEVER", "foo.csv") def test_get_remote_file(self): with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 2f396d39d6..0d4793d3f9 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -212,22 +212,25 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): mocked_paths = { input_dir: "/tmp/", self._file_fullpath( - "pending", direction="input", fname=file_names[0] + "pending", fname=file_names[0], checker=self.checker_input ): self.fakepath_input_pending_1, self._file_fullpath( - "pending", direction="input", fname=file_names[1] + "pending", fname=file_names[1], checker=self.checker_input ): self.fakepath_input_pending_2, } - existing = self.env["edi.exchange.record"].search_count( + existing_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) - self.assertEqual(existing, 0) # Run cron action: found_files = [input_dir + fname for fname in file_names] with self._mock_storage_backend_find_files(found_files): self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( - [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + [ + ("backend_id", "=", self.backend.id), + ("type_id", "=", exch_type.id), + ("id", "not in", existing_records.ids), + ] ) self.assertEqual(len(new_records), 2) for rec in new_records: diff --git a/edi_storage_oca/tests/test_exchange_type.py b/edi_storage_oca/tests/test_exchange_type.py new file mode 100644 index 0000000000..6b886eab3c --- /dev/null +++ b/edi_storage_oca/tests/test_exchange_type.py @@ -0,0 +1,67 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + + +class EDIExchangeTypeTestCase(EDIBackendCommonTestCase): + def _check_test_storage_fullpath(self, wanted_fullpath, directory, filename): + fullpath = self.exchange_type_out._storage_fullpath(directory, filename) + self.assertEqual(fullpath.as_posix(), wanted_fullpath) + + def _do_test_storage_fullpath(self, prefix=""): + # Test with no directory and no filename + wanted_fullpath = prefix or "." + self._check_test_storage_fullpath(wanted_fullpath, None, None) + + # Test with directory + directory = "test_directory" + wanted_fullpath = f"{prefix}/{directory}" if prefix else directory + self._check_test_storage_fullpath(wanted_fullpath, directory, None) + + # Test with filename + filename = "test_filename.csv" + wanted_fullpath = f"{prefix}/{filename}" if prefix else filename + self._check_test_storage_fullpath(wanted_fullpath, None, filename) + + # Test with directory and filename + wanted_fullpath = ( + f"{prefix}/{directory}/{filename}" if prefix else f"{directory}/{filename}" + ) + self._check_test_storage_fullpath(wanted_fullpath, directory, filename) + + def test_storage_fullpath(self): + """ + Test storage fullpath defined into advanced settings. + Example of pattern: + storage: + # simple string + path: path/to/file + # name of the param containing the path + path_config_param: path/to/file + """ + + # Test without any prefix + self._do_test_storage_fullpath() + + # Force path on advanced settings + prefix = "prefix/path" + self.exchange_type_out.advanced_settings_edit = f""" + storage: + path: {prefix} + """ + self._do_test_storage_fullpath(prefix=prefix) + + # Force path on advanced settings using config param, but not defined + self.exchange_type_out.advanced_settings_edit = """ + storage: + path_config_param: prefix_path_config_param + """ + self._do_test_storage_fullpath() + + # Define config param + prefix = "prefix/path/by/config/param" + self.env["ir.config_parameter"].sudo().set_param( + "prefix_path_config_param", prefix + ) + self._do_test_storage_fullpath(prefix=prefix) From 3747393526cee8f97d1ed4c3092552f910b8b3b2 Mon Sep 17 00:00:00 2001 From: Jasmin Solanki Date: Tue, 12 Apr 2022 16:45:22 +0530 Subject: [PATCH 51/69] [IMP] edi_storage_oca: Update development status --- edi_storage_oca/README.rst | 6 +++--- edi_storage_oca/__manifest__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index dc3a04a126..426802b679 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -7,9 +7,9 @@ EDI Storage backend support !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status - :alt: Alpha + :alt: Beta .. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/14.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index b0b2c16e81..b6c5b1426e 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -8,7 +8,7 @@ Base module to allow exchanging files via storage backend (eg: SFTP). """, "version": "14.0.1.5.1", - "development_status": "Alpha", + "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", From fbf5313f282f2110a421fcbb1b5afda2968e7fd6 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 13 Jul 2022 08:45:41 +0000 Subject: [PATCH 52/69] [UPD] README.rst --- edi_storage_oca/README.rst | 7 +------ edi_storage_oca/static/description/index.html | 8 +------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 426802b679..874b8938ad 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/14.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -49,11 +49,6 @@ Inside this folder you have this hierarchy:: The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly. -.. IMPORTANT:: - This is an alpha version, the data model and design can change at any time without warning. - Only for development or testing purpose, do not use in production. - `More details on development status `_ - **Table of contents** .. contents:: diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 257c2d221a..d8755c4215 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Beta License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -389,12 +389,6 @@

EDI Storage backend support

The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly.

-
-

Important

-

This is an alpha version, the data model and design can change at any time without warning. -Only for development or testing purpose, do not use in production. -More details on development status

-

Table of contents

    From 159d4af34d4165a1e99597c68e008f5951efddf0 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 13 Jul 2022 08:45:41 +0000 Subject: [PATCH 53/69] edi_storage_oca 14.0.1.5.2 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index b6c5b1426e..5241c20bf7 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.5.1", + "version": "14.0.1.5.2", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From e0e883285c39e470a44742e9b83934142dc59217 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Thu, 28 Jul 2022 07:15:20 +0000 Subject: [PATCH 54/69] edi_storage_oca 14.0.1.6.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 5241c20bf7..63b5f7fddc 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.5.2", + "version": "14.0.1.6.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 9087ec73bb3e73726c17d1a5fbc0c73121c35cfb Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sun, 28 Aug 2022 12:23:49 +0200 Subject: [PATCH 55/69] edi: fix _cron_check_output_exchange_sync Skip 'output_sent' records by default. Reason: most of the times when a message is sent out you don't care about its state anymore. Any subsequent update from outside should come with a new input record instead. Yet, if for any reason this feature is required you can still enable it by passing 'skip_sent=False'. --- edi_storage_oca/models/edi_backend.py | 6 ++++++ edi_storage_oca/tests/test_edi_backend_storage.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 3669eabf92..e396ae69df 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -167,3 +167,9 @@ def _storage_get_input_filenames(self, exchange_type): def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} + + def _check_output_exchange_sync(self, skip_send=False, skip_sent=True): + # Do not skip sent records when dealing w/ storage related exchanges, + # because we want to update the file state + # depending on where they are in the external folder. + return super()._check_output_exchange_sync(skip_sent=not self.storage_id) diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 0d4793d3f9..fb37ae7115 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -148,6 +148,8 @@ def test_cron_full_flow(self): "model": partner2._name, "res_id": partner2.id, "exchange_filename": "rec2.csv", + "exchange_file": rec1.exchange_file, + "edi_exchange_state": rec1.edi_exchange_state, } ) rec3 = self.record.copy( @@ -156,6 +158,7 @@ def test_cron_full_flow(self): "res_id": partner3.id, "exchange_filename": "rec3.csv", "edi_exchange_state": "output_sent_and_error", + "exchange_file": rec1.exchange_file, } ) mocked_paths = { From 2f4d17299da34cc1a14c5cdb6daa5c5673a15782 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 29 Aug 2022 15:25:27 +0000 Subject: [PATCH 56/69] edi_storage_oca 14.0.1.7.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 63b5f7fddc..3821c38622 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.6.0", + "version": "14.0.1.7.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From e32ade2423488fc98e9c2d9ca99a7a656c5036c2 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Tue, 6 Sep 2022 12:05:16 +0200 Subject: [PATCH 57/69] edi_storage: adapt _check_output_exchange_sync override --- edi_storage_oca/models/edi_backend.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index e396ae69df..e3eb2d700f 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -168,8 +168,10 @@ def _storage_get_input_filenames(self, exchange_type): def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} - def _check_output_exchange_sync(self, skip_send=False, skip_sent=True): + def _check_output_exchange_sync(self, **kw): # Do not skip sent records when dealing w/ storage related exchanges, # because we want to update the file state # depending on where they are in the external folder. - return super()._check_output_exchange_sync(skip_sent=not self.storage_id) + if self.storage_id: + kw["skip_sent"] = False + return super()._check_output_exchange_sync(**kw) From fb99c66f413007181ba85f1100e107550d70d2e0 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 6 Sep 2022 12:48:14 +0000 Subject: [PATCH 58/69] edi_storage_oca 14.0.1.8.0 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 3821c38622..a974c39c99 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.7.0", + "version": "14.0.1.8.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 3fa35ad957d9379e3d8496c58b8d342afc4c1ca8 Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Tue, 7 Feb 2023 17:20:38 +0100 Subject: [PATCH 59/69] [IMP] edi_storage_oca: Delete processed files if you want it --- edi_storage_oca/components/listener.py | 19 ++++++++++++++++--- edi_storage_oca/models/edi_backend.py | 1 + .../tests/test_edi_storage_listener.py | 15 +++++++++++++++ edi_storage_oca/views/edi_backend_views.xml | 6 +++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index 54ad1ec01c..24ffb97aaa 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -18,15 +18,23 @@ def _move_file(self, storage, from_dir_str, to_dir_str, filename): # The file might have been moved after a previous error. return False self._add_after_commit_hook( - storage._move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + storage.move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() ) return True - def _add_after_commit_hook(self, move_func, sftp_filepath, sftp_destination_path): + def _remove_file(self, storage, from_dir_str, filename): + from_dir = PurePath(from_dir_str) + if filename not in storage.list_files(from_dir.as_posix()): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook(storage.delete, (from_dir / filename).as_posix()) + return True + + def _add_after_commit_hook(self, partial_func, *args): """Add hook after commit to move the file when transaction is over.""" self.env.cr.after( "commit", - functools.partial(move_func, sftp_filepath, sftp_destination_path), + functools.partial(partial_func, *args), ) def on_edi_exchange_done(self, record): @@ -43,6 +51,11 @@ def on_edi_exchange_done(self, record): error_dir = record.type_id._storage_fullpath( record.backend_id.input_dir_error ).as_posix() + if record.backend_id.input_dir_remove: + res = self._remove_file(storage, pending_dir, file) + if not res: + res = self._remove_file(storage, error_dir, file) + return res if not done_dir: return res res = self._move_file(storage, pending_dir, done_dir, file) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index e3eb2d700f..0d04e569b3 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -42,6 +42,7 @@ class EDIBackend(models.Model): input_dir_pending = fields.Char( "Input pending directory", help="Path to folder for pending operations" ) + input_dir_remove = fields.Boolean("Remove input after done") input_dir_done = fields.Char( "Input done directory", help="Path to folder for doneful operations" ) diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py index 4f12f3311f..3ab50ebac9 100644 --- a/edi_storage_oca/tests/test_edi_storage_listener.py +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -47,6 +47,9 @@ def _move_file_mocked(self, *args): def _mock_listener_move_file(self): return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + def _mock_listener_remove_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._remove_file", self._move_file_mocked) + def test_01_process_record_success(self): with self._mock_listener_move_file(): self.record.write({"edi_exchange_state": "input_received"}) @@ -69,3 +72,15 @@ def test_02_process_record_with_error(self): self.assertEqual(from_dir_str, self.backend.input_dir_pending) self.assertEqual(to_dir_str, self.backend.input_dir_error) self.assertEqual(filename, self.record.exchange_filename) + + def test_03_process_record_success_delete(self): + self.backend.write({"input_dir_remove": True, "input_dir_done": False}) + self.backend.flush() + + with self._mock_listener_remove_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(filename, self.record.exchange_filename) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index ead51ef513..59ed64b0c7 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -9,7 +9,11 @@ - + + From 80a620354197ad4865b8aec0be966efdd35e94bc Mon Sep 17 00:00:00 2001 From: oca-ci Date: Wed, 15 Feb 2023 17:12:45 +0000 Subject: [PATCH 60/69] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index eb9e806888..f012629aec 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -108,6 +108,11 @@ msgstr "" msgid "Path to folder for pending operations" msgstr "" +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_remove +msgid "Remove input after done" +msgstr "" + #. module: edi_storage_oca #: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form msgid "Storage" From 4334ac5e471640044f1a1ab8c2019c3d20adf828 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 15 Feb 2023 17:21:05 +0000 Subject: [PATCH 61/69] edi_storage_oca 14.0.1.8.1 --- edi_storage_oca/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index a974c39c99..261634f193 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.8.0", + "version": "14.0.1.8.1", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", From 908f9e266dd0bed5f0e58afea8cc4c3c80846f7f Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Sun, 3 Sep 2023 12:42:17 +0000 Subject: [PATCH 62/69] [UPD] README.rst --- edi_storage_oca/README.rst | 15 +++++--- edi_storage_oca/static/description/index.html | 38 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 874b8938ad..e032248fe7 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -2,10 +2,13 @@ EDI Storage backend support =========================== -.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:8b943f104b4d54ed51a4fb55f4e17ffe1ec3995e962bb7654ea4c5837d952673 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status @@ -19,11 +22,11 @@ EDI Storage backend support .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca :alt: Translate me on Weblate -.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/14.0 - :alt: Try me on Runbot +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi&target_branch=14.0 + :alt: Try me on Runboat -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -64,7 +67,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed +If you spotted it first, help us to smash it by providing a detailed and welcomed `feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index d8755c4215..0268d818c2 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -1,20 +1,20 @@ - + - + EDI Storage backend support