From e939391c415917f64b6b84062e807e1d8c7427ad Mon Sep 17 00:00:00 2001 From: marination Date: Fri, 9 Feb 2024 14:55:24 +0100 Subject: [PATCH 01/22] fix: Use doctype setting to set auto-extracted file as private - Use `make_attachments_public` to determine the file privacy while auto creating files from the text editor field - Currently, all files in the text editor field are automatically public (cherry picked from commit 7445de9b1c101ea57e13944a852e529189f4085b) --- frappe/core/doctype/file/test_file.py | 8 +++++--- frappe/core/doctype/file/utils.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 5ceb050fc061..2bbb513e5078 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -709,6 +709,8 @@ def tearDown(self) -> None: class TestFileUtils(FrappeTestCase): def test_extract_images_from_doc(self): + is_private = not frappe.db.get_value("DocType", "ToDo", "make_attachments_public") + # with filename in data URI todo = frappe.get_doc( { @@ -716,9 +718,9 @@ def test_extract_images_from_doc(self): "description": 'Test ', } ).insert() - self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name})) - self.assertIn('', todo.description) - self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/files/pix.png"]) + self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name, "is_private": is_private})) + self.assertIn('', todo.description) + self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/private/files/pix.png"]) # without filename in data URI todo = frappe.get_doc( diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 5f245707936b..eedf2fb1fced 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -214,7 +214,7 @@ def get_file_name(fname: str, optional_suffix: str | None = None) -> str: def extract_images_from_doc(doc: "Document", fieldname: str): content = doc.get(fieldname) - content = extract_images_from_html(doc, content) + content = extract_images_from_html(doc, content, is_private=(not doc.meta.make_attachments_public)) if frappe.flags.has_dataurl: doc.set(fieldname, content) From c3bb7b2658f89285cd8b58937aca44c93e9410bd Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 9 Feb 2024 19:58:41 +0530 Subject: [PATCH 02/22] test: use meta for fetching settings (cherry picked from commit ed6613ce8e3ad6393419828a3cb94392ec45efed) --- frappe/core/doctype/file/test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2bbb513e5078..02a668fc3b96 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -709,7 +709,7 @@ def tearDown(self) -> None: class TestFileUtils(FrappeTestCase): def test_extract_images_from_doc(self): - is_private = not frappe.db.get_value("DocType", "ToDo", "make_attachments_public") + is_private = not frappe.get_meta("ToDo").make_attachments_public # with filename in data URI todo = frappe.get_doc( From c9dbdc5ab6f6ee9bb335e54812a654ef6f28d221 Mon Sep 17 00:00:00 2001 From: marination Date: Fri, 9 Feb 2024 19:14:40 +0100 Subject: [PATCH 03/22] test: Comment + Comm.n file extraction & Attachments default to public for Communication - Note: DocType Communication's records are used for emails that have the file link embedded in its content - fix: The files extracted from a Communication must be public so that they are visible in an email (link in email) - Test: Check if file is created appropriately from a Comment and a Communication (cherry picked from commit 65c8a376361910a09d4df14c76f83ee121950b32) # Conflicts: # frappe/core/doctype/communication/communication.json # frappe/core/doctype/file/test_file.py --- .../doctype/communication/communication.json | 5 ++ frappe/core/doctype/file/test_file.py | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index e5f090e2f7f3..8ce39dad6cbc 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -395,7 +395,12 @@ "icon": "fa fa-comment", "idx": 1, "links": [], +<<<<<<< HEAD "modified": "2023-03-16 12:04:18.113817", +======= + "make_attachments_public": 1, + "modified": "2024-02-09 12:10:01.200845", +>>>>>>> 65c8a37636 (test: Comment + Comm.n file extraction & Attachments default to public for Communication) "modified_by": "Administrator", "module": "Core", "name": "Communication", diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 02a668fc3b96..d6477166463f 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -17,6 +17,12 @@ move_file, unzip_file, ) +<<<<<<< HEAD +======= +from frappe.core.doctype.file.exceptions import FileTypeNotAllowed +from frappe.core.doctype.file.utils import get_extension +from frappe.desk.form.utils import add_comment +>>>>>>> 65c8a37636 (test: Comment + Comm.n file extraction & Attachments default to public for Communication) from frappe.exceptions import ValidationError from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path @@ -732,6 +738,48 @@ def test_extract_images_from_doc(self): filename = frappe.db.exists("File", {"attached_to_name": todo.name}) self.assertIn(f'', + frappe.session.user, + frappe.session.user, + ) + + self.assertTrue( + frappe.db.exists("File", {"attached_to_name": test_doc.name, "is_private": is_private}) + ) + self.assertRegex(comment.content, r"") + + def test_extract_images_from_communication(self): + """ + Ensure that images are extracted from communication and become public attachments. + """ + is_private = not frappe.get_meta("Communication").make_attachments_public + communication = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "communication_medium": "Email", + "content": '
', + "recipients": "to ", + "cc": None, + "bcc": None, + "sender": "sender@test.com", + } + ).insert(ignore_permissions=True) + + self.assertTrue( + frappe.db.exists("File", {"attached_to_name": communication.name, "is_private": is_private}) + ) + self.assertRegex(communication.content, r"") + def test_create_new_folder(self): folder = create_new_folder("test_folder", "Home") self.assertTrue(folder.is_folder) From 3d7e572360121b3e2721c9da0f98038386c497e3 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 13 Feb 2024 13:14:22 +0530 Subject: [PATCH 04/22] fix: Use regex in failing test (cherry picked from commit 1f3e32a98decfba4562ef66a9e27ff4638849325) --- frappe/core/doctype/file/test_file.py | 34 +++++++++++---------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index d6477166463f..8d9c208dabb2 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -719,21 +719,17 @@ def test_extract_images_from_doc(self): # with filename in data URI todo = frappe.get_doc( - { - "doctype": "ToDo", - "description": 'Test ', - } + doctype="ToDo", + description='Test ', ).insert() self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name, "is_private": is_private})) - self.assertIn('', todo.description) + self.assertRegex(todo.description, r"") self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/private/files/pix.png"]) # without filename in data URI todo = frappe.get_doc( - { - "doctype": "ToDo", - "description": 'Test ', - } + doctype="ToDo", + description='Test ', ).insert() filename = frappe.db.exists("File", {"attached_to_name": todo.name}) self.assertIn(f'', - "recipients": "to ", - "cc": None, - "bcc": None, - "sender": "sender@test.com", - } + doctype="Communication", + communication_type="Communication", + communication_medium="Email", + content='
', + recipients="to ", + cc=None, + bcc=None, + sender="sender@test.com", ).insert(ignore_permissions=True) self.assertTrue( From ca5daed9ab3e94abb36bd36966937e038f6da2eb Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:11:15 +0000 Subject: [PATCH 05/22] fix: invalid filter on email acccount (#25674) (#25675) (cherry picked from commit 59b95a4d19f36fe37e47446d3b9eef0e45c63210) Co-authored-by: Ankush Menat --- frappe/email/doctype/email_account/email_account_list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account_list.js b/frappe/email/doctype/email_account/email_account_list.js index 5913706cbf96..0e8fc0c0dd1c 100644 --- a/frappe/email/doctype/email_account/email_account_list.js +++ b/frappe/email/doctype/email_account/email_account_list.js @@ -16,7 +16,7 @@ frappe.listview_settings["Email Account"] = { return [__("Default Sending"), color, "default_outgoing,=,Yes"]; } else { color = doc.enable_incoming ? "blue" : "gray"; - return [__("Inbox"), color, "is_global,=,No|is_default=No"]; + return [__("Inbox"), color, "default_outgoing,=,No|default_incoming=No"]; } }, }; From 4e7dd037a8fc8dcf67a304fd5946ecbd1abc79f4 Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 27 Mar 2024 12:21:42 +0100 Subject: [PATCH 06/22] fix: Merge conflicts --- frappe/core/doctype/communication/communication.json | 6 +----- frappe/core/doctype/file/test_file.py | 5 ----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 8ce39dad6cbc..ecd9fc9ee3f7 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -395,12 +395,8 @@ "icon": "fa fa-comment", "idx": 1, "links": [], -<<<<<<< HEAD - "modified": "2023-03-16 12:04:18.113817", -======= "make_attachments_public": 1, - "modified": "2024-02-09 12:10:01.200845", ->>>>>>> 65c8a37636 (test: Comment + Comm.n file extraction & Attachments default to public for Communication) + "modified": "2024-03-27 12:10:01.200845", "modified_by": "Administrator", "module": "Core", "name": "Communication", diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 8d9c208dabb2..c30bfa5be9f4 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -17,12 +17,7 @@ move_file, unzip_file, ) -<<<<<<< HEAD -======= -from frappe.core.doctype.file.exceptions import FileTypeNotAllowed -from frappe.core.doctype.file.utils import get_extension from frappe.desk.form.utils import add_comment ->>>>>>> 65c8a37636 (test: Comment + Comm.n file extraction & Attachments default to public for Communication) from frappe.exceptions import ValidationError from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path From fa1f3fc2c00d4da5737fc44a483a6cd91e93b2d8 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 26 Mar 2024 10:33:35 +0100 Subject: [PATCH 07/22] fix: Use CssParser to correctly pass options to wkhtmltopdf - Regex incorrectly fetches .print-format's child styles and also extracts the wrong attribute value - A CssParser is more maintainable and more readable as well as less prone to errors while extracting values - Method: We extract style tag contents out of the html and tokenize them. We then filter the styles for the right selector and extract the attributes we want from them. - This way we make sure that the right value is extracted and only the ones applicable to .print-format directly (cherry picked from commit 5dbcbbb915bdbc8e3b895c71333d4ec683073067) --- frappe/utils/pdf.py | 53 +++++++++++++++++++++++++++++++++++++-------- pyproject.toml | 1 + 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 16cbca23c43b..98415a966a38 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -5,11 +5,11 @@ import io import mimetypes import os -import re import subprocess from distutils.version import LooseVersion from urllib.parse import parse_qs, urlparse +import cssutils import pdfkit from bs4 import BeautifulSoup from PyPDF2 import PdfReader, PdfWriter @@ -160,7 +160,8 @@ def read_options_from_html(html): toggle_visible_pdf(soup) - # use regex instead of soup-parser + valid_styles = get_print_format_styles(soup) + for attr in ( "margin-top", "margin-bottom", @@ -172,17 +173,51 @@ def read_options_from_html(html): "page-width", "page-height", ): - try: - pattern = re.compile(r"(\.print-format)([\S|\s][^}]*?)(" + str(attr) + r":)(.+)(mm;)") - match = pattern.findall(html) - if match: - options[attr] = str(match[-1][3]).strip() - except Exception: - pass + for style in valid_styles: + if attr == style.name: + options[attr] = style.value return str(soup), options +def get_print_format_styles(soup: BeautifulSoup) -> list[cssutils.css.Property]: + """ + Get styles purely on class 'print-format'. + Valid: + 1) .print-format { ... } + 2) .print-format, p { ... } | p, .print-format { ... } + + Invalid (applied on child elements): + 1) .print-format p { ... } | .print-format > p { ... } + 2) .print-format #abc { ... } + + Returns: + [cssutils.css.Property(name='margin-top', value='50mm', priority=''), ...] + """ + stylesheet = "" + style_tags = soup.find_all("style") + + # Prepare a css stylesheet from all the style tags' contents + for style_tag in style_tags: + stylesheet += style_tag.string + + # Use css parser to tokenize the classes and their styles + parsed_sheet = cssutils.parseString(stylesheet) + + # Get all styles that are only for .print-format + valid_styles = [] + for rule in parsed_sheet: + if not isinstance(rule, cssutils.css.CSSStyleRule): + continue + + # Allow only .print-format { ... } and .print-format, p { ... } + # Disallow .print-format p { ... } and .print-format > p { ... } + if ".print-format" in [x.strip() for x in rule.selectorText.split(",")]: + valid_styles.extend(entry for entry in rule.style) + + return valid_styles + + def inline_private_images(html) -> str: soup = BeautifulSoup(html, "html.parser") for img in soup.find_all("img"): diff --git a/pyproject.toml b/pyproject.toml index 0251efb1fc65..83f2b98d8162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "chardet~=5.1.0", "croniter~=1.3.5", "cryptography~=42.0.0", + "cssutils~=2.9.0", "email-reply-parser~=0.5.12", "git-url-parse~=1.2.2", "gitdb~=4.0.7", From ac0076a88e056f2b6e7c7621e99e4d8b400213be Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 26 Mar 2024 18:16:53 +0100 Subject: [PATCH 08/22] chore: Test valid options extraction (cherry picked from commit 8814fe0beb63e5e2b7a4448af4377a06927bfad0) --- frappe/tests/test_pdf.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index 335bd2c8ee5f..c99c2d32ea9c 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -37,9 +37,32 @@ def runTest(self): def test_read_options_from_html(self): _, html_options = pdfgen.read_options_from_html(self.html) self.assertTrue(html_options["margin-top"] == "0") - self.assertTrue(html_options["margin-left"] == "10") + self.assertTrue(html_options["margin-left"] == "10mm") self.assertTrue(html_options["margin-right"] == "0") + html_1 = """ +
Hello
+ """ + _, options = pdfgen.read_options_from_html(html_1) + + self.assertTrue(options["margin-top"] == "0") + self.assertTrue(options["margin-left"] == "10mm") + self.assertTrue(options["margin-bottom"] == "20mm") + # margin-right was for .more-info (child of .print-format) + # so it should not be extracted into options + self.assertFalse(options.get("margin-right")) + def test_pdf_encryption(self): password = "qwe" pdf = pdfgen.get_pdf(self.html, options={"password": password}) From 51929fac43ba9bc46eb59309d308788773763454 Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 27 Mar 2024 12:00:49 +0100 Subject: [PATCH 09/22] chore: Use dict comprehension instead of nested loops (cherry picked from commit 96667b1babc4d6dcce3378961aa51a6746705e19) --- frappe/utils/pdf.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 98415a966a38..d7c67573bc81 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -162,7 +162,7 @@ def read_options_from_html(html): valid_styles = get_print_format_styles(soup) - for attr in ( + attrs = ( "margin-top", "margin-bottom", "margin-left", @@ -172,11 +172,8 @@ def read_options_from_html(html): "orientation", "page-width", "page-height", - ): - for style in valid_styles: - if attr == style.name: - options[attr] = style.value - + ) + options |= {style.name: style.value for style in valid_styles if style.name in attrs} return str(soup), options From 0737e8496982b7955480a5f5a85316786204c029 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 13:31:50 +0000 Subject: [PATCH 10/22] fix: preserve original error message (#25682) (#25684) (cherry picked from commit 85f66c083e22d5708588d944ecfc3dcd4f5b3af0) Co-authored-by: Ankush Menat --- frappe/email/smtp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 0e5a268177b9..b0ce04bc2154 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -7,7 +7,7 @@ import frappe from frappe import _ from frappe.email.oauth import Oauth -from frappe.utils import cint, cstr +from frappe.utils import cint, cstr, get_traceback class InvalidEmailCredentials(frappe.ValidationError): @@ -115,8 +115,9 @@ def quit(self): @classmethod def throw_invalid_credentials_exception(cls): + original_exception = get_traceback() or "\n" frappe.throw( - _("Please check your email login credentials."), + _("Please check your email login credentials.") + " " + original_exception.splitlines()[-1], title=_("Invalid Credentials"), exc=InvalidEmailCredentials, ) From 8042ef827e83ad225ac9a0b69a9dbca30cd29384 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:59:10 +0530 Subject: [PATCH 11/22] fix: incorrect UI icon for desc sort (#25687) (#25688) (cherry picked from commit 8dd10b3a03084fede6c1fc07b9621b3ef193c392) Co-authored-by: Ankush Menat --- frappe/public/js/frappe/ui/sort_selector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/sort_selector.js b/frappe/public/js/frappe/ui/sort_selector.js index eb59bec8c75a..b6051ec5deff 100644 --- a/frappe/public/js/frappe/ui/sort_selector.js +++ b/frappe/public/js/frappe/ui/sort_selector.js @@ -154,7 +154,7 @@ frappe.ui.SortSelector = class SortSelector { // set default this.sort_by = this.args.sort_by; - this.sort_order = this.args.sort_order; + this.sort_order = this.args.sort_order = this.args.sort_order.toLowerCase(); } get_meta_sort_field() { var meta = frappe.get_meta(this.doctype); From 2b35e4bf8e51ba677aaa873aa9d7834b8a94d2c7 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:56:09 +0530 Subject: [PATCH 12/22] fix: incorrect status on data import (backport #25660) (#25702) * fix: incorrect status on data import - Used to show the status as "Not Started" if a prior import session was partially successful - `data_import_list.js` needs some rethinking just displaying the past few logs isn't super helpful the import logs aren't individually identifiable on that screen - could start deleting the logs of a previously failed imported record, if it's successful on a subsequent run or atleast flag it to not display it Refer to internal ticket 9486 and 12166 for further information (cherry picked from commit f75dd9d1ebb8f3cf55d5cac1192c84648606c081) * fix: Correct logic for comlete success (cherry picked from commit f22120190d28241aac963df291de1db8bb04b7f1) * fix(importer): set `Pending` status as a fallback Will match scenarios like success == failed == 0 Signed-off-by: Akhil Narang (cherry picked from commit 2710c288707171b9e1c04ca19bf1958e5bbc98d6) * fix: erase logs in case of complete failure (cherry picked from commit 9f7c385b112c26a36860deb7c457e627baafb700) --------- Co-authored-by: Arjun Choudhary Co-authored-by: Ankush Menat Co-authored-by: Akhil Narang --- .../doctype/data_import/data_import_list.js | 4 ---- frappe/core/doctype/data_import/importer.py | 24 ++++++++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index a16478cdb166..5aa2ece6d514 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -27,10 +27,6 @@ frappe.listview_settings["Data Import"] = { if (imports_in_progress.includes(doc.name)) { status = "In Progress"; } - if (status === "Pending") { - status = "Not Started"; - } - return [__(status), colors[status], "status,=," + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 8b21a3eda755..c4cf158e955d 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -103,9 +103,13 @@ def import_data(self): log_index = 0 # Do not remove rows in case of retry after an error or pending data import - if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + if ( + self.data_import.status in ("Partial Success", "Error") + and len(import_log) >= self.data_import.payload_count + ): # remove previous failures from import log only in case of retry after partial success import_log = [log for log in import_log if log.get("success")] + frappe.db.delete("Data Import Log", {"success": 0, "data_import": self.data_import.name}) # get successfully imported rows imported_rows = [] @@ -214,13 +218,21 @@ def import_data(self): ) # set status - failures = [log for log in import_log if not log.get("success")] - if len(failures) == total_payload_count: - status = "Pending" - elif len(failures) > 0: + successes = [] + failures = [] + for log in import_log: + if log.get("success"): + successes.append(log) + else: + failures.append(log) + if len(failures) >= total_payload_count and len(successes) == 0: + status = "Error" + elif len(failures) > 0 and len(successes) > 0: status = "Partial Success" - else: + elif len(successes) == total_payload_count: status = "Success" + else: + status = "Pending" if self.console: self.print_import_log(import_log) From 85e5f2f85f273b79ddbf22e1ec7283a60b7f102f Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:19:53 +0100 Subject: [PATCH 13/22] fix: make insights ad translatable (cherry picked from commit 449d7d3b3d6af9fb3414270b21f42c9102dff581) --- frappe/public/js/frappe/list/list_sidebar.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/list/list_sidebar.js b/frappe/public/js/frappe/list/list_sidebar.js index 9c395972a7ef..5ef0873c651e 100644 --- a/frappe/public/js/frappe/list/list_sidebar.js +++ b/frappe/public/js/frappe/list/list_sidebar.js @@ -268,9 +268,9 @@ frappe.views.ListSidebar = class ListSidebar { this.insights_banner.remove(); } - const message = "Get more insights with"; + const message = __("Get more insights with"); const link = "https://frappe.io/s/insights"; - const cta = "Frappe Insights"; + const cta = __("Frappe Insights"); this.insights_banner = $(`
From 9ad5e58118b1e0da86bcd004f09d7c75fcea56f0 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:55:38 +0100 Subject: [PATCH 14/22] fix: advertise insights to system manager only (cherry picked from commit 57bcfe548eb2be283f0d2ba864e5bce1f0b7cafe) --- frappe/public/js/frappe/list/list_sidebar.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_sidebar.js b/frappe/public/js/frappe/list/list_sidebar.js index 9c395972a7ef..effd5b7d779e 100644 --- a/frappe/public/js/frappe/list/list_sidebar.js +++ b/frappe/public/js/frappe/list/list_sidebar.js @@ -39,7 +39,9 @@ frappe.views.ListSidebar = class ListSidebar { }); } - this.add_insights_banner(); + if (frappe.user.has_role("System Manager")) { + this.add_insights_banner(); + } } setup_views() { From 7558acfcf75902ba2c101342cac0a760d6fa2c73 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Wed, 27 Mar 2024 13:05:50 +0100 Subject: [PATCH 15/22] fix: translatable web footer (cherry picked from commit b056147fea43082d17024e4b0e86883448d2d969) --- frappe/templates/includes/footer/footer_powered.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/includes/footer/footer_powered.html b/frappe/templates/includes/footer/footer_powered.html index d71c537fb0c1..8cca3d9a8ecb 100644 --- a/frappe/templates/includes/footer/footer_powered.html +++ b/frappe/templates/includes/footer/footer_powered.html @@ -1 +1 @@ -Built on Frappe \ No newline at end of file +{{ _("Built on {0}").format('Frappe') }} From 53d28adbaedb341c5f69316eb02b7b26b58fbb0e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:06:40 +0530 Subject: [PATCH 16/22] fix: make ads translatable (#25710) (cherry picked from commit 7426805425493660f3f9792d750a2e0326d3812a) Co-authored-by: barredterra <14891507+barredterra@users.noreply.github.com> --- frappe/core/doctype/server_script/server_script_list.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/server_script/server_script_list.js b/frappe/core/doctype/server_script/server_script_list.js index 0df447a0eba2..1d4cdfdfe666 100644 --- a/frappe/core/doctype/server_script/server_script_list.js +++ b/frappe/core/doctype/server_script/server_script_list.js @@ -15,9 +15,9 @@ function add_github_star_cta(listview) { listview.github_star_banner.remove(); } - const message = "Loving Frappe Framework?"; + const message = __("Loving Frappe Framework?"); const link = "https://github.com/frappe/frappe"; - const cta = "Star us on GitHub"; + const cta = __("Star us on GitHub"); listview.github_star_banner = $(`
From 7a99b75d1f5fb1ba25bb1a4d0360fc28c595765f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:55:50 +0000 Subject: [PATCH 17/22] fix: reserved keywords as col name (#25718) (#25725) (cherry picked from commit 87ffe25e710c20f7a5e18394dad90a48eb338a7f) Co-authored-by: Ankush Menat --- frappe/model/delete_doc.py | 2 +- frappe/model/document.py | 4 +++- frappe/model/dynamic_links.py | 2 +- frappe/tests/test_linked_with.py | 37 ++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index cb7b7a617091..cae395603194 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -338,7 +338,7 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"): df["table"] = ", `parent`, `parenttype`, `idx`" if meta.istable else "" for refdoc in frappe.db.sql( """select `name`, `docstatus` {table} from `tab{parent}` where - {options}=%s and {fieldname}=%s""".format(**df), + `{options}`=%s and `{fieldname}`=%s""".format(**df), (doc.doctype, doc.name), as_dict=True, ): diff --git a/frappe/model/document.py b/frappe/model/document.py index cae1ccd63f7a..34d3160f6588 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -178,9 +178,11 @@ def load_from_db(self): if hasattr(self, "__setup__"): self.__setup__() + return self + def reload(self): """Reload document from database""" - self.load_from_db() + return self.load_from_db() def get_latest(self): if not getattr(self, "_doc_before_save", None): diff --git a/frappe/model/dynamic_links.py b/frappe/model/dynamic_links.py index e4b86ac14551..ccd6baacc3d0 100644 --- a/frappe/model/dynamic_links.py +++ b/frappe/model/dynamic_links.py @@ -43,7 +43,7 @@ def get_dynamic_link_map(for_delete=False): else: try: links = frappe.db.sql_list( - """select distinct {options} from `tab{parent}`""".format(**df) + """select distinct `{options}` from `tab{parent}`""".format(**df) ) for doctype in links: dynamic_link_map.setdefault(doctype, []).append(df) diff --git a/frappe/tests/test_linked_with.py b/frappe/tests/test_linked_with.py index d47d3fd18a94..ee13485ef7ec 100644 --- a/frappe/tests/test_linked_with.py +++ b/frappe/tests/test_linked_with.py @@ -1,5 +1,9 @@ +import random +import string + import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.database import savepoint from frappe.desk.form import linked_with from frappe.tests.utils import FrappeTestCase @@ -148,3 +152,36 @@ def test_check_delete_integrity(self): amendment.submit() self.assertRaises(frappe.LinkExistsError, doc.delete) + + def test_reserved_keywords(self): + dt_name = "Test " + "".join(random.sample(string.ascii_lowercase, 10)) + new_doctype( + dt_name, + fields=[ + { + "fieldname": "from", + "fieldtype": "Link", + "options": "DocType", + }, + { + "fieldname": "order", + "fieldtype": "Dynamic Link", + "options": "from", + }, + ], + is_submittable=True, + ).insert() + + linked_doc = frappe.new_doc(dt_name).insert().submit() + + second_doc = ( + frappe.get_doc(doctype=dt_name, **{"from": linked_doc.doctype, "order": linked_doc.name}) + .insert() + .submit() + ) + + with savepoint(frappe.LinkExistsError): + linked_doc.cancel() and self.fail("Cancellation shouldn't have worked") + + second_doc.cancel() + linked_doc.reload().cancel() From 2b7bb06180fd0e44da765e75eb183d153e92c353 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Mon, 1 Apr 2024 19:08:11 +0530 Subject: [PATCH 18/22] feat(customize_form): allow setting `creation` as a default sort field (#25760) Signed-off-by: Akhil Narang (cherry picked from commit 2bf0ab079ac02c4a91a275c96c957f577be4ef09) --- frappe/custom/doctype/customize_form/customize_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 247526aaf947..af42debd69fe 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -208,7 +208,7 @@ frappe.ui.form.on("Customize Form", { var fields = $.map(frm.doc.fields, function (df) { return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null; }); - fields = ["", "name", "modified"].concat(fields); + fields = ["", "name", "creation", "modified"].concat(fields); frm.set_df_property("sort_field", "options", fields); } }, From 2aa939ae4bbbb7e412bec4e622857243a01121d6 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Sat, 30 Sep 2023 10:03:13 +0200 Subject: [PATCH 19/22] fix: non-html notifications from files (cherry picked from commit fe8cc7a5c2eb7997850e4b15b6903dc6610bacba) Signed-off-by: Akhil Narang --- .../doctype/notification/notification.py | 31 +++++++++++++------ frappe/utils/jinja.py | 2 +- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 65685faf30ae..da5291e81ecb 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -12,7 +12,7 @@ from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message from frappe.model.document import Document from frappe.modules.utils import export_module_json, get_doc_module -from frappe.utils import add_to_date, cast, is_html, nowdate, validate_email_address +from frappe.utils import add_to_date, cast, nowdate, validate_email_address from frappe.utils.jinja import validate_template from frappe.utils.safe_exec import get_safe_globals @@ -48,8 +48,16 @@ def on_update(self): frappe.cache().hdel("notifications", self.document_type) path = export_module_json(self, self.is_standard, self.module) if path: - # js - if not os.path.exists(path + ".md") and not os.path.exists(path + ".html"): + formats = [".md", ".html", ".txt"] + + for ext in formats: + file_path = path + ext + if os.path.exists(file_path): + with open(file_path, "w") as f: + f.write(self.message) + break + + else: with open(path + ".md", "w") as f: f.write(self.message) @@ -349,7 +357,7 @@ def get_attachment(self, doc): } ] - def get_template(self): + def get_template(self, md_as_html=False): module = get_doc_module(self.module, self.doctype, self.name) def load_template(extn): @@ -360,7 +368,15 @@ def load_template(extn): template = f.read() return template - return load_template(".html") or load_template(".md") + formats = [".html", ".md", ".txt"] + + for format in formats: + template = load_template(format) + if template: + if format == ".md" and md_as_html: + return frappe.utils.md_to_html(template) + else: + return template def load_standard_properties(self, context): """load templates and run get_context""" @@ -371,10 +387,7 @@ def load_standard_properties(self, context): if out: context.update(out) - self.message = self.get_template() - - if not is_html(self.message): - self.message = frappe.utils.md_to_html(self.message) + self.message = self.get_template(md_as_html=True) def on_trash(self): frappe.cache().hdel("notifications", self.document_type) diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index e9161cd85b90..9fad56963cec 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -104,7 +104,7 @@ def guess_is_path(template): # if its single line and ends with a html, then its probably a path if "\n" not in template and "." in template: extn = template.rsplit(".")[-1] - if extn in ("html", "css", "scss", "py", "md", "json", "js", "xml"): + if extn in ("html", "css", "scss", "py", "md", "json", "js", "xml", "txt"): return True return False From fe6cbddb2b6f5154f6cd3221cef2f5ea3a782343 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Fri, 17 Nov 2023 12:52:44 +0100 Subject: [PATCH 20/22] feat(notification): specify message type (html, md, txt) (cherry picked from commit 90b076a75a4920c23a30a094b36e7ff1ba786f8c) Signed-off-by: Akhil Narang --- .../doctype/notification/notification.json | 13 ++++- .../doctype/notification/notification.py | 54 +++++++++---------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index 8b6900a3c938..3626cc29bc58 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -36,6 +36,7 @@ "send_to_all_assignees", "recipients", "message_sb", + "message_type", "message", "message_examples", "view_properties", @@ -277,15 +278,24 @@ "fieldname": "send_to_all_assignees", "fieldtype": "Check", "label": "Send To All Assignees" + }, + { + "default": "Markdown", + "depends_on": "is_standard", + "fieldname": "message_type", + "fieldtype": "Select", + "label": "Message Type", + "options": "Markdown\nHTML\nPlain Text" } ], "icon": "fa fa-envelope", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-05-04 11:17:11.882314", + "modified": "2023-11-17 08:48:25.616203", "modified_by": "Administrator", "module": "Email", "name": "Notification", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -301,6 +311,7 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index da5291e81ecb..2423fc17440f 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -16,6 +16,8 @@ from frappe.utils.jinja import validate_template from frappe.utils.safe_exec import get_safe_globals +FORMATS = {"HTML": ".html", "Markdown": ".md", "Plain Text": ".txt"} + class Notification(Document): def onload(self): @@ -47,19 +49,11 @@ def validate(self): def on_update(self): frappe.cache().hdel("notifications", self.document_type) path = export_module_json(self, self.is_standard, self.module) - if path: - formats = [".md", ".html", ".txt"] - - for ext in formats: - file_path = path + ext - if os.path.exists(file_path): - with open(file_path, "w") as f: - f.write(self.message) - break - - else: - with open(path + ".md", "w") as f: - f.write(self.message) + if path and self.message: + extension = FORMATS.get(self.message_type, ".md") + file_path = path + extension + with open(file_path, "w") as f: + f.write(self.message) # py if not os.path.exists(path + ".py"): @@ -360,23 +354,23 @@ def get_attachment(self, doc): def get_template(self, md_as_html=False): module = get_doc_module(self.module, self.doctype, self.name) - def load_template(extn): - template = "" - template_path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name) + extn) - if os.path.exists(template_path): - with open(template_path) as f: - template = f.read() - return template - - formats = [".html", ".md", ".txt"] - - for format in formats: - template = load_template(format) - if template: - if format == ".md" and md_as_html: - return frappe.utils.md_to_html(template) - else: - return template + path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name)) + extension = FORMATS.get(self.message_type, ".md") + file_path = path + extension + + template = "" + + if os.path.exists(file_path): + with open(file_path) as f: + template = f.read() + + if not template: + return + + if extension == ".md": + return frappe.utils.md_to_html(template) + + return template def load_standard_properties(self, context): """load templates and run get_context""" From 8f2408d115bdecd89e2eab12b1a70b79f7f5bdef Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Tue, 2 Apr 2024 16:40:17 +0530 Subject: [PATCH 21/22] fix(query_report): don't crash if undefined Mismerged in #25562 Signed-off-by: Akhil Narang --- frappe/public/js/frappe/views/reports/query_report.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index ff6c0231c428..0fd8f3f192ca 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1492,8 +1492,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { ); } - const visible_idx = this.datatable.bodyRenderer.visibleRowIndices; - if (visible_idx.length + 1 === this.data.length) { + const visible_idx = this.datatable?.bodyRenderer.visibleRowIndices || []; + if (visible_idx.length + 1 === this.data?.length) { visible_idx.push(visible_idx.length); } From c3350b8fa862a80a27147afcea6c2f43db9554f2 Mon Sep 17 00:00:00 2001 From: Frappe PR Bot Date: Tue, 2 Apr 2024 17:08:44 +0000 Subject: [PATCH 22/22] chore(release): Bumped to Version 14.70.0 # [14.70.0](https://github.com/frappe/frappe/compare/v14.69.0...v14.70.0) (2024-04-02) ### Bug Fixes * advertise insights to system manager only ([9ad5e58](https://github.com/frappe/frappe/commit/9ad5e58118b1e0da86bcd004f09d7c75fcea56f0)) * incorrect status on data import (backport [#25660](https://github.com/frappe/frappe/issues/25660)) ([#25702](https://github.com/frappe/frappe/issues/25702)) ([2b35e4b](https://github.com/frappe/frappe/commit/2b35e4bf8e51ba677aaa873aa9d7834b8a94d2c7)) * incorrect UI icon for desc sort ([#25687](https://github.com/frappe/frappe/issues/25687)) ([#25688](https://github.com/frappe/frappe/issues/25688)) ([8042ef8](https://github.com/frappe/frappe/commit/8042ef827e83ad225ac9a0b69a9dbca30cd29384)) * invalid filter on email acccount ([#25674](https://github.com/frappe/frappe/issues/25674)) ([#25675](https://github.com/frappe/frappe/issues/25675)) ([ca5daed](https://github.com/frappe/frappe/commit/ca5daed9ab3e94abb36bd36966937e038f6da2eb)) * make ads translatable ([#25710](https://github.com/frappe/frappe/issues/25710)) ([53d28ad](https://github.com/frappe/frappe/commit/53d28adbaedb341c5f69316eb02b7b26b58fbb0e)) * make insights ad translatable ([85e5f2f](https://github.com/frappe/frappe/commit/85e5f2f85f273b79ddbf22e1ec7283a60b7f102f)) * Merge conflicts ([4e7dd03](https://github.com/frappe/frappe/commit/4e7dd037a8fc8dcf67a304fd5946ecbd1abc79f4)) * non-html notifications from files ([2aa939a](https://github.com/frappe/frappe/commit/2aa939ae4bbbb7e412bec4e622857243a01121d6)) * preserve original error message ([#25682](https://github.com/frappe/frappe/issues/25682)) ([#25684](https://github.com/frappe/frappe/issues/25684)) ([0737e84](https://github.com/frappe/frappe/commit/0737e8496982b7955480a5f5a85316786204c029)) * **query_report:** don't crash if undefined ([8f2408d](https://github.com/frappe/frappe/commit/8f2408d115bdecd89e2eab12b1a70b79f7f5bdef)), closes [#25562](https://github.com/frappe/frappe/issues/25562) * reserved keywords as col name ([#25718](https://github.com/frappe/frappe/issues/25718)) ([#25725](https://github.com/frappe/frappe/issues/25725)) ([7a99b75](https://github.com/frappe/frappe/commit/7a99b75d1f5fb1ba25bb1a4d0360fc28c595765f)) * translatable web footer ([7558acf](https://github.com/frappe/frappe/commit/7558acfcf75902ba2c101342cac0a760d6fa2c73)) * Use CssParser to correctly pass options to wkhtmltopdf ([fa1f3fc](https://github.com/frappe/frappe/commit/fa1f3fc2c00d4da5737fc44a483a6cd91e93b2d8)) * Use doctype setting to set auto-extracted file as private ([e939391](https://github.com/frappe/frappe/commit/e939391c415917f64b6b84062e807e1d8c7427ad)) * Use regex in failing test ([3d7e572](https://github.com/frappe/frappe/commit/3d7e572360121b3e2721c9da0f98038386c497e3)) ### Features * **customize_form:** allow setting `creation` as a default sort field ([#25760](https://github.com/frappe/frappe/issues/25760)) ([2b7bb06](https://github.com/frappe/frappe/commit/2b7bb06180fd0e44da765e75eb183d153e92c353)) * **notification:** specify message type (html, md, txt) ([fe6cbdd](https://github.com/frappe/frappe/commit/fe6cbddb2b6f5154f6cd3221cef2f5ea3a782343)) --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 492a9b00b141..60f1e4283522 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -47,7 +47,7 @@ ) from .utils.lazy_loader import lazy_import -__version__ = "14.69.0" +__version__ = "14.70.0" __title__ = "Frappe Framework" controllers = {}