From 406bd01928d5cad37f86112e99fdf515747e9734 Mon Sep 17 00:00:00 2001 From: mihirpatel1510 Date: Thu, 30 Apr 2026 17:16:18 +0530 Subject: [PATCH 1/4] TASK-2026-00536 & TASK-2026-00657 - Done --- nextiq/api.py | 92 ++++++++++++++++--- nextiq/constants.py | 2 +- .../doctype/card_scan_log/card_scan_log.json | 9 ++ 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/nextiq/api.py b/nextiq/api.py index cea158f..d00b210 100644 --- a/nextiq/api.py +++ b/nextiq/api.py @@ -24,6 +24,35 @@ }) _MAX_FIELD_LEN = 500 # max characters per Lead field value +# Map Frappe DocType names (as they appear in "Could not find X: Y" errors) to +# the Lead field name, so _find_bad_field can strip the offending field. +_LINK_DOCTYPE_TO_FIELD = { + "country": "country", + "salutation": "salutation", + "Country": "country", + "Salutation": "salutation", +} + + +def _find_bad_field(error_msg, data): + """ + Parse a Frappe ValidationError message and return the Lead field name + that caused it, or None if it cannot be determined. + """ + import re + # "Could not find {DocType}: {value}" — Link field resolution failure + m = re.search(r"Could not find ([\w ]+):", error_msg) + if m: + doctype = m.group(1).strip() + field = _LINK_DOCTYPE_TO_FIELD.get(doctype) or _LINK_DOCTYPE_TO_FIELD.get(doctype.lower()) + if field and field in data: + return field + # Fallback: if any field's current value appears verbatim in the error, that's the culprit + for field, value in data.items(): + if value and str(value) in error_msg: + return field + return None + class _QuotaExceededError(Exception): pass @@ -127,6 +156,7 @@ def submit_card_scan(merged_image_base64, filename="business_card.jpg"): "submitted_at": frappe.utils.now(), "job_id": job_id, "cb_secret": cb_secret, + "scanned_by": frappe.session.user, }) log.insert(ignore_permissions=True) frappe.db.commit() @@ -186,9 +216,13 @@ def scan_callback(job_id, cb_secret, success, data=None, error=None, if not job_id or not cb_secret: return {"success": False, "error": "missing_params"} - log_name = frappe.db.get_value("Card Scan Log", {"job_id": job_id}, "name") - if not log_name: + log_data = frappe.db.get_value( + "Card Scan Log", {"job_id": job_id}, ["name", "scanned_by"], as_dict=True + ) + if not log_data: return {"success": False, "error": "invalid_job_id"} + log_name = log_data.name + scanned_by = log_data.scanned_by # Constant-time secret comparison — prevents timing-based enumeration stored_secret = frappe.db.get_value("Card Scan Log", log_name, "cb_secret") or "" @@ -218,11 +252,51 @@ def scan_callback(job_id, cb_secret, success, data=None, error=None, if k in _ALLOWED_LEAD_FIELDS and v not in (None, "") } if data: + skipped_fields = {} try: - lead = frappe.get_doc({"doctype": "Lead", **data}) - lead.insert(ignore_permissions=True) - frappe.db.commit() - lead_name = lead.name + # Retry loop: on ValidationError, strip the offending field and try again. + # This handles AI values that don't match ERPNext options (e.g. country="BHARAT"). + for _attempt in range(len(data) + 1): + try: + lead_doc_data = {"doctype": "Lead", **data} + if scanned_by and scanned_by != "Guest": + lead_doc_data["lead_owner"] = scanned_by + lead = frappe.get_doc(lead_doc_data) + lead.insert(ignore_permissions=True) + frappe.db.commit() + lead_name = lead.name + break + except frappe.exceptions.DuplicateEntryError: + raise + except frappe.ValidationError as e: + frappe.db.rollback() + bad_field = _find_bad_field(str(e), data) + if bad_field: + skipped_fields[bad_field] = data.pop(bad_field) + else: + raise # can't identify which field — propagate + else: + raise frappe.ValidationError("All fields were invalid; no lead could be created.") + + # Add a comment listing any skipped fields so the sales rep knows what was dropped + if skipped_fields: + # Null out the skipped fields — without this, Frappe applies doctype + # defaults (e.g. country defaults to "India") when the field is absent. + frappe.db.set_value("Lead", lead_name, + {f: None for f in skipped_fields}) + lines = ["NextIQ: the following fields were skipped (invalid values):") + frappe.get_doc({ + "doctype": "Comment", + "comment_type": "Info", + "reference_doctype": "Lead", + "reference_name": lead_name, + "content": "".join(lines), + }).insert(ignore_permissions=True) + frappe.db.commit() + except frappe.exceptions.DuplicateEntryError as e: err_msg = str(e)[:500] or "A lead with this email address already exists." frappe.db.rollback() @@ -236,11 +310,7 @@ def scan_callback(job_id, cb_secret, success, data=None, error=None, _send_scan_notification(log_name, "duplicate_lead", message=err_msg) return {"success": False, "error": "duplicate_lead"} except frappe.ValidationError as e: - # AI processed successfully and returned data, but the data has - # values that Frappe cannot accept (e.g. country="USA" instead of - # "United States", invalid select option, bad link value, etc.). - # Scan is already charged — this is a data quality issue, not a failure. - err_msg = str(e)[:500] or "AI data could not be saved as a Lead — one or more field values were invalid." + err_msg = str(e)[:500] or "AI data could not be saved as a Lead — all field values were invalid." frappe.db.rollback() frappe.db.set_value("Card Scan Log", log_name, { "status": "Invalid Data", diff --git a/nextiq/constants.py b/nextiq/constants.py index f01da3f..e8c73f9 100644 --- a/nextiq/constants.py +++ b/nextiq/constants.py @@ -1,4 +1,4 @@ # Copyright (c) 2026, krushang.patel@satat.tech and contributors # For license information, please see license.txt -SERVICE_URL = "https://nextiq.in" +SERVICE_URL = "http://nextiq.service:8004" \ No newline at end of file diff --git a/nextiq/nextiq/doctype/card_scan_log/card_scan_log.json b/nextiq/nextiq/doctype/card_scan_log/card_scan_log.json index 35eaed1..2415079 100644 --- a/nextiq/nextiq/doctype/card_scan_log/card_scan_log.json +++ b/nextiq/nextiq/doctype/card_scan_log/card_scan_log.json @@ -7,6 +7,7 @@ "field_order": [ "status", "submitted_at", + "scanned_by", "column_break_pxss", "merged_image", "processed_at", @@ -22,6 +23,14 @@ "cb_secret" ], "fields": [ + { + "fieldname": "scanned_by", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Scanned By", + "options": "User", + "read_only": 1 + }, { "fieldname": "status", "fieldtype": "Select", From 506dd0349849538559203292c16f9019b542d6f8 Mon Sep 17 00:00:00 2001 From: mihirpatel1510 Date: Fri, 1 May 2026 10:08:52 +0530 Subject: [PATCH 2/4] add production url --- nextiq/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextiq/constants.py b/nextiq/constants.py index e8c73f9..2145f9e 100644 --- a/nextiq/constants.py +++ b/nextiq/constants.py @@ -1,4 +1,4 @@ # Copyright (c) 2026, krushang.patel@satat.tech and contributors # For license information, please see license.txt -SERVICE_URL = "http://nextiq.service:8004" \ No newline at end of file +SERVICE_URL = "https://nextiq.in" \ No newline at end of file From bdcf0588db51f54c5d32bcd642f21e6a55c6265e Mon Sep 17 00:00:00 2001 From: mihirpatel1510 Date: Fri, 1 May 2026 10:56:08 +0530 Subject: [PATCH 3/4] put validation before lead creating --- nextiq/api.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/nextiq/api.py b/nextiq/api.py index d00b210..5aca121 100644 --- a/nextiq/api.py +++ b/nextiq/api.py @@ -5,6 +5,7 @@ import hashlib import hmac import ipaddress +import re import secrets import traceback @@ -33,13 +34,40 @@ "Salutation": "salutation", } +# Pre-validation patterns — applied before attempting lead insert +_EMAIL_RE = re.compile(r'^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$') +_URL_RE = re.compile(r'^https?://', re.IGNORECASE) +_GENDER_OPTIONS = frozenset({"Male", "Female", "Other", "Prefer Not To Say"}) +_SALUTATION_OPTIONS = frozenset({"Mr", "Ms", "Mrs", "Dr", "Prof"}) + + +def _is_valid_field_value(field, value): + """Return True if value passes format validation for this Lead field.""" + if field == "email_id": + return bool(_EMAIL_RE.match(value)) + if field in ("mobile_no", "whatsapp_no", "phone", "fax"): + # Strip formatting characters and require at least 3 digits + digits = re.sub(r'[\s\(\)\-\.\+x#*,]', '', value) + return digits.isdigit() and len(digits) >= 3 + if field == "phone_ext": + digits = re.sub(r'\D', '', value) + return 1 <= len(digits) <= 10 + if field == "website": + return bool(_URL_RE.match(value)) + if field == "gender": + return value in _GENDER_OPTIONS + if field == "salutation": + return value in _SALUTATION_OPTIONS + # Text fields (first_name, last_name, job_title, company_name, city, state, country, …) + # have no format constraint beyond being non-empty — already guaranteed by the caller. + return True + def _find_bad_field(error_msg, data): """ Parse a Frappe ValidationError message and return the Lead field name that caused it, or None if it cannot be determined. """ - import re # "Could not find {DocType}: {value}" — Link field resolution failure m = re.search(r"Could not find ([\w ]+):", error_msg) if m: @@ -253,6 +281,14 @@ def scan_callback(job_id, cb_secret, success, data=None, error=None, } if data: skipped_fields = {} + + # Pre-validate every field before attempting insert. + # Strip any value that fails format checks into skipped_fields so the + # lead is still created and a comment records what was dropped. + for _field in list(data.keys()): + if not _is_valid_field_value(_field, data[_field]): + skipped_fields[_field] = data.pop(_field) + try: # Retry loop: on ValidationError, strip the offending field and try again. # This handles AI values that don't match ERPNext options (e.g. country="BHARAT"). From 401ad9796b008398e1bc1ca37d20277fac412dd7 Mon Sep 17 00:00:00 2001 From: mihirpatel1510 Date: Fri, 1 May 2026 11:18:51 +0530 Subject: [PATCH 4/4] done --- nextiq/api.py | 70 ++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/nextiq/api.py b/nextiq/api.py index 5aca121..9f3057e 100644 --- a/nextiq/api.py +++ b/nextiq/api.py @@ -5,7 +5,6 @@ import hashlib import hmac import ipaddress -import re import secrets import traceback @@ -34,33 +33,31 @@ "Salutation": "salutation", } -# Pre-validation patterns — applied before attempting lead insert -_EMAIL_RE = re.compile(r'^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$') -_URL_RE = re.compile(r'^https?://', re.IGNORECASE) -_GENDER_OPTIONS = frozenset({"Male", "Female", "Other", "Prefer Not To Say"}) -_SALUTATION_OPTIONS = frozenset({"Mr", "Ms", "Mrs", "Dr", "Prof"}) - - -def _is_valid_field_value(field, value): - """Return True if value passes format validation for this Lead field.""" - if field == "email_id": - return bool(_EMAIL_RE.match(value)) - if field in ("mobile_no", "whatsapp_no", "phone", "fax"): - # Strip formatting characters and require at least 3 digits - digits = re.sub(r'[\s\(\)\-\.\+x#*,]', '', value) - return digits.isdigit() and len(digits) >= 3 - if field == "phone_ext": - digits = re.sub(r'\D', '', value) - return 1 <= len(digits) <= 10 - if field == "website": - return bool(_URL_RE.match(value)) - if field == "gender": - return value in _GENDER_OPTIONS - if field == "salutation": - return value in _SALUTATION_OPTIONS - # Text fields (first_name, last_name, job_title, company_name, city, state, country, …) - # have no format constraint beyond being non-empty — already guaranteed by the caller. - return True +# Maps lowercase field labels (as Frappe uses them in error messages) to Lead field names. +# Lets _find_bad_field identify any field from a ValidationError, not just link fields. +_FIELD_LABEL_TO_NAME = { + "salutation": "salutation", + "first name": "first_name", + "middle name": "middle_name", + "last name": "last_name", + "gender": "gender", + "job title": "job_title", + "email id": "email_id", + "email": "email_id", + "mobile no": "mobile_no", + "mobile": "mobile_no", + "whatsapp no": "whatsapp_no", + "whatsapp": "whatsapp_no", + "phone": "phone", + "phone ext": "phone_ext", + "company name": "company_name", + "company": "company_name", + "website": "website", + "fax": "fax", + "city": "city", + "state": "state", + "country": "country", +} def _find_bad_field(error_msg, data): @@ -68,6 +65,7 @@ def _find_bad_field(error_msg, data): Parse a Frappe ValidationError message and return the Lead field name that caused it, or None if it cannot be determined. """ + import re # "Could not find {DocType}: {value}" — Link field resolution failure m = re.search(r"Could not find ([\w ]+):", error_msg) if m: @@ -75,10 +73,16 @@ def _find_bad_field(error_msg, data): field = _LINK_DOCTYPE_TO_FIELD.get(doctype) or _LINK_DOCTYPE_TO_FIELD.get(doctype.lower()) if field and field in data: return field - # Fallback: if any field's current value appears verbatim in the error, that's the culprit + # Check if any field's current value appears verbatim in the error message for field, value in data.items(): if value and str(value) in error_msg: return field + # Check if any field label appears in the error message + # (e.g. "Value for Gender must be one of …", "Invalid Email Id") + err_lower = error_msg.lower() + for label, field in _FIELD_LABEL_TO_NAME.items(): + if label in err_lower and field in data: + return field return None @@ -281,14 +285,6 @@ def scan_callback(job_id, cb_secret, success, data=None, error=None, } if data: skipped_fields = {} - - # Pre-validate every field before attempting insert. - # Strip any value that fails format checks into skipped_fields so the - # lead is still created and a comment records what was dropped. - for _field in list(data.keys()): - if not _is_valid_field_value(_field, data[_field]): - skipped_fields[_field] = data.pop(_field) - try: # Retry loop: on ValidationError, strip the offending field and try again. # This handles AI values that don't match ERPNext options (e.g. country="BHARAT").