From 81f004177f2810e530675ddce1141d3b2122b9cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 15:34:22 +0000 Subject: [PATCH 1/4] Initial plan From 48ee982ef978ae335d7c5670ab04541e1a8a79ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 15:58:07 +0000 Subject: [PATCH 2/4] Implement batch commit optimization for Gmail sync - Add BATCH_COMMIT_SIZE constant (20 emails per commit) - Track emails_processed counter across sync operations - Batch commits every N emails in both initial and incremental sync - Update history ID during batch commits for crash recovery - Add rollback in exception handler for failed transactions - Keep final commits at end of sync operations Co-authored-by: mrrobot47 <25586785+mrrobot47@users.noreply.github.com> --- .../doctype/gmail_thread/gmail_thread.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py index 88f90ff..23a2d09 100644 --- a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py +++ b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py @@ -19,6 +19,7 @@ ) SCOPES = "https://www.googleapis.com/auth/gmail.readonly" +BATCH_COMMIT_SIZE = 20 # Commit every N emails for performance optimization class GmailThread(Document): @@ -133,6 +134,7 @@ def sync(user=None): # Always store the maximum history id seen, to avoid skipping emails last_history_id = int(gmail_account.last_historyid or 0) max_history_id = last_history_id + emails_processed = 0 # Track emails for batch commits for label_id in label_ids: try: @@ -219,7 +221,17 @@ def sync(user=None): replace_inline_images(email, email_object) gmail_thread.append("emails", email) gmail_thread.save(ignore_permissions=True) - frappe.db.commit() # nosemgrep + emails_processed += 1 + + # Batch commit every BATCH_COMMIT_SIZE emails + if emails_processed % BATCH_COMMIT_SIZE == 0: + frappe.db.commit() + # Update history ID periodically for crash recovery + gmail_account.reload() + gmail_account.last_historyid = max_history_id + gmail_account.save(ignore_permissions=True) + frappe.db.commit() + frappe.db.set_value( "Gmail Thread", gmail_thread.name, @@ -341,6 +353,17 @@ def sync(user=None): replace_inline_images(email, email_object) gmail_thread.append("emails", email) gmail_thread.save(ignore_permissions=True) + emails_processed += 1 + + # Batch commit every BATCH_COMMIT_SIZE emails + if emails_processed % BATCH_COMMIT_SIZE == 0: + frappe.db.commit() + # Update history ID periodically for crash recovery + gmail_account.reload() + gmail_account.last_historyid = max_history_id + gmail_account.save(ignore_permissions=True) + frappe.db.commit() + frappe.db.set_value( "Gmail Thread", gmail_thread.name, @@ -378,6 +401,7 @@ def sync(user=None): docname=docname, ) except Exception: + frappe.db.rollback() frappe.log_error(frappe.get_traceback(), "Gmail Thread Sync Error") continue From 913447aed96d6ad12a04df548d09475740b4d9a8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 15:59:48 +0000 Subject: [PATCH 3/4] Refactor batch commit logic into helper function - Extract _batch_commit_if_needed() helper to reduce code duplication - Optimize to single commit per batch (removed duplicate commit) - Update both initial and incremental sync to use helper - Improve code maintainability and readability Co-authored-by: mrrobot47 <25586785+mrrobot47@users.noreply.github.com> --- .../doctype/gmail_thread/gmail_thread.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py index 23a2d09..c966bfe 100644 --- a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py +++ b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py @@ -115,6 +115,15 @@ def sync_labels(account_name, should_save=True): gmail_account.save(ignore_permissions=True) +def _batch_commit_if_needed(emails_processed, gmail_account, max_history_id): + """Commit database changes in batches for performance optimization.""" + if emails_processed % BATCH_COMMIT_SIZE == 0: + gmail_account.reload() + gmail_account.last_historyid = max_history_id + gmail_account.save(ignore_permissions=True) + frappe.db.commit() + + def sync(user=None): if user: frappe.set_user(user) @@ -223,14 +232,10 @@ def sync(user=None): gmail_thread.save(ignore_permissions=True) emails_processed += 1 - # Batch commit every BATCH_COMMIT_SIZE emails - if emails_processed % BATCH_COMMIT_SIZE == 0: - frappe.db.commit() - # Update history ID periodically for crash recovery - gmail_account.reload() - gmail_account.last_historyid = max_history_id - gmail_account.save(ignore_permissions=True) - frappe.db.commit() + # Batch commit every BATCH_COMMIT_SIZE emails for performance + _batch_commit_if_needed( + emails_processed, gmail_account, max_history_id + ) frappe.db.set_value( "Gmail Thread", @@ -355,14 +360,10 @@ def sync(user=None): gmail_thread.save(ignore_permissions=True) emails_processed += 1 - # Batch commit every BATCH_COMMIT_SIZE emails - if emails_processed % BATCH_COMMIT_SIZE == 0: - frappe.db.commit() - # Update history ID periodically for crash recovery - gmail_account.reload() - gmail_account.last_historyid = max_history_id - gmail_account.save(ignore_permissions=True) - frappe.db.commit() + # Batch commit every BATCH_COMMIT_SIZE emails for performance + _batch_commit_if_needed( + emails_processed, gmail_account, max_history_id + ) frappe.db.set_value( "Gmail Thread", From e9360a8a8dc223bb83fd27261e5656c168d51f86 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 16:01:00 +0000 Subject: [PATCH 4/4] Fix batch commit to avoid unnecessary commit at zero emails - Add check for emails_processed > 0 to prevent commit when counter is zero - Ensures batch commits only happen after processing actual emails Co-authored-by: mrrobot47 <25586785+mrrobot47@users.noreply.github.com> --- .../frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py index c966bfe..e0a360d 100644 --- a/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py +++ b/frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py @@ -117,7 +117,7 @@ def sync_labels(account_name, should_save=True): def _batch_commit_if_needed(emails_processed, gmail_account, max_history_id): """Commit database changes in batches for performance optimization.""" - if emails_processed % BATCH_COMMIT_SIZE == 0: + if emails_processed > 0 and emails_processed % BATCH_COMMIT_SIZE == 0: gmail_account.reload() gmail_account.last_historyid = max_history_id gmail_account.save(ignore_permissions=True)