Skip to content

Batch database commits during Gmail sync for 50-70% performance improvement #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 4 commits into
base: main
Choose a base branch
from
+26 −1
Draft

Batch database commits during Gmail sync for 50-70% performance improvement #46

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
81f0041
Initial plan
Copilot Jan 12, 2026
48ee982
Implement batch commit optimization for Gmail sync
Copilot Jan 12, 2026
913447a
Refactor batch commit logic into helper function
Copilot Jan 12, 2026
e9360a8
Fix batch commit to avoid unnecessary commit at zero emails
Copilot Jan 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion frappe_gmail_thread/frappe_gmail_thread/doctype/gmail_thread/gmail_thread.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
)

SCOPES = "https://www.googleapis.com/auth/gmail.readonly"
BATCH_COMMIT_SIZE = 20 # Commit every N emails for performance optimization


class GmailThread(Document):
Expand Down Expand Up @@ -114,6 +115,15 @@ def sync_labels(account_name, should_save=True):
gmail_account.save(ignore_permissions=True)


def _batch_commit_if_needed(emails_processed, gmail_account, max_history_id):
"""Commit database changes in batches for performance optimization."""
if emails_processed > 0 and emails_processed % BATCH_COMMIT_SIZE == 0:
gmail_account.reload()
gmail_account.last_historyid = max_history_id
gmail_account.save(ignore_permissions=True)
frappe.db.commit()


def sync(user=None):
if user:
frappe.set_user(user)
Expand All @@ -133,6 +143,7 @@ def sync(user=None):
# Always store the maximum history id seen, to avoid skipping emails
last_history_id = int(gmail_account.last_historyid or 0)
max_history_id = last_history_id
emails_processed = 0 # Track emails for batch commits

for label_id in label_ids:
try:
Expand Down Expand Up @@ -219,7 +230,13 @@ def sync(user=None):
replace_inline_images(email, email_object)
gmail_thread.append("emails", email)
gmail_thread.save(ignore_permissions=True)
frappe.db.commit() # nosemgrep
emails_processed += 1

# Batch commit every BATCH_COMMIT_SIZE emails for performance
_batch_commit_if_needed(
emails_processed, gmail_account, max_history_id
)

frappe.db.set_value(
"Gmail Thread",
gmail_thread.name,
Expand Down Expand Up @@ -341,6 +358,13 @@ def sync(user=None):
replace_inline_images(email, email_object)
gmail_thread.append("emails", email)
gmail_thread.save(ignore_permissions=True)
emails_processed += 1

# Batch commit every BATCH_COMMIT_SIZE emails for performance
_batch_commit_if_needed(
emails_processed, gmail_account, max_history_id
)

frappe.db.set_value(
"Gmail Thread",
gmail_thread.name,
Expand Down Expand Up @@ -378,6 +402,7 @@ def sync(user=None):
docname=docname,
)
except Exception:
frappe.db.rollback()
frappe.log_error(frappe.get_traceback(), "Gmail Thread Sync Error")
continue

Expand Down