feat: Reconcile Payments in background (#34596)

* feat: auto reconcile in background

* chore: Option to enable auto reconciliation in settings

* refactor: validate if feature is enabled in settings

* refactor: check for running job while using reconciliation tool

* chore: using doc to get filter values

* chore: use frappe.db.get_value in validations

* chore: cleanup commented out code

* chore: replace get_list with get_all

* chore: use block scope variable

* chore: type information for functions

* refactor: flag to ignore job validation check

* refactor: update parent doc status if all reconciled

* chore: create test_records file

* test: create a bunch of vouchers for testing auto reconcile

* chore: renamed auto_reconcile to process_payment_reconciliation

* chore: another child doctype to hold payments

* chore: remove duplicate field

* chore: add fetched payments to log

* chore: Popup comment message update

* chore: replace get_all with get_value

* chore: replace label in settings page

* chore: remove unit test and records

* refactor: status in reconciliation log

* refactor: set status in log as well

* chore: fix field name

* chore: change triggered job name

* chore: use status field in list view of log

* chore: status while there are no allocations

* refactor: split trigger function into two

* chore: adding cancelled status

* refactor: function trigger queued docs

* chore: cron job scheduled

* chore: fixing accouts settings json file

* chore: typos and variable scope

* chore: use 'pluck' in db call

* chore: remove redundant whitelist decorator

* chore: use single DB call to fetch values

* chore: replace get_all with get_value

* refactor: use raw db calls to fetch reconciliation log records

Using get_doc on `Process Payment Reconciliation Log` is costly when
handling large volumes of invoices.

Use raw frappe.db.get_all to selectively pull status and reconciled count

* chore: update status on successful batch operation

* chore: make payment table readonly

* chore: ability to pause the background job

* chore: remove isolate_each_allocation

* chore: more description in progress bar

* refactor: partially working state

* refactor: update reconcile flag and setting hard limits for fetching

* chore: make allocation editable -- NEED TO REVERT

* chore: pause button

* refactor: skip setter function in Payment Entry for better performan

* refactor: split reconcile function and skip a setter function

1. Split reconcile function into 2
2. While reconciling against payment entry, skip a
set_missing_ref_details setter method

* chore: increase payment limit

* refactor: replace frappe.db.get_all with frappe.db.get_value

* chore: remove unwanted doctypes

* refactor: make allocation table readonly

* perf: update ref_details only for newly linked invoices

* chore: rename skip flag

* refactor(UI): receivable_payable field should auto populate

* refactor: no control statements in finally block

* chore: cleanup section and rename checkbox

* chore: update new fieldname in code

* chore: update error msg

* refactor: start and pause integrated into status

pause checkbox has been removed

* refactor: added cancelled status to the log doctype

1. Moved the status section to the bottom in parent doc
2. Using alerts to indicate Job trigger status

(cherry picked from commit ed14d1c)

erpnext/accounts/doctype/accounts_settings/accounts_settings.json

@@ -40,6 +40,8 @@
   "show_payment_schedule_in_print",
   "currency_exchange_section",
   "allow_stale",
+  "section_break_jpd0",
+  "auto_reconcile_payments",
   "stale_days",
   "invoicing_settings_tab",
   "accounts_transactions_settings_section",
@@ -170,11 +172,6 @@
    "fieldtype": "Int",
    "label": "Stale Days"
   },
-  {
-   "fieldname": "report_settings_sb",
-   "fieldtype": "Section Break",
-   "label": "Report Settings"
-  },
   {
    "default": "0",
    "description": "Only select this if you have set up the Cash Flow Mapper documents",
@@ -370,14 +367,25 @@
    "fieldname": "merge_similar_account_heads",
    "fieldtype": "Check",
    "label": "Merge Similar Account Heads"
+  },
+  {
+   "fieldname": "section_break_jpd0",
+   "fieldtype": "Section Break",
+   "label": "Payment Reconciliations"
+  },
+  {
+   "default": "0",
+   "fieldname": "auto_reconcile_payments",
+   "fieldtype": "Check",
+   "label": "Auto Reconcile Payments"
   }
  ],
  "icon": "icon-cog",
  "idx": 1,
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-04-17 11:45:42.049247",
+ "modified": "2023-04-21 13:11:37.130743",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Accounts Settings",

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.js

@@ -82,6 +82,32 @@ erpnext.accounts.PaymentReconciliationController = class PaymentReconciliationCo
 			this.frm.change_custom_button_type('Get Unreconciled Entries', null, 'default');
 			this.frm.change_custom_button_type('Allocate', null, 'default');
 		}
+
+		// check for any running reconciliation jobs
+		if (this.frm.doc.receivable_payable_account) {
+			frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments").then((enabled) => {
+ 				if(enabled) {
+					this.frm.call({
+						'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.is_any_doc_running",
+						"args": {
+							for_filter: {
+								company: this.frm.doc.company,
+								party_type: this.frm.doc.party_type,
+								party: this.frm.doc.party,
+								receivable_payable_account: this.frm.doc.receivable_payable_account
+							}
+						}
+					}).then(r => {
+						if (r.message) {
+							let doc_link = frappe.utils.get_form_link("Process Payment Reconciliation", r.message, true);
+							let msg = __("Payment Reconciliation Job: {0} is running for this party. Can't reconcile now.", [doc_link]);
+							this.frm.dashboard.add_comment(msg, "yellow");
+						}
+					});
+				}
+			});
+		}
+
 	}
 
 	company() {

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.py

@@ -7,9 +7,12 @@
 from frappe.model.document import Document
 from frappe.query_builder.custom import ConstantColumn
 from frappe.query_builder.functions import IfNull
-from frappe.utils import flt, getdate, nowdate, today
+from frappe.utils import flt, get_link_to_form, getdate, nowdate, today
 
 import erpnext
+from erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation import (
+	is_any_doc_running,
+)
 from erpnext.accounts.utils import (
 	QueryPaymentLedger,
 	get_outstanding_invoices,
@@ -304,9 +307,7 @@ def get_allocated_entry(self, pay, inv, allocated_amount):
 			}
 		)
 
-	@frappe.whitelist()
-	def reconcile(self):
-		self.validate_allocation()
+	def reconcile_allocations(self, skip_ref_details_update_for_pe=False):
 		dr_or_cr = (
 			"credit_in_account_currency"
 			if erpnext.get_party_account_type(self.party_type) == "Receivable"
@@ -330,12 +331,35 @@ def reconcile(self):
 					self.make_difference_entry(payment_details)
 
 		if entry_list:
-			reconcile_against_document(entry_list)
+			reconcile_against_document(entry_list, skip_ref_details_update_for_pe)
 
 		if dr_or_cr_notes:
 			reconcile_dr_cr_note(dr_or_cr_notes, self.company)
 
+	@frappe.whitelist()
+	def reconcile(self):
+		if frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
+			running_doc = is_any_doc_running(
+				dict(
+					company=self.company,
+					party_type=self.party_type,
+					party=self.party,
+					receivable_payable_account=self.receivable_payable_account,
+				)
+			)
+
+			if running_doc:
+				frappe.throw(
+					_("A Reconciliation Job {0} is running for the same filters. Cannot reconcile now").format(
+						get_link_to_form("Auto Reconcile", running_doc)
+					)
+				)
+				return
+
+		self.validate_allocation()
+		self.reconcile_allocations()
 		msgprint(_("Successfully Reconciled"))
+
 		self.get_unreconciled_entries()
 
 	def make_difference_entry(self, row):

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.js

@@ -0,0 +1,130 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Process Payment Reconciliation", {
+	onload: function(frm) {
+		// set queries
+		frm.set_query("party_type", function() {
+			return {
+				"filters": {
+					"name": ["in", Object.keys(frappe.boot.party_account_types)],
+				}
+			}
+		});
+		frm.set_query('receivable_payable_account',  function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+					"account_type": frappe.boot.party_account_types[doc.party_type]
+				}
+			};
+		});
+		frm.set_query('cost_center', function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+				}
+			};
+		});
+		frm.set_query('bank_cash_account', function(doc) {
+			return {
+				filters:[
+					['Account', 'company', '=', doc.company],
+					['Account', 'is_group', '=', 0],
+					['Account', 'account_type', 'in', ['Bank', 'Cash']]
+				]
+			};
+		});
+
+	},
+	refresh: function(frm) {
+		if (frm.doc.docstatus==1 && ['Queued', 'Paused'].find(x => x == frm.doc.status)) {
+			let execute_btn = __("Start / Resume")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					method: 'erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_job_for_doc',
+					args: {
+						docname: frm.doc.name
+					}
+				}).then(r => {
+					if(!r.exc) {
+						frappe.show_alert(__("Job Started"));
+						frm.reload_doc();
+					}
+				});
+			});
+		}
+		if (frm.doc.docstatus==1 && ['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
+			frm.call({
+				'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.get_reconciled_count",
+				args: {
+					"docname": frm.docname,
+				}
+			}).then(r => {
+				if (r.message) {
+					let progress = 0;
+					let description = "";
+
+					if (r.message.processed) {
+						progress = (r.message.processed/r.message.total) * 100;
+						description = r.message.processed + "/" + r.message.total +  " processed";
+					} else if (r.message.total == 0 && frm.doc.status == "Completed") {
+						progress = 100;
+					}
+
+
+					frm.dashboard.add_progress('Reconciliation Progress', progress, description);
+				}
+			})
+		}
+		if (frm.doc.docstatus==1 && frm.doc.status == 'Running') {
+			let execute_btn = __("Pause")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.pause_job_for_doc",
+					args: {
+						"docname": frm.docname,
+					}
+				}).then(r => {
+					if (!r.exc) {
+						frappe.show_alert(__("Job Paused"));
+						frm.reload_doc()
+					}
+				});
+
+			});
+		}
+	},
+	company(frm) {
+		frm.set_value('party', '');
+		frm.set_value('receivable_payable_account', '');
+	},
+	party_type(frm) {
+		frm.set_value('party', '');
+	},
+
+	party(frm) {
+		frm.set_value('receivable_payable_account', '');
+		if (!frm.doc.receivable_payable_account && frm.doc.party_type && frm.doc.party) {
+			return frappe.call({
+				method: "erpnext.accounts.party.get_party_account",
+				args: {
+					company: frm.doc.company,
+					party_type: frm.doc.party_type,
+					party: frm.doc.party
+				},
+				callback: (r) => {
+					if (!r.exc && r.message) {
+						frm.set_value("receivable_payable_account", r.message);
+					}
+					frm.refresh();
+
+				}
+			});
+		}
+	}
+});