feat: Reconcile Payments in background (#34596)

* feat: auto reconcile in background

* chore: Option to enable auto reconciliation in settings

* refactor: validate if feature is enabled in settings

* refactor: check for running job while using reconciliation tool

* chore: using doc to get filter values

* chore: use frappe.db.get_value in validations

* chore: cleanup commented out code

* chore: replace get_list with get_all

* chore: use block scope variable

* chore: type information for functions

* refactor: flag to ignore job validation check

* refactor: update parent doc status if all reconciled

* chore: create test_records file

* test: create a bunch of vouchers for testing auto reconcile

* chore: renamed auto_reconcile to process_payment_reconciliation

* chore: another child doctype to hold payments

* chore: remove duplicate field

* chore: add fetched payments to log

* chore: Popup comment message update

* chore: replace get_all with get_value

* chore: replace label in settings page

* chore: remove unit test and records

* refactor: status in reconciliation log

* refactor: set status in log as well

* chore: fix field name

* chore: change triggered job name

* chore: use status field in list view of log

* chore: status while there are no allocations

* refactor: split trigger function into two

* chore: adding cancelled status

* refactor: function trigger queued docs

* chore: cron job scheduled

* chore: fixing accouts settings json file

* chore: typos and variable scope

* chore: use 'pluck' in db call

* chore: remove redundant whitelist decorator

* chore: use single DB call to fetch values

* chore: replace get_all with get_value

* refactor: use raw db calls to fetch reconciliation log records

Using get_doc on `Process Payment Reconciliation Log` is costly when
handling large volumes of invoices.

Use raw frappe.db.get_all to selectively pull status and reconciled count

* chore: update status on successful batch operation

* chore: make payment table readonly

* chore: ability to pause the background job

* chore: remove isolate_each_allocation

* chore: more description in progress bar

* refactor: partially working state

* refactor: update reconcile flag and setting hard limits for fetching

* chore: make allocation editable -- NEED TO REVERT

* chore: pause button

* refactor: skip setter function in Payment Entry for better performan

* refactor: split reconcile function and skip a setter function

1. Split reconcile function into 2
2. While reconciling against payment entry, skip a
set_missing_ref_details setter method

* chore: increase payment limit

* refactor: replace frappe.db.get_all with frappe.db.get_value

* chore: remove unwanted doctypes

* refactor: make allocation table readonly

* perf: update ref_details only for newly linked invoices

* chore: rename skip flag

* refactor(UI): receivable_payable field should auto populate

* refactor: no control statements in finally block

* chore: cleanup section and rename checkbox

* chore: update new fieldname in code

* chore: update error msg

* refactor: start and pause integrated into status

pause checkbox has been removed

* refactor: added cancelled status to the log doctype

1. Moved the status section to the bottom in parent doc
2. Using alerts to indicate Job trigger status

erpnext/accounts/doctype/accounts_settings/accounts_settings.json

@@ -40,6 +40,8 @@
   "show_payment_schedule_in_print",
   "currency_exchange_section",
   "allow_stale",
+  "section_break_jpd0",
+  "auto_reconcile_payments",
   "stale_days",
   "invoicing_settings_tab",
   "accounts_transactions_settings_section",
@@ -59,7 +61,6 @@
   "acc_frozen_upto",
   "column_break_25",
   "frozen_accounts_modifier",
-  "report_settings_sb",
   "tab_break_dpet",
   "show_balance_in_coa"
  ],
@@ -172,11 +173,6 @@
    "fieldtype": "Int",
    "label": "Stale Days"
   },
-  {
-   "fieldname": "report_settings_sb",
-   "fieldtype": "Section Break",
-   "label": "Report Settings"
-  },
   {
    "default": "0",
    "description": "Only select this if you have set up the Cash Flow Mapper documents",
@@ -383,14 +379,25 @@
    "fieldname": "merge_similar_account_heads",
    "fieldtype": "Check",
    "label": "Merge Similar Account Heads"
+  },
+  {
+   "fieldname": "section_break_jpd0",
+   "fieldtype": "Section Break",
+   "label": "Payment Reconciliations"
+  },
+  {
+   "default": "0",
+   "fieldname": "auto_reconcile_payments",
+   "fieldtype": "Check",
+   "label": "Auto Reconcile Payments"
   }
  ],
  "icon": "icon-cog",
  "idx": 1,
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-04-17 11:45:42.049247",
+ "modified": "2023-04-21 13:11:37.130743",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Accounts Settings",

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.js

@@ -82,6 +82,32 @@ erpnext.accounts.PaymentReconciliationController = class PaymentReconciliationCo
 			this.frm.change_custom_button_type('Get Unreconciled Entries', null, 'default');
 			this.frm.change_custom_button_type('Allocate', null, 'default');
 		}
+
+		// check for any running reconciliation jobs
+		if (this.frm.doc.receivable_payable_account) {
+			frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments").then((enabled) => {
+ 				if(enabled) {
+					this.frm.call({
+						'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.is_any_doc_running",
+						"args": {
+							for_filter: {
+								company: this.frm.doc.company,
+								party_type: this.frm.doc.party_type,
+								party: this.frm.doc.party,
+								receivable_payable_account: this.frm.doc.receivable_payable_account
+							}
+						}
+					}).then(r => {
+						if (r.message) {
+							let doc_link = frappe.utils.get_form_link("Process Payment Reconciliation", r.message, true);
+							let msg = __("Payment Reconciliation Job: {0} is running for this party. Can't reconcile now.", [doc_link]);
+							this.frm.dashboard.add_comment(msg, "yellow");
+						}
+					});
+				}
+			});
+		}
+
 	}
 
 	company() {

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.py

@@ -7,9 +7,12 @@
 from frappe.model.document import Document
 from frappe.query_builder.custom import ConstantColumn
 from frappe.query_builder.functions import IfNull
-from frappe.utils import flt, getdate, nowdate, today
+from frappe.utils import flt, get_link_to_form, getdate, nowdate, today
 
 import erpnext
+from erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation import (
+	is_any_doc_running,
+)
 from erpnext.accounts.utils import (
 	QueryPaymentLedger,
 	get_outstanding_invoices,
@@ -304,9 +307,7 @@ def get_allocated_entry(self, pay, inv, allocated_amount):
 			}
 		)
 
-	@frappe.whitelist()
-	def reconcile(self):
-		self.validate_allocation()
+	def reconcile_allocations(self, skip_ref_details_update_for_pe=False):
 		dr_or_cr = (
 			"credit_in_account_currency"
 			if erpnext.get_party_account_type(self.party_type) == "Receivable"
@@ -330,12 +331,35 @@ def reconcile(self):
 					self.make_difference_entry(payment_details)
 
 		if entry_list:
-			reconcile_against_document(entry_list)
+			reconcile_against_document(entry_list, skip_ref_details_update_for_pe)
 
 		if dr_or_cr_notes:
 			reconcile_dr_cr_note(dr_or_cr_notes, self.company)
 
+	@frappe.whitelist()
+	def reconcile(self):
+		if frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
+			running_doc = is_any_doc_running(
+				dict(
+					company=self.company,
+					party_type=self.party_type,
+					party=self.party,
+					receivable_payable_account=self.receivable_payable_account,
+				)
+			)
+
+			if running_doc:
+				frappe.throw(
+					_("A Reconciliation Job {0} is running for the same filters. Cannot reconcile now").format(
+						get_link_to_form("Auto Reconcile", running_doc)
+					)
+				)
+				return
+
+		self.validate_allocation()
+		self.reconcile_allocations()
 		msgprint(_("Successfully Reconciled"))
+
 		self.get_unreconciled_entries()
 
 	def make_difference_entry(self, row):

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.js

@@ -0,0 +1,130 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Process Payment Reconciliation", {
+	onload: function(frm) {
+		// set queries
+		frm.set_query("party_type", function() {
+			return {
+				"filters": {
+					"name": ["in", Object.keys(frappe.boot.party_account_types)],
+				}
+			}
+		});
+		frm.set_query('receivable_payable_account',  function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+					"account_type": frappe.boot.party_account_types[doc.party_type]
+				}
+			};
+		});
+		frm.set_query('cost_center', function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+				}
+			};
+		});
+		frm.set_query('bank_cash_account', function(doc) {
+			return {
+				filters:[
+					['Account', 'company', '=', doc.company],
+					['Account', 'is_group', '=', 0],
+					['Account', 'account_type', 'in', ['Bank', 'Cash']]
+				]
+			};
+		});
+
+	},
+	refresh: function(frm) {
+		if (frm.doc.docstatus==1 && ['Queued', 'Paused'].find(x => x == frm.doc.status)) {
+			let execute_btn = __("Start / Resume")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					method: 'erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_job_for_doc',
+					args: {
+						docname: frm.doc.name
+					}
+				}).then(r => {
+					if(!r.exc) {
+						frappe.show_alert(__("Job Started"));
+						frm.reload_doc();
+					}
+				});
+			});
+		}
+		if (frm.doc.docstatus==1 && ['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
+			frm.call({
+				'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.get_reconciled_count",
+				args: {
+					"docname": frm.docname,
+				}
+			}).then(r => {
+				if (r.message) {
+					let progress = 0;
+					let description = "";
+
+					if (r.message.processed) {
+						progress = (r.message.processed/r.message.total) * 100;
+						description = r.message.processed + "/" + r.message.total +  " processed";
+					} else if (r.message.total == 0 && frm.doc.status == "Completed") {
+						progress = 100;
+					}
+
+
+					frm.dashboard.add_progress('Reconciliation Progress', progress, description);
+				}
+			})
+		}
+		if (frm.doc.docstatus==1 && frm.doc.status == 'Running') {
+			let execute_btn = __("Pause")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.pause_job_for_doc",
+					args: {
+						"docname": frm.docname,
+					}
+				}).then(r => {
+					if (!r.exc) {
+						frappe.show_alert(__("Job Paused"));
+						frm.reload_doc()
+					}
+				});
+
+			});
+		}
+	},
+	company(frm) {
+		frm.set_value('party', '');
+		frm.set_value('receivable_payable_account', '');
+	},
+	party_type(frm) {
+		frm.set_value('party', '');
+	},
+
+	party(frm) {
+		frm.set_value('receivable_payable_account', '');
+		if (!frm.doc.receivable_payable_account && frm.doc.party_type && frm.doc.party) {
+			return frappe.call({
+				method: "erpnext.accounts.party.get_party_account",
+				args: {
+					company: frm.doc.company,
+					party_type: frm.doc.party_type,
+					party: frm.doc.party
+				},
+				callback: (r) => {
+					if (!r.exc && r.message) {
+						frm.set_value("receivable_payable_account", r.message);
+					}
+					frm.refresh();
+
+				}
+			});
+		}
+	}
+});