Merge dashpay#6429: backport: merge bitcoin#24794, bitcoin#23524, bit…

…coin#24902, bitcoin#24915, bitcoin#24916, bitcoin#24929, bitcoin#23506, bitcoin#24840, bitcoin#24982, partial bitcoin#25288 (lint backports: part 2)

2fa480a partial bitcoin#25288: Reliably don't start itself (lint-all.py runs all tests twice) (Kittywhiskers Van Gogh)
bda1e03 merge bitcoin#24982: Port `lint-all.sh` to `lint-all.py` (Kittywhiskers Van Gogh)
b054a0d merge bitcoin#24840: port `lint-shell.sh` to python (Kittywhiskers Van Gogh)
973ca7b merge bitcoin#23506: Make more shell scripts verifiable by the `shellcheck` tool (Kittywhiskers Van Gogh)
694c1a4 merge bitcoin#24929: convert shell locale linter test to Python (Kittywhiskers Van Gogh)
2a7d32a merge bitcoin#24916: Convert lint-python-utf8-encoding.sh to Python (Kittywhiskers Van Gogh)
0321fa0 merge bitcoin#24915: Convert lint-circular-dependencies.sh to Python (Kittywhiskers Van Gogh)
e3dc4b1 merge bitcoin#24902: Convert lint-include-guards.sh to Python (Kittywhiskers Van Gogh)
fc48a13 merge bitcoin#23524: Fix typos in endif header comments (Kittywhiskers Van Gogh)
1f8c3b5 merge bitcoin#24794: Convert Python linter to Python (Kittywhiskers Van Gogh)
110b6ac partial revert dash#4807: enable more multi-threading and caching in linters (Kittywhiskers Van Gogh)

Pull request description:

  ## Additional Information

  * Depends on dashpay#6428

  * The introduction in `flake8-cached` for `lint-python.sh` in [dash#4807](dashpay#4807) was reverted as this logic wasn't going to be ported over to the Python replacement as the `flake8-cached` repo has been archived since April 2023 ([source](https://github.com/jnoortheen/flake8-cached)) and we don't use it in CI through GitLab ([build](https://gitlab.com/dashpay/dash/-/jobs/8456994796#L144)) or GitHub Actions ([build](https://github.com/dashpay/dash/actions/runs/11981121905/job/33406844883#step:7:75)).

  * [bitcoin#25288](bitcoin#25288) has been marked as partial as the change of the glob pattern from `{mod_path}/lint-*` to `{mod_path}/lint-*.py` as we still have `lint-cppcheck-dash.sh` around ([source](https://github.com/dashpay/dash/blob/b88d9910a8362ad7a47cf2ea822a00649793350b/test/lint/lint-cppcheck-dash.sh)) (and the original `cppcheck` linter upstream was removed in [bitcoin#25091](bitcoin#25091)).

    A Python port of that linter would allow for completing [bitcoin#25288](bitcoin#25288).

  ## Breaking Changes

  None expected.

  ## Checklist

  - [x] I have performed a self-review of my own code
  - [x] I have commented my code, particularly in hard-to-understand areas **(note: N/A)**
  - [x] I have added or updated relevant unit/integration/functional/e2e tests
  - [x] I have made corresponding changes to the documentation
  - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_

ACKs for top commit:
  UdjinM6:
    utACK 2fa480a
  PastaPastaPasta:
    utACK 2fa480a

Tree-SHA512: 48ddf11be11232df26051b39dfadac9f363d2f201b9f303cad6ddd54550e2f1881947061155da9d4eaf3f5a87cdd371368dc36b4d70eb81ff4c48a7a93af63ae

ci/dash/build_src.sh

@@ -25,7 +25,7 @@ if [ "$CHECK_DOC" = 1 ]; then
     # TODO: Check docs (re-enable after all Bitcoin PRs have been merged and docs fully fixed)
     #test/lint/check-doc.py
     # Run all linters
-    test/lint/lint-all.sh
+    test/lint/all-lint.py
 fi
 
 ccache --zero-stats --max-size=$CCACHE_SIZE

ci/lint/06_script.sh

@@ -21,7 +21,7 @@ test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/check-doc.py
-test/lint/lint-all.sh
+test/lint/all-lint.py
 
 if [ "$CIRRUS_REPO_FULL_NAME" = "dashpay/dash" ] && [ -n "$CIRRUS_CRON" ]; then
     git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit

contrib/guix/libexec/prelude.bash

@@ -2,10 +2,10 @@
 export LC_ALL=C
 set -e -o pipefail
 
-# shellcheck source=../../shell/realpath.bash
+# shellcheck source=contrib/shell/realpath.bash
 source contrib/shell/realpath.bash
 
-# shellcheck source=../../shell/git-utils.bash
+# shellcheck source=contrib/shell/git-utils.bash
 source contrib/shell/git-utils.bash
 
 ################

src/banman.h

@@ -97,4 +97,4 @@ class BanMan
     CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001};
 };
 
-#endif
+#endif // BITCOIN_BANMAN_H

src/batchedlogger.h

@@ -35,4 +35,4 @@ class CBatchedLogger
     void Flush();
 };
 
-#endif//BITCOIN_BATCHEDLOGGER_H
+#endif // BITCOIN_BATCHEDLOGGER_H

src/coinjoin/common.h

@@ -132,4 +132,4 @@ constexpr int CalculateAmountPriority(CAmount nInputAmount)
 
 } // namespace CoinJoin
 
-#endif
+#endif // BITCOIN_COINJOIN_COMMON_H

src/consensus/amount.h

@@ -26,4 +26,4 @@ static constexpr CAmount COIN = 100000000;
 static constexpr CAmount MAX_MONEY = 21000000 * COIN;
 inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); }
 
-#endif //  BITCOIN_CONSENSUS_AMOUNT_H
+#endif // BITCOIN_CONSENSUS_AMOUNT_H

src/context.h

@@ -33,4 +33,4 @@ T* GetContext(const CoreContext& context) noexcept
                 : nullptr;
 }
 
-#endif // BITCOIN_CONTEXT_VARIANT_H
+#endif // BITCOIN_CONTEXT_H

src/evo/creditpool.h

@@ -141,4 +141,4 @@ std::optional<CCreditPoolDiff> GetCreditPoolDiffForBlock(CCreditPoolManager& cpo
                                                          const CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams,
                                                          const CAmount blockSubsidy, BlockValidationState& state);
 
-#endif
+#endif // BITCOIN_EVO_CREDITPOOL_H

src/evo/dmnstate.h

@@ -377,4 +377,4 @@ class CDeterministicMNStateDiff
 };
 
 
-#endif //BITCOIN_EVO_DMNSTATE_H
+#endif // BITCOIN_EVO_DMNSTATE_H

src/governance/common.h

@@ -81,6 +81,6 @@ class Object
         }
     }
 };
-
 } // namespace Governance
-#endif
+
+#endif // BITCOIN_GOVERNANCE_COMMON_H

src/httprpc.h

@@ -31,4 +31,4 @@ void InterruptREST();
  */
 void StopREST();
 
-#endif
+#endif // BITCOIN_HTTPRPC_H

src/i2p.h

@@ -286,4 +286,4 @@ class Session
 } // namespace sam
 } // namespace i2p
 
-#endif /* BITCOIN_I2P_H */
+#endif // BITCOIN_I2P_H

src/llmq/snapshot.h

@@ -233,4 +233,4 @@ extern std::unique_ptr<CQuorumSnapshotManager> quorumSnapshotManager;
 
 } // namespace llmq
 
-#endif //BITCOIN_LLMQ_SNAPSHOT_H
+#endif // BITCOIN_LLMQ_SNAPSHOT_H

src/node/psbt.h

@@ -50,4 +50,4 @@ struct PSBTAnalysis {
  */
 PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx);
 
-#endif // BITCOIN_PSBT_H
+#endif // BITCOIN_NODE_PSBT_H

src/policy/feerate.h

@@ -66,4 +66,4 @@ class CFeeRate
     SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); }
 };
 
-#endif //  BITCOIN_POLICY_FEERATE_H
+#endif // BITCOIN_POLICY_FEERATE_H

src/randomenv.h

@@ -14,4 +14,4 @@ void RandAddDynamicEnv(CSHA512& hasher);
 /** Gather non-cryptographic environment data that does not change over time. */
 void RandAddStaticEnv(CSHA512& hasher);
 
-#endif
+#endif // BITCOIN_RANDOMENV_H

src/rpc/blockchain.h

@@ -64,4 +64,4 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry,
  */
 UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile);
 
-#endif
+#endif // BITCOIN_RPC_BLOCKCHAIN_H

src/rpc/index_util.h

@@ -42,4 +42,4 @@ bool GetTimestampIndex(CBlockTreeDB& block_tree_db, const uint32_t high, const u
                        std::vector<uint256>& hashes)
     EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 
-#endif // BITCOIN_RPC_CLIENT_H
+#endif // BITCOIN_RPC_INDEX_UTIL_H

src/scheduler.h

@@ -152,4 +152,4 @@ class SingleThreadedSchedulerClient
     size_t CallbacksPending() EXCLUSIVE_LOCKS_REQUIRED(!m_callbacks_mutex);
 };
 
-#endif
+#endif // BITCOIN_SCHEDULER_H

src/shutdown.h

@@ -36,4 +36,4 @@ bool RestartRequested();
  */
 void WaitForShutdown();
 
-#endif
+#endif // BITCOIN_SHUTDOWN_H

src/span.h

@@ -295,4 +295,4 @@ template<typename C>
     return span.end();
 }
 
-#endif
+#endif // BITCOIN_SPAN_H

src/stacktraces.h

@@ -37,4 +37,4 @@ inline std::string GetExceptionWhat(const T& e)
 void RegisterPrettyTerminateHander();
 void RegisterPrettySignalHandlers();
 
-#endif//BITCOIN_STACKTRACES_H
+#endif // BITCOIN_STACKTRACES_H

src/test/scriptnum10.h

@@ -179,4 +179,4 @@ class CScriptNum10
 };
 
 
-#endif // BITCOIN_TEST_BIGNUM_H
+#endif // BITCOIN_TEST_SCRIPTNUM10_H

src/test/util/setup_common.h

@@ -250,4 +250,4 @@ class HasReason
 // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_*
 std::ostream& operator<<(std::ostream& os, const uint256& num);
 
-#endif
+#endif // BITCOIN_TEST_UTIL_SETUP_COMMON_H

src/threadinterrupt.h

@@ -32,4 +32,4 @@ class CThreadInterrupt
     std::atomic<bool> flag;
 };
 
-#endif //BITCOIN_THREADINTERRUPT_H
+#endif // BITCOIN_THREADINTERRUPT_H

src/torcontrol.h

@@ -159,4 +159,4 @@ class TorController
     static void reconnect_cb(evutil_socket_t fd, short what, void *arg);
 };
 
-#endif /* BITCOIN_TORCONTROL_H */
+#endif // BITCOIN_TORCONTROL_H

src/util/edge.h

@@ -59,4 +59,4 @@ class EdgeTriggeredEvents
     int m_fd{-1};
 };
 
-#endif /* BITCOIN_UTIL_EDGE_H */
+#endif // BITCOIN_UTIL_EDGE_H

src/util/readwritefile.h

@@ -25,4 +25,4 @@ std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxs
  */
 bool WriteBinaryFile(const fs::path &filename, const std::string &data);
 
-#endif /* BITCOIN_UTIL_READWRITEFILE_H */
+#endif // BITCOIN_UTIL_READWRITEFILE_H

src/util/trace.h

@@ -55,4 +55,4 @@
 #endif
 
 
-#endif /* BITCOIN_UTIL_TRACE_H */
+#endif // BITCOIN_UTIL_TRACE_H

src/util/underlying.h

@@ -11,4 +11,4 @@ template <typename E>
     return static_cast<typename std::underlying_type<E>::type>(e);
 }
 
-#endif //BITCOIN_UTIL_UNDERLYING_H
+#endif // BITCOIN_UTIL_UNDERLYING_H

src/util/wpipe.h

@@ -56,4 +56,4 @@ class WakeupPipe
     EdgeTriggeredEvents* m_edge_trig_events{nullptr};
 };
 
-#endif /* BITCOIN_UTIL_WPIPE_H */
+#endif // BITCOIN_UTIL_WPIPE_H

src/wallet/rpcwallet.h

@@ -41,4 +41,4 @@ RPCHelpMan getaddressinfo();
 RPCHelpMan getrawchangeaddress();
 RPCHelpMan addmultisigaddress();
 RPCHelpMan signrawtransactionwithwallet();
-#endif //BITCOIN_WALLET_RPCWALLET_H
+#endif // BITCOIN_WALLET_RPCWALLET_H

src/warnings.h

@@ -20,4 +20,4 @@ void SetfLargeWorkInvalidChainFound(bool flag);
  */
 bilingual_str GetWarnings(bool verbose);
 
-#endif //  BITCOIN_WARNINGS_H
+#endif // BITCOIN_WARNINGS_H

src/zmq/zmqrpc.h

@@ -9,4 +9,4 @@ class CRPCTable;
 
 void RegisterZMQRPCCommands(CRPCTable& t);
 
-#endif // BITCOIN_ZMQ_ZMRRPC_H
+#endif // BITCOIN_ZMQ_ZMQRPC_H

test/README.md

@@ -310,11 +310,11 @@ Use the `-v` option for verbose output.
 
 | Lint test | Dependency |
 |-----------|:----------:|
-| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8)
-| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy)
-| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq)
+| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8)
+| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy)
+| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq)
 | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture)
-| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck)
+| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck)
 | [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell)
 
 In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh).
@@ -332,7 +332,7 @@ test/lint/lint-files.py
 You can run all the shell-based lint tests by running:
 
 ```
-test/lint/lint-all.sh
+test/lint/all-lint.py
 ```
 
 # Writing functional tests

test/lint/README.md

@@ -39,6 +39,6 @@ To do so, add the upstream repository as remote:
 git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git
 ```
 
-lint-all.sh
+all-lint.py
 ===========
 Calls other scripts with the `lint-` prefix.

test/lint/all-lint.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2017-2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# This script runs all test/lint/lint-* files, and fails if any exit
+# with a non-zero status code.
+
+from glob import glob
+from os import path as os_path, remove
+from pathlib import Path
+from shutil import which
+from subprocess import run
+
+exit_code = 0
+mod_path = Path(__file__).parent
+lints = glob(f"{mod_path}/lint-*")
+if which("parallel") and which("column"):
+    logfile = "parallel_out.log"
+    command = ["parallel", "--jobs", "100%", "--will-cite", "--joblog", logfile, ":::"] + lints
+    result = run(command)
+    if result.returncode != 0:
+        print(f"^---- failure generated")
+        exit_code = result.returncode
+    result = run(["column", "-t", logfile])
+    if os_path.isfile(logfile):
+        remove(logfile)
+else:
+    for lint in lints:
+        result = run([lint])
+        if result.returncode != 0:
+            print(f"^---- failure generated from {lint.split('/')[-1]}")
+            exit_code |= result.returncode
+
+exit(exit_code)