backport: bitcoin#18933, #23123, #23147, #23549, #26074, #26207, #26307, #26341, #26352, #26656, #26886, #26996, #27035

doc/REST-interface.md

@@ -125,11 +125,15 @@ Returns various information about the transaction mempool.
 Only supports JSON as output format.
 Refer to the `getmempoolinfo` RPC help for details.
 
-`GET /rest/mempool/contents.json`
+`GET /rest/mempool/contents.json?verbose=<true|false>&mempool_sequence=<false|true>`
 
 Returns the transactions in the mempool.
 Only supports JSON as output format.
-Refer to the `getrawmempool` RPC help for details.
+Refer to the `getrawmempool` RPC help for details. Defaults to setting
+`verbose=true` and `mempool_sequence=false`.
+
+*Query parameters for `verbose` and `mempool_sequence` available in 25.0 and up.*
+
 
 Risks
 -------------

doc/descriptors.md

@@ -21,6 +21,8 @@ Supporting RPCs are:
 - `importdescriptors` takes as input descriptors to import into a descriptor wallet
   (since v0.21).
 - `listdescriptors` outputs descriptors imported into a descriptor wallet (since v22).
+- `scanblocks` takes as input descriptors to scan for in blocks and returns the
+   relevant blockhashes (since v24).
 
 This document describes the language. For the specifics on usage, see the RPC
 documentation for the functions mentioned above.

src/rest.cpp

@@ -645,7 +645,20 @@ static bool rest_mempool(const CoreContext& context, HTTPRequest* req, const std
 
         std::string str_json;
         if (param == "contents") {
-            str_json = MempoolToJSON(*mempool, llmq_ctx->isman.get(), true).write() + "\n";
+            const std::string raw_verbose{req->GetQueryParameter("verbose").value_or("true")};
+            if (raw_verbose != "true" && raw_verbose != "false") {
+                return RESTERR(req, HTTP_BAD_REQUEST, "The \"verbose\" query parameter must be either \"true\" or \"false\".");
+            }
+            const std::string raw_mempool_sequence{req->GetQueryParameter("mempool_sequence").value_or("false")};
+            if (raw_mempool_sequence != "true" && raw_mempool_sequence != "false") {
+                return RESTERR(req, HTTP_BAD_REQUEST, "The \"mempool_sequence\" query parameter must be either \"true\" or \"false\".");
+            }
+            const bool verbose{raw_verbose == "true"};
+            const bool mempool_sequence{raw_mempool_sequence == "true"};
+            if (verbose && mempool_sequence) {
+                return RESTERR(req, HTTP_BAD_REQUEST, "Verbose results cannot contain mempool sequence values. (hint: set \"verbose=false\")");
+            }
+            str_json = MempoolToJSON(*mempool, llmq_ctx->isman.get(), verbose, mempool_sequence).write() + "\n";
         } else {
             str_json = MempoolInfoToJSON(*mempool, *llmq_ctx->isman).write() + "\n";
         }

src/rpc/client.cpp

@@ -39,6 +39,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "generatetodescriptor", 0, "num_blocks" },
     { "generatetodescriptor", 2, "maxtries" },
     { "generateblock", 1, "transactions" },
+    { "generateblock", 2, "submit" },
 #endif // ENABLE_MINER
     { "getnetworkhashps", 0, "nblocks" },
     { "getnetworkhashps", 1, "height" },
@@ -104,6 +105,9 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "sendmany", 10, "fee_rate" },
     { "sendmany", 11, "verbose" },
     { "deriveaddresses", 1, "range" },
+    { "scanblocks", 1, "scanobjects" },
+    { "scanblocks", 2, "start_height" },
+    { "scanblocks", 3, "stop_height" },
     { "scantxoutset", 1, "scanobjects" },
     { "addmultisigaddress", 0, "nrequired" },
     { "addmultisigaddress", 1, "keys" },

src/rpc/evo.cpp

@@ -161,7 +161,7 @@ static RPCArg GetRpcArg(const std::string& strParamName)
                             {"reward", RPCArg::Type::NUM, RPCArg::Optional::NO, "The payout share in basis points."},
                         }},
                 },
-                "\"payoutAddress\"|[{\"address\",\"reward\"},...]", {"string or array", "string or array"}}
+                RPCArgOptions{.oneline_description={"\"payoutAddress\" | [{\"address\",\"reward\"},...] (string or array)"}}}
         },
         {"payoutAddress_update",
             {"payoutAddress", RPCArg::Type::ARR, RPCArg::Optional::NO,
@@ -175,7 +175,7 @@ static RPCArg GetRpcArg(const std::string& strParamName)
                             {"reward", RPCArg::Type::NUM, RPCArg::Optional::NO, "The payout share in basis points."},
                         }},
                 },
-                "\"payoutAddress\"|[{\"address\",\"reward\"},...]", {"string or array", "string or array"}}
+                RPCArgOptions{.oneline_description={"\"payoutAddress\" | [{\"address\",\"reward\"},...] (string or array)"}}}
         },
         {"proTxHash",
             {"proTxHash", RPCArg::Type::STR, RPCArg::Optional::NO,

src/rpc/mining.cpp

@@ -123,9 +123,9 @@ static RPCHelpMan getnetworkhashps()
 }
 
 #if ENABLE_MINER
-static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, uint256& block_hash)
+static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, std::shared_ptr<const CBlock>& block_out, bool process_new_block)
 {
-    block_hash.SetNull();
+    block_out.reset();
     block.hashMerkleRoot = BlockMerkleRoot(block);
 
     while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainman.GetConsensus()) && !ShutdownRequested()) {
@@ -139,12 +139,14 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t&
         return true;
     }
 
-    std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block);
-    if (!chainman.ProcessNewBlock(shared_pblock, true, nullptr)) {
+    block_out = std::make_shared<const CBlock>(block);
+
+    if (!process_new_block) return true;
+
+    if (!chainman.ProcessNewBlock(block_out, /*force_processing=*/true, nullptr)) {
         throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted");
     }
 
-    block_hash = block.GetHash();
     return true;
 }
 
@@ -158,16 +160,15 @@ static UniValue generateBlocks(ChainstateManager& chainman, const NodeContext& n
         std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(chainman.ActiveChainstate(), node, &mempool).CreateNewBlock(coinbase_script));
         if (!pblocktemplate.get())
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block");
-        CBlock *pblock = &pblocktemplate->block;
 
-        uint256 block_hash;
-        if (!GenerateBlock(chainman, *pblock, nMaxTries, block_hash)) {
+        std::shared_ptr<const CBlock> block_out;
+        if (!GenerateBlock(chainman, pblocktemplate->block, nMaxTries, block_out, /*process_new_block=*/true)) {
             break;
         }
 
-        if (!block_hash.IsNull()) {
+        if (block_out) {
             --nGenerate;
-            blockHashes.push_back(block_hash.GetHex());
+            blockHashes.push_back(block_out->GetHash().GetHex());
         }
     }
     return blockHashes;
@@ -299,11 +300,13 @@ static RPCHelpMan generateblock()
                     {"rawtx/txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""},
                 },
             },
+            {"submit", RPCArg::Type::BOOL, RPCArg::Default{true}, "Whether to submit the block before the RPC call returns or to return it as hex."},
         },
         RPCResult{
             RPCResult::Type::OBJ, "", "",
             {
                 {RPCResult::Type::STR_HEX, "hash", "hash of generated block"},
+                {RPCResult::Type::STR_HEX, "hex", /*optional=*/true, "hex of generated block, only present when submit=false"},
             }
         },
         RPCExamples{
@@ -352,10 +355,12 @@ static RPCHelpMan generateblock()
         }
     }
 
+    const bool process_new_block{request.params[2].isNull() ? true : request.params[2].get_bool()};
+    CBlock block;
+
     ChainstateManager& chainman = EnsureChainman(node);
     Chainstate& active_chainstate = chainman.ActiveChainstate();
 
-    CBlock block;
     {
         LOCK(cs_main);
 
@@ -382,15 +387,20 @@ static RPCHelpMan generateblock()
         }
     }
 
-    uint256 block_hash;
+    std::shared_ptr<const CBlock> block_out;
     uint64_t max_tries{DEFAULT_MAX_TRIES};
 
-    if (!GenerateBlock(chainman, block, max_tries, block_hash) || block_hash.IsNull()) {
+    if (!GenerateBlock(chainman, block, max_tries, block_out, process_new_block) || !block_out) {
         throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block.");
     }
 
     UniValue obj(UniValue::VOBJ);
-    obj.pushKV("hash", block_hash.GetHex());
+    obj.pushKV("hash", block_out->GetHash().GetHex());
+    if (!process_new_block) {
+        CDataStream block_ser{SER_NETWORK, PROTOCOL_VERSION};
+        block_ser << *block_out;
+        obj.pushKV("hex", HexStr(block_ser));
+    }
     return obj;
 },
     };
@@ -544,20 +554,18 @@ static RPCHelpMan getblocktemplate()
         "    https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki#getblocktemplate_changes\n",
         {
             {"template_request", RPCArg::Type::OBJ, RPCArg::Default{UniValue::VOBJ}, "Format of the template",
+            {
+                {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"},
+                {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings",
                 {
-                    {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"},
-                    {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings",
-                        {
-                            {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"},
-                        },
-                        },
-                    {"rules", RPCArg::Type::ARR, RPCArg::Optional::NO, "A list of strings",
-                        {
-                            {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported softfork deployment"},
-                        },
-                        },
-                },
-                "\"template_request\""},
+                    {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"},
+                }},
+                {"rules", RPCArg::Type::ARR, RPCArg::Optional::NO, "A list of strings",
+                {
+                        {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported softfork deployment"},
+                }},
+            },
+            RPCArgOptions{.oneline_description="\"template_request\""}},
         },
         {
             RPCResult{"If the proposal was accepted with mode=='proposal'", RPCResult::Type::NONE, "", ""},

src/rpc/server.cpp

@@ -183,7 +183,7 @@ static RPCHelpMan stop()
     // to the client (intended for testing)
                 "\nRequest a graceful shutdown of " PACKAGE_NAME ".",
                 {
-                    {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", "", {}, /*hidden=*/true},
+                    {"wait", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "how long to wait in ms", RPCArgOptions{.hidden=true}},
                 },
                 RPCResult{RPCResult::Type::STR, "", "A string with the content '" + RESULT + "'"},
                 RPCExamples{""},

src/rpc/util.cpp

@@ -361,8 +361,8 @@ struct Sections {
         case RPCArg::Type::BOOL: {
             if (is_top_level_arg) return; // Nothing more to do for non-recursive types on first recursion
             auto left = indent;
-            if (arg.m_type_str.size() != 0 && push_name) {
-                left += "\"" + arg.GetName() + "\": " + arg.m_type_str.at(0);
+            if (arg.m_opts.type_str.size() != 0 && push_name) {
+                left += "\"" + arg.GetName() + "\": " + arg.m_opts.type_str.at(0);
             } else {
                 left += push_name ? arg.ToStringObj(/*oneline=*/false) : arg.ToString(/*oneline=*/false);
             }
@@ -607,7 +607,7 @@ std::string RPCHelpMan::ToString() const
     ret += m_name;
     bool was_optional{false};
     for (const auto& arg : m_args) {
-        if (arg.m_hidden) break; // Any arg that follows is also hidden
+        if (arg.m_opts.hidden) break; // Any arg that follows is also hidden
         const bool optional = arg.IsOptional();
         ret += " ";
         if (optional) {
@@ -628,7 +628,7 @@ std::string RPCHelpMan::ToString() const
     Sections sections;
     for (size_t i{0}; i < m_args.size(); ++i) {
         const auto& arg = m_args.at(i);
-        if (arg.m_hidden) break; // Any arg that follows is also hidden
+        if (arg.m_opts.hidden) break; // Any arg that follows is also hidden
 
         if (i == 0) ret += "\nArguments:\n";
 
@@ -693,8 +693,8 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const
 {
     std::string ret;
     ret += "(";
-    if (m_type_str.size() != 0) {
-        ret += m_type_str.at(1);
+    if (m_opts.type_str.size() != 0) {
+        ret += m_opts.type_str.at(1);
     } else {
         switch (m_type) {
         case Type::STR_HEX:
@@ -936,7 +936,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const
 
 std::string RPCArg::ToString(const bool oneline) const
 {
-    if (oneline && !m_oneline_description.empty()) return m_oneline_description;
+    if (oneline && !m_opts.oneline_description.empty()) return m_opts.oneline_description;
 
     switch (m_type) {
     case Type::STR_HEX:

src/rpc/util.h

@@ -150,6 +150,12 @@ enum class OuterType {
 /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */
 std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider);
 
+struct RPCArgOptions {
+    std::string oneline_description{};   //!< Should be empty unless it is supposed to override the auto-generated summary line
+    std::vector<std::string> type_str{}; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_opts.type_str.at(0) will override the type of the value in a key-value pair, m_opts.type_str.at(1) will override the type in the argument description.
+    bool hidden{false};                  //!< For testing only
+};
+
 struct RPCArg {
     enum class Type {
         OBJ,
@@ -188,28 +194,22 @@ struct RPCArg {
 
     const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments)
     const Type m_type;
-    const bool m_hidden;
     const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts
     const Fallback m_fallback;
     const std::string m_description;
-    const std::string m_oneline_description; //!< Should be empty unless it is supposed to override the auto-generated summary line
-    const std::vector<std::string> m_type_str; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_type_str.at(0) will override the type of the value in a key-value pair, m_type_str.at(1) will override the type in the argument description.
+    const RPCArgOptions m_opts;
 
     RPCArg(
         std::string name,
         Type type,
         Fallback fallback,
         std::string description,
-        std::string oneline_description = "",
-        std::vector<std::string> type_str = {},
-        bool hidden = false)
+        RPCArgOptions opts = {})
         : m_names{std::move(name)},
           m_type{std::move(type)},
-          m_hidden{hidden},
           m_fallback{std::move(fallback)},
           m_description{std::move(description)},
-          m_oneline_description{std::move(oneline_description)},
-          m_type_str{std::move(type_str)}
+          m_opts{std::move(opts)}
     {
         CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_USER_KEYS);
     }
@@ -220,16 +220,13 @@ struct RPCArg {
         Fallback fallback,
         std::string description,
         std::vector<RPCArg> inner,
-        std::string oneline_description = "",
-        std::vector<std::string> type_str = {})
+        RPCArgOptions opts = {})
         : m_names{std::move(name)},
           m_type{std::move(type)},
-          m_hidden{false},
           m_inner{std::move(inner)},
           m_fallback{std::move(fallback)},
           m_description{std::move(description)},
-          m_oneline_description{std::move(oneline_description)},
-          m_type_str{std::move(type_str)}
+          m_opts{std::move(opts)}
     {
         CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_USER_KEYS);
     }
@@ -244,7 +241,7 @@ struct RPCArg {
 
     /**
      * Return the type string of the argument.
-     * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description).
+     * Set oneline to allow it to be overridden by a custom oneline type string (m_opts.oneline_description).
      */
     std::string ToString(bool oneline) const;
     /**

src/test/fuzz/rpc.cpp

@@ -152,6 +152,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "prioritisetransaction",
     "pruneblockchain",
     "reconsiderblock",
+    "scanblocks",
     "scantxoutset",
     "sendmsgtopeer", // when no peers are connected, no p2p message is sent
     "sendrawtransaction",