Richer IDL generation with compilation

.gitignore

@@ -14,6 +14,7 @@ tests/*/Cargo.lock
 tests/**/Cargo.lock
 tests/*/yarn.lock
 tests/**/yarn.lock
+!tests/new-idl/programs/new-idl/src/bin
 .DS_Store
 docs/yarn.lock
 ts/docs/

cli/src/lib.rs

@@ -1543,7 +1543,7 @@ fn idl(cfg_override: &ConfigOverride, subcmd: IdlCommand) -> Result<()> {
             out,
             out_ts,
             no_docs,
-        } => idl_parse(cfg_override, file, out, out_ts, no_docs),
+        } => idl_compile(cfg_override, file, out, out_ts, no_docs),
         IdlCommand::Fetch { address, out } => idl_fetch(cfg_override, address, out),
     }
 }
@@ -1823,6 +1823,62 @@ fn idl_parse(
     Ok(())
 }
 
+fn idl_compile(
+    cfg_override: &ConfigOverride,
+    file: String,
+    out: Option<String>,
+    _out_ts: Option<String>,
+    no_docs: bool,
+) -> Result<()> {
+    let cfg = Config::discover(cfg_override)?.expect("Not in workspace.");
+
+    // extract the idl bin src
+    let file = PathBuf::from(&*shellexpand::tilde(&file));
+    let manifest_from_path = std::env::current_dir()?.join(file.parent().unwrap());
+    let cargo = Manifest::discover_from_path(manifest_from_path)?
+        .ok_or_else(|| anyhow!("Cargo.toml not found"))?;
+
+    let bin_src =
+        anchor_syn::idl::bin::gen_src(&*file, cargo.version(), cfg.features.seeds, no_docs, false)?
+            .ok_or_else(|| anyhow!("IDL not parsed"))?;
+
+    // TODO: make the command generate idl based on program name instead of referencing a file:
+    // `anchor idl compile <program-name>`
+    let program_name = file
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .file_name()
+        .unwrap()
+        .to_str()
+        .unwrap();
+
+    let bin_dir = file.parent().unwrap().join("bin");
+    if !bin_dir.exists() {
+        fs::create_dir(bin_dir.clone())?;
+    }
+    let bin_path = bin_dir.join("idl.rs");
+    fs::write(bin_path.clone(), bin_src)?;
+
+    // this might not be necessary or at least don't panic if rustfmt is not installed
+    std::process::Command::new("rustfmt")
+        .arg(bin_path.to_str().unwrap())
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .output()
+        .expect("error running rustfmt on the generated idl.rs");
+
+    std::process::Command::new("cargo")
+        .args(["run", "-p", program_name, "--bin", "idl"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .output()
+        .expect("error running running the generated idl.rs");
+
+    Ok(())
+}
+
 fn idl_fetch(cfg_override: &ConfigOverride, address: Pubkey, out: Option<String>) -> Result<()> {
     let idl = fetch_idl(cfg_override, address)?;
     let out = match out {

lang/syn/src/idl/bin.rs

@@ -0,0 +1,164 @@
+use crate::idl::*;
+use crate::parser::context::CrateContext;
+use crate::parser::{self, accounts, docs, error, program};
+use crate::Ty;
+use crate::{AccountField, AccountsStruct, StateIx};
+use anyhow::Result;
+use heck::MixedCase;
+use quote::{ToTokens, quote, format_ident};
+use std::collections::{HashMap, HashSet};
+use std::path::Path;
+
+const DERIVE_NAME: &str = "Accounts";
+// TODO: share this with `anchor_lang` crate.
+const ERROR_CODE_OFFSET: u32 = 6000;
+
+// Generate the source of the idl binary
+pub fn gen_src(
+    filename: impl AsRef<Path>,
+    version: String,
+    seeds_feature: bool,
+    no_docs: bool,
+    safety_checks: bool,
+) -> Result<Option<String>> {
+    let ctx = CrateContext::parse(filename)?;
+    if safety_checks {
+        ctx.safety_checks()?;
+    }
+
+    let program_mod = match parse_program_mod(&ctx) {
+        None => return Ok(None),
+        Some(m) => m,
+    };
+    let mut p = program::parse(program_mod)?;
+
+    if no_docs {
+        p.docs = None;
+        for ix in &mut p.ixs {
+            ix.docs = None;
+        }
+    }
+
+    let accs = parse_account_derives(&ctx);
+
+    let account_struct = accs.get("Initialize").unwrap();
+    let accounts = account_struct.fields.iter().map(|acc: &AccountField| {
+        match acc {
+            AccountField::CompositeField(_) => panic!("TODO"),
+            AccountField::Field(acc) => {
+                let name = acc.ident.to_string();
+                let is_mut = acc.constraints.is_mutable();
+                let is_signer = match acc.ty {
+                    Ty::Signer => true,
+                    _ => acc.constraints.is_signer()
+                };
+
+                let mut fields = vec![
+                    quote!("name": #name),
+                    quote!("isMut": #is_mut),
+                    quote!("isSigner": #is_signer),
+                    // TODO: docs
+                ];
+
+                // pubkey
+                // TODO: also handle `Sysvar` and `address = <>` constraint
+                let pubkey = match &acc.ty {
+                    // transform from `Program<'info, SomeType>` to `SomeType::id().to_string()`
+                    Ty::Program(program) => program.account_type_path.path.get_ident().map(|i| quote!{#i::id().to_string()}),
+                    _ => None
+                };
+                pubkey.map(|pubkey| fields.push(quote!{"pubkey": #pubkey}));
+
+                // seeds
+                let seeds: Option<Vec<proc_macro2::TokenStream>> = acc.constraints.seeds.as_ref().map(|seeds| {
+                    // TODO: cover the cases when seed expression referencess instruction args or accounts
+                    seeds.seeds.iter().map(|seed| quote!{
+                        {
+                            "kind": "const",
+                            "type": "base58",
+                            "value": bs58::encode(#seed).into_string()
+                        }
+                    }).collect()
+                });
+                // TODO handle `seeds::program = <>` constraint
+                seeds.map(|seeds| fields.push(quote!("pda": {
+                    "seeds": [#(#seeds),*]
+                })));
+
+
+                quote!{
+                    {
+                       #(#fields),*
+                    }
+                }
+            }
+        }
+    });
+
+
+    let ret = quote!{
+        use anchor_lang::prelude::*;
+        use std::str::FromStr;
+
+        const MY_SEED_U64: u64 = 3;
+
+        fn main() {
+            let instructions = serde_json::json!({
+                "instructions": [
+                    {
+                        "name": "initialize",
+                        "accounts": [#(#accounts),*],
+                        "args": []
+                    }
+                ]
+            });
+
+            println!("{}", serde_json::to_string_pretty(&instructions).unwrap());
+        }
+    };
+
+    Ok(Some(format!("{}", ret)))
+}
+
+// Parse the main program mod.
+fn parse_program_mod(ctx: &CrateContext) -> Option<syn::ItemMod> {
+    let root = ctx.root_module();
+    let mods = root
+        .items()
+        .filter_map(|i| match i {
+            syn::Item::Mod(item_mod) => {
+                let mod_count = item_mod
+                    .attrs
+                    .iter()
+                    .filter(|attr| attr.path.segments.last().unwrap().ident == "program")
+                    .count();
+                if mod_count != 1 {
+                    return None;
+                }
+                Some(item_mod)
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    if mods.len() != 1 {
+        return None;
+    }
+    Some(mods[0].clone())
+}
+
+// Parse all structs implementing the `Accounts` trait.
+fn parse_account_derives(ctx: &CrateContext) -> HashMap<String, AccountsStruct> {
+    // TODO: parse manual implementations. Currently we only look
+    //       for derives.
+    ctx.structs()
+        .filter_map(|i_strct| {
+            for attr in &i_strct.attrs {
+                if attr.path.is_ident("derive") && attr.tokens.to_string().contains(DERIVE_NAME) {
+                    let strct = accounts::parse(i_strct).expect("Code not parseable");
+                    return Some((strct.ident.to_string(), strct));
+                }
+            }
+            None
+        })
+        .collect()
+}

lang/syn/src/idl/mod.rs

@@ -3,6 +3,7 @@ use serde_json::Value as JsonValue;
 
 pub mod file;
 pub mod pda;
+pub mod bin;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct Idl {

tests/new-idl/Anchor.toml

@@ -0,0 +1,14 @@
+[features]
+seeds = true
+[programs.localnet]
+new_idl = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
+
+[registry]
+url = "https://anchor.projectserum.com"
+
+[provider]
+cluster = "localnet"
+wallet = "/home/work/.config/solana/id.json"
+
+[scripts]
+test = "yarn run ts-mocha -p ./tsconfig.json -t 1000000 tests/**/*.ts"

tests/new-idl/Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+members = [
+    "programs/*"
+]

tests/new-idl/idl.json

@@ -0,0 +1,55 @@
+{
+  "instructions": [
+    {
+      "name": "initialize",
+      "accounts": [
+        {
+          "name": "state",
+          "isMut": true,
+          "isSigner": false,
+          "pda": {
+            "seeds": [
+              {
+                "kind": "const",
+                "type": "base58",
+                "value": "11111111111111111111111111111111"
+              },
+              {
+                "kind": "const",
+                "type": "base58",
+                "value": "3tMg6nFceRK19FX3WY1Cbtu6DboaabhdVfeYP5BKqkuH"
+              },
+              {
+                "kind": "const",
+                "type": "base58",
+                "value": "W723RTUpoZ"
+              },
+              {
+                "kind": "const",
+                "type": "base58",
+                "value": "2UDrs33K4gNG3"
+              },
+              {
+                "kind": "const",
+                "type": "base58",
+                "value": "cM"
+              }
+            ]
+          }
+        },
+        {
+          "name": "payer",
+          "isMut": true,
+          "isSigner": true
+        },
+        {
+          "name": "system_program",
+          "isMut": false,
+          "isSigner": false,
+          "pubkey": "11111111111111111111111111111111"
+        }
+      ],
+      "args": []
+    }
+  ]
+}

tests/new-idl/package.json

@@ -0,0 +1,19 @@
+{
+    "scripts": {
+        "lint:fix": "prettier */*.js \"*/**/*{.js,.ts}\" -w",
+        "lint": "prettier */*.js \"*/**/*{.js,.ts}\" --check"
+    },
+    "dependencies": {
+        "@project-serum/anchor": "^0.24.2"
+    },
+    "devDependencies": {
+        "chai": "^4.3.4",
+        "mocha": "^9.0.3",
+        "ts-mocha": "^8.0.0",
+        "@types/bn.js": "^5.1.0",
+        "@types/chai": "^4.3.0",
+        "@types/mocha": "^9.0.0",
+        "typescript": "^4.3.5",
+        "prettier": "^2.6.2"
+    }
+}

tests/new-idl/programs/new-idl/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "new-idl"
+version = "0.1.0"
+description = "Created with Anchor"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib", "lib"]
+name = "new_idl"
+
+[features]
+no-entrypoint = []
+no-idl = []
+no-log-ix-name = []
+cpi = ["no-entrypoint"]
+default = []
+
+[profile.release]
+overflow-checks = true
+
+[dependencies]
+anchor-lang = { path = "../../../../lang" }
+serde_json = { version = "1.0", features = ["preserve_order"] }
+bs58 = "0.4.0"

tests/new-idl/programs/new-idl/Xargo.toml

@@ -0,0 +1,2 @@
+[target.bpfel-unknown-unknown.dependencies.std]
+features = []

tests/new-idl/programs/new-idl/src/bin/idl.rs

@@ -0,0 +1,7 @@
+use anchor_lang::prelude::*;
+use std::str::FromStr;
+const MY_SEED_U64: u64 = 3;
+fn main() {
+    let instructions = serde_json :: json ! ({ "instructions" : [{ "name" : "initialize" , "accounts" : [{ "name" : "state" , "isMut" : true , "isSigner" : false , "pda" : { "seeds" : [{ "kind" : "const" , "type" : "base58" , "value" : bs58 :: encode (anchor_lang :: solana_program :: system_program :: ID . as_ref ()) . into_string () } , { "kind" : "const" , "type" : "base58" , "value" : bs58 :: encode (Pubkey :: from_str ("3tMg6nFceRK19FX3WY1Cbtu6DboaabhdVfeYP5BKqkuH") . unwrap () . as_ref ()) . into_string () } , { "kind" : "const" , "type" : "base58" , "value" : bs58 :: encode (& MY_SEED_U64 . to_le_bytes ()) . into_string () } , { "kind" : "const" , "type" : "base58" , "value" : bs58 :: encode (b"some-seed" . as_ref ()) . into_string () } , { "kind" : "const" , "type" : "base58" , "value" : bs58 :: encode (& [8 , 2]) . into_string () }] } } , { "name" : "payer" , "isMut" : true , "isSigner" : true } , { "name" : "system_program" , "isMut" : false , "isSigner" : false , "pubkey" : System :: id () . to_string () }] , "args" : [] }] });
+    println!("{}", serde_json::to_string_pretty(&instructions).unwrap());
+}