From 523d44408914d55dc68ca32defad53d72d12fb8a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 4 Mar 2020 14:43:00 +1030 Subject: [PATCH 01/25] wire: add fromwire_tal_bytes() helper. Does the allocation and copying; this is useful because we can avoid being fooled into doing giant allocations. Signed-off-by: Rusty Russell --- common/onionreply.c | 4 ++-- devtools/print_wire.c | 5 ++--- lightningd/gossip_msg.c | 3 +-- wire/fromwire.c | 28 ++++++++++++++++++++-------- wire/wire.h | 2 ++ 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/common/onionreply.c b/common/onionreply.c index 445b5e2ea84f..94cf1127d94d 100644 --- a/common/onionreply.c +++ b/common/onionreply.c @@ -12,8 +12,8 @@ struct onionreply *fromwire_onionreply(const tal_t *ctx, const u8 **cursor, size_t *max) { struct onionreply *r = tal(ctx, struct onionreply); - r->contents = tal_arr(r, u8, fromwire_u16(cursor, max)); - fromwire_u8_array(cursor, max, r->contents, tal_count(r->contents)); + r->contents = fromwire_tal_bytes(r, cursor, max, + fromwire_u16(cursor, max)); if (!*cursor) return tal_free(r); return r; diff --git a/devtools/print_wire.c b/devtools/print_wire.c index 64e395ca6e0a..d8dc8c5bbde2 100644 --- a/devtools/print_wire.c +++ b/devtools/print_wire.c @@ -104,10 +104,9 @@ static void printwire_addresses(const u8 **cursor, size_t *plen, size_t len) static void printwire_encoded_short_ids(const u8 **cursor, size_t *plen, size_t len) { struct short_channel_id *scids; - u8 *arr = tal_arr(tmpctx, u8, len); + u8 *arr = fromwire_tal_bytes(tmpctx, cursor, plen, len); - fromwire_u8_array(cursor, plen, arr, len); - if (!*cursor) + if (!arr) return; printf("["); diff --git a/lightningd/gossip_msg.c b/lightningd/gossip_msg.c index 9b000f666e09..fce0d31b21e1 100644 --- a/lightningd/gossip_msg.c +++ b/lightningd/gossip_msg.c @@ -21,8 +21,7 @@ struct gossip_getnodes_entry *fromwire_gossip_getnodes_entry(const tal_t *ctx, } flen = fromwire_u16(pptr, max); - entry->features = tal_arr(entry, u8, flen); - fromwire_u8_array(pptr, max, entry->features, flen); + entry->features = fromwire_tal_bytes(entry, pptr, max, flen); numaddresses = fromwire_u8(pptr, max); diff --git a/wire/fromwire.c b/wire/fromwire.c index 80a6babfb9e2..11be488c9187 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -308,6 +308,19 @@ void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num) fromwire(cursor, max, arr, num); } +u8 *fromwire_tal_bytes(const tal_t *ctx, + const u8 **cursor, size_t *max, size_t num) +{ + u8 *arr; + if (num > *max) { + fromwire_fail(cursor, max); + return NULL; + } + arr = tal_arr(ctx, u8, num); + fromwire_u8_array(cursor, max, arr, num); + return arr; +} + void fromwire_pad(const u8 **cursor, size_t *max, size_t num) { fromwire(cursor, max, NULL, num); @@ -393,20 +406,19 @@ struct bitcoin_tx_output *fromwire_bitcoin_tx_output(const tal_t *ctx, struct bitcoin_tx_output *output = tal(ctx, struct bitcoin_tx_output); output->amount = fromwire_amount_sat(cursor, max); u16 script_len = fromwire_u16(cursor, max); - output->script = tal_arr(output, u8, script_len); - fromwire_u8_array(cursor, max, output->script, script_len); + output->script = fromwire_tal_bytes(output, cursor, max, script_len); + if (!*cursor) + return tal_free(output); return output; } struct witscript *fromwire_witscript(const tal_t *ctx, const u8 **cursor, size_t *max) { - struct witscript *retval; + struct witscript *retval = tal(ctx, struct witscript); u16 len = fromwire_u16(cursor, max); - if (!len) - return NULL; - retval = tal(ctx, struct witscript); - retval->ptr = tal_arr(retval, u8, len); - fromwire_u8_array(cursor, max, retval->ptr, len); + retval->ptr = fromwire_tal_bytes(retval, cursor, max, len); + if (!*cursor) + return tal_free(retval); return retval; } diff --git a/wire/wire.h b/wire/wire.h index db383c3085f2..8577c7ef32e6 100644 --- a/wire/wire.h +++ b/wire/wire.h @@ -136,6 +136,8 @@ struct amount_sat fromwire_amount_sat(const u8 **cursor, size_t *max); void fromwire_pad(const u8 **cursor, size_t *max, size_t num); void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num); +u8 *fromwire_tal_bytes(const tal_t *ctx, + const u8 **cursor, size_t *max, size_t num); char *fromwire_wirestring(const tal_t *ctx, const u8 **cursor, size_t *max); struct bitcoin_tx *fromwire_bitcoin_tx(const tal_t *ctx, const u8 **cursor, size_t *max); From 0cf5ac6bcfad760aa9c9261df4fbe9f2ea775935 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Feb 2020 16:31:04 +0100 Subject: [PATCH 02/25] pytest: Add tests for the sphinx onion generation and processing These just run the test vectors and add a test for the devtools/onion tool so we don't accidentally break them. --- tests/test_onion.py | 60 ++++++++++++++++++++++ tests/vectors/onion-error-test.json | 43 ++++++++++++++++ tests/vectors/onion-test-multi-frame.json | 42 +++++++++++++++ tests/vectors/onion-test-v0.json | 62 +++++++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 tests/test_onion.py create mode 100644 tests/vectors/onion-error-test.json create mode 100644 tests/vectors/onion-test-multi-frame.json create mode 100644 tests/vectors/onion-test-v0.json diff --git a/tests/test_onion.py b/tests/test_onion.py new file mode 100644 index 000000000000..8933f909b5cb --- /dev/null +++ b/tests/test_onion.py @@ -0,0 +1,60 @@ +import subprocess +import pytest +import os +from fixtures import * # noqa: F401,F403 + + +@pytest.fixture +def oniontool(): + path = os.path.join(os.path.dirname(__file__), "..", "devtools", "onion") + return path + + +privkeys = [ + '4141414141414141414141414141414141414141414141414141414141414141', + '4242424242424242424242424242424242424242424242424242424242424242', + '4343434343434343434343434343434343434343434343434343434343434343', + '4444444444444444444444444444444444444444444444444444444444444444', + '4545454545454545454545454545454545454545454545454545454545454545' +] + +pubkeys = [ + '02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619', + '0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c', + '027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007', + '032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991', + '02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145' +] + + +def test_onion(directory, oniontool): + """ Generate a 5 hop onion and then decode it. + """ + os.makedirs(directory) + tempfile = os.path.join(directory, 'onion') + out = subprocess.check_output( + [oniontool, 'generate'] + pubkeys + ).decode('ASCII').strip().split('\n') + assert(len(out) == 1) + + def store_onion(o): + with open(tempfile, 'w') as f: + f.write(o) + + store_onion(out[0]) + + for i, pk in enumerate(privkeys): + out = subprocess.check_output([oniontool, 'decode', tempfile, pk]).decode('ASCII').strip().split('\n') + store_onion(out[-1][5:]) + + assert(out == ['payload=000000000000000000000000000000000400000004000000000000000000000000']) + + +def test_onion_vectors(oniontool): + tests = [ + 'onion-test-multi-frame.json', + 'onion-test-v0.json'] + + for t in tests: + p = os.path.join(os.path.dirname(__file__), 'vectors', t) + print(subprocess.check_output([oniontool, 'runtest', p]).decode('ASCII')) diff --git a/tests/vectors/onion-error-test.json b/tests/vectors/onion-error-test.json new file mode 100644 index 000000000000..7d416b4d7049 --- /dev/null +++ b/tests/vectors/onion-error-test.json @@ -0,0 +1,43 @@ +{ + "comment": "A simple error returned by node hops[4], the public/private keys and shared_secrets are identical to the ones used in `onion-test-v0.json`", + "generate": { + "session_key": "4141414141414141414141414141414141414141414141414141414141414141", + "failure_message": "2002", + "hops": [ + { + "realm": 0, + "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", + "hop_shared_secret": "53eb63ea8a3fec3b3cd433b85cd62a4b145e1dda09391b348c4e1cd36a03ea66", + "ammag_key": "3761ba4d3e726d8abb16cba5950ee976b84937b61b7ad09e741724d7dee12eb5" + }, + { + "realm": 0, + "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c", + "hop_shared_secret": "a6519e98832a0b179f62123b3567c106db99ee37bef036e783263602f3488fae", + "ammag_key": "59ee5867c5c151daa31e36ee42530f429c433836286e63744f2020b980302564" + }, + { + "realm": 0, + "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007", + "hop_shared_secret": "3a6b412548762f0dbccce5c7ae7bb8147d1caf9b5471c34120b30bc9c04891cc", + "ammag_key": "1bf08df8628d452141d56adfd1b25c1530d7921c23cecfc749ac03a9b694b0d3" + }, + { + "realm": 0, + "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991", + "hop_shared_secret": "21e13c2d7cfe7e18836df50872466117a295783ab8aab0e7ecc8c725503ad02d", + "ammag_key": "cd9ac0e09064f039fa43a31dea05f5fe5f6443d40a98be4071af4a9d704be5ad" + }, + { + "realm": 0, + "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145", + "hop_shared_secret": "b5756b9b542727dbafc6765a49488b023a725d631af688fc031217e90770c328", + "um_key": "4da7f2923edce6c2d85987d1d9fa6d88023e6c3a9c3d20f07d3b10b61a78d646", + "ammag_key": "2f36bb8822e1f0d04c27b7d8bb7d7dd586e032a3218b8d414afbba6f169a4d68", + "payload": "0002200200fe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "comment": "hops[4] is the only node that needs a mu_key since it is the only node that computes an HMAC of the original error message, the payload is omits the HMAC." + } + ] + }, + "errorpacket": "9c5add3963fc7f6ed7f148623c84134b5647e1306419dbe2174e523fa9e2fbed3a06a19f899145610741c83ad40b7712aefaddec8c6baf7325d92ea4ca4d1df8bce517f7e54554608bf2bd8071a4f52a7a2f7ffbb1413edad81eeea5785aa9d990f2865dc23b4bc3c301a94eec4eabebca66be5cf638f693ec256aec514620cc28ee4a94bd9565bc4d4962b9d3641d4278fb319ed2b84de5b665f307a2db0f7fbb757366067d88c50f7e829138fde4f78d39b5b5802f1b92a8a820865af5cc79f9f30bc3f461c66af95d13e5e1f0381c184572a91dee1c849048a647a1158cf884064deddbf1b0b88dfe2f791428d0ba0f6fb2f04e14081f69165ae66d9297c118f0907705c9c4954a199bae0bb96fad763d690e7daa6cfda59ba7f2c8d11448b604d12d" +} diff --git a/tests/vectors/onion-test-multi-frame.json b/tests/vectors/onion-test-multi-frame.json new file mode 100644 index 000000000000..5d20be295fe8 --- /dev/null +++ b/tests/vectors/onion-test-multi-frame.json @@ -0,0 +1,42 @@ +{ + "comment": "A testcase for a variable length hop_payload. The third payload is 256 bytes long.", + "generate": { + "session_key": "4141414141414141414141414141414141414141414141414141414141414141", + "associated_data": "4242424242424242424242424242424242424242424242424242424242424242", + "hops": [ + { + "type": "legacy", + "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", + "payload": "0000000000000000000000000000000000000000" + }, + { + "type": "tlv", + "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c", + "payload": "0101010101010101000000000000000100000001" + }, + { + "type": "raw", + "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007", + "payload": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff" + }, + { + "type": "tlv", + "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991", + "payload": "0303030303030303000000000000000300000003" + }, + { + "type": "legacy", + "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145", + "payload": "0404040404040404000000000000000400000004" + } + ] + }, + "onion": "0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a710f8eaf9ccc768f66bb5dec1f7827f33c43fe2ddd05614c8283aa78e9e7573f87c50f7d61ab590531cf08000178a333a347f8b4072e1cea42da7552402b10765adae3f581408f35ff0a71a34b78b1d8ecae77df96c6404bae9a8e8d7178977d7094a1ae549f89338c0777551f874159eb42d3a59fb9285ad4e24883f27de23942ec966611e99bee1cee503455be9e8e642cef6cef7b9864130f692283f8a973d47a8f1c1726b6e59969385975c766e35737c8d76388b64f748ee7943ffb0e2ee45c57a1abc40762ae598723d21bd184e2b338f68ebff47219357bd19cd7e01e2337b806ef4d717888e129e59cd3dc31e6201ccb2fd6d7499836f37a993262468bcb3a4dcd03a22818aca49c6b7b9b8e9e870045631d8e039b066ff86e0d1b7291f71cefa7264c70404a8e538b566c17ccc5feab231401e6c08a01bd5edfc1aa8e3e533b96e82d1f91118d508924b923531929aea889fcdf057f5995d9731c4bf796fb0e41c885d488dcbc68eb742e27f44310b276edc6f652658149e7e9ced4edde5d38c9b8f92e16f6b4ab13d710ee5c193921909bdd75db331cd9d7581a39fca50814ed8d9d402b86e7f8f6ac2f3bca8e6fe47eb45fbdd3be21a8a8d200797eae3c9a0497132f92410d804977408494dff49dd3d8bce248e0b74fd9e6f0f7102c25ddfa02bd9ad9f746abbfa3379834bc2380d58e9d23237821475a1874484783a15d68f47d3dc339f38d9bf925655d5c946778680fd6d1f062f84128895aff09d35d6c92cca63d3f95a9ee8f2a84f383b4d6a087533e65de12fc8dcaf85777736a2088ff4b22462265028695b37e70963c10df8ef2458756c73007dc3e544340927f9e9f5ea4816a9fd9832c311d122e9512739a6b4714bba590e31caa143ce83cb84b36c738c60c3190ff70cd9ac286a9fd2ab619399b68f1f7447be376ce884b5913c8496d01cbf7a44a60b6e6747513f69dc538f340bc1388e0fde5d0c1db50a4dcb9cc0576e0e2474e4853af9623212578d502757ffb2e0e749695ed70f61c116560d0d4154b64dcf3cbf3c91d89fb6dd004dc19588e3479fcc63c394a4f9e8a3b8b961fce8a532304f1337f1a697a1bb14b94d2953f39b73b6a3125d24f27fcd4f60437881185370bde68a5454d816e7a70d4cea582effab9a4f1b730437e35f7a5c4b769c7b72f0346887c1e63576b2f1e2b3706142586883f8cf3a23595cc8e35a52ad290afd8d2f8bcd5b4c1b891583a4159af7110ecde092079209c6ec46d2bda60b04c519bb8bc6dffb5c87f310814ef2f3003671b3c90ddf5d0173a70504c2280d31f17c061f4bb12a978122c8a2a618bb7d1edcf14f84bf0fa181798b826a254fca8b6d7c81e0beb01bd77f6461be3c8647301d02b04753b0771105986aa0cbc13f7718d64e1b3437e8eef1d319359914a7932548c91570ef3ea741083ca5be5ff43c6d9444d29df06f76ec3dc936e3d180f4b6d0fbc495487c7d44d7c8fe4a70d5ff1461d0d9593f3f898c919c363fa18341ce9dae54f898ccf3fe792136682272941563387263c51b2a2f32363b804672cc158c9230472b554090a661aa81525d11876eefdcc45442249e61e07284592f1606491de5c0324d3af4be035d7ede75b957e879e9770cdde2e1bbc1ef75d45fe555f1ff6ac296a2f648eeee59c7c08260226ea333c285bcf37a9bbfa57ba2ab8083c4be6fc2ebe279537d22da96a07392908cf22b233337a74fe5c603b51712b43c3ee55010ee3d44dd9ba82bba3145ec358f863e04bbfa53799a7a9216718fd5859da2f0deb77b8e315ad6868fdec9400f45a48e6dc8ddbaeb3", + "decode": [ + "4141414141414141414141414141414141414141414141414141414141414141", + "4242424242424242424242424242424242424242424242424242424242424242", + "4343434343434343434343434343434343434343434343434343434343434343", + "4444444444444444444444444444444444444444444444444444444444444444", + "4545454545454545454545454545454545454545454545454545454545454545" + ] +} diff --git a/tests/vectors/onion-test-v0.json b/tests/vectors/onion-test-v0.json new file mode 100644 index 000000000000..66755f4fedb2 --- /dev/null +++ b/tests/vectors/onion-test-v0.json @@ -0,0 +1,62 @@ +{ + "comment": "This is a simple testcase in which we only use v0 payloads, and all hops have single frame payloads", + "generate": { + "session_key": "4141414141414141414141414141414141414141414141414141414141414141", + "associated_data": "4242424242424242424242424242424242424242424242424242424242424242", + "hops": [ + { + "realm": 0, + "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", + "hop_shared_secret": "53eb63ea8a3fec3b3cd433b85cd62a4b145e1dda09391b348c4e1cd36a03ea66", + "payload": "0000000000000000000000000000000000000000000000000000000000000000", + "rhokey": "ce496ec94def95aadd4bec15cdb41a740c9f2b62347c4917325fcc6fb0453986", + "mukey": "b57061dc6d0a2b9f261ac410c8b26d64ac5506cbba30267a649c28c179400eba", + "hmac": "b8640887e027e946df96488b47fbc4a4fadaa8beda4abe446fafea5403fae2ef" + }, + { + "realm": 0, + "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c", + "hop_shared_secret": "a6519e98832a0b179f62123b3567c106db99ee37bef036e783263602f3488fae", + "payload": "0101010101010101000000000000000100000001000000000000000000000000", + "rhokey": "450ffcabc6449094918ebe13d4f03e433d20a3d28a768203337bc40b6e4b2c59", + "mukey": "05ed2b4a3fb023c2ff5dd6ed4b9b6ea7383f5cfe9d59c11d121ec2c81ca2eea9", + "hmac": "a93aa4f40241cef3e764e24b28570a0db39af82ab5102c3a04e51bec8cca9394" + }, + { + "realm": 0, + "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007", + "hop_shared_secret": "3a6b412548762f0dbccce5c7ae7bb8147d1caf9b5471c34120b30bc9c04891cc", + "payload": "0202020202020202000000000000000200000002000000000000000000000000", + "rhokey": "11bf5c4f960239cb37833936aa3d02cea82c0f39fd35f566109c41f9eac8deea", + "mukey": "caafe2820fa00eb2eeb78695ae452eba38f5a53ed6d53518c5c6edf76f3f5b78", + "hmac": "5d1b11f1efeaa9be32eb1c74b113c0b46f056bb49e2a35a51ceaece6bd31332c" + }, + { + "realm": 0, + "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991", + "hop_shared_secret": "21e13c2d7cfe7e18836df50872466117a295783ab8aab0e7ecc8c725503ad02d", + "payload": "0303030303030303000000000000000300000003000000000000000000000000", + "rhokey": "cbe784ab745c13ff5cffc2fbe3e84424aa0fd669b8ead4ee562901a4a4e89e9e", + "mukey": "5052aa1b3d9f0655a0932e50d42f0c9ba0705142c25d225515c45f47c0036ee9", + "hmac": "19ca6357b5552b28e50ae226854eec874bbbf7025cf290a34c06b4eff5d2bac0" + }, + { + "realm": 0, + "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145", + "hop_shared_secret": "b5756b9b542727dbafc6765a49488b023a725d631af688fc031217e90770c328", + "payload": "0404040404040404000000000000000400000004000000000000000000000000", + "rhokey": "034e18b8cc718e8af6339106e706c52d8df89e2b1f7e9142d996acf88df8799b", + "mukey": "8e45e5c61c2b24cb6382444db6698727afb063adecd72aada233d4bf273d975a", + "hmac": "16d4553c6084b369073d259381bb5b02c16bb2c590bbd9e69346cf7ebd563229" + } + ] + }, + "onion": "0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a71e87f9aab8f6378c6ff744c1f34b393ad28d065b535c1a8668d85d3b34a1b3befd10f7d61ab590531cf08000178a333a347f8b4072e216400406bdf3bf038659793a1f9e7abc789266cc861cabd95818c0fc8efbdfdc14e3f7c2bc7eb8d6a79ef75ce721caad69320c3a469a202f3e468c67eaf7a7cda226d0fd32f7b48084dca885d014698cf05d742557763d9cb743faeae65dcc79dddaecf27fe5942be5380d15e9a1ec866abe044a9ad635778ba61fc0776dc832b39451bd5d35072d2269cf9b040a2a2fba158a0d8085926dc2e44f0c88bf487da56e13ef2d5e676a8589881b4869ed4c7f0218ff8c6c7dd7221d189c65b3b9aaa71a01484b122846c7c7b57e02e679ea8469b70e14fe4f70fee4d87b910cf144be6fe48eef24da475c0b0bcc6565a9f99728426ce2380a9580e2a9442481ceae7679906c30b1a0e21a10f26150e0645ab6edfdab1ce8f8bea7b1dee511c5fd38ac0e702c1c15bb86b52bca1b71e15b96982d262a442024c33ceb7dd8f949063c2e5e613e873250e2f8708bd4e1924abd45f65c2fa5617bfb10ee9e4a42d6b5811acc8029c16274f937dac9e8817c7e579fdb767ffe277f26d413ced06b620ede8362081da21cf67c2ca9d6f15fe5bc05f82f5bb93f8916bad3d63338ca824f3bbc11b57ce94a5fa1bc239533679903d6fec92a8c792fd86e2960188c14f21e399cfd72a50c620e10aefc6249360b463df9a89bf6836f4f26359207b765578e5ed76ae9f31b1cc48324be576e3d8e44d217445dba466f9b6293fdf05448584eb64f61e02903f834518622b7d4732471c6e0e22e22d1f45e31f0509eab39cdea5980a492a1da2aaac55a98a01216cd4bfe7abaa682af0fbff2dfed030ba28f1285df750e4d3477190dd193f8643b61d8ac1c427d590badb1f61a05d480908fbdc7c6f0502dd0c4abb51d725e92f95da2a8facb79881a844e2026911adcc659d1fb20a2fce63787c8bb0d9f6789c4b231c76da81c3f0718eb7156565a081d2be6b4170c0e0bcebddd459f53db2590c974bca0d705c055dee8c629bf854a5d58edc85228499ec6dde80cce4c8910b81b1e9e8b0f43bd39c8d69c3a80672729b7dc952dd9448688b6bd06afc2d2819cda80b66c57b52ccf7ac1a86601410d18d0c732f69de792e0894a9541684ef174de766fd4ce55efea8f53812867be6a391ac865802dbc26d93959df327ec2667c7256aa5a1d3c45a69a6158f285d6c97c3b8eedb09527848500517995a9eae4cd911df531544c77f5a9a2f22313e3eb72ca7a07dba243476bc926992e0d1e58b4a2fc8c7b01e0cad726237933ea319bad7537d39f3ed635d1e6c1d29e97b3d2160a09e30ee2b65ac5bce00996a73c008bcf351cecb97b6833b6d121dcf4644260b2946ea204732ac9954b228f0beaa15071930fd9583dfc466d12b5f0eeeba6dcf23d5ce8ae62ee5796359d97a4a15955c778d868d0ef9991d9f2833b5bb66119c5f8b396fd108baed7906cbb3cc376d13551caed97fece6f42a4c908ee279f1127fda1dd3ee77d8de0a6f3c135fa3f1cffe38591b6738dc97b55f0acc52be9753ce53e64d7e497bb00ca6123758df3b68fad99e35c04389f7514a8e36039f541598a417275e77869989782325a15b5342ac5011ff07af698584b476b35d941a4981eac590a07a092bb50342da5d3341f901aa07964a8d02b623c7b106dd0ae50bfa007a22d46c8772fa55558176602946cb1d11ea5460db7586fb89c6d3bcd3ab6dd20df4a4db63d2e7d52380800ad812b8640887e027e946df96488b47fbc4a4fadaa8beda4abe446fafea5403fae2ef", + "decode": [ + "4141414141414141414141414141414141414141414141414141414141414141", + "4242424242424242424242424242424242424242424242424242424242424242", + "4343434343434343434343434343434343434343434343434343434343434343", + "4444444444444444444444444444444444444444444444444444444444444444", + "4545454545454545454545454545454545454545454545454545454545454545" + ] +} From aff6c01e27cf1a6d62983d3f91052917b07012ab Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 20 Feb 2020 15:53:34 +0100 Subject: [PATCH 03/25] sphinx: Working onion wrapping with filler cancellation --- common/sphinx.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/common/sphinx.c b/common/sphinx.c index 80ab1adae39b..9155dd247f02 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -57,6 +57,14 @@ struct sphinx_path { /* The individual hops on this route. */ struct sphinx_hop *hops; + + /* If this is a rendez-vous onion, then the following node_id tells us + * which node will be processing this onion and decompressing the + * onion. It is used to generate the prefill obfuscation stream to + * hide the fact that the onion was compressed from the next + * node. NULL if this is not a rendez-vous onion, and shouldn't be + * compressible. */ + struct pubkey *rendezvous_id; }; struct sphinx_path *sphinx_path_new(const tal_t *ctx, const u8 *associated_data) @@ -64,6 +72,7 @@ struct sphinx_path *sphinx_path_new(const tal_t *ctx, const u8 *associated_data) struct sphinx_path *sp = tal(ctx, struct sphinx_path); sp->associated_data = tal_dup_talarr(sp, u8, associated_data); sp->session_key = NULL; + sp->rendezvous_id = NULL; sp->hops = tal_arr(sp, struct sphinx_hop, 0); return sp; } @@ -259,6 +268,36 @@ static bool generate_header_padding(void *dst, size_t dstlen, return true; } +static bool generate_prefill(void *dst, size_t dstlen, + const struct sphinx_path *path, + struct hop_params *params) +{ + u8 stream[2 * ROUTING_INFO_SIZE]; + u8 key[KEY_LEN]; + size_t fillerStart, fillerSize; + + memset(dst, 0, dstlen); + for (int i = 0; i < tal_count(path->hops); i++) { + if (!generate_key(&key, RHO_KEYTYPE, strlen(RHO_KEYTYPE), + ¶ms[i].secret)) + return false; + + generate_cipher_stream(stream, key, sizeof(stream)); + + /* Sum up how many bytes have been used by previous hops, + * that gives us the start in the stream */ + fillerSize = 0; + for (int j = 0; j < i; j++) + fillerSize += sphinx_hop_size(&path->hops[j]); + fillerStart = ROUTING_INFO_SIZE - fillerSize - dstlen; + + /* Apply the cipher-stream to the part of the filler that'll + * be added by this hop */ + xorbytes(dst, dst, stream + fillerStart, dstlen); + } + return true; +} + static void compute_blinding_factor(const struct pubkey *key, const struct secret *sharedsecret, u8 res[BLINDING_FACTOR_SIZE]) @@ -386,6 +425,39 @@ static void sphinx_write_frame(u8 *dest, const struct sphinx_hop *hop) memcpy(dest + tal_bytelen(hop->raw_payload), hop->hmac, HMAC_SIZE); } +static void sphinx_prefill_stream_xor(u8 *dst, size_t dstlen, + const struct secret *shared_secret) +{ + u8 padkey[KEY_LEN]; + generate_key(padkey, "prefill", 7, shared_secret); + xor_cipher_stream(dst, padkey, dstlen); +} + +static void sphinx_prefill(u8 *routinginfo, const struct sphinx_path *sp, + size_t prefill_size, struct hop_params *params) +{ + int num_hops = tal_count(sp->hops); + size_t fillerSize = sphinx_path_payloads_size(sp) - + sphinx_hop_size(&sp->hops[num_hops - 1]); + size_t last_hop_size = sphinx_hop_size(&sp->hops[num_hops - 1]); + int prefill_offset = + ROUTING_INFO_SIZE - fillerSize - last_hop_size - prefill_size; + u8 prefill[prefill_size]; + struct secret shared_secret; + + /* Generate the prefill stream, which cancels out the layers of + * encryption that will be applied while wrapping the onion. This + * leaves the middle, unused, section with all 0x00 bytes after + * encrypting. */ + generate_prefill(prefill, prefill_size, sp, params); + memcpy(routinginfo + prefill_offset, prefill, prefill_size); + + /* Now fill in the obfuscation stream, which can be regenerated by the + * node processing this onion. */ + create_shared_secret(&shared_secret, sp->rendezvous_id, sp->session_key); + sphinx_prefill_stream_xor(routinginfo + prefill_offset, prefill_size, &shared_secret); +} + struct onionpacket *create_onionpacket( const tal_t *ctx, struct sphinx_path *sp, @@ -402,6 +474,8 @@ struct onionpacket *create_onionpacket( u8 nexthmac[HMAC_SIZE]; struct hop_params *params; struct secret *secrets = tal_arr(ctx, struct secret, num_hops); + size_t payloads_size = sphinx_path_payloads_size(sp); + size_t max_prefill = ROUTING_INFO_SIZE - payloads_size; if (sphinx_path_payloads_size(sp) > ROUTING_INFO_SIZE) { tal_free(packet); @@ -435,6 +509,11 @@ struct onionpacket *create_onionpacket( generate_header_padding(filler, sizeof(filler), sp, params); + if (sp->rendezvous_id != NULL) + /* FIXME: Fuzz this or expose to the caller to hide encoded + * route length. */ + sphinx_prefill(packet->routinginfo, sp, max_prefill, params); + for (i = num_hops - 1; i >= 0; i--) { memcpy(sp->hops[i].hmac, nexthmac, HMAC_SIZE); generate_key_set(¶ms[i].secret, &keys); From 9ab41b999e347f00713520fecba77ea0cc70186a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Feb 2020 14:56:59 +0100 Subject: [PATCH 04/25] sphinx: Functions to enable RV mode and serialize compressed onions We will later use these to generate RV compressed onions and to opt into the rendezvous style generation. --- common/sphinx.c | 37 +++++++++++++++++++++++++++++++++++++ common/sphinx.h | 19 +++++++++++++++++++ common/test/run-sphinx.c | 3 +++ 3 files changed, 59 insertions(+) diff --git a/common/sphinx.c b/common/sphinx.c index 9155dd247f02..972d84279719 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -86,6 +86,19 @@ struct sphinx_path *sphinx_path_new_with_key(const tal_t *ctx, return sp; } +bool sphinx_path_set_rendezvous(struct sphinx_path *sp, + const struct node_id *rendezvous_id) +{ + if (rendezvous_id == NULL) { + sp->rendezvous_id = tal_free(sp->rendezvous_id); + return true; + } else { + sp->rendezvous_id = tal_free(sp->rendezvous_id); + sp->rendezvous_id = tal(sp, struct pubkey); + return pubkey_from_node_id(sp->rendezvous_id, rendezvous_id); + } +} + static size_t sphinx_hop_size(const struct sphinx_hop *hop) { return tal_bytelen(hop->raw_payload) + HMAC_SIZE; @@ -143,6 +156,30 @@ u8 *serialize_onionpacket( return dst; } +u8 *serialize_compressed_onion(const tal_t *ctx, + const struct sphinx_path *sp, + const struct onionpacket *packet) +{ + u8 *dst; + u8 der[PUBKEY_CMPR_LEN]; + size_t payloads_size = sphinx_path_payloads_size(sp); + size_t max_prefill = ROUTING_INFO_SIZE - payloads_size; + size_t rv_onion_size = TOTAL_PACKET_SIZE - max_prefill; + int p = 0; + + assert(sp->rendezvous_id != NULL); + + dst = tal_arr(ctx, u8, rv_onion_size); + + pubkey_to_der(der, &packet->ephemeralkey); + write_buffer(dst, &packet->version, 1, &p); + write_buffer(dst, der, sizeof(der), &p); + write_buffer(dst, packet->routinginfo, ROUTING_INFO_SIZE - max_prefill, &p); + write_buffer(dst, packet->mac, sizeof(packet->mac), &p); + return dst; +} + + enum onion_type parse_onionpacket(const u8 *src, const size_t srclen, struct onionpacket *dest) diff --git a/common/sphinx.h b/common/sphinx.h index 045167d123c4..28fd588e186c 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -227,6 +227,25 @@ void sphinx_add_hop(struct sphinx_path *path, const struct pubkey *pubkey, */ size_t sphinx_path_payloads_size(const struct sphinx_path *path); +/** + * Compress a rendez-vous onion by removing the unused blinded middle + * part. This middle part can be regenerated by the node processing this + * onion. + */ +u8 *serialize_compressed_onion(const tal_t *ctx, + const struct sphinx_path *sp, + const struct onionpacket *packet); + +/** + * Set the rendez-vous node_id and make the onion generated from the + * sphinx_path compressible. To unset pass in a NULL rendezvous_id. + * + * Returns false if there was an error converting from the node_id to a public + * key. + */ +bool sphinx_path_set_rendezvous(struct sphinx_path *sp, + const struct node_id *rendezvous_id); + #if DEVELOPER /* Override to force us to reject valid onion packets */ extern bool dev_fail_process_onionpacket; diff --git a/common/test/run-sphinx.c b/common/test/run-sphinx.c index 6499088184b7..66659e2b1030 100644 --- a/common/test/run-sphinx.c +++ b/common/test/run-sphinx.c @@ -45,6 +45,9 @@ size_t bigsize_get(const u8 *p UNNEEDED, size_t max UNNEEDED, bigsize_t *val UNN /* Generated stub for bigsize_put */ size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED) { fprintf(stderr, "bigsize_put called!\n"); abort(); } +/* Generated stub for pubkey_from_node_id */ +bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED) +{ fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); } /* AUTOGENERATED MOCKS END */ extern secp256k1_context *secp256k1_ctx; From 4049d9a17ce2962ce83352ad20a8153843347c01 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Feb 2020 15:12:30 +0100 Subject: [PATCH 05/25] onion: Allow devtool/onion to generate rendezvous onions Adds the `--rendezvous-id` option allowing the caller to specify the node_id of the rendez-vous node, and opting into the compressed onion generation. --- devtools/onion.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/devtools/onion.c b/devtools/onion.c index 5a8d53eaf20f..f58756394df8 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -16,12 +16,14 @@ #include #include #include +#include #include #define ASSOC_DATA_SIZE 32 static void do_generate(int argc, char **argv, - const u8 assocdata[ASSOC_DATA_SIZE]) + const u8 assocdata[ASSOC_DATA_SIZE], + const struct node_id *rvnode_id) { const tal_t *ctx = talz(NULL, tal_t); int num_hops = argc - 2; @@ -36,6 +38,7 @@ static void do_generate(int argc, char **argv, memset(&session_key, 'A', sizeof(struct secret)); sp = sphinx_path_new_with_key(ctx, tmp_assocdata, &session_key); + sphinx_path_set_rendezvous(sp, rvnode_id); for (int i = 0; i < num_hops; i++) { size_t klen = strcspn(argv[2 + i], "/"); @@ -92,6 +95,10 @@ static void do_generate(int argc, char **argv, struct onionpacket *res = create_onionpacket(ctx, sp, &shared_secrets); + if (rvnode_id != NULL) + printf("Rendezvous onion: %s\n", + tal_hex(ctx, serialize_compressed_onion(ctx, sp, res))); + u8 *serialized = serialize_onionpacket(ctx, res); if (!serialized) errx(1, "Error serializing message."); @@ -172,6 +179,12 @@ static void opt_show_ad(char buf[OPT_SHOW_LEN], const u8 *assocdata) hex_encode(assocdata, ASSOC_DATA_SIZE, buf, OPT_SHOW_LEN); } +static char *opt_set_node_id(const char *arg, struct node_id *node_id) +{ + node_id_from_hexstr(arg, strlen(arg), node_id); + return NULL; +} + /** * Run an onion encoding/decoding unit-test from a file */ @@ -301,15 +314,18 @@ int main(int argc, char **argv) setup_locale(); const char *method; u8 assocdata[ASSOC_DATA_SIZE]; + struct node_id rendezvous_id; memset(&assocdata, 'B', sizeof(assocdata)); + memset(&rendezvous_id, 0, sizeof(struct node_id)); secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | SECP256K1_CONTEXT_SIGN); opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn); - opt_register_arg("--assoc-data", opt_set_ad, opt_show_ad, - assocdata, + opt_register_arg("--assoc-data", opt_set_ad, opt_show_ad, assocdata, "Associated data (usu. payment_hash of payment)"); + opt_register_arg("--rendezvous-id", opt_set_node_id, NULL, + &rendezvous_id, "Node ID of the rendez-vous node"); opt_register_noarg("--help|-h", opt_usage_and_exit, "\n\n\tdecode \n" "\tgenerate ...\n" @@ -333,7 +349,10 @@ int main(int argc, char **argv) errx(1, "'runtest' requires a filename argument"); runtest(argv[2]); } else if (streq(method, "generate")) { - do_generate(argc, argv, assocdata); + if (memeqzero(&rendezvous_id, sizeof(rendezvous_id))) + do_generate(argc, argv, assocdata, NULL); + else + do_generate(argc, argv, assocdata, &rendezvous_id); } else if (streq(method, "decode")) { do_decode(argc, argv, assocdata); } else { From 3070cf60af6ad8f07d4e1dc0eebdf3247ac39a2c Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Feb 2020 16:32:11 +0100 Subject: [PATCH 06/25] pytest: Add test for compressed onion This one generates a compressed onion, decompresses it, and then proceeds with normal processing. --- tests/test_onion.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_onion.py b/tests/test_onion.py index 8933f909b5cb..80f85d830f99 100644 --- a/tests/test_onion.py +++ b/tests/test_onion.py @@ -50,6 +50,43 @@ def store_onion(o): assert(out == ['payload=000000000000000000000000000000000400000004000000000000000000000000']) +def test_rendezvous_onion(directory, oniontool): + """Create a compressed onion, decompress it at the RV node and then forward normally. + """ + os.makedirs(directory) + tempfile = os.path.join(directory, 'onion') + out = subprocess.check_output( + [oniontool, '--rendezvous-id', pubkeys[0], 'generate'] + pubkeys + ).decode('ASCII').strip().split('\n') + assert(len(out) == 2) + compressed = out[0].split(' ')[-1] + uncompressed = out[1] + + assert(len(compressed) < len(uncompressed)) + + # Now decompress the onion to get back the original onion + out2 = subprocess.check_output( + [oniontool, 'decompress', privkeys[0], compressed] + ).decode('ASCII').strip().split('\n') + decompressed = out2[-1].split(' ')[-1] + + assert(uncompressed == decompressed) + + # And now just for safety make sure the following nodes can still process + # and forward the onion. + def store_onion(o): + with open(tempfile, 'w') as f: + f.write(o) + + store_onion(decompressed) + + for i, pk in enumerate(privkeys): + out = subprocess.check_output([oniontool, 'decode', tempfile, pk]).decode('ASCII').strip().split('\n') + store_onion(out[-1][5:]) + + assert(out == ['payload=000000000000000000000000000000000400000004000000000000000000000000']) + + def test_onion_vectors(oniontool): tests = [ 'onion-test-multi-frame.json', From 6296915eabb5746d8514e0a051334c2866b785ca Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Feb 2020 17:55:43 +0100 Subject: [PATCH 07/25] sphinx: Add functions to decompress Also implements a way to decompress an onion using the devtools/onion tool Changelog-Added: devtools: The `onion` tool can now generate, compress and decompress onions for rendez-vous routing --- common/sphinx.c | 24 ++++++++++++++++++++++++ common/sphinx.h | 7 +++++++ devtools/onion.c | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/common/sphinx.c b/common/sphinx.c index 972d84279719..e43c01ba97e4 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -785,3 +785,27 @@ u8 *unwrap_onionreply(const tal_t *ctx, return tal_free(final); return final; } + +u8 *sphinx_decompress(const tal_t *ctx, const u8 *compressed, + struct secret *shared_secret) +{ + size_t compressedlen = tal_bytelen(compressed); + size_t prefill_size = TOTAL_PACKET_SIZE - compressedlen; + u8 *dst; + int p = 0; + + assert(prefill_size >= 0); + assert(compressedlen >= VERSION_SIZE + PUBKEY_SIZE + HMAC_SIZE); + dst = tal_arrz(ctx, u8, TOTAL_PACKET_SIZE); + write_buffer( + dst, compressed, + VERSION_SIZE + PUBKEY_SIZE + ROUTING_INFO_SIZE - prefill_size, &p); + + /* We can just XOR here since we initialized the array with zeros. */ + sphinx_prefill_stream_xor(dst + p, prefill_size, shared_secret); + p += prefill_size; + + write_buffer(dst, compressed + compressedlen - HMAC_SIZE, HMAC_SIZE, + &p); + return dst; +} diff --git a/common/sphinx.h b/common/sphinx.h index 28fd588e186c..fe5b946a9848 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -246,6 +246,13 @@ u8 *serialize_compressed_onion(const tal_t *ctx, bool sphinx_path_set_rendezvous(struct sphinx_path *sp, const struct node_id *rendezvous_id); +/** + * Given a compressed onion expand it by re-generating the prefiller and + * inserting it in the appropriate place. + */ +u8 *sphinx_decompress(const tal_t *ctx, const u8 *compressed, + struct secret *shared_secret); + #if DEVELOPER /* Override to force us to reject valid onion packets */ extern bool dev_fail_process_onionpacket; diff --git a/devtools/onion.c b/devtools/onion.c index f58756394df8..b20e33a5159d 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -290,6 +290,30 @@ static void runtest(const char *filename) tal_free(ctx); } +static void decompress(char *hexprivkey, char *hexonion) +{ + struct privkey rendezvous_key; + size_t onionlen = hex_data_size(strlen(hexonion)); + u8 *compressed, *decompressed; + struct pubkey ephkey; + struct secret shared_secret; + + if (!hex_decode(hexprivkey, strlen(hexprivkey), &rendezvous_key, sizeof(rendezvous_key))) + errx(1, "Invalid private key hex '%s'", hexprivkey); + + compressed = tal_arr(NULL, u8, onionlen); + if (!hex_decode(hexonion, strlen(hexonion), compressed, onionlen)) + errx(1, "Invalid onion hex '%s'", hexonion); + + if (onionlen < HMAC_SIZE + 1 + PUBKEY_SIZE) + errx(1, "Onion is too short to contain the version, ephemeral key and HMAC"); + + pubkey_from_der(compressed + 1, PUBKEY_SIZE, &ephkey); + + decompressed = sphinx_decompress(NULL, compressed, &shared_secret); + printf("Decompressed Onion: %s\n", tal_hex(NULL, decompressed)); +} + /* Tal wrappers for opt. */ static void *opt_allocfn(size_t size) { @@ -353,6 +377,15 @@ int main(int argc, char **argv) do_generate(argc, argv, assocdata, NULL); else do_generate(argc, argv, assocdata, &rendezvous_id); + } else if (streq(method, "decompress")) { + if (argc != 4) { + errx(2, + "'%s decompress' requires a private key and a " + "compressed onion", + argv[0]); + } + + decompress(argv[2], argv[3]); } else if (streq(method, "decode")) { do_decode(argc, argv, assocdata); } else { From d829cc54bb73f32d2489aebb684276206dd1c354 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 2 Mar 2020 19:39:16 +0100 Subject: [PATCH 08/25] sphinx: Expose the shared secret creation function --- common/sphinx.c | 18 +++++++++--------- common/sphinx.h | 11 +++++++++++ devtools/onion.c | 19 +++++++++++++++++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/common/sphinx.c b/common/sphinx.c index e43c01ba97e4..6c29972ee92b 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -364,12 +364,12 @@ static bool blind_group_element(struct pubkey *blindedelement, return true; } -static bool create_shared_secret(struct secret *secret, +bool sphinx_create_shared_secret(struct secret *privkey, const struct pubkey *pubkey, - const struct secret *session_key) + const struct secret *secret) { - if (secp256k1_ecdh(secp256k1_ctx, secret->data, &pubkey->pubkey, - session_key->data, NULL, NULL) != 1) + if (secp256k1_ecdh(secp256k1_ctx, privkey->data, &pubkey->pubkey, + secret->data, NULL, NULL) != 1) return false; return true; } @@ -379,8 +379,8 @@ bool onion_shared_secret( const struct onionpacket *packet, const struct privkey *privkey) { - return create_shared_secret(secret, &packet->ephemeralkey, - &privkey->secret); + return sphinx_create_shared_secret(secret, &packet->ephemeralkey, + &privkey->secret); } static void generate_key_set(const struct secret *secret, @@ -408,8 +408,8 @@ static struct hop_params *generate_hop_params( path->session_key->data) != 1) return NULL; - if (!create_shared_secret(¶ms[0].secret, &path->hops[0].pubkey, - path->session_key)) + if (!sphinx_create_shared_secret( + ¶ms[0].secret, &path->hops[0].pubkey, path->session_key)) return NULL; compute_blinding_factor( @@ -491,7 +491,7 @@ static void sphinx_prefill(u8 *routinginfo, const struct sphinx_path *sp, /* Now fill in the obfuscation stream, which can be regenerated by the * node processing this onion. */ - create_shared_secret(&shared_secret, sp->rendezvous_id, sp->session_key); + sphinx_create_shared_secret(&shared_secret, sp->rendezvous_id, sp->session_key); sphinx_prefill_stream_xor(routinginfo + prefill_offset, prefill_size, &shared_secret); } diff --git a/common/sphinx.h b/common/sphinx.h index fe5b946a9848..63189dff4e36 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -253,6 +253,17 @@ bool sphinx_path_set_rendezvous(struct sphinx_path *sp, u8 *sphinx_decompress(const tal_t *ctx, const u8 *compressed, struct secret *shared_secret); +/** + * Use ECDH to generate a shared secret from a privkey and a pubkey. + * + * Sphinx uses shared secrets derived from a private key and a public key + * using ECDH in a number of places. This is a simple wrapper around the + * secp256k1 functions, with our internal types. + */ +bool sphinx_create_shared_secret(struct secret *privkey, + const struct pubkey *pubkey, + const struct secret *secret); + #if DEVELOPER /* Override to force us to reject valid onion packets */ extern bool dev_fail_process_onionpacket; diff --git a/devtools/onion.c b/devtools/onion.c index b20e33a5159d..75039fca210c 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -310,8 +310,23 @@ static void decompress(char *hexprivkey, char *hexonion) pubkey_from_der(compressed + 1, PUBKEY_SIZE, &ephkey); - decompressed = sphinx_decompress(NULL, compressed, &shared_secret); - printf("Decompressed Onion: %s\n", tal_hex(NULL, decompressed)); + tinyonion = sphinx_compressed_onion_deserialize(NULL, compressed); + if (tinyonion == NULL) + errx(1, "Could not deserialize compressed onion"); + + if (!sphinx_create_shared_secret(&shared_secret, + &tinyonion->ephemeralkey, + &rendezvous_key.secret)) + errx(1, + "Could not generate shared secret from ephemeral key %s " + "and private key %s", + pubkey_to_hexstr(NULL, &ephkey), hexprivkey); + + onion = sphinx_decompress(NULL, tinyonion, &shared_secret); + if (onion == NULL) + errx(1, "Could not decompress compressed onion"); + + printf("Decompressed Onion: %s\n", tal_hex(NULL, serialize_onionpacket(NULL, onion))); } /* Tal wrappers for opt. */ From 144192388b62ff476cf49198dfdba0457635b94a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 2 Mar 2020 15:00:25 +0100 Subject: [PATCH 09/25] sphinx: Treat compressed onions as a standalone struct Expands the interface to play with onions a bit more. Potentially a bit slower due to allocations, but that's a small price to pay. It also allows us to avoid serializing a compressed onion to `u8*` if we process it right away. --- common/sphinx.c | 102 +++++++++++++++++++++++++++++++++++++++-------- common/sphinx.h | 27 ++++++++++++- devtools/onion.c | 4 +- 3 files changed, 114 insertions(+), 19 deletions(-) diff --git a/common/sphinx.c b/common/sphinx.c index 6c29972ee92b..1b444117852c 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -786,26 +786,96 @@ u8 *unwrap_onionreply(const tal_t *ctx, return final; } -u8 *sphinx_decompress(const tal_t *ctx, const u8 *compressed, - struct secret *shared_secret) +struct onionpacket *sphinx_decompress(const tal_t *ctx, + const struct sphinx_compressed_onion *src, + const struct secret *shared_secret) { - size_t compressedlen = tal_bytelen(compressed); - size_t prefill_size = TOTAL_PACKET_SIZE - compressedlen; - u8 *dst; + struct onionpacket *res = tal(ctx, struct onionpacket); + size_t srclen = tal_bytelen(src->routinginfo); + size_t prefill_size = ROUTING_INFO_SIZE - srclen; + + res->version = src->version; + res->ephemeralkey = src->ephemeralkey; + memcpy(res->mac, src->mac, HMAC_SIZE); + + /* Decompress routinginfo by copying the unmodified prefix, setting + * the compressed suffix to 0x00 bytes and then xoring the obfuscation + * stream in place. */ + memset(res->routinginfo, 0, ROUTING_INFO_SIZE); + memcpy(res->routinginfo, src->routinginfo, srclen); + sphinx_prefill_stream_xor(res->routinginfo + srclen, prefill_size, + shared_secret); + + return res; +} + +struct sphinx_compressed_onion * +sphinx_compress(const tal_t *ctx, const struct onionpacket *packet, + const struct sphinx_path *path) +{ + struct sphinx_compressed_onion *res; + size_t payloads_size = sphinx_path_payloads_size(path); + + /* We can't compress an onion that doesn't have a rendez-vous node. */ + if (path->rendezvous_id) + return NULL; + + res = tal(ctx, struct sphinx_compressed_onion); + res->version = packet->version; + res->ephemeralkey = packet->ephemeralkey; + memcpy(res->mac, packet->mac, HMAC_SIZE); + + res->routinginfo = tal_arr(res, u8, payloads_size); + memcpy(res->routinginfo, packet->routinginfo, payloads_size); + + return res; +} + +u8 *sphinx_compressed_onion_serialize(const tal_t *ctx, const struct sphinx_compressed_onion *onion) +{ + size_t routelen = tal_bytelen(onion->routinginfo); + size_t len = VERSION_SIZE + PUBKEY_SIZE + routelen + HMAC_SIZE; + u8 *dst = tal_arr(ctx, u8, len); + u8 der[PUBKEY_CMPR_LEN]; + int p = 0; + + pubkey_to_der(der, &onion->ephemeralkey); + + write_buffer(dst, &onion->version, VERSION_SIZE, &p); + write_buffer(dst, der, PUBKEY_SIZE, &p); + write_buffer(dst, onion->routinginfo, routelen, &p); + write_buffer(dst, onion->mac, HMAC_SIZE, &p); + + assert(p == len); + return dst; +} + +struct sphinx_compressed_onion * +sphinx_compressed_onion_deserialize(const tal_t *ctx, const u8 *src) +{ + size_t srclen = tal_bytelen(src); + size_t routelen = srclen - VERSION_SIZE - PUBKEY_SIZE - HMAC_SIZE; + struct sphinx_compressed_onion *dst = + tal(ctx, struct sphinx_compressed_onion); int p = 0; + u8 ephkey[PUBKEY_SIZE]; + + assert(srclen <= TOTAL_PACKET_SIZE); - assert(prefill_size >= 0); - assert(compressedlen >= VERSION_SIZE + PUBKEY_SIZE + HMAC_SIZE); - dst = tal_arrz(ctx, u8, TOTAL_PACKET_SIZE); - write_buffer( - dst, compressed, - VERSION_SIZE + PUBKEY_SIZE + ROUTING_INFO_SIZE - prefill_size, &p); + read_buffer(&dst->version, src, 1, &p); + if (dst->version != 0x00) + return tal_free(dst); + + read_buffer(ephkey, src, PUBKEY_SIZE, &p); + + if (!pubkey_from_der(ephkey, PUBKEY_SIZE, &dst->ephemeralkey)) { + return tal_free(dst); + } - /* We can just XOR here since we initialized the array with zeros. */ - sphinx_prefill_stream_xor(dst + p, prefill_size, shared_secret); - p += prefill_size; + dst->routinginfo = tal_arr(dst, u8, routelen); + read_buffer(dst->routinginfo, src, routelen, &p); + read_buffer(&dst->mac, src, HMAC_SIZE, &p); + assert(p == srclen); - write_buffer(dst, compressed + compressedlen - HMAC_SIZE, HMAC_SIZE, - &p); return dst; } diff --git a/common/sphinx.h b/common/sphinx.h index 63189dff4e36..2f359c123def 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -30,6 +30,14 @@ struct onionpacket { u8 routinginfo[ROUTING_INFO_SIZE]; }; +struct sphinx_compressed_onion { + u8 version; + struct pubkey ephemeralkey; + u8 *routinginfo; + u8 mac[HMAC_SIZE]; +}; + + enum route_next_case { ONION_END = 0, ONION_FORWARD = 1, @@ -250,8 +258,9 @@ bool sphinx_path_set_rendezvous(struct sphinx_path *sp, * Given a compressed onion expand it by re-generating the prefiller and * inserting it in the appropriate place. */ -u8 *sphinx_decompress(const tal_t *ctx, const u8 *compressed, - struct secret *shared_secret); +struct onionpacket *sphinx_decompress(const tal_t *ctx, + const struct sphinx_compressed_onion *src, + const struct secret *shared_secret); /** * Use ECDH to generate a shared secret from a privkey and a pubkey. @@ -264,6 +273,20 @@ bool sphinx_create_shared_secret(struct secret *privkey, const struct pubkey *pubkey, const struct secret *secret); + +/** + * Given a compressible onionpacket, return the compressed version. + */ +struct sphinx_compressed_onion * +sphinx_compress(const tal_t *ctx, const struct onionpacket *packet, + const struct sphinx_path *path); + +u8 *sphinx_compressed_onion_serialize( + const tal_t *ctx, const struct sphinx_compressed_onion *onion); + +struct sphinx_compressed_onion * +sphinx_compressed_onion_deserialize(const tal_t *ctx, const u8 *src); + #if DEVELOPER /* Override to force us to reject valid onion packets */ extern bool dev_fail_process_onionpacket; diff --git a/devtools/onion.c b/devtools/onion.c index 75039fca210c..49008e6ae96b 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -294,9 +294,11 @@ static void decompress(char *hexprivkey, char *hexonion) { struct privkey rendezvous_key; size_t onionlen = hex_data_size(strlen(hexonion)); - u8 *compressed, *decompressed; + u8 *compressed; struct pubkey ephkey; struct secret shared_secret; + struct onionpacket *onion; + struct sphinx_compressed_onion *tinyonion; if (!hex_decode(hexprivkey, strlen(hexprivkey), &rendezvous_key, sizeof(rendezvous_key))) errx(1, "Invalid private key hex '%s'", hexprivkey); From 860abdace1c8fc9c0de94ae0449a0e5b23840840 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 3 Mar 2020 18:15:55 +0100 Subject: [PATCH 10/25] =?UTF-8?q?sphinx:=20Kill=20read=5Fbuffer=20with=20f?= =?UTF-8?q?ire=20=F0=9F=94=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suggested-by: Rusty Russell <@rustyrussell> Signed-off-by: Christian Decker <@cdecker> --- common/sphinx.c | 52 ++++++++++++++++++++----------------------------- common/sphinx.h | 1 - 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/common/sphinx.c b/common/sphinx.c index 1b444117852c..4adea1e0cce7 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -17,6 +17,8 @@ #include #include +#include + #define BLINDING_FACTOR_SIZE 32 #define KEY_LEN 32 @@ -130,15 +132,6 @@ static void write_buffer(u8 *dst, const void *src, const size_t len, int *pos) *pos += len; } -/* Read len bytes from the source at position pos into dst and update - * the position pos accordingly. - */ -static void read_buffer(void *dst, const u8 *src, const size_t len, int *pos) -{ - memcpy(dst, src + *pos, len); - *pos += len; -} - u8 *serialize_onionpacket( const tal_t *ctx, const struct onionpacket *m) @@ -184,25 +177,25 @@ enum onion_type parse_onionpacket(const u8 *src, const size_t srclen, struct onionpacket *dest) { - int p = 0; - u8 rawEphemeralkey[PUBKEY_CMPR_LEN]; + const u8 *cursor = src; + size_t max = srclen; assert(srclen == TOTAL_PACKET_SIZE); - read_buffer(&dest->version, src, 1, &p); + dest->version = fromwire_u8(&cursor, &max); if (dest->version != 0x00) { // FIXME add logging return WIRE_INVALID_ONION_VERSION; } - read_buffer(rawEphemeralkey, src, sizeof(rawEphemeralkey), &p); - if (!pubkey_from_der(rawEphemeralkey, sizeof(rawEphemeralkey), - &dest->ephemeralkey)) { + fromwire_pubkey(&cursor, &max, &dest->ephemeralkey); + if (cursor == NULL) { return WIRE_INVALID_ONION_KEY; } - read_buffer(&dest->routinginfo, src, ROUTING_INFO_SIZE, &p); - read_buffer(&dest->mac, src, HMAC_SIZE, &p); + fromwire_u8_array(&cursor, &max, dest->routinginfo, ROUTING_INFO_SIZE); + fromwire_u8_array(&cursor, &max, dest->mac, HMAC_SIZE); + assert(max == 0); return 0; } @@ -853,29 +846,26 @@ u8 *sphinx_compressed_onion_serialize(const tal_t *ctx, const struct sphinx_comp struct sphinx_compressed_onion * sphinx_compressed_onion_deserialize(const tal_t *ctx, const u8 *src) { - size_t srclen = tal_bytelen(src); - size_t routelen = srclen - VERSION_SIZE - PUBKEY_SIZE - HMAC_SIZE; + const u8 *cursor = src; + size_t max = tal_bytelen(src); + size_t routelen = max - VERSION_SIZE - PUBKEY_SIZE - HMAC_SIZE; struct sphinx_compressed_onion *dst = tal(ctx, struct sphinx_compressed_onion); - int p = 0; - u8 ephkey[PUBKEY_SIZE]; - assert(srclen <= TOTAL_PACKET_SIZE); + /* This is not a compressed onion, so let's not parse it. */ + if (routelen > ROUTING_INFO_SIZE) + return tal_free(dst); - read_buffer(&dst->version, src, 1, &p); + dst->version = fromwire_u8(&cursor, &max); if (dst->version != 0x00) return tal_free(dst); - read_buffer(ephkey, src, PUBKEY_SIZE, &p); - - if (!pubkey_from_der(ephkey, PUBKEY_SIZE, &dst->ephemeralkey)) { - return tal_free(dst); - } + fromwire_pubkey(&cursor, &max, &dst->ephemeralkey); dst->routinginfo = tal_arr(dst, u8, routelen); - read_buffer(dst->routinginfo, src, routelen, &p); - read_buffer(&dst->mac, src, HMAC_SIZE, &p); - assert(p == srclen); + fromwire_u8_array(&cursor, &max, dst->routinginfo, routelen); + fromwire_u8_array(&cursor, &max, dst->mac, HMAC_SIZE); + assert(max == 0); return dst; } diff --git a/common/sphinx.h b/common/sphinx.h index 2f359c123def..dc9caeb2f3b8 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -10,7 +10,6 @@ #include #include #include -#include #define VERSION_SIZE 1 #define REALM_SIZE 1 From 75ef940b09c3cc3783d4c37a48f33a4629d827b5 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 3 Mar 2020 18:32:06 +0100 Subject: [PATCH 11/25] sphinx: Migrate sphinx compression to new interface It also removes the duplicate compression code and serialization code. --- common/sphinx.c | 26 +------------------------- common/sphinx.h | 9 --------- devtools/onion.c | 18 ++++++++++++------ 3 files changed, 13 insertions(+), 40 deletions(-) diff --git a/common/sphinx.c b/common/sphinx.c index 4adea1e0cce7..7b51625e1c38 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -149,30 +149,6 @@ u8 *serialize_onionpacket( return dst; } -u8 *serialize_compressed_onion(const tal_t *ctx, - const struct sphinx_path *sp, - const struct onionpacket *packet) -{ - u8 *dst; - u8 der[PUBKEY_CMPR_LEN]; - size_t payloads_size = sphinx_path_payloads_size(sp); - size_t max_prefill = ROUTING_INFO_SIZE - payloads_size; - size_t rv_onion_size = TOTAL_PACKET_SIZE - max_prefill; - int p = 0; - - assert(sp->rendezvous_id != NULL); - - dst = tal_arr(ctx, u8, rv_onion_size); - - pubkey_to_der(der, &packet->ephemeralkey); - write_buffer(dst, &packet->version, 1, &p); - write_buffer(dst, der, sizeof(der), &p); - write_buffer(dst, packet->routinginfo, ROUTING_INFO_SIZE - max_prefill, &p); - write_buffer(dst, packet->mac, sizeof(packet->mac), &p); - return dst; -} - - enum onion_type parse_onionpacket(const u8 *src, const size_t srclen, struct onionpacket *dest) @@ -810,7 +786,7 @@ sphinx_compress(const tal_t *ctx, const struct onionpacket *packet, size_t payloads_size = sphinx_path_payloads_size(path); /* We can't compress an onion that doesn't have a rendez-vous node. */ - if (path->rendezvous_id) + if (path->rendezvous_id == NULL) return NULL; res = tal(ctx, struct sphinx_compressed_onion); diff --git a/common/sphinx.h b/common/sphinx.h index dc9caeb2f3b8..ccb57cd1b2d8 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -234,15 +234,6 @@ void sphinx_add_hop(struct sphinx_path *path, const struct pubkey *pubkey, */ size_t sphinx_path_payloads_size(const struct sphinx_path *path); -/** - * Compress a rendez-vous onion by removing the unused blinded middle - * part. This middle part can be regenerated by the node processing this - * onion. - */ -u8 *serialize_compressed_onion(const tal_t *ctx, - const struct sphinx_path *sp, - const struct onionpacket *packet); - /** * Set the rendez-vous node_id and make the onion generated from the * sphinx_path compressible. To unset pass in a NULL rendezvous_id. diff --git a/devtools/onion.c b/devtools/onion.c index 49008e6ae96b..e9e13ef0184a 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -32,7 +32,9 @@ static void do_generate(int argc, char **argv, struct secret session_key; struct secret *shared_secrets; struct sphinx_path *sp; - + struct sphinx_compressed_onion *comp; + u8 *serialized; + struct onionpacket *packet; const u8* tmp_assocdata =tal_dup_arr(ctx, u8, assocdata, ASSOC_DATA_SIZE, 0); memset(&session_key, 'A', sizeof(struct secret)); @@ -93,13 +95,17 @@ static void do_generate(int argc, char **argv, } } - struct onionpacket *res = create_onionpacket(ctx, sp, &shared_secrets); + packet = create_onionpacket(ctx, sp, &shared_secrets); - if (rvnode_id != NULL) - printf("Rendezvous onion: %s\n", - tal_hex(ctx, serialize_compressed_onion(ctx, sp, res))); + if (rvnode_id != NULL) { + comp = sphinx_compress(ctx, packet, sp); + serialized = sphinx_compressed_onion_serialize(ctx, comp); + printf("Rendezvous onion: %s\n", tal_hex(ctx, serialized)); + } else { + assert(sphinx_compress(ctx, packet, sp) == NULL); + } - u8 *serialized = serialize_onionpacket(ctx, res); + serialized = serialize_onionpacket(ctx, packet); if (!serialized) errx(1, "Error serializing message."); printf("%s\n", tal_hex(ctx, serialized)); From 71b9da25b44f09a0f619c5bf17a3e9c27e60da4e Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 4 Mar 2020 10:16:35 +0100 Subject: [PATCH 12/25] sphinx: Use fromwire_tal_bytes() to deserialize compressed onions Suggested-by: Rusty Russell <@rustyrussell> Signed-off-by: Christian Decker <@cdecker> --- common/sphinx.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/sphinx.c b/common/sphinx.c index 7b51625e1c38..4c54394ac617 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -824,12 +824,11 @@ sphinx_compressed_onion_deserialize(const tal_t *ctx, const u8 *src) { const u8 *cursor = src; size_t max = tal_bytelen(src); - size_t routelen = max - VERSION_SIZE - PUBKEY_SIZE - HMAC_SIZE; struct sphinx_compressed_onion *dst = tal(ctx, struct sphinx_compressed_onion); /* This is not a compressed onion, so let's not parse it. */ - if (routelen > ROUTING_INFO_SIZE) + if (max > TOTAL_PACKET_SIZE) return tal_free(dst); dst->version = fromwire_u8(&cursor, &max); @@ -837,11 +836,12 @@ sphinx_compressed_onion_deserialize(const tal_t *ctx, const u8 *src) return tal_free(dst); fromwire_pubkey(&cursor, &max, &dst->ephemeralkey); - - dst->routinginfo = tal_arr(dst, u8, routelen); - fromwire_u8_array(&cursor, &max, dst->routinginfo, routelen); + dst->routinginfo = fromwire_tal_bytes(dst, &cursor, &max, max - HMAC_SIZE); fromwire_u8_array(&cursor, &max, dst->mac, HMAC_SIZE); - assert(max == 0); + /* If at any point we failed to pull from the serialized compressed + * onion the entire deserialization is considered to have failed. */ + if (cursor == NULL) + return tal_free(dst); return dst; } From e7dacaeefb414c450143b0da08b56d078bde0c66 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:25:59 +1030 Subject: [PATCH 13/25] devtools/onion: rename --rendezvous-id to --partial-to We're going to use this for messages, not just rendezvous. Signed-off-by: Rusty Russell --- devtools/onion.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/devtools/onion.c b/devtools/onion.c index e9e13ef0184a..fa9966bc8399 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -96,19 +96,22 @@ static void do_generate(int argc, char **argv, } packet = create_onionpacket(ctx, sp, &shared_secrets); + if (!packet) + errx(1, "Onion is too long?"); if (rvnode_id != NULL) { comp = sphinx_compress(ctx, packet, sp); serialized = sphinx_compressed_onion_serialize(ctx, comp); - printf("Rendezvous onion: %s\n", tal_hex(ctx, serialized)); + if (!serialized) + errx(1, "Error serializing partial onion."); } else { assert(sphinx_compress(ctx, packet, sp) == NULL); + serialized = serialize_onionpacket(ctx, packet); + if (!serialized) + errx(1, "Error serializing message."); } - - serialized = serialize_onionpacket(ctx, packet); - if (!serialized) - errx(1, "Error serializing message."); printf("%s\n", tal_hex(ctx, serialized)); + tal_free(ctx); } @@ -371,8 +374,8 @@ int main(int argc, char **argv) opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn); opt_register_arg("--assoc-data", opt_set_ad, opt_show_ad, assocdata, "Associated data (usu. payment_hash of payment)"); - opt_register_arg("--rendezvous-id", opt_set_node_id, NULL, - &rendezvous_id, "Node ID of the rendez-vous node"); + opt_register_arg("--partial-to", opt_set_node_id, NULL, + &rendezvous_id, "Make partial onion for this node id"); opt_register_noarg("--help|-h", opt_usage_and_exit, "\n\n\tdecode \n" "\tgenerate ...\n" From e8bfc2ee1825fe961879bab3d282c33eabd3f196 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:26:59 +1030 Subject: [PATCH 14/25] devtools/onion: allow '-' input file so you can pipe from stdin. This avoid the requirement to use a temporary file. Signed-off-by: Rusty Russell --- devtools/onion.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devtools/onion.c b/devtools/onion.c index fa9966bc8399..11b9b2992b0f 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -148,7 +148,8 @@ static void do_decode(int argc, char **argv, const u8 assocdata[ASSOC_DATA_SIZE] if (argc != 4) opt_usage_exit_fail("Expect an filename and privkey with 'decode' method"); - char *hextemp = grab_file(ctx, argv[2]); + /* "-" means stdin, which is NULL for grab_file */ + char *hextemp = grab_file(ctx, streq(argv[2], "-") ? NULL : argv[2]); size_t hexlen = strlen(hextemp); // trim trailing whitespace From f33b88ac8bf35ef54876b7d035397039171feb05 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:27:59 +1030 Subject: [PATCH 15/25] devtools/onion: change defile assocdata to empty. This is in preparation for messages, which want this as their assocdata. Plus, it's a bit cleaner rather than creating a tmp tal array. Signed-off-by: Rusty Russell --- devtools/onion.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/devtools/onion.c b/devtools/onion.c index 11b9b2992b0f..c31c35c938f6 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -19,10 +19,8 @@ #include #include -#define ASSOC_DATA_SIZE 32 - static void do_generate(int argc, char **argv, - const u8 assocdata[ASSOC_DATA_SIZE], + const u8 *assocdata, const struct node_id *rvnode_id) { const tal_t *ctx = talz(NULL, tal_t); @@ -35,11 +33,10 @@ static void do_generate(int argc, char **argv, struct sphinx_compressed_onion *comp; u8 *serialized; struct onionpacket *packet; - const u8* tmp_assocdata =tal_dup_arr(ctx, u8, assocdata, - ASSOC_DATA_SIZE, 0); + memset(&session_key, 'A', sizeof(struct secret)); - sp = sphinx_path_new_with_key(ctx, tmp_assocdata, &session_key); + sp = sphinx_path_new_with_key(ctx, assocdata, &session_key); sphinx_path_set_rendezvous(sp, rvnode_id); for (int i = 0; i < num_hops; i++) { @@ -139,7 +136,7 @@ static struct route_step *decode_with_privkey(const tal_t *ctx, const u8 *onion, } -static void do_decode(int argc, char **argv, const u8 assocdata[ASSOC_DATA_SIZE]) +static void do_decode(int argc, char **argv, const u8 *assocdata) { const tal_t *ctx = talz(NULL, tal_t); u8 serialized[TOTAL_PACKET_SIZE]; @@ -160,9 +157,7 @@ static void do_decode(int argc, char **argv, const u8 assocdata[ASSOC_DATA_SIZE] errx(1, "Invalid onion hex '%s'", hextemp); } - const u8* tmp_assocdata =tal_dup_arr(ctx, u8, assocdata, - ASSOC_DATA_SIZE, 0); - step = decode_with_privkey(ctx, serialized, tal_strdup(ctx, argv[3]), tmp_assocdata); + step = decode_with_privkey(ctx, serialized, tal_strdup(ctx, argv[3]), assocdata); if (!step || !step->next) errx(1, "Error processing message."); @@ -177,16 +172,17 @@ static void do_decode(int argc, char **argv, const u8 assocdata[ASSOC_DATA_SIZE] tal_free(ctx); } -static char *opt_set_ad(const char *arg, u8 *assocdata) +static char *opt_set_ad(const char *arg, u8 **assocdata) { - if (!hex_decode(arg, strlen(arg), assocdata, ASSOC_DATA_SIZE)) + *assocdata = tal_hexdata(NULL, arg, strlen(arg)); + if (!*assocdata) return "Bad hex string"; return NULL; } -static void opt_show_ad(char buf[OPT_SHOW_LEN], const u8 *assocdata) +static void opt_show_ad(char buf[OPT_SHOW_LEN], u8 *const *assocdata) { - hex_encode(assocdata, ASSOC_DATA_SIZE, buf, OPT_SHOW_LEN); + hex_encode(*assocdata, tal_bytelen(*assocdata), buf, OPT_SHOW_LEN); } static char *opt_set_node_id(const char *arg, struct node_id *node_id) @@ -364,16 +360,15 @@ int main(int argc, char **argv) { setup_locale(); const char *method; - u8 assocdata[ASSOC_DATA_SIZE]; + u8 *assocdata = NULL; struct node_id rendezvous_id; - memset(&assocdata, 'B', sizeof(assocdata)); memset(&rendezvous_id, 0, sizeof(struct node_id)); secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | SECP256K1_CONTEXT_SIGN); opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn); - opt_register_arg("--assoc-data", opt_set_ad, opt_show_ad, assocdata, + opt_register_arg("--assoc-data", opt_set_ad, opt_show_ad, &assocdata, "Associated data (usu. payment_hash of payment)"); opt_register_arg("--partial-to", opt_set_node_id, NULL, &rendezvous_id, "Make partial onion for this node id"); From 1267d9737c4cad3801af89f3d77dbb8754ef18d9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:28:59 +1030 Subject: [PATCH 16/25] tests: note the private keys of our test nodes. I needed them to debug the onion messages API, so might as well record them somewhere. Signed-off-by: Rusty Russell --- tests/test_misc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_misc.py b/tests/test_misc.py index 1e03ae0e96a1..e82935ff0e03 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -51,6 +51,11 @@ def test_stop_pending_fundchannel(node_factory, executor): def test_names(node_factory): + # Note: + # private keys: + # l1: 41bfd2660762506c9933ade59f1debf7e6495b10c14a92dbcd2d623da2507d3d01, + # l2: c4a813f81ffdca1da6864db81795ad2d320add274452cafa1fb2ac2d07d062bd01 + # l3: dae24b3853e1443a176daba5544ee04f7db33ebe38e70bdfdb1da34e89512c1001 configs = [ ('0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518', 'JUNIORBEAM', '0266e4'), ('022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59', 'SILENTARTIST', '022d22'), From 79ed54f89fdc0f7284ccd868e94b0dc4d68749e1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:29:22 +1030 Subject: [PATCH 17/25] sphinx: add (unused) infrastructure for end-to-end payload. The Sphinx paper calls this "payload", but we've already used that liberally in "per-hop payload": The payload of the message is kept separate from the mix header used to perform the routing. It is decrypted at each stage of mixing using a block cipher with a large block size (the size of the entire message), such as LIONESS [1]. In case the adversary modifies the payload in transit, any information contained in it becomes irrecoverable. Sender-anonymous messages contain the final address of the message, as well as the message itself as part of the payload, and so any modification destroys this information. Since we don't want to add a block cypher, we use chacha20poly1305 with the shared secret as the key instead. This is described in a BOLT proposal: https://github.com/lightningnetwork/lightning-rfc/pull/755 Signed-off-by: Rusty Russell --- common/sphinx.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ common/sphinx.h | 40 ++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/common/sphinx.c b/common/sphinx.c index 4c54394ac617..5bef2c06f19b 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include #include @@ -14,6 +16,7 @@ #include +#include #include #include @@ -550,6 +553,94 @@ struct onionpacket *create_onionpacket( return packet; } +u8 *create_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *secret) +{ + u8 *enc; + + /* Sphinx paper uses a block cypher here, with the first + * bytes being the receivers' key. We use AD instead, + * since we already rely on that. */ + static unsigned char npub[crypto_aead_chacha20poly1305_ietf_NPUBBYTES]; + unsigned long long clen; + u8 key[crypto_aead_chacha20poly1305_IETF_KEYBYTES]; + int ret; + size_t e2e_len = tal_bytelen(e2e_payload); + + BUILD_ASSERT(sizeof(key) == KEY_LEN); + + /* If taken, reuse in-place */ + if (taken(e2e_payload)) { + tal_resize(&e2e_payload, + e2e_len + crypto_aead_chacha20poly1305_IETF_ABYTES); + enc = cast_const(u8 *, e2e_payload); + } else { + enc = tal_arr(ctx, u8, + e2e_len + crypto_aead_chacha20poly1305_IETF_ABYTES); + } + + generate_key(key, "pi", 2, secret); + + ret = crypto_aead_chacha20poly1305_ietf_encrypt(enc, &clen, + e2e_payload, + e2e_len, + NULL, 0, + NULL, npub, key); + assert(ret == 0); + assert(clen == tal_bytelen(enc)); + + return enc; +} + +u8 *unwrap_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *shared_secret) +{ + u8 key[KEY_LEN]; + u8 *dec = tal_dup_talarr(ctx, u8, e2e_payload); + + generate_key(key, "pi", 2, shared_secret); + xor_cipher_stream(dec, key, tal_bytelen(dec)); + return dec; +} + +u8 *final_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *shared_secret) +{ + /* Final hop needs to check decryption */ + u8 key[KEY_LEN]; + unsigned long long dlen; + static unsigned char npub[crypto_aead_chacha20poly1305_ietf_NPUBBYTES]; + u8 *ret; + + if (tal_bytelen(e2e_payload) < crypto_aead_chacha20poly1305_IETF_ABYTES) { + if (taken(e2e_payload)) + tal_free(e2e_payload); + return NULL; + } + + if (taken(e2e_payload)) + ret = cast_const(u8 *, tal_steal(ctx, e2e_payload)); + else + ret = tal_arr(ctx, u8, tal_bytelen(e2e_payload) - crypto_aead_chacha20poly1305_IETF_ABYTES); + + generate_key(key, "pi", 2, shared_secret); + if (crypto_aead_chacha20poly1305_ietf_decrypt(ret, &dlen, + NULL, + e2e_payload, + tal_bytelen(e2e_payload), + NULL, 0, + npub, key) != 0) + return tal_free(ret); + + /* Trim to length if we are reusing buffer. */ + if (ret == e2e_payload) + tal_resize(&ret, dlen); + return ret; +} + #if DEVELOPER bool dev_fail_process_onionpacket; #endif diff --git a/common/sphinx.h b/common/sphinx.h index ccb57cd1b2d8..40ce380faaa8 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -204,6 +204,46 @@ u8 *unwrap_onionreply(const tal_t *ctx, const struct onionreply *reply, int *origin_index); +/** + * create_e2e_payload - Encrypt end-to-end payload for onion. + * + * @ctx: tal context to allocate from + * @e2e_payload: tal_bytelen(@e2e_payload) of payload (copied unless taken()) + * @shared_secrets: shared secret for recipient + */ +u8 *create_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *shared_secret); + +/** + * unwrap_e2e_payload - process an incoming end-to-end-paylod by stripping one + * + * @ctx: tal context to allocate from + * @e2e_payload: tal_bytelen(@e2e_payload) of payload (copied unless taken()) + * @shared_secret: the result of onion_shared_secret. + * + * Never fails. Note that this is *not* to be used for the final + * destination: see final_e2e_payload. + */ +u8 *unwrap_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *shared_secret); + +/* This is perfectly symmetrical */ +#define wrap_e2e_payload unwrap_e2e_payload + +/** + * final_e2e_payload - process a final end-to-end payload + * + * @ctx: tal context to allocate from + * @e2e_payload: tal_bytelen(@e2e_payload) of payload (copied unless taken()) + * @shared_secret: the result of onion_shared_secret. + * + * Returns NULL if it was corrupt */ +u8 *final_e2e_payload(const tal_t *ctx, + const u8 *e2e_payload TAKES, + const struct secret *shared_secret); + /** * Create a new empty sphinx_path. * From c224ccabba7b93fb835d04f5ca09c7c2009888d7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:30:22 +1030 Subject: [PATCH 18/25] common/sphinx: add realm flag so we can avoid legacy parsing. For messages, we use the onion but payload lengths 0 and 1 aren't special. Create a flag to disable that logic. Signed-off-by: Rusty Russell --- common/onion.c | 20 +++++++++++++------- common/onion.h | 2 ++ common/sphinx.c | 5 +++-- common/sphinx.h | 4 +++- devtools/onion.c | 4 ++-- lightningd/peer_htlcs.c | 2 +- wallet/test/run-wallet.c | 3 ++- 7 files changed, 26 insertions(+), 14 deletions(-) diff --git a/common/onion.c b/common/onion.c index 4b0f57cc777d..c74c9a1b26dc 100644 --- a/common/onion.c +++ b/common/onion.c @@ -143,6 +143,7 @@ u8 *onion_final_hop(const tal_t *ctx, /* Returns true if valid, and fills in type. */ static bool pull_payload_length(const u8 **cursor, size_t *max, + bool has_realm, enum onion_payload_type *type, size_t *len) { @@ -163,7 +164,7 @@ static bool pull_payload_length(const u8 **cursor, * length. In this case the `hop_payload_length` is defined to be 32 * bytes. */ - if (*len == 0) { + if (has_realm && *len == 0) { if (type) *type = ONION_V0_PAYLOAD; assert(*cursor - start == 1); @@ -176,10 +177,15 @@ static bool pull_payload_length(const u8 **cursor, * case the `hop_payload_length` is equal to the numeric value of * `length`. */ - if (*len > 1) { + if (!has_realm || *len > 1) { /* It's still invalid if it claims to be too long! */ - if (*len > ROUTING_INFO_SIZE - HMAC_SIZE) - return false; + if (has_realm) { + if (*len > ROUTING_INFO_SIZE - HMAC_SIZE) + return false; + } else { + if (*len > *max) + return false; + } if (type) *type = ONION_TLV_PAYLOAD; @@ -190,12 +196,12 @@ static bool pull_payload_length(const u8 **cursor, return false; } -size_t onion_payload_length(const u8 *raw_payload, size_t len, +size_t onion_payload_length(const u8 *raw_payload, size_t len, bool has_realm, bool *valid, enum onion_payload_type *type) { size_t max = len, payload_len; - *valid = pull_payload_length(&raw_payload, &max, type, &payload_len); + *valid = pull_payload_length(&raw_payload, &max, has_realm, type, &payload_len); /* If it's not valid, copy the entire thing. */ if (!*valid) @@ -214,7 +220,7 @@ struct onion_payload *onion_decode(const tal_t *ctx, size_t max = tal_bytelen(cursor), len; struct tlv_tlv_payload *tlv; - if (!pull_payload_length(&cursor, &max, &p->type, &len)) + if (!pull_payload_length(&cursor, &max, true, &p->type, &len)) return tal_free(p); switch (p->type) { diff --git a/common/onion.h b/common/onion.h index 1ec8d4eec4b1..7f7c38feb387 100644 --- a/common/onion.h +++ b/common/onion.h @@ -39,6 +39,7 @@ u8 *onion_final_hop(const tal_t *ctx, * onion_payload_length: measure payload length in decrypted onion. * @raw_payload: payload to look at. * @len: length of @raw_payload in bytes. + * @has_realm: used for HTLCs, where first byte 0 is magical. * @valid: set to true if it is valid, false otherwise. * @type: if non-NULL, set to type of payload if *@valid is true. * @@ -47,6 +48,7 @@ u8 *onion_final_hop(const tal_t *ctx, * the return value is @len (i.e. the entire payload). */ size_t onion_payload_length(const u8 *raw_payload, size_t len, + bool has_realm, bool *valid, enum onion_payload_type *type); diff --git a/common/sphinx.c b/common/sphinx.c index 5bef2c06f19b..787c65ba1118 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -654,7 +654,8 @@ struct route_step *process_onionpacket( const struct onionpacket *msg, const struct secret *shared_secret, const u8 *assocdata, - const size_t assocdatalen + const size_t assocdatalen, + bool has_realm ) { struct route_step *step = talz(ctx, struct route_step); @@ -687,7 +688,7 @@ struct route_step *process_onionpacket( if (!blind_group_element(&step->next->ephemeralkey, &msg->ephemeralkey, blind)) return tal_free(step); - payload_size = onion_payload_length(paddedheader, ROUTING_INFO_SIZE, + payload_size = onion_payload_length(paddedheader, ROUTING_INFO_SIZE, has_realm, &valid, NULL); /* Can't decode? Treat it as terminal. */ diff --git a/common/sphinx.h b/common/sphinx.h index 40ce380faaa8..a90f379883cd 100644 --- a/common/sphinx.h +++ b/common/sphinx.h @@ -133,13 +133,15 @@ bool onion_shared_secret( * @hoppayload: the per-hop payload destined for the processing node. * @assocdata: associated data to commit to in HMACs * @assocdatalen: length of the assocdata + * @has_realm: used for HTLCs, where first byte 0 is magical. */ struct route_step *process_onionpacket( const tal_t * ctx, const struct onionpacket *packet, const struct secret *shared_secret, const u8 *assocdata, - const size_t assocdatalen + const size_t assocdatalen, + bool has_realm ); /** diff --git a/devtools/onion.c b/devtools/onion.c index c31c35c938f6..4eeb94a217e5 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -131,7 +131,7 @@ static struct route_step *decode_with_privkey(const tal_t *ctx, const u8 *onion, errx(1, "Error creating shared secret."); step = process_onionpacket(ctx, &packet, &shared_secret, assocdata, - tal_bytelen(assocdata)); + tal_bytelen(assocdata), true); return step; } @@ -285,7 +285,7 @@ static void runtest(const char *filename) errx(1, "Error serializing message."); onion_payload_length(step->raw_payload, tal_bytelen(step->raw_payload), - &valid, &type); + true, &valid, &type); assert(valid); printf(" Type: %d\n", type); printf(" Payload: %s\n", tal_hex(ctx, step->raw_payload)); diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index cfed09ca0248..a5a9994e0427 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -1108,7 +1108,7 @@ static bool peer_accepted_htlc(const tal_t *ctx, rs = process_onionpacket(tmpctx, &op, hin->shared_secret, hin->payment_hash.u.u8, - sizeof(hin->payment_hash)); + sizeof(hin->payment_hash), true); if (!rs) { *badonion = WIRE_INVALID_ONION_HMAC; log_debug(channel->log, diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index b555bc7e1747..bd185443c3d1 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -550,7 +550,8 @@ struct route_step *process_onionpacket( const struct onionpacket *packet UNNEEDED, const struct secret *shared_secret UNNEEDED, const u8 *assocdata UNNEEDED, - const size_t assocdatalen + const size_t assocdatalen UNNEEDED, + bool has_realm ) { fprintf(stderr, "process_onionpacket called!\n"); abort(); } /* Generated stub for serialize_onionpacket */ From 0163b694372eb25ae2d5109fe9b9b940b5c8cb21 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:22 +1030 Subject: [PATCH 19/25] devtools/onion: print shared secrets. We'll need these to encrypt the end-to-end payload. Signed-off-by: Rusty Russell --- devtools/onion.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/devtools/onion.c b/devtools/onion.c index 4eeb94a217e5..8c194ba9ce1c 100644 --- a/devtools/onion.c +++ b/devtools/onion.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -109,6 +110,10 @@ static void do_generate(int argc, char **argv, } printf("%s\n", tal_hex(ctx, serialized)); + printf("secrets:"); + for (int i = 0; i < num_hops; i++) + printf(" %s", type_to_string(ctx, struct secret, &shared_secrets[i])); + printf("\n"); tal_free(ctx); } From 98c5a43284c0f0be0064957a8d9f30f7877ac71c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 20/25] devtools/onionmessage: new tool for encrypting/decrypting payloads. Signed-off-by: Rusty Russell --- devtools/Makefile | 6 ++- devtools/onionmessage.c | 89 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 devtools/onionmessage.c diff --git a/devtools/Makefile b/devtools/Makefile index 7a8bbb22c383..4704b95e3992 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -1,6 +1,6 @@ DEVTOOLS_SRC := devtools/gen_print_wire.c devtools/gen_print_onion_wire.c devtools/print_wire.c DEVTOOLS_OBJS := $(DEVTOOLS_SRC:.c=.o) -DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/checkchannels devtools/mkquery devtools/lightning-checkmessage +DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/onionmessage devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/checkchannels devtools/mkquery devtools/lightning-checkmessage DEVTOOLS_TOOL_SRC := $(DEVTOOLS:=.c) DEVTOOLS_TOOL_OBJS := $(DEVTOOLS_TOOL_SRC:.c=.o) @@ -65,6 +65,10 @@ devtools/onion.c: ccan/config.h devtools/onion: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o +devtools/onionmessage.c: ccan/config.h + +devtools/onionmessage: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/onionmessage.o common/sphinx.o + devtools/gossipwith: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/gen_peer_wire.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o common/crypto_sync.o $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h devtools/gen_print_wire.h devtools/gen_print_onion_wire.h diff --git a/devtools/onionmessage.c b/devtools/onionmessage.c new file mode 100644 index 000000000000..5eb76c8e9dc4 --- /dev/null +++ b/devtools/onionmessage.c @@ -0,0 +1,89 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Tal wrappers for opt. */ +static void *opt_allocfn(size_t size) +{ + return tal_arr_label(NULL, char, size, TAL_LABEL("opt_allocfn", "")); +} + +static void *tal_reallocfn(void *ptr, size_t size) +{ + if (!ptr) + return opt_allocfn(size); + tal_resize_(&ptr, 1, size, false); + return ptr; +} + +static void tal_freefn(void *ptr) +{ + tal_free(ptr); +} + +int main(int argc, char **argv) +{ + setup_locale(); + const char *method; + u8 *e2e; + struct secret *secrets; + size_t num_secrets; + + secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | + SECP256K1_CONTEXT_SIGN); + + opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn); + opt_register_noarg("--help|-h", opt_usage_and_exit, + "\n\n\tencrypt ...\n" + "\t[un]wrap ...\n" + "\tdecrypt ...", "Show this message"); + opt_register_version(); + + opt_parse(&argc, argv, opt_log_stderr_exit); + + if (argc < 4) + errx(1, "You must specify a method, payload and at least 1 secret"); + + e2e = tal_hexdata(NULL, argv[2], strlen(argv[2])); + if (!e2e) + errx(1, "Bad hex data %s", argv[2]); + + num_secrets = argc - 3; + secrets = tal_arr(NULL, struct secret, num_secrets); + for (size_t i = 0; i < num_secrets; i++) { + if (!hex_decode(argv[3+i], strlen(argv[3+i]), + &secrets[i], sizeof(secrets[i]))) + errx(1, "Bad hex secret %s", argv[3+i]); + } + + method = argv[1]; + if (streq(method, "encrypt")) { + e2e = create_e2e_payload(NULL, e2e, &secrets[num_secrets-1]); + for (int i = num_secrets - 2; i >= 0; i--) + e2e = wrap_e2e_payload(NULL, take(e2e), &secrets[i]); + } else if (streq(method, "wrap") || streq(method, "unwrap")) { + for (size_t i = 0; i < num_secrets; i++) + e2e = unwrap_e2e_payload(NULL, take(e2e), &secrets[i]); + } else if (streq(method, "decrypt")) { + for (size_t i = 0; i < num_secrets - 1; i++) + e2e = unwrap_e2e_payload(NULL, take(e2e), &secrets[i]); + e2e = final_e2e_payload(NULL, e2e, &secrets[num_secrets-1]); + if (!e2e) + errx(1, "decryption failed"); + } else + errx(1, "Unknown method %s", method); + + printf("%s\n", tal_hex(NULL, e2e)); + return 0; +} From c74944efa8f2c51ecb50f75bcdea003c30a849f4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 21/25] channeld: messages to handle 'onion message' EXPERIMENTAL_FEATURES Signed-off-by: Rusty Russell --- channeld/channel_wire.csv | 24 +++ channeld/channeld.c | 177 ++++++++++++++++++ gossipd/gossipd.c | 3 + lightningd/channel_control.c | 5 + ...l_daa0a499c300056dea24a3afd24681c310e1904c | 17 ++ ...l_daa0a499c300056dea24a3afd24681c310e1904c | 10 + wire/peer_wire.c | 6 + 7 files changed, 242 insertions(+) create mode 100644 wire/extracted_onion_experimental_daa0a499c300056dea24a3afd24681c310e1904c create mode 100644 wire/extracted_peer_experimental_daa0a499c300056dea24a3afd24681c310e1904c diff --git a/channeld/channel_wire.csv b/channeld/channel_wire.csv index 6aed51038d05..c1375c21317c 100644 --- a/channeld/channel_wire.csv +++ b/channeld/channel_wire.csv @@ -2,6 +2,7 @@ #include #include #include +#include #include # Begin! (passes gossipd-client fd) @@ -206,3 +207,26 @@ msgdata,channel_send_error,reason,wirestring, # Tell master channeld has sent the error message. msgtype,channel_send_error_reply,1108 + +# Tell lightningd we got a onion message (for us, or to fwd) +msgtype,got_onionmsg_to_us,1142 +msgdata,got_onionmsg_to_us,next_scid,?short_channel_id, +msgdata,got_onionmsg_to_us,next_node_id,?node_id, +msgdata,got_onionmsg_to_us,next_onion_len,u16, +msgdata,got_onionmsg_to_us,next_onion,u8,next_onion_len +msgdata,got_onionmsg_to_us,shared_secret,secret, +msgdata,got_onionmsg_to_us,e2e_len,u16, +msgdata,got_onionmsg_to_us,e2e_payload,u8,e2e_len + +msgtype,got_onionmsg_forward,1143 +msgdata,got_onionmsg_forward,next_scid,?short_channel_id, +msgdata,got_onionmsg_forward,next_node_id,?node_id, +msgdata,got_onionmsg_forward,onion,u8,1366 +msgdata,got_onionmsg_forward,len,u16, +msgdata,got_onionmsg_forward,e2e_payload,u8,len + +# Lightnignd tells us to send a onion message. +msgtype,send_onionmsg,1040 +msgdata,send_onionmsg,onion,u8,1366 +msgdata,send_onionmsg,len,u16, +msgdata,send_onionmsg,e2e_payload,u8,len diff --git a/channeld/channeld.c b/channeld/channeld.c index 9dc7ae3ac1b0..798fa2ed0b46 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -1622,6 +1622,167 @@ static bool channeld_handle_custommsg(const u8 *msg) #endif } +#if EXPERIMENTAL_FEATURES +static u8 *unpack_onion(const tal_t *ctx, const u8 *partial_onion) +{ + struct secret ss; + struct sphinx_compressed_onion *c; + const u8 *msg; + + c = sphinx_compressed_onion_deserialize(tmpctx, partial_onion); + if (!c) { + status_debug("onion msg: invalid next_onion %s", + tal_hex(tmpctx, partial_onion)); + return NULL; + } + + /* We need to ecdh again to get shared secret here. + * FIXME: Can't we share? */ + msg = hsm_req(tmpctx, towire_hsm_ecdh_req(tmpctx, &c->ephemeralkey)); + if (!fromwire_hsm_ecdh_resp(msg, &ss)) + status_failed(STATUS_FAIL_HSM_IO, "Reading ecdh response"); + + return serialize_onionpacket(ctx, sphinx_decompress(tmpctx, c, &ss)); +} + +/* Peer sends onion msg. */ +static void handle_onionmsg(struct peer *peer, const u8 *msg) +{ + enum onion_type badreason; + struct onionpacket op; + struct secret ss; + struct route_step *rs; + struct sha256 hash; + u8 onion[TOTAL_PACKET_SIZE]; + const u8 *cursor; + u8 *e2e_payload, *next_onion; + size_t max, maxlen; + struct tlv_tlv_dm_payload *dm; + const struct short_channel_id *next_scid; + struct node_id *next_node; + + if (!fromwire_onionmsg(msg, msg, onion, &e2e_payload)) + peer_failed(peer->pps, + &peer->channel_id, + "Bad onionmsg %s", tal_hex(peer, msg)); + + sha256(&hash, onion, sizeof(onion)); + + /* We unwrap the onion now. */ + badreason = parse_onionpacket(onion, TOTAL_PACKET_SIZE, &op); + if (badreason != 0) { + status_debug("onion msg: can't parse onionpacket: %s", + onion_type_name(badreason)); + return; + } + + /* Because wire takes struct pubkey. */ + msg = hsm_req(tmpctx, towire_hsm_ecdh_req(tmpctx, &op.ephemeralkey)); + if (!fromwire_hsm_ecdh_resp(msg, &ss)) + status_failed(STATUS_FAIL_HSM_IO, "Reading ecdh response"); + + /* We make sure we can parse onion packet, so we know if shared secret + * is actually valid (this checks hmac). */ + rs = process_onionpacket(tmpctx, &op, &ss, NULL, 0, false); + if (!rs) { + status_debug("onion msg: can't process onionpacket ss=%s", + type_to_string(tmpctx, struct secret, &ss)); + return; + } + + /* The raw payload is prepended with length in the TLV world. */ + cursor = rs->raw_payload; + max = tal_bytelen(rs->raw_payload); + maxlen = fromwire_bigsize(&cursor, &max); + if (!cursor) { + status_debug("onion msg: Invalid hop payload %s", + tal_hex(tmpctx, rs->raw_payload)); + return; + } + if (maxlen > max) { + status_debug("onion msg: overlong hop payload %s", + tal_hex(tmpctx, rs->raw_payload)); + return; + } + + dm = tlv_tlv_dm_payload_new(msg); + if (!fromwire_tlv_dm_payload(&cursor, &maxlen, dm)) { + status_debug("onion msg: invalid dm_payload %s", + tal_hex(tmpctx, rs->raw_payload)); + return; + } + + if (dm->next_short_channel_id) + next_scid = &dm->next_short_channel_id->short_channel_id; + else + next_scid = NULL; + + if (dm->next_node_id) { + next_node = tal(msg, struct node_id); + node_id_from_pubkey(next_node, &dm->next_node_id->node_id); + } else + next_node = NULL; + + if (dm->forward_onion) { + /* If this has a forward onion, we use that, and forward */ + next_onion = unpack_onion(msg, dm->forward_onion->partial_onion); + rs->nextcase = ONION_FORWARD; + } else + next_onion = NULL; + + if (rs->nextcase == ONION_END) { + u8 *reply_onion; + if (dm->reply_onion) + reply_onion = unpack_onion(msg, dm->reply_onion->partial_onion); + else + reply_onion = NULL; + + /* payload may not be valid, so we hand it pre-decrypted to lightningd */ + wire_sync_write(MASTER_FD, + take(towire_got_onionmsg_to_us(NULL, + next_scid, + next_node, + reply_onion, + &ss, + e2e_payload))); + } else { + e2e_payload = unwrap_e2e_payload(tmpctx, e2e_payload, &ss); + + /* If they don't override, we just send next onion */ + if (!next_onion) + next_onion = serialize_onionpacket(tmpctx, rs->next); + + /* This *MUST* have instructions on where to go next. */ + if (!next_scid && !next_node) { + status_debug("onion msg: no next field in %s", + tal_hex(tmpctx, rs->raw_payload)); + return; + } + + wire_sync_write(MASTER_FD, + take(towire_got_onionmsg_forward(NULL, + next_scid, + next_node, + next_onion, + e2e_payload))); + } +} + +/* We send onion msg. */ +static void send_onionmsg(struct peer *peer, const u8 *msg) +{ + u8 onion_routing_packet[TOTAL_PACKET_SIZE]; + u8 *e2e_payload; + + if (!fromwire_send_onionmsg(msg, msg, + onion_routing_packet, &e2e_payload)) + master_badmsg(WIRE_SEND_ONIONMSG, msg); + + sync_crypto_write(peer->pps, + take(towire_onionmsg(NULL, onion_routing_packet, e2e_payload))); +} +#endif /* EXPERIMENTAL_FEATURES */ + static void peer_in(struct peer *peer, const u8 *msg) { enum wire_type type = fromwire_peektype(msg); @@ -1693,6 +1854,11 @@ static void peer_in(struct peer *peer, const u8 *msg) case WIRE_SHUTDOWN: handle_peer_shutdown(peer, msg); return; +#if EXPERIMENTAL_FEATURES + case WIRE_ONIONMSG: + handle_onionmsg(peer, msg); + return; +#endif case WIRE_INIT: case WIRE_OPEN_CHANNEL: @@ -2681,6 +2847,15 @@ static void req_in(struct peer *peer, const u8 *msg) case WIRE_CHANNEL_SEND_ERROR: handle_send_error(peer, msg); return; +#if EXPERIMENTAL_FEATURES + case WIRE_SEND_ONIONMSG: + send_onionmsg(peer, msg); + return; +#else + case WIRE_SEND_ONIONMSG: + break; +#endif /* !EXPERIMENTAL_FEATURES */ + #if DEVELOPER case WIRE_CHANNEL_DEV_REENABLE_COMMIT: handle_dev_reenable_commit(peer); @@ -2708,6 +2883,8 @@ static void req_in(struct peer *peer, const u8 *msg) case WIRE_CHANNEL_FAIL_FALLEN_BEHIND: case WIRE_CHANNEL_DEV_MEMLEAK_REPLY: case WIRE_CHANNEL_SEND_ERROR_REPLY: + case WIRE_GOT_ONIONMSG_TO_US: + case WIRE_GOT_ONIONMSG_FORWARD: break; } diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index 882cbbed5070..8886b92e8af3 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -488,6 +488,9 @@ static struct io_plan *peer_msg_in(struct io_conn *conn, case WIRE_CHANNEL_REESTABLISH: case WIRE_ANNOUNCEMENT_SIGNATURES: case WIRE_GOSSIP_TIMESTAMP_FILTER: +#if EXPERIMENTAL_FEATURES + case WIRE_ONIONMSG: +#endif status_broken("peer %s: relayed unexpected msg of type %s", type_to_string(tmpctx, struct node_id, &peer->id), wire_type_name(fromwire_peektype(msg))); diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index d5c075528a62..3712e8ab7f94 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -319,6 +319,10 @@ static unsigned channel_msg(struct subd *sd, const u8 *msg, const int *fds) case WIRE_CHANNEL_SEND_ERROR_REPLY: handle_error_channel(sd->channel, msg); break; + case WIRE_GOT_ONIONMSG_TO_US: + case WIRE_GOT_ONIONMSG_FORWARD: + /* FIXME */ + break; /* And we never get these from channeld. */ case WIRE_CHANNEL_INIT: @@ -334,6 +338,7 @@ static unsigned channel_msg(struct subd *sd, const u8 *msg, const int *fds) case WIRE_CHANNEL_FEERATES: case WIRE_CHANNEL_SPECIFIC_FEERATES: case WIRE_CHANNEL_DEV_MEMLEAK: + case WIRE_SEND_ONIONMSG: /* Replies go to requests. */ case WIRE_CHANNEL_OFFER_HTLC_REPLY: case WIRE_CHANNEL_DEV_REENABLE_COMMIT_REPLY: diff --git a/wire/extracted_onion_experimental_daa0a499c300056dea24a3afd24681c310e1904c b/wire/extracted_onion_experimental_daa0a499c300056dea24a3afd24681c310e1904c new file mode 100644 index 000000000000..1ed077dcc1f4 --- /dev/null +++ b/wire/extracted_onion_experimental_daa0a499c300056dea24a3afd24681c310e1904c @@ -0,0 +1,17 @@ +--- wire/extracted_onion_wire_csv 2020-02-25 05:52:39.612291156 +1030 ++++ - 2020-03-04 16:26:35.787772217 +1030 +@@ -8,6 +8,14 @@ + tlvtype,tlv_payload,payment_data,8 + tlvdata,tlv_payload,payment_data,payment_secret,byte,32 + tlvdata,tlv_payload,payment_data,total_msat,tu64, ++tlvtype,tlv_dm_payload,next_short_channel_id,2 ++tlvdata,tlv_dm_payload,next_short_channel_id,short_channel_id,short_channel_id, ++tlvtype,tlv_dm_payload,next_node_id,4 ++tlvdata,tlv_dm_payload,next_node_id,node_id,point, ++tlvtype,tlv_dm_payload,forward_onion,6 ++tlvdata,tlv_dm_payload,forward_onion,partial_onion,byte,... ++tlvtype,tlv_dm_payload,reply_onion,8 ++tlvdata,tlv_dm_payload,reply_onion,partial_onion,byte,... + msgtype,invalid_realm,PERM|1 + msgtype,temporary_node_failure,NODE|2 + msgtype,permanent_node_failure,PERM|NODE|2 diff --git a/wire/extracted_peer_experimental_daa0a499c300056dea24a3afd24681c310e1904c b/wire/extracted_peer_experimental_daa0a499c300056dea24a3afd24681c310e1904c new file mode 100644 index 000000000000..8b6fa551b982 --- /dev/null +++ b/wire/extracted_peer_experimental_daa0a499c300056dea24a3afd24681c310e1904c @@ -0,0 +1,10 @@ +--- wire/extracted_peer_wire_csv 2020-02-25 05:52:39.612291156 +1030 ++++ - 2020-03-04 16:26:35.725479720 +1030 +@@ -208,3 +205,7 @@ + msgdata,gossip_timestamp_filter,chain_hash,chain_hash, + msgdata,gossip_timestamp_filter,first_timestamp,u32, + msgdata,gossip_timestamp_filter,timestamp_range,u32, ++msgtype,onionmsg,385,option_onion_messages ++msgdata,onionmsg,onion_routing_packet,byte,1366 ++msgdata,onionmsg,len,u16, ++msgdata,onionmsg,end_to_end_payload,byte,len diff --git a/wire/peer_wire.c b/wire/peer_wire.c index e9210d7d8434..9d9525b8b3a5 100644 --- a/wire/peer_wire.c +++ b/wire/peer_wire.c @@ -31,6 +31,9 @@ static bool unknown_type(enum wire_type t) case WIRE_QUERY_CHANNEL_RANGE: case WIRE_REPLY_CHANNEL_RANGE: case WIRE_GOSSIP_TIMESTAMP_FILTER: +#if EXPERIMENTAL_FEATURES + case WIRE_ONIONMSG: +#endif return false; } return true; @@ -68,6 +71,9 @@ bool is_msg_for_gossipd(const u8 *cursor) case WIRE_CHANNEL_REESTABLISH: case WIRE_ANNOUNCEMENT_SIGNATURES: case WIRE_GOSSIP_TIMESTAMP_FILTER: +#if EXPERIMENTAL_FEATURES + case WIRE_ONIONMSG: +#endif break; } return false; From 3be02b4b69e122c6471b6ededca2e2347c4320b9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 22/25] lightningd: forward onion messages. Signed-off-by: Rusty Russell --- lightningd/Makefile | 1 + lightningd/channel_control.c | 11 +++- lightningd/onion_message.c | 98 ++++++++++++++++++++++++++++++++++++ lightningd/onion_message.h | 11 ++++ 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 lightningd/onion_message.c create mode 100644 lightningd/onion_message.h diff --git a/lightningd/Makefile b/lightningd/Makefile index 480806594eb5..8971f50bdd99 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -74,6 +74,7 @@ LIGHTNINGD_SRC := \ lightningd/channel_control.c \ lightningd/closing_control.c \ lightningd/connect_control.c \ + lightningd/onion_message.c \ lightningd/gossip_control.c \ lightningd/gossip_msg.c \ lightningd/hsm_control.c \ diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index 3712e8ab7f94..2bdd9d95ca7b 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -319,11 +320,17 @@ static unsigned channel_msg(struct subd *sd, const u8 *msg, const int *fds) case WIRE_CHANNEL_SEND_ERROR_REPLY: handle_error_channel(sd->channel, msg); break; +#if EXPERIMENTAL_FEATURES case WIRE_GOT_ONIONMSG_TO_US: + handle_onionmsg_to_us(sd->channel, msg); + break; case WIRE_GOT_ONIONMSG_FORWARD: - /* FIXME */ + handle_onionmsg_forward(sd->channel, msg); break; - +#else + case WIRE_GOT_ONIONMSG_TO_US: + case WIRE_GOT_ONIONMSG_FORWARD: +#endif /* And we never get these from channeld. */ case WIRE_CHANNEL_INIT: case WIRE_CHANNEL_FUNDING_DEPTH: diff --git a/lightningd/onion_message.c b/lightningd/onion_message.c new file mode 100644 index 000000000000..4947abbb4895 --- /dev/null +++ b/lightningd/onion_message.c @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include +#include + +#if EXPERIMENTAL_FEATURES +/* Returns false if we can't tell it */ +static bool make_peer_send(struct lightningd *ld, + struct channel *dst, const u8 *msg TAKES) +{ + /* Take ownership of msg (noop if it's taken) */ + msg = tal_dup_talarr(tmpctx, u8, msg); + + if (!dst) { + log_debug(ld->log, "Can't send %s: no channel", + channel_wire_type_name(fromwire_peektype(msg))); + return false; + } + + if (!dst->owner) { + log_debug(ld->log, "Can't send %s: not connected", + channel_wire_type_name(fromwire_peektype(msg))); + return false; + } + + /* FIXME: We should allow this for closingd too, and we should + * allow incoming via openingd!. */ + if (!streq(dst->owner->name, "channeld")) { + log_debug(ld->log, "Can't send %s: owned by %s", + channel_wire_type_name(fromwire_peektype(msg)), + dst->owner->name); + return false; + } + subd_send_msg(dst->owner, take(msg)); + return true; +} + +void handle_onionmsg_to_us(struct channel *channel, const u8 *msg) +{ + u8 *e2e_payload, *plaintext; + struct short_channel_id *next_scid; + struct node_id *next_node; + u8 *next_onion; + struct secret ss; + + if (!fromwire_got_onionmsg_to_us(msg, msg, &next_scid, &next_node, + &next_onion, &ss, &e2e_payload)) { + channel_internal_error(channel, "bad got_onionmsg_tous: %s", + tal_hex(tmpctx, msg)); + return; + } + + /* FIXME: Wire up onion message handling! */ + plaintext = final_e2e_payload(msg, e2e_payload, &ss); + if (!plaintext) { + log_info(channel->log, "Received invalid onion message %s%s", + tal_hex(tmpctx, e2e_payload), + next_onion ? " (with next_onion)": ""); + } else { + log_info(channel->log, "Received onion message %s%s", + tal_hex(tmpctx, plaintext), + next_onion ? " (with next_onion)": ""); + } +} + +void handle_onionmsg_forward(struct channel *channel, const u8 *msg) +{ + struct lightningd *ld = channel->peer->ld; + u8 *e2e_payload; + struct short_channel_id *next_scid; + struct node_id *next_node; + u8 onion[TOTAL_PACKET_SIZE]; + struct channel *outchan; + + if (!fromwire_got_onionmsg_forward(msg, msg, &next_scid, &next_node, + onion, &e2e_payload)) { + channel_internal_error(channel, "bad got_onionmsg_forward: %s", + tal_hex(tmpctx, msg)); + return; + } + + if (next_scid) + outchan = active_channel_by_scid(ld, next_scid); + else if (next_node) { + struct peer *p = peer_by_id(ld, next_node); + if (p) + outchan = peer_active_channel(p); + else + outchan = NULL; + } else + outchan = NULL; + + make_peer_send(ld, outchan, + take(towire_send_onionmsg(NULL, onion, e2e_payload))); +} +#endif /* EXPERIMENTAL_FEATURES */ diff --git a/lightningd/onion_message.h b/lightningd/onion_message.h new file mode 100644 index 000000000000..c2a2099a8ebb --- /dev/null +++ b/lightningd/onion_message.h @@ -0,0 +1,11 @@ +#ifndef LIGHTNING_LIGHTNINGD_ONION_MESSAGE_H +#define LIGHTNING_LIGHTNINGD_ONION_MESSAGE_H +#include "config.h" +#include + +struct channel; + +void handle_onionmsg_to_us(struct channel *channel, const u8 *msg); +void handle_onionmsg_forward(struct channel *channel, const u8 *msg); + +#endif /* LIGHTNING_LIGHTNINGD_ONION_MESSAGE_H */ From 676bcc8bf3226fbb35d5b0d365372b89faba48a3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 23/25] plugins: expose onion_message hook (EXPERIMENTAL_FEATURES) Signed-off-by: Rusty Russell --- lightningd/onion_message.c | 83 +++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/lightningd/onion_message.c b/lightningd/onion_message.c index 4947abbb4895..eb06e4267f67 100644 --- a/lightningd/onion_message.c +++ b/lightningd/onion_message.c @@ -1,11 +1,62 @@ #include +#include #include #include #include #include +#include #include #if EXPERIMENTAL_FEATURES +struct onion_message_hook_payload { + /* plaintext is NULL if we couldn't decrypt. */ + const u8 *plaintext; + struct secret ss; + u8 *e2e_payload; + /* These are *optional* */ + struct short_channel_id *next_scid; + struct node_id *next_node; + u8 *reply_onion; +}; + +static void +onion_message_serialize(struct onion_message_hook_payload *payload, + struct json_stream *stream) +{ + json_object_start(stream, "onion_message"); + if (payload->plaintext) { + json_add_hex_talarr(stream, "plaintext", payload->plaintext); + } else { + json_add_hex_talarr(stream, "payload", payload->e2e_payload); + } + json_add_secret(stream, "shared_secret", &payload->ss); + if (payload->next_scid) + json_add_short_channel_id(stream, "next_short_channel_id", + payload->next_scid); + if (payload->next_node) + json_add_node_id(stream, "next_node_id", payload->next_node); + if (payload->reply_onion) + json_add_hex_talarr(stream, "reply_onion", payload->reply_onion); + json_object_end(stream); +} + +static void +onion_message_hook_cb(struct onion_message_hook_payload *payload, + const char *buffer, + const jsmntok_t *toks) +{ + /* The core infra checks the "result"; anything other than continue + * just stops. */ + tal_free(payload); +} + +REGISTER_PLUGIN_HOOK(onion_message, + PLUGIN_HOOK_CHAIN, + onion_message_hook_cb, + struct onion_message_hook_payload *, + onion_message_serialize, + struct onion_message_hook_payload *); + /* Returns false if we can't tell it */ static bool make_peer_send(struct lightningd *ld, struct channel *dst, const u8 *msg TAKES) @@ -39,30 +90,34 @@ static bool make_peer_send(struct lightningd *ld, void handle_onionmsg_to_us(struct channel *channel, const u8 *msg) { - u8 *e2e_payload, *plaintext; - struct short_channel_id *next_scid; - struct node_id *next_node; - u8 *next_onion; - struct secret ss; + struct lightningd *ld = channel->peer->ld; + struct onion_message_hook_payload *payload; - if (!fromwire_got_onionmsg_to_us(msg, msg, &next_scid, &next_node, - &next_onion, &ss, &e2e_payload)) { + payload = tal(ld, struct onion_message_hook_payload); + if (!fromwire_got_onionmsg_to_us(payload, msg, + &payload->next_scid, + &payload->next_node, + &payload->reply_onion, + &payload->ss, + &payload->e2e_payload)) { channel_internal_error(channel, "bad got_onionmsg_tous: %s", tal_hex(tmpctx, msg)); return; } - /* FIXME: Wire up onion message handling! */ - plaintext = final_e2e_payload(msg, e2e_payload, &ss); - if (!plaintext) { + payload->plaintext = final_e2e_payload(payload, + payload->e2e_payload, + &payload->ss); + if (!payload->plaintext) { log_info(channel->log, "Received invalid onion message %s%s", - tal_hex(tmpctx, e2e_payload), - next_onion ? " (with next_onion)": ""); + tal_hex(tmpctx, payload->e2e_payload), + payload->reply_onion ? " (with reply_onion)": ""); } else { log_info(channel->log, "Received onion message %s%s", - tal_hex(tmpctx, plaintext), - next_onion ? " (with next_onion)": ""); + tal_hex(tmpctx, payload->plaintext), + payload->reply_onion ? " (with reply_onion)": ""); } + plugin_hook_call_onion_message(ld, payload, payload); } void handle_onionmsg_forward(struct channel *channel, const u8 *msg) From dd2873f18716baea62121f947e4b76d877298ddf Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 24/25] lightningd: EXPERIMENTAL_FEATURES JSON command to sendonionmessage. The message construction is subtle, so I left it to userspace (after several attempts). Signed-off-by: Rusty Russell --- lightningd/onion_message.c | 75 +++++++++++++++++++++++++++++++++ tests/test_misc.py | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/lightningd/onion_message.c b/lightningd/onion_message.c index eb06e4267f67..20e0f9d105b4 100644 --- a/lightningd/onion_message.c +++ b/lightningd/onion_message.c @@ -150,4 +150,79 @@ void handle_onionmsg_forward(struct channel *channel, const u8 *msg) make_peer_send(ld, outchan, take(towire_send_onionmsg(NULL, onion, e2e_payload))); } + +static struct command_result *param_first_hop(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct channel **channel) +{ + struct short_channel_id scid; + struct node_id node_id; + + if (json_to_short_channel_id(buffer, tok, &scid)) { + *channel = active_channel_by_scid(cmd->ld, &scid); + if (!*channel) { + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "No active channel with scid %s", + type_to_string(tmpctx, + struct short_channel_id, + &scid)); + } + } else if (json_to_node_id(buffer, tok, &node_id)) { + *channel = active_channel_by_id(cmd->ld, &node_id, NULL); + if (!*channel) { + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "No active channel with peer %s", + type_to_string(tmpctx, + struct node_id, + &node_id)); + } + } else + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%s' needs to be a short_channel_id or node_id", + name); + return NULL; +} + +static struct command_result *json_send_onion_message(struct command *cmd, + const char *buffer, + const jsmntok_t *obj UNNEEDED, + const jsmntok_t *params) +{ + const u8 *ser; + u8 *onion, *e2e; + struct channel *first_hop; + enum onion_type failcode; + struct onionpacket packet; + + if (!param(cmd, buffer, params, + p_req("onion", param_bin_from_hex, &onion), + p_req("first_hop", param_first_hop, &first_hop), + p_req("payload", param_bin_from_hex, &e2e), + NULL)) + return command_param_failed(); + + failcode = parse_onionpacket(onion, tal_bytelen(onion), &packet); + if (failcode != 0) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Could not parse the onion. Parsing failed " + "with failcode=%d", + failcode); + + ser = serialize_onionpacket(cmd, &packet); + if (!make_peer_send(cmd->ld, first_hop, + take(towire_send_onionmsg(NULL, ser, e2e)))) + return command_fail(cmd, LIGHTNINGD, "First peer not ready"); + + return command_success(cmd, json_stream_success(cmd)); +} + +static const struct json_command send_onion_message_command = { + "sendonionmessage", + "utility", + json_send_onion_message, + "Send {onion} as onion message to {first_hop} (node_id or short_channel_id) with {payload}" +}; +AUTODATA(json_command, &send_onion_message_command); #endif /* EXPERIMENTAL_FEATURES */ diff --git a/tests/test_misc.py b/tests/test_misc.py index e82935ff0e03..08b3d34a6290 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -10,6 +10,7 @@ TailableProc, env ) from ephemeral_port_reserve import reserve +from utils import EXPERIMENTAL_FEATURES import json import os @@ -2157,6 +2158,91 @@ def test_sendcustommsg(node_factory): r'Got a custom message {serialized} from peer {peer_id}'.format( serialized=serialized, peer_id=l2.info['id'])) +# FIXME: A real python library to make TLVs would be nice! +def bigsize_hex(value): + if value < 0xfd: + return "{:02x}".format(value) + elif value <= 0xffff: + return "fd" + "{:04x}".format(value) + elif value <= 0xffffffff: + return "fe" + "{:08x}".format(value) + else: + return "ff" + "{:016x}".format(value) + +def prefix_len_hex(hexvalue): + return bigsize_hex(len(hexvalue) // 2) + hexvalue + +def tlv_hex(typenum, hexvalue): + return bigsize_hex(typenum) + prefix_len_hex(hexvalue) + + +@unittest.skipIf(not EXPERIMENTAL_FEATURES, "Needs sendonionmessage") +def test_sendonionmessage(node_factory): + l1, l2, l3 = node_factory.line_graph(3) + + oniontool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onion") + onionmsgtool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onionmessage") + + hexmsg = bytes("Test message", encoding="utf8").hex() + # 1. type: 4 (`next_node_id`) + l2tlvs = prefix_len_hex(tlv_hex(4, l3.info['id'])) + l3tlvs = prefix_len_hex('') + output = subprocess.check_output( + [oniontool, 'generate', l2.info['id'] + '/' + l2tlvs, l3.info['id'] + '/' + l3tlvs] + ).decode('ASCII') + + onion = output.split('\n')[0] + secrets = output.split('\n')[1].split(':')[1].split() + payload = subprocess.check_output( + [onionmsgtool, 'encrypt', hexmsg, secrets[0], secrets[1]] + ).decode('ASCII').strip() + + l1.rpc.call('sendonionmessage', [onion, l2.info['id'], payload]) + assert l3.daemon.wait_for_log('Received onion message ' + hexmsg) + + # Now by SCID. + # 1. type: 2 (`next_short_channel_id`) + scid = l2.get_channel_scid(l3).split('x') + hexscid = "{:06x}{:06x}{:04x}".format(int(scid[0]), int(scid[1]), int(scid[2])) + l2tlvs = prefix_len_hex(tlv_hex(2, hexscid)) + l3tlvs = prefix_len_hex('') + output = subprocess.check_output( + [oniontool, 'generate', l2.info['id'] + '/' + l2tlvs, l3.info['id'] + '/' + l3tlvs] + ).decode('ASCII') + + onion = output.split('\n')[0] + secrets = output.split('\n')[1].split(':')[1].split() + payload = subprocess.check_output( + [onionmsgtool, 'encrypt', hexmsg, secrets[0], secrets[1]] + ).decode('ASCII').strip() + + l1.rpc.call('sendonionmessage', [onion, l2.info['id'], payload]) + assert l3.daemon.wait_for_log('Received onion message ' + hexmsg) + + # Now with forward onion! + output = subprocess.check_output( + [oniontool, '--partial-to', l2.info['id'], 'generate', l3.info['id'] + '/' + l3tlvs] + ).decode('ASCII') + + partonion = output.split('\n')[0].strip() + final_secret = output.split('\n')[1].split(':')[1].strip() + + # 1. type: 6 (`forward_onion`) + l2tlvs = prefix_len_hex(tlv_hex(2, hexscid) + tlv_hex(6, partonion)) + output = subprocess.check_output( + [oniontool, 'generate', l2.info['id'] + '/' + l2tlvs] + ).decode('ASCII') + + onion = output.split('\n')[0] + first_secret = output.split('\n')[1].split(':')[1].strip() + + payload = subprocess.check_output( + [onionmsgtool, 'encrypt', hexmsg, first_secret, final_secret] + ).decode('ASCII').strip() + + l1.rpc.call('sendonionmessage', [onion, l2.info['id'], payload]) + assert l3.daemon.wait_for_log('Received onion message ' + hexmsg) + @unittest.skipIf(not DEVELOPER, "needs --dev-force-privkey") def test_getsharedsecret(node_factory): From 721c8b7ec7866faab63842bf424898fcd0a92983 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 6 Mar 2020 16:31:24 +1030 Subject: [PATCH 25/25] pytest: test the reply functionality using a plugin. Signed-off-by: Rusty Russell --- tests/plugins/onionmessage-reply.py | 45 ++++++++++++++++++++++++ tests/test_misc.py | 54 +++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100755 tests/plugins/onionmessage-reply.py diff --git a/tests/plugins/onionmessage-reply.py b/tests/plugins/onionmessage-reply.py new file mode 100755 index 000000000000..0b6df39bea31 --- /dev/null +++ b/tests/plugins/onionmessage-reply.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +This plugin is used to test the `onion_message` hook. +""" +from lightning import Plugin +import subprocess + +plugin = Plugin() + + +@plugin.hook("onion_message") +def on_onion_message(plugin, onion_message, **kwargs): + onionmsgtool = plugin.get_option('onionmsgtool') + + if 'plaintext' not in onion_message: + plugin.log("payload:{}".format(onion_message['payload'])) + return + + plaintext = bytearray.fromhex(onion_message['plaintext']).decode() + plugin.log("plaintext:{}".format(plaintext)) + + if 'reply_onion' not in onion_message: + plugin.log("no reply onion") + return + + if 'next_node_id' in onion_message: + nextpeer = onion_message['next_node_id'] + elif 'next_short_channel_id' in onion_message: + nextpeer = onion_message['next_short_channel_id'] + else: + plugin.log("No next_node_id or next_short_channel_id?") + return + + hexmsg = bytes("Acknowledge: {}".format(plaintext), encoding="utf8").hex() + payload = subprocess.check_output( + [onionmsgtool, 'encrypt', hexmsg, onion_message['shared_secret']] + ).decode('ASCII').strip() + + plugin.rpc.call('sendonionmessage', [onion_message['reply_onion'], nextpeer, payload]) + plugin.log("sent reply encrypted using {} ({})".format(onion_message['shared_secret'], payload)) + return {"result": "continue"} + + +plugin.add_option('onionmsgtool', None, 'Location of the "onionmessage" binary.') +plugin.run() diff --git a/tests/test_misc.py b/tests/test_misc.py index 08b3d34a6290..2341812376ad 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -2244,6 +2244,60 @@ def test_sendonionmessage(node_factory): assert l3.daemon.wait_for_log('Received onion message ' + hexmsg) +@unittest.skipIf(not EXPERIMENTAL_FEATURES, "Needs sendonionmessage") +def test_sendonionmessage_reply(node_factory): + oniontool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onion") + onionmsgtool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onionmessage") + + plugin = os.path.join(os.path.dirname(__file__), "plugins", "onionmessage-reply.py") + l1, l2, l3 = node_factory.line_graph(3, opts={'plugin': plugin, 'onionmsgtool': onionmsgtool}) + + # Make reply onion. + # 1. type: 4 (`next_node_id`) + l2tlvs = prefix_len_hex(tlv_hex(4, l1.info['id'])) + l1tlvs = prefix_len_hex('') + output = subprocess.check_output( + [oniontool, '--partial-to', l3.info['id'], 'generate', + l2.info['id'] + '/' + l2tlvs, + l1.info['id'] + '/' + l1tlvs] + ).decode('ASCII') + replyonion = output.split('\n')[0] + replysecrets = output.split('\n')[1].split(':')[1].split() + + hexmsg = bytes("Test message", encoding="utf8").hex() + # 1. type: 4 (`next_node_id`) + l2tlvs = prefix_len_hex(tlv_hex(4, l3.info['id'])) + # 1. type: 8 (`reply_onion`) + l3tlvs = prefix_len_hex(tlv_hex(4, l2.info['id']) + tlv_hex(8, replyonion)) + output = subprocess.check_output( + [oniontool, 'generate', l2.info['id'] + '/' + l2tlvs, l3.info['id'] + '/' + l3tlvs] + ).decode('ASCII') + print(output) + + onion = output.split('\n')[0] + secrets = output.split('\n')[1].split(':')[1].split() + payload = subprocess.check_output( + [onionmsgtool, 'encrypt', hexmsg, secrets[0], secrets[1]] + ).decode('ASCII').strip() + + l1.rpc.call('sendonionmessage', [onion, l2.info['id'], payload]) + assert l3.daemon.wait_for_log('Received onion message ' + hexmsg) + + assert l3.daemon.wait_for_log('sent reply') + # We will get reply, which we can't decode. + line = l1.daemon.wait_for_log('payload:') + assert line + payload = re.search('payload:([0-9a-fA-F]*)', line).group(1) + payload = subprocess.check_output( + [onionmsgtool, 'unwrap', payload, secrets[0]] + ).decode('ASCII').strip() + print("unwrapping with {} gave {}".format(secrets[0], payload)) + plaintext = subprocess.check_output( + [onionmsgtool, 'decrypt', payload, secrets[1]] + ).decode('ASCII').strip() + assert plaintext == bytes('Acknowledge: ', encoding="utf8").hex() + hexmsg + + @unittest.skipIf(not DEVELOPER, "needs --dev-force-privkey") def test_getsharedsecret(node_factory): """