diff --git a/Cargo.lock b/Cargo.lock index 4515713e5db..4d304c0bcfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -851,7 +851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8235645834fbc6832939736ce2f2d08192652269e11010a6240f61b908a1c6" dependencies = [ "hybrid-array", - "rand_core 0.9.3", + "rand_core", ] [[package]] @@ -897,7 +897,7 @@ dependencies = [ "curve25519-dalek-derive", "digest", "fiat-crypto", - "rand_core 0.9.3", + "rand_core", "rustc_version", "serde", "subtle", @@ -1123,7 +1123,7 @@ checksum = "ad207ed88a133091f83224265eac21109930db09bedcad05d5252f2af2de20a1" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core 0.9.3", + "rand_core", "serde", "sha2", "signature", @@ -1192,6 +1192,18 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "fastbloom" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18c1ddb9231d8554c2d6bdf4cfaabf0c59251658c68b6c95cd52dd0c513a912a" +dependencies = [ + "getrandom 0.3.3", + "libm", + "rand", + "siphasher", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1631,7 +1643,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.2", + "rand", "smallvec", "spinning_top", "web-time", @@ -1747,7 +1759,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand", "ring", "rustls", "serde", @@ -1772,7 +1784,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand", "resolv-conf", "rustls", "serde", @@ -1983,7 +1995,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2", ] [[package]] @@ -2117,7 +2129,7 @@ dependencies = [ "hyper", "hyper-util", "log", - "rand 0.9.2", + "rand", "tokio", "url", "xmltree", @@ -2244,8 +2256,8 @@ dependencies = [ "n0-snafu", "n0-watcher", "nested_enum_utils", - "netdev 0.38.2", - "netwatch 0.10.0", + "netdev", + "netwatch", "parse-size", "pin-project", "pkarr", @@ -2253,13 +2265,14 @@ dependencies = [ "portmapper", "postcard", "pretty_assertions", - "rand 0.9.2", - "rand_chacha 0.9.0", + "rand", + "rand_chacha", "reqwest", "ring", + "rustc-hash", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.5.3", "rustls-webpki", "serde", "serde_json", @@ -2295,9 +2308,9 @@ dependencies = [ "nested_enum_utils", "postcard", "proptest", - "rand 0.9.2", - "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand", + "rand_chacha", + "rand_core", "serde", "serde_json", "serde_test", @@ -2319,7 +2332,7 @@ dependencies = [ "iroh-quinn", "n0-future 0.3.0", "n0-snafu", - "rand 0.9.2", + "rand", "rcgen 0.14.5", "rustls", "tokio", @@ -2353,8 +2366,8 @@ dependencies = [ "n0-future 0.3.0", "n0-snafu", "pkarr", - "rand 0.9.2", - "rand_chacha 0.9.0", + "rand", + "rand_chacha", "rcgen 0.14.5", "redb", "regex", @@ -2415,8 +2428,7 @@ dependencies = [ [[package]] name = "iroh-quinn" version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde160ebee7aabede6ae887460cd303c8b809054224815addf1469d54a6fcf7" +source = "git+https://github.com/n0-computer/quinn?branch=multipath-quinn-0.11.x#a4597406bf649a8eb38a5f8a1861979b6cee2ef4" dependencies = [ "bytes", "cfg_aliases", @@ -2425,9 +2437,10 @@ dependencies = [ "pin-project-lite", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.0", "thiserror 2.0.17", "tokio", + "tokio-stream", "tracing", "web-time", ] @@ -2435,17 +2448,18 @@ dependencies = [ [[package]] name = "iroh-quinn-proto" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "929d5d8fa77d5c304d3ee7cae9aede31f13908bd049f9de8c7c0094ad6f7c535" +source = "git+https://github.com/n0-computer/quinn?branch=multipath-quinn-0.11.x#a4597406bf649a8eb38a5f8a1861979b6cee2ef4" dependencies = [ "bytes", - "getrandom 0.2.16", - "rand 0.8.5", + "fastbloom", + "getrandom 0.3.3", + "lru-slab", + "rand", "ring", "rustc-hash", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.6.1", "slab", "thiserror 2.0.17", "tinyvec", @@ -2455,16 +2469,15 @@ dependencies = [ [[package]] name = "iroh-quinn-udp" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c53afaa1049f7c83ea1331f5ebb9e6ebc5fdd69c468b7a22dd598b02c9bcc973" +version = "0.5.12" +source = "git+https://github.com/n0-computer/quinn?branch=multipath-quinn-0.11.x#a4597406bf649a8eb38a5f8a1861979b6cee2ef4" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2501,8 +2514,8 @@ dependencies = [ "pkarr", "postcard", "proptest", - "rand 0.9.2", - "rand_chacha 0.9.0", + "rand", + "rand_chacha", "rcgen 0.14.5", "regex", "reloadable-state", @@ -2601,6 +2614,12 @@ version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "libredox" version = "0.1.10" @@ -2886,23 +2905,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "netdev" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa1e3eaf125c54c21e6221df12dd2a0a682784a068782dd564c836c0f281b6d" -dependencies = [ - "dlopen2", - "ipnet", - "libc", - "netlink-packet-core 0.7.0", - "netlink-packet-route 0.22.0", - "netlink-sys", - "once_cell", - "system-configuration", - "windows-sys 0.59.0", -] - [[package]] name = "netdev" version = "0.38.2" @@ -2912,25 +2914,14 @@ dependencies = [ "dlopen2", "ipnet", "libc", - "netlink-packet-core 0.8.1", - "netlink-packet-route 0.25.1", + "netlink-packet-core", + "netlink-packet-route", "netlink-sys", "once_cell", "system-configuration", "windows-sys 0.59.0", ] -[[package]] -name = "netlink-packet-core" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4" -dependencies = [ - "anyhow", - "byteorder", - "netlink-packet-utils", -] - [[package]] name = "netlink-packet-core" version = "0.8.1" @@ -2940,36 +2931,6 @@ dependencies = [ "paste", ] -[[package]] -name = "netlink-packet-route" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e7987b28514adf555dc1f9a5c30dfc3e50750bbaffb1aec41ca7b23dcd8e4" -dependencies = [ - "anyhow", - "bitflags", - "byteorder", - "libc", - "log", - "netlink-packet-core 0.7.0", - "netlink-packet-utils", -] - -[[package]] -name = "netlink-packet-route" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d83370a96813d7c977f8b63054f1162df6e5784f1c598d689236564fb5a6f2" -dependencies = [ - "anyhow", - "bitflags", - "byteorder", - "libc", - "log", - "netlink-packet-core 0.7.0", - "netlink-packet-utils", -] - [[package]] name = "netlink-packet-route" version = "0.25.1" @@ -2979,33 +2940,7 @@ dependencies = [ "bitflags", "libc", "log", - "netlink-packet-core 0.8.1", -] - -[[package]] -name = "netlink-packet-utils" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" -dependencies = [ - "anyhow", - "byteorder", - "paste", - "thiserror 1.0.69", -] - -[[package]] -name = "netlink-proto" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60" -dependencies = [ - "bytes", - "futures", - "log", - "netlink-packet-core 0.7.0", - "netlink-sys", - "thiserror 2.0.17", + "netlink-packet-core", ] [[package]] @@ -3017,7 +2952,7 @@ dependencies = [ "bytes", "futures", "log", - "netlink-packet-core 0.8.1", + "netlink-packet-core", "netlink-sys", "thiserror 2.0.17", ] @@ -3035,46 +2970,10 @@ dependencies = [ "tokio", ] -[[package]] -name = "netwatch" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a63d76f52f3f15ebde3ca751a2ab73a33ae156662bc04383bac8e824f84e9bb" -dependencies = [ - "atomic-waker", - "bytes", - "cfg_aliases", - "derive_more 2.0.1", - "iroh-quinn-udp", - "js-sys", - "libc", - "n0-future 0.1.3", - "n0-watcher", - "nested_enum_utils", - "netdev 0.37.3", - "netlink-packet-core 0.7.0", - "netlink-packet-route 0.24.0", - "netlink-proto 0.11.5", - "netlink-sys", - "pin-project-lite", - "serde", - "snafu", - "socket2 0.6.0", - "time", - "tokio", - "tokio-util", - "tracing", - "web-sys", - "windows 0.61.3", - "windows-result 0.3.4", - "wmi", -] - [[package]] name = "netwatch" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29acc9361df4a91bde6d2ec1ce110de7c826e574eeb3662ec1f574af00b96c48" +source = "git+https://github.com/n0-computer/net-tools?branch=feat-multipath#18e022817e2a45552b8acbce581f9b50a5374e6c" dependencies = [ "atomic-waker", "bytes", @@ -3086,10 +2985,10 @@ dependencies = [ "n0-future 0.2.0", "n0-watcher", "nested_enum_utils", - "netdev 0.38.2", - "netlink-packet-core 0.8.1", - "netlink-packet-route 0.25.1", - "netlink-proto 0.12.0", + "netdev", + "netlink-packet-core", + "netlink-packet-route", + "netlink-proto", "netlink-sys", "pin-project-lite", "serde", @@ -3506,9 +3405,9 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portmapper" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f7313cafd74e95e6a358c1d0a495112f175502cc2e69870d0a5b12b6553059" +checksum = "2b3a9274d533a9554cccbc791b8a016b6574aa68c2b2f9bda68463348791330e" dependencies = [ "base64", "bytes", @@ -3520,9 +3419,9 @@ dependencies = [ "iroh-metrics", "libc", "nested_enum_utils", - "netwatch 0.9.0", + "netwatch", "num_enum", - "rand 0.9.2", + "rand", "serde", "smallvec", "snafu", @@ -3633,8 +3532,8 @@ dependencies = [ "bitflags", "lazy_static", "num-traits", - "rand 0.9.2", - "rand_chacha 0.9.0", + "rand", + "rand_chacha", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -3692,7 +3591,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand 0.9.2", + "rand", "ring", "rustc-hash", "rustls", @@ -3733,35 +3632,14 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -3771,16 +3649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", + "rand_core", ] [[package]] @@ -3798,7 +3667,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "rand_core 0.9.3", + "rand_core", ] [[package]] @@ -4137,6 +4006,27 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be59af91596cac372a6942530653ad0c3a246cdd491aaa9dcaee47f88d67d5a0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs 1.0.2", + "windows-sys 0.59.0", +] + [[package]] name = "rustls-platform-verifier-android" version = "0.1.1" @@ -4457,6 +4347,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.11" @@ -4628,7 +4524,7 @@ dependencies = [ "hex", "parking_lot", "pnet_packet", - "rand 0.9.2", + "rand", "socket2 0.5.10", "thiserror 1.0.69", "tokio", @@ -4643,7 +4539,7 @@ checksum = "4eae338a4551897c6a50fa2c041c4b75f578962d9fca8adb828cf81f7158740f" dependencies = [ "acto", "hickory-proto", - "rand 0.9.2", + "rand", "socket2 0.5.10", "thiserror 2.0.17", "tokio", @@ -4968,7 +4864,7 @@ dependencies = [ "getrandom 0.3.3", "http 1.3.1", "httparse", - "rand 0.9.2", + "rand", "ring", "rustls-pki-types", "simdutf8", diff --git a/Cargo.toml b/Cargo.toml index 0d0681861f5..048c9667157 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,3 +40,12 @@ unexpected_cfgs = { level = "warn", check-cfg = ["cfg(iroh_docsrs)", "cfg(iroh_l [workspace.lints.clippy] unused-async = "warn" + + +[patch.crates-io] +netwatch = { git = "https://github.com/n0-computer/net-tools", branch = "feat-multipath" } + +[patch."https://github.com/n0-computer/quinn"] +# iroh-quinn = { path = "../iroh-quinn/quinn" } +# iroh-quinn-proto = { path = "../iroh-quinn/quinn-proto" } +# iroh-quinn-udp = { path = "../iroh-quinn/quinn-udp" } diff --git a/iroh-base/src/endpoint_addr.rs b/iroh-base/src/endpoint_addr.rs index 342f472ec41..56ffbcc5d38 100644 --- a/iroh-base/src/endpoint_addr.rs +++ b/iroh-base/src/endpoint_addr.rs @@ -36,7 +36,9 @@ use crate::{EndpointId, PublicKey, RelayUrl}; /// [discovery]: https://docs.rs/iroh/*/iroh/index.html#endpoint-discovery /// [home relay]: https://docs.rs/iroh/*/iroh/relay/index.html /// [Relay server]: https://docs.rs/iroh/*/iroh/index.html#relay-servers -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + derive_more::Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, +)] pub struct EndpointAddr { /// The endpoint's identifier. pub id: EndpointId, @@ -54,6 +56,18 @@ pub enum TransportAddr { Ip(SocketAddr), } +impl TransportAddr { + /// Whether this is a transport address via a relay server. + pub fn is_relay(&self) -> bool { + matches!(self, Self::Relay(_)) + } + + /// Whether this is an IP transport address. + pub fn is_ip(&self) -> bool { + matches!(self, Self::Ip(_)) + } +} + impl EndpointAddr { /// Creates a new [`EndpointAddr`] with no network level addresses. /// diff --git a/iroh-relay/Cargo.toml b/iroh-relay/Cargo.toml index d9a5399092b..63f785ad186 100644 --- a/iroh-relay/Cargo.toml +++ b/iroh-relay/Cargo.toml @@ -42,8 +42,8 @@ postcard = { version = "1", default-features = false, features = [ "use-std", "experimental-derive", ] } -quinn = { package = "iroh-quinn", version = "0.14.0", default-features = false, features = ["rustls-ring"] } -quinn-proto = { package = "iroh-quinn-proto", version = "0.13.0" } +quinn = { package = "iroh-quinn", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x", default-features = false, features = ["rustls-ring"] } +quinn-proto = { package = "iroh-quinn-proto", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x" } rand = "0.9.2" reqwest = { version = "0.12", default-features = false, features = [ "rustls-tls", diff --git a/iroh-relay/src/client.rs b/iroh-relay/src/client.rs index e398b2fa401..a0fbf2a063d 100644 --- a/iroh-relay/src/client.rs +++ b/iroh-relay/src/client.rs @@ -281,7 +281,7 @@ impl ClientBuilder { let conn = Conn::new(conn, self.key_cache.clone(), &self.secret_key).await?; event!( - target: "events.net.relay.connected", + target: "iroh::_events::net::relay::connected", Level::DEBUG, url = %self.url, ); @@ -337,7 +337,7 @@ impl ClientBuilder { let conn = Conn::new(ws_stream, self.key_cache.clone(), &self.secret_key).await?; event!( - target: "events.net.relay.connected", + target: "iroh::_events::net::relay::connected", Level::DEBUG, url = %self.url, ); diff --git a/iroh-relay/src/client/conn.rs b/iroh-relay/src/client/conn.rs index 7a76bcff21a..6e0a68e813a 100644 --- a/iroh-relay/src/client/conn.rs +++ b/iroh-relay/src/client/conn.rs @@ -11,7 +11,7 @@ use iroh_base::SecretKey; use n0_future::{Sink, Stream}; use nested_enum_utils::common_fields; use snafu::{Backtrace, Snafu}; -use tracing::debug; +use tracing::{debug, trace}; use super::KeyCache; #[cfg(not(wasm_browser))] @@ -99,9 +99,9 @@ impl Conn { let mut conn = WsBytesFramed { io }; // exchange information with the server - debug!("server_handshake: started"); + trace!("server_handshake: started"); handshake::clientside(&mut conn, secret_key).await?; - debug!("server_handshake: done"); + trace!("server_handshake: done"); Ok(Self { conn, key_cache }) } diff --git a/iroh-relay/src/client/tls.rs b/iroh-relay/src/client/tls.rs index ba62e9cc28a..a83752e15e3 100644 --- a/iroh-relay/src/client/tls.rs +++ b/iroh-relay/src/client/tls.rs @@ -138,7 +138,6 @@ impl MaybeTlsStreamBuilder { async fn dial_url_direct(&self) -> Result { use tokio::net::TcpStream; - debug!(%self.url, "dial url"); let dst_ip = self .dns_resolver .resolve_host(&self.url, self.prefer_ipv6, DNS_TIMEOUT) @@ -147,7 +146,7 @@ impl MaybeTlsStreamBuilder { let port = url_port(&self.url).context(InvalidTargetPortSnafu)?; let addr = SocketAddr::new(dst_ip, port); - debug!("connecting to {}", addr); + trace!("connecting to {}", addr); let tcp_stream = time::timeout(DIAL_ENDPOINT_TIMEOUT, async move { TcpStream::connect(addr).await }) diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 328f80c2519..84dd91f0280 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -46,15 +46,16 @@ nested_enum_utils = "0.2.1" netwatch = { version = "0.10" } pin-project = "1" pkarr = { version = "5", default-features = false, features = ["relays"] } -quinn = { package = "iroh-quinn", version = "0.14.0", default-features = false, features = ["rustls-ring"] } -quinn-proto = { package = "iroh-quinn-proto", version = "0.13.0" } -quinn-udp = { package = "iroh-quinn-udp", version = "0.5.7" } +quinn = { package = "iroh-quinn", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x", default-features = false, features = ["rustls-ring"] } +quinn-proto = { package = "iroh-quinn-proto", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x" } +quinn-udp = { package = "iroh-quinn-udp", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x" } rand = "0.9.2" reqwest = { version = "0.12", default-features = false, features = [ "rustls-tls", "stream", ] } ring = "0.17" +rustc-hash = "2" rustls = { version = "0.23.33", default-features = false, features = ["ring"] } serde = { version = "1.0.219", features = ["derive", "rc"] } smallvec = "1.11.1" @@ -97,7 +98,7 @@ hickory-resolver = "0.25.1" igd-next = { version = "0.16", features = ["aio_tokio"] } netdev = { version = "0.38.1" } portmapper = { version = "0.10", default-features = false } -quinn = { package = "iroh-quinn", version = "0.14.0", default-features = false, features = ["runtime-tokio", "rustls-ring"] } +quinn = { package = "iroh-quinn", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x", default-features = false, features = ["runtime-tokio", "rustls-ring"] } tokio = { version = "1", features = [ "io-util", "macros", diff --git a/iroh/bench/Cargo.toml b/iroh/bench/Cargo.toml index c5df9439638..1f2294f6b76 100644 --- a/iroh/bench/Cargo.toml +++ b/iroh/bench/Cargo.toml @@ -12,7 +12,7 @@ iroh = { path = ".." } iroh-metrics = "0.36" n0-future = "0.3.0" n0-snafu = "0.2.0" -quinn = { package = "iroh-quinn", version = "0.14" } +quinn = { package = "iroh-quinn", git = "https://github.com/n0-computer/quinn", branch = "multipath-quinn-0.11.x" } rand = "0.9.2" rcgen = "0.14" rustls = { version = "0.23.33", default-features = false, features = ["ring"] } diff --git a/iroh/src/disco.rs b/iroh/src/disco.rs index 75e10aa7664..c25d8fe0b3a 100644 --- a/iroh/src/disco.rs +++ b/iroh/src/disco.rs @@ -24,7 +24,7 @@ use std::{ }; use data_encoding::HEXLOWER; -use iroh_base::{PublicKey, RelayUrl}; +use iroh_base::{EndpointId, PublicKey, RelayUrl}; use nested_enum_utils::common_fields; use rand::Rng; use serde::{Deserialize, Serialize}; @@ -119,12 +119,50 @@ pub struct Ping { /// Random client-generated per-ping transaction ID. pub tx_id: TransactionId, - /// Allegedly the ping sender's wireguard public key. - /// It shouldn't be trusted by itself, but can be combined with - /// netmap data to reduce the discokey:endpointkey relation from 1:N to 1:1. + /// Allegedly the ping sender's public key. + /// + /// It shouldn't be trusted by itself. pub endpoint_key: PublicKey, } +impl Ping { + /// Creates a ping message to ping `node_id`. + /// + /// Uses a randomly generated STUN transaction ID. + pub(crate) fn new(endpoint_id: EndpointId) -> Self { + Self { + tx_id: TransactionId::default(), + endpoint_key: endpoint_id, + } + } + + fn from_bytes(p: &[u8]) -> Result { + // Deliberately lax on longer-than-expected messages, for future compatibility. + ensure!(p.len() >= PING_LEN, TooShortSnafu); + let tx_id: [u8; TX_LEN] = p[..TX_LEN].try_into().expect("length checked"); + let raw_key = &p[TX_LEN..TX_LEN + iroh_base::PublicKey::LENGTH]; + let endpoint_key = + PublicKey::try_from(raw_key).map_err(|_| InvalidEncodingSnafu.build())?; + let tx_id = TransactionId::from(tx_id); + + Ok(Ping { + tx_id, + endpoint_key, + }) + } + + fn as_bytes(&self) -> Vec { + let header = msg_header(MessageType::Ping, V0); + let mut out = vec![0u8; PING_LEN + HEADER_LEN]; + + out[..HEADER_LEN].copy_from_slice(&header); + out[HEADER_LEN..HEADER_LEN + TX_LEN].copy_from_slice(&self.tx_id); + out[HEADER_LEN + TX_LEN..].copy_from_slice(self.endpoint_key.as_ref()); + + out + } +} + /// A response a Ping. /// /// It includes the sender's source IP + port, so it's effectively a STUN response. @@ -146,21 +184,6 @@ pub enum SendAddr { Relay(RelayUrl), } -impl SendAddr { - /// Returns if this is a `relay` addr. - pub fn is_relay(&self) -> bool { - matches!(self, Self::Relay(_)) - } - - /// Returns the `Some(Url)` if it is a relay addr. - pub fn relay_url(&self) -> Option { - match self { - Self::Relay(url) => Some(url.clone()), - Self::Udp(_) => None, - } - } -} - impl From for SendAddr { fn from(addr: transports::Addr) -> Self { match addr { @@ -214,34 +237,6 @@ pub struct CallMeMaybe { pub my_numbers: Vec, } -impl Ping { - fn from_bytes(p: &[u8]) -> Result { - // Deliberately lax on longer-than-expected messages, for future compatibility. - ensure!(p.len() >= PING_LEN, TooShortSnafu); - let tx_id: [u8; TX_LEN] = p[..TX_LEN].try_into().expect("length checked"); - let raw_key = &p[TX_LEN..TX_LEN + iroh_base::PublicKey::LENGTH]; - let endpoint_key = - PublicKey::try_from(raw_key).map_err(|_| InvalidEncodingSnafu.build())?; - let tx_id = TransactionId::from(tx_id); - - Ok(Ping { - tx_id, - endpoint_key, - }) - } - - fn as_bytes(&self) -> Vec { - let header = msg_header(MessageType::Ping, V0); - let mut out = vec![0u8; PING_LEN + HEADER_LEN]; - - out[..HEADER_LEN].copy_from_slice(&header); - out[HEADER_LEN..HEADER_LEN + TX_LEN].copy_from_slice(&self.tx_id); - out[HEADER_LEN + TX_LEN..].copy_from_slice(self.endpoint_key.as_ref()); - - out - } -} - #[allow(missing_docs)] #[common_fields({ backtrace: Option, diff --git a/iroh/src/discovery.rs b/iroh/src/discovery.rs index d2a27c7caac..da3c1488998 100644 --- a/iroh/src/discovery.rs +++ b/iroh/src/discovery.rs @@ -124,12 +124,11 @@ use snafu::{IntoError, Snafu, ensure}; use tokio::sync::oneshot; use tracing::{Instrument, debug, error_span, warn}; -use crate::Endpoint; pub use crate::endpoint_info::{EndpointData, EndpointInfo, ParseError, UserData}; +use crate::{Endpoint, magicsock::endpoint_map::Source}; #[cfg(not(wasm_browser))] pub mod dns; - #[cfg(feature = "discovery-local-network")] pub mod mdns; pub mod pkarr; @@ -492,11 +491,7 @@ impl Discovery for ConcurrentDiscovery { } } -/// Maximum duration since the last control or data message received from an endpoint to make us -/// start a discovery task. -const MAX_AGE: Duration = Duration::from_secs(10); - -/// A wrapper around a tokio task which runs an endpoint discovery. +/// A wrapper around a tokio task which runs a node discovery. pub(super) struct DiscoveryTask { on_first_rx: oneshot::Receiver>, _task: AbortOnDropHandle<()>, @@ -527,30 +522,19 @@ impl DiscoveryTask { /// If `delay` is set, the [`DiscoveryTask`] will first wait for `delay` and then check again /// if we recently received messages from remote endpoint. If true, the task will abort. /// Otherwise, or if no `delay` is set, the discovery will be started. - pub(super) fn maybe_start_after_delay( + pub(super) fn start_after_delay( ep: &Endpoint, endpoint_id: EndpointId, - delay: Option, + delay: Duration, ) -> Result, DiscoveryError> { // If discovery is not needed, don't even spawn a task. - if !ep.needs_discovery(endpoint_id, MAX_AGE) { - return Ok(None); - } ensure!(!ep.discovery().is_empty(), NoServiceConfiguredSnafu); let (on_first_tx, on_first_rx) = oneshot::channel(); let ep = ep.clone(); let me = ep.id(); let task = task::spawn( async move { - // If delay is set, wait and recheck if discovery is needed. If not, early-exit. - if let Some(delay) = delay { - time::sleep(delay).await; - if !ep.needs_discovery(endpoint_id, MAX_AGE) { - debug!("no discovery needed, abort"); - on_first_tx.send(Ok(())).ok(); - return; - } - } + time::sleep(delay).await; Self::run(ep, endpoint_id, on_first_tx).await } .instrument( @@ -606,10 +590,10 @@ impl DiscoveryTask { continue; } debug!(%provenance, addr = ?endpoint_addr, "new address found"); - let source = crate::magicsock::Source::Discovery { + let source = Source::Discovery { name: provenance.to_string(), }; - ep.add_endpoint_addr(endpoint_addr, source).ok(); + ep.add_endpoint_addr(endpoint_addr, source).await.ok(); if let Some(tx) = on_first_tx.take() { tx.send(Ok(())).ok(); diff --git a/iroh/src/endpoint.rs b/iroh/src/endpoint.rs index b18b0e97290..71c372f5bbd 100644 --- a/iroh/src/endpoint.rs +++ b/iroh/src/endpoint.rs @@ -13,6 +13,7 @@ use std::{ any::Any, + collections::HashMap, future::{Future, IntoFuture}, net::{IpAddr, SocketAddr, SocketAddrV4, SocketAddrV6}, pin::Pin, @@ -27,10 +28,28 @@ use n0_future::time::Duration; use n0_watcher::Watcher; use nested_enum_utils::common_fields; use pin_project::pin_project; +// Missing still: SendDatagram and ConnectionClose::frame_type's Type. +pub use quinn::{ + AcceptBi, AcceptUni, AckFrequencyConfig, ApplicationClose, Chunk, ClosedStream, + ConnectionClose, ConnectionError, ConnectionStats, MtuDiscoveryConfig, OpenBi, OpenUni, + ReadDatagram, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, RetryError, + SendDatagramError, SendStream, ServerConfig, StoppedError, StreamId, TransportConfig, VarInt, + WeakConnectionHandle, WriteError, +}; +use quinn_proto::PathId; +pub use quinn_proto::{ + FrameStats, PathStats, TransportError, TransportErrorCode, UdpStats, Written, + congestion::{Controller, ControllerFactory}, + crypto::{ + AeadKey, CryptoError, ExportKeyingMaterialError, HandshakeTokenKey, + ServerConfig as CryptoServerConfig, UnsupportedVersion, + }, +}; use snafu::{ResultExt, Snafu, ensure}; use tracing::{debug, instrument, trace, warn}; use url::Url; +pub use super::magicsock::{AddEndpointAddrError, ConnectionType, DirectAddr, DirectAddrType}; #[cfg(wasm_browser)] use crate::discovery::pkarr::PkarrResolver; #[cfg(not(wasm_browser))] @@ -40,45 +59,27 @@ use crate::{ ConcurrentDiscovery, DiscoveryError, DiscoveryTask, DynIntoDiscovery, IntoDiscovery, UserData, }, - endpoint::presets::Preset, - magicsock::{self, EndpointIdMappedAddr, Handle, OwnAddressSnafu}, + magicsock::{ + self, HEARTBEAT_INTERVAL, Handle, MAX_MULTIPATH_PATHS, OwnAddressSnafu, + PATH_MAX_IDLE_TIMEOUT, PathInfo, + endpoint_map::Source, + mapped_addrs::{EndpointIdMappedAddr, MappedAddr}, + }, metrics::EndpointMetrics, net_report::Report, tls::{self, DEFAULT_MAX_TLS_TICKETS}, }; pub mod presets; -mod rtt_actor; -// Missing still: SendDatagram and ConnectionClose::frame_type's Type. -pub use quinn::{ - AcceptBi, AcceptUni, AckFrequencyConfig, ApplicationClose, Chunk, ClosedStream, - ConnectionClose, ConnectionError, ConnectionStats, MtuDiscoveryConfig, OpenBi, OpenUni, - ReadDatagram, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, RetryError, - SendDatagramError, SendStream, ServerConfig, StoppedError, StreamId, TransportConfig, VarInt, - WeakConnectionHandle, WriteError, -}; -pub use quinn_proto::{ - FrameStats, PathStats, TransportError, TransportErrorCode, UdpStats, Written, - congestion::{Controller, ControllerFactory}, - crypto::{ - AeadKey, CryptoError, ExportKeyingMaterialError, HandshakeTokenKey, - ServerConfig as CryptoServerConfig, UnsupportedVersion, - }, -}; - -use self::rtt_actor::RttMessage; -pub use super::magicsock::{ - AddEndpointAddrError, ConnectionType, ControlMsg, DirectAddr, DirectAddrInfo, DirectAddrType, - Source, -}; +use self::presets::Preset; /// The delay to fall back to discovery when direct addresses fail. /// -/// When a connection is attempted with a [`EndpointAddr`] containing direct addresses the -/// [`Endpoint`] assumes one of those addresses probably works. If after this delay there -/// is still no connection the configured [`crate::discovery::Discovery`] will be used however. -const DISCOVERY_WAIT_PERIOD: Duration = Duration::from_millis(500); +/// When a connection is attempted and we have some addressing info for the remote, we +/// assume that one of these probably works. If after this delay there is still no +/// connection, discovery will be started. +const DISCOVERY_WAIT_PERIOD: Duration = Duration::from_millis(150); /// Defines the mode of path selection for all traffic flowing through /// the endpoint. @@ -145,7 +146,7 @@ impl Builder { secret_key: Default::default(), relay_mode, alpn_protocols: Default::default(), - transport_config, + transport_config: quinn::TransportConfig::default(), keylog: Default::default(), discovery: Default::default(), discovery_user_data: Default::default(), @@ -165,12 +166,23 @@ impl Builder { // # The final constructor that everyone needs. /// Binds the magic endpoint. - pub async fn bind(self) -> Result { + pub async fn bind(mut self) -> Result { let mut rng = rand::rng(); let relay_map = self.relay_mode.relay_map(); let secret_key = self .secret_key .unwrap_or_else(move || SecretKey::generate(&mut rng)); + + // Override some transport config settings. + self.transport_config + .keep_alive_interval(Some(HEARTBEAT_INTERVAL)); + self.transport_config + .default_path_keep_alive_interval(Some(HEARTBEAT_INTERVAL)); + self.transport_config + .default_path_max_idle_timeout(Some(PATH_MAX_IDLE_TIMEOUT)); + self.transport_config + .max_concurrent_multipath_paths(MAX_MULTIPATH_PATHS); + let static_config = StaticConfig { transport_config: Arc::new(self.transport_config), tls_config: tls::TlsConfig::new(secret_key.clone(), self.max_tls_tickets), @@ -204,10 +216,8 @@ impl Builder { trace!("created magicsock"); debug!(version = env!("CARGO_PKG_VERSION"), "iroh Endpoint created"); - let metrics = msock.metrics.magicsock.clone(); let ep = Endpoint { msock, - rtt_actor: Arc::new(rtt_actor::RttHandle::new(metrics)), static_config: Arc::new(static_config), }; @@ -475,9 +485,7 @@ impl StaticConfig { #[derive(Clone, Debug)] pub struct Endpoint { /// Handle to the magicsocket/actor - msock: Handle, - /// Handle to the actor that resets the quinn RTT estimator - rtt_actor: Arc, + pub(crate) msock: Handle, /// Configuration structs for quinn, holds the transport config, certificate setup, secret key etc. static_config: Arc, } @@ -695,16 +703,24 @@ impl Endpoint { ensure!(endpoint_addr.id != self.id(), SelfConnectSnafu); if !endpoint_addr.is_empty() { - self.add_endpoint_addr(endpoint_addr.clone(), Source::App)?; + self.add_endpoint_addr(endpoint_addr.clone(), Source::App) + .await?; } let endpoint_id = endpoint_addr.id; let ip_addresses: Vec<_> = endpoint_addr.ip_addrs().cloned().collect(); let relay_url = endpoint_addr.relay_urls().next().cloned(); + trace!( + dst_endpoint_id = %endpoint_id.fmt_short(), + ?relay_url, + ?ip_addresses, + "connecting", + ); - // Get the mapped IPv6 address from the magic socket. Quinn will connect to this - // address. Start discovery for this endpoint if it's enabled and we have no valid or - // verified address information for this endpoint. Dropping the discovery cancels any - // still running task. + // When we start a connection we want to send the QUIC Initial packets on all the + // known paths for the remote endpoint. For this we use an EndpointIdMappedAddr as + // destination for Quinn. Start discovery for this endpoint if it's enabled and we have + // no valid or verified address information for this endpoint. Dropping the discovery + // cancels any still running task. let (mapped_addr, _discovery_drop_guard) = self .get_mapping_addr_and_maybe_start_discovery(endpoint_addr) .await @@ -717,12 +733,6 @@ impl Endpoint { // Start connecting via quinn. This will time out after 10 seconds if no reachable // address is available. - debug!( - ?mapped_addr, - ?ip_addresses, - ?relay_url, - "Attempting connection..." - ); let client_config = { let mut alpn_protocols = vec![alpn.to_vec()]; alpn_protocols.extend(options.additional_alpns); @@ -735,15 +745,12 @@ impl Endpoint { client_config }; + let dest_addr = mapped_addr.private_socket_addr(); let server_name = &tls::name::encode(endpoint_id); let connect = self .msock .endpoint() - .connect_with( - client_config, - mapped_addr.private_socket_addr(), - server_name, - ) + .connect_with(client_config, dest_addr, server_name) .context(QuinnSnafu)?; Ok(Connecting { @@ -793,14 +800,14 @@ impl Endpoint { /// /// [`StaticProvider`]: crate::discovery::static_provider::StaticProvider /// [`RelayUrl`]: crate::RelayUrl - pub(crate) fn add_endpoint_addr( + pub(crate) async fn add_endpoint_addr( &self, endpoint_addr: EndpointAddr, source: Source, ) -> Result<(), AddEndpointAddrError> { // Connecting to ourselves is not supported. snafu::ensure!(endpoint_addr.id != self.id(), OwnAddressSnafu); - self.msock.add_endpoint_addr(endpoint_addr, source) + self.msock.add_endpoint_addr(endpoint_addr, source).await } // # Getter methods for properties of this Endpoint itself. @@ -1008,8 +1015,8 @@ impl Endpoint { /// Returns the currently lowest latency for this endpoint. /// /// Will return `None` if we do not have any address information for the given `endpoint_id`. - pub fn latency(&self, endpoint_id: EndpointId) -> Option { - self.msock.latency(endpoint_id) + pub async fn latency(&self, endpoint_id: EndpointId) -> Option { + self.msock.latency(endpoint_id).await } /// Returns the DNS resolver used in this [`Endpoint`]. @@ -1225,29 +1232,6 @@ impl Endpoint { // # Remaining private methods - /// Checks if the given `EndpointId` needs discovery. - pub(crate) fn needs_discovery(&self, endpoint_id: EndpointId, max_age: Duration) -> bool { - match self.msock.remote_info(endpoint_id) { - // No info means no path to endpoint -> start discovery. - None => true, - Some(info) => { - match ( - info.last_received(), - info.relay_url.as_ref().and_then(|r| r.last_alive), - ) { - // No path to endpoint -> start discovery. - (None, None) => true, - // If we haven't received on direct addresses or the relay for MAX_AGE, - // start discovery. - (Some(elapsed), Some(elapsed_relay)) => { - elapsed > max_age && elapsed_relay > max_age - } - (Some(elapsed), _) | (_, Some(elapsed)) => elapsed > max_age, - } - } - } - } - /// Return the quic mapped address for this `endpoint_id` and possibly start discovery /// services if discovery is enabled on this magic endpoint. /// @@ -1269,25 +1253,20 @@ impl Endpoint { // Only return a mapped addr if we have some way of dialing this endpoint, in other // words, we have either a relay URL or at least one direct address. - let addr = if self.msock.has_send_address(endpoint_id) { - self.msock.get_mapping_addr(endpoint_id) + let addr = if self.msock.has_send_address(endpoint_id).await { + Some(self.msock.get_endpoint_mapped_addr(endpoint_id)) } else { None }; match addr { - Some(addr) => { - // We have some way of dialing this endpoint, but that doesn't actually mean - // we can actually connect to any of these addresses. - // Therefore, we will invoke the discovery service if we haven't received from the - // endpoint on any of the existing paths recently. - // If the user provided addresses in this connect call, we will add a delay - // followed by a recheck before starting the discovery, to give the magicsocket a - // chance to test the newly provided addresses. - let delay = (!endpoint_addr.is_empty()).then_some(DISCOVERY_WAIT_PERIOD); - let discovery = DiscoveryTask::maybe_start_after_delay(self, endpoint_id, delay) - .ok() - .flatten(); - Ok((addr, discovery)) + Some(maddr) => { + // We have some way of dialing this endpoint, but that doesn't mean we can + // connect to any of these addresses. Start discovery after a small delay. + let discovery = + DiscoveryTask::start_after_delay(self, endpoint_id, DISCOVERY_WAIT_PERIOD) + .ok() + .flatten(); + Ok((maddr, discovery)) } None => { @@ -1301,11 +1280,8 @@ impl Endpoint { .first_arrived() .await .context(get_mapping_address_error::DiscoverSnafu)?; - if let Some(addr) = self.msock.get_mapping_addr(endpoint_id) { - Ok((addr, Some(discovery))) - } else { - Err(get_mapping_address_error::NoAddressSnafu.build()) - } + let addr = self.msock.get_endpoint_mapped_addr(endpoint_id); + Ok((addr, Some(discovery))) } } } @@ -1510,8 +1486,7 @@ impl Future for IncomingFuture { Poll::Pending => Poll::Pending, Poll::Ready(Err(err)) => Poll::Ready(Err(err)), Poll::Ready(Ok(inner)) => { - let conn = Connection { inner }; - try_send_rtt_msg(&conn, this.ep, None); + let conn = Connection::new(inner, None, this.ep); Poll::Ready(Ok(conn)) } } @@ -1598,18 +1573,18 @@ impl Connecting { pub fn into_0rtt(self) -> Result<(Connection, ZeroRttAccepted), Self> { match self.inner.into_0rtt() { Ok((inner, zrtt_accepted)) => { - let conn = Connection { inner }; + // This call is why `self.remote_node_id` was introduced. + // When we `Connecting::into_0rtt`, then we don't yet have `handshake_data` + // in our `Connection`, thus we won't be able to pick up + // `Connection::remote_node_id`. + // Instead, we provide `self.remote_node_id` here - we know it in advance, + // after all. + let conn = Connection::new(inner, self.remote_endpoint_id, &self.ep); let zrtt_accepted = ZeroRttAccepted { inner: zrtt_accepted, _discovery_drop_guard: self._discovery_drop_guard, }; - // This call is why `self.remote_endpoint_id` was introduced. - // When we `Connecting::into_0rtt`, then we don't yet have `handshake_data` - // in our `Connection`, thus `try_send_rtt_msg` won't be able to pick up - // `Connection::remote_endpoint_id`. - // Instead, we provide `self.remote_endpoint_id` here - we know it in advance, - // after all. - try_send_rtt_msg(&conn, &self.ep, self.remote_endpoint_id); + Ok((conn, zrtt_accepted)) } Err(inner) => Err(Self { @@ -1648,8 +1623,7 @@ impl Future for Connecting { Poll::Pending => Poll::Pending, Poll::Ready(Err(err)) => Poll::Ready(Err(err)), Poll::Ready(Ok(inner)) => { - let conn = Connection { inner }; - try_send_rtt_msg(&conn, this.ep, *this.remote_endpoint_id); + let conn = Connection::new(inner, *this.remote_endpoint_id, this.ep); Poll::Ready(Ok(conn)) } } @@ -1697,6 +1671,7 @@ impl Future for ZeroRttAccepted { #[derive(Debug, Clone)] pub struct Connection { inner: quinn::Connection, + paths_info: n0_watcher::Direct>, } #[allow(missing_docs)] @@ -1707,6 +1682,42 @@ pub struct RemoteEndpointIdError { } impl Connection { + fn new(inner: quinn::Connection, remote_id: Option, ep: &Endpoint) -> Self { + let mut paths_info = HashMap::with_capacity(5); + if let Some(path0) = inner.path(PathId::ZERO) { + // This all is supposed to be infallible, but anyway. + if let Ok(remote) = path0.remote_address() { + if let Some(remote) = ep.msock.endpoint_map.transport_addr_from_mapped(remote) { + paths_info.insert( + remote.clone(), + PathInfo { + remote, + path_id: PathId::ZERO, + }, + ); + } + } + } + let paths_info_watcher = n0_watcher::Watchable::new(paths_info); + let conn = Connection { + inner, + paths_info: paths_info_watcher.watch(), + }; + + // Grab the remote identity and register this connection + if let Some(remote) = remote_id { + ep.msock + .register_connection(remote, &conn.inner, paths_info_watcher); + } else if let Ok(remote) = conn.remote_id() { + ep.msock + .register_connection(remote, &conn.inner, paths_info_watcher); + } else { + warn!("unable to determine node id for the remote"); + } + + conn + } + /// Initiates a new outgoing unidirectional stream. /// /// Streams are cheap and instantaneous to open unless blocked by flow control. As a @@ -1962,6 +1973,15 @@ impl Connection { self.inner.stable_id() } + /// Returns information about the network paths in use by this connection. + /// + /// A connection can have several network paths to the remote endpoint, commonly there + /// will be a path via the relay server and a holepunched path. This returns all the + /// paths in use by this connection. + pub fn paths_info(&self) -> impl Watcher> { + self.paths_info.clone() + } + /// Derives keying material from this connection's TLS session secrets. /// /// When both peers call this method with the same `label` and `context` @@ -2005,34 +2025,6 @@ impl Connection { } } -/// Try send a message to the rtt-actor. -/// -/// If we can't notify the actor that will impact performance a little, but we can still -/// function. -fn try_send_rtt_msg( - conn: &Connection, - magic_ep: &Endpoint, - remote_endpoint_id: Option, -) { - // If we can't notify the rtt-actor that's not great but not critical. - let Some(endpoint_id) = remote_endpoint_id.or_else(|| conn.remote_id().ok()) else { - warn!(?conn, "failed to get remote endpoint id"); - return; - }; - let Some(conn_type_changes) = magic_ep.conn_type(endpoint_id) else { - warn!(?conn, "failed to create conn_type stream"); - return; - }; - let rtt_msg = RttMessage::NewConnection { - connection: conn.inner.weak_handle(), - conn_type_changes: conn_type_changes.stream(), - endpoint_id, - }; - if let Err(err) = magic_ep.rtt_actor.msg_tx.try_send(rtt_msg) { - warn!(?conn, "rtt-actor not reachable: {err:#}"); - } -} - /// Read a proxy url from the environment, in this order /// /// - `HTTP_PROXY` @@ -2139,12 +2131,13 @@ mod tests { use n0_watcher::Watcher; use quinn::ConnectionError; use rand::SeedableRng; - use tracing::{Instrument, error_span, info, info_span}; + use tokio::sync::oneshot; + use tracing::{Instrument, error_span, info, info_span, instrument}; use tracing_test::traced_test; use super::Endpoint; use crate::{ - RelayMode, + RelayMap, RelayMode, discovery::static_provider::StaticProvider, endpoint::{ConnectOptions, Connection, ConnectionType}, protocol::{AcceptError, ProtocolHandler, Router}, @@ -2166,6 +2159,7 @@ mod tests { assert!(res.is_err()); let err = res.err().unwrap(); assert!(err.to_string().starts_with("Connecting to ourself")); + Ok(()) } @@ -2419,6 +2413,67 @@ mod tests { Ok(()) } + #[tokio::test] + #[traced_test] + async fn endpoint_two_direct_only() -> Result { + // Connect two endpoints on the same network, without a relay server, without + // discovery. + let ep1 = { + let span = info_span!("server"); + let _guard = span.enter(); + Endpoint::builder() + .alpns(vec![TEST_ALPN.to_vec()]) + .relay_mode(RelayMode::Disabled) + .bind() + .await? + }; + let ep2 = { + let span = info_span!("client"); + let _guard = span.enter(); + Endpoint::builder() + .alpns(vec![TEST_ALPN.to_vec()]) + .relay_mode(RelayMode::Disabled) + .bind() + .await? + }; + let ep1_nodeaddr = ep1.addr(); + + #[instrument(name = "client", skip_all)] + async fn connect(ep: Endpoint, dst: EndpointAddr) -> Result { + info!(me = %ep.id().fmt_short(), "client starting"); + let conn = ep.connect(dst, TEST_ALPN).await?; + let mut send = conn.open_uni().await.e()?; + send.write_all(b"hello").await.e()?; + send.finish().e()?; + Ok(conn.closed().await) + } + + #[instrument(name = "server", skip_all)] + async fn accept(ep: Endpoint, src: EndpointId) -> Result { + info!(me = %ep.id().fmt_short(), "server starting"); + let conn = ep.accept().await.e()?.await.e()?; + let node_id = conn.remote_id()?; + assert_eq!(node_id, src); + let mut recv = conn.accept_uni().await.e()?; + let msg = recv.read_to_end(100).await.e()?; + assert_eq!(msg, b"hello"); + // Dropping the connection closes it just fine. + Ok(()) + } + + let ep1_accept = tokio::spawn(accept(ep1.clone(), ep2.id())); + let ep2_connect = tokio::spawn(connect(ep2.clone(), ep1_nodeaddr)); + + ep1_accept.await.e()??; + let conn_closed = dbg!(ep2_connect.await.e()??); + assert!(matches!( + conn_closed, + ConnectionError::ApplicationClosed(quinn::ApplicationClose { .. }) + )); + + Ok(()) + } + #[tokio::test] #[traced_test] async fn endpoint_relay_map_change() -> Result { @@ -2528,6 +2583,97 @@ mod tests { Ok(()) } + #[tokio::test] + #[traced_test] + async fn endpoint_two_relay_only() -> Result { + // Connect two endpoints on the same network, via a relay server, without + // discovery. + let (relay_map, _relay_url, _relay_server_guard) = run_relay_server().await?; + let (node_addr_tx, node_addr_rx) = oneshot::channel(); + + #[instrument(name = "client", skip_all)] + async fn connect( + relay_map: RelayMap, + node_addr_rx: oneshot::Receiver, + ) -> Result { + let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); + let secret = SecretKey::generate(&mut rng); + let ep = Endpoint::builder() + .secret_key(secret) + .alpns(vec![TEST_ALPN.to_vec()]) + .insecure_skip_relay_cert_verify(true) + .relay_mode(RelayMode::Custom(relay_map)) + .bind() + .await?; + info!(me = %ep.id().fmt_short(), "client starting"); + let dst = node_addr_rx.await.e()?; + + info!(me = %ep.id().fmt_short(), "client connecting"); + let conn = ep.connect(dst, TEST_ALPN).await?; + let mut send = conn.open_uni().await.e()?; + send.write_all(b"hello").await.e()?; + let mut paths = conn.paths_info().stream(); + info!("Waiting for direct connection"); + while let Some(infos) = paths.next().await { + info!(?infos, "new PathInfos"); + if infos.keys().any(|addr| addr.is_ip()) { + break; + } + } + info!("Have direct connection"); + send.write_all(b"close please").await.e()?; + send.finish().e()?; + Ok(conn.closed().await) + } + + #[instrument(name = "server", skip_all)] + async fn accept( + relay_map: RelayMap, + node_addr_tx: oneshot::Sender, + ) -> Result { + let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(1u64); + let secret = SecretKey::generate(&mut rng); + let ep = Endpoint::builder() + .secret_key(secret) + .alpns(vec![TEST_ALPN.to_vec()]) + .insecure_skip_relay_cert_verify(true) + .relay_mode(RelayMode::Custom(relay_map)) + .bind() + .await?; + ep.online().await; + let mut node_addr = ep.addr(); + node_addr.addrs.retain(|addr| addr.is_relay()); + node_addr_tx.send(node_addr).unwrap(); + + info!(me = %ep.id().fmt_short(), "server starting"); + let conn = ep.accept().await.e()?.await.e()?; + // let node_id = conn.remote_node_id()?; + // assert_eq!(node_id, src); + let mut recv = conn.accept_uni().await.e()?; + let mut msg = [0u8; 5]; + recv.read_exact(&mut msg).await.e()?; + assert_eq!(&msg, b"hello"); + info!("received hello"); + let msg = recv.read_to_end(100).await.e()?; + assert_eq!(msg, b"close please"); + info!("received 'close please'"); + // Dropping the connection closes it just fine. + Ok(()) + } + + let server_task = tokio::spawn(accept(relay_map.clone(), node_addr_tx)); + let client_task = tokio::spawn(connect(relay_map, node_addr_rx)); + + server_task.await.e()??; + let conn_closed = dbg!(client_task.await.e()??); + assert!(matches!( + conn_closed, + ConnectionError::ApplicationClosed(quinn::ApplicationClose { .. }) + )); + + Ok(()) + } + #[tokio::test] #[traced_test] async fn endpoint_bidi_send_recv() -> Result { diff --git a/iroh/src/endpoint/rtt_actor.rs b/iroh/src/endpoint/rtt_actor.rs deleted file mode 100644 index 4dddc7955af..00000000000 --- a/iroh/src/endpoint/rtt_actor.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! Actor which coordinates the congestion controller for the magic socket - -use std::{pin::Pin, sync::Arc, task::Poll}; - -use iroh_base::EndpointId; -use n0_future::{ - MergeUnbounded, Stream, StreamExt, - task::{self, AbortOnDropHandle}, -}; -use tokio::sync::mpsc; -use tracing::{Instrument, debug, info_span}; - -use crate::{magicsock::ConnectionType, metrics::MagicsockMetrics}; - -#[derive(Debug)] -pub(super) struct RttHandle { - // We should and some point use this to propagate panics and errors. - pub(super) _handle: AbortOnDropHandle<()>, - pub(super) msg_tx: mpsc::Sender, -} - -impl RttHandle { - pub(super) fn new(metrics: Arc) -> Self { - let mut actor = RttActor { - connection_events: Default::default(), - metrics, - }; - let (msg_tx, msg_rx) = mpsc::channel(16); - let handle = task::spawn( - async move { - actor.run(msg_rx).await; - } - .instrument(info_span!("rtt-actor")), - ); - Self { - _handle: AbortOnDropHandle::new(handle), - msg_tx, - } - } -} - -/// Messages to send to the [`RttActor`]. -#[derive(Debug)] -pub(super) enum RttMessage { - /// Informs the [`RttActor`] of a new connection is should monitor. - NewConnection { - /// The connection. - connection: quinn::WeakConnectionHandle, - /// Path changes for this connection from the magic socket. - conn_type_changes: n0_watcher::Stream>, - /// For reporting-only, the Endpoint ID of this connection. - endpoint_id: EndpointId, - }, -} - -/// Actor to coordinate congestion controller state with magic socket state. -/// -/// The magic socket can change the underlying network path, between two endpoints. If we can -/// inform the QUIC congestion controller of this event it will work much more efficiently. -#[derive(derive_more::Debug)] -struct RttActor { - /// Stream of connection type changes. - #[debug("MergeUnbounded>")] - connection_events: MergeUnbounded, - metrics: Arc, -} - -#[derive(Debug)] -struct MappedStream { - stream: n0_watcher::Stream>, - endpoint_id: EndpointId, - /// Reference to the connection. - connection: quinn::WeakConnectionHandle, - /// This an indiciator of whether this connection was direct before. - /// This helps establish metrics on number of connections that became direct. - was_direct_before: bool, -} - -struct ConnectionEvent { - became_direct: bool, -} - -impl Stream for MappedStream { - type Item = ConnectionEvent; - - /// Performs the congestion controller reset for a magic socket path change. - /// - /// Regardless of which kind of path we are changed to, the congestion controller needs - /// resetting. Even when switching to mixed we should reset the state as e.g. switching - /// from direct to mixed back to direct should be a rare exception and is a bug if this - /// happens commonly. - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll> { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Ready(Some(new_conn_type)) => { - let mut became_direct = false; - if self.connection.network_path_changed() { - debug!( - endpoint_id = %self.endpoint_id.fmt_short(), - new_type = ?new_conn_type, - "Congestion controller state reset", - ); - if !self.was_direct_before && matches!(new_conn_type, ConnectionType::Direct(_)) - { - self.was_direct_before = true; - became_direct = true - } - }; - Poll::Ready(Some(ConnectionEvent { became_direct })) - } - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -impl RttActor { - /// Runs the actor main loop. - /// - /// The main loop will finish when the sender is dropped. - async fn run(&mut self, mut msg_rx: mpsc::Receiver) { - loop { - tokio::select! { - biased; - msg = msg_rx.recv() => { - match msg { - Some(msg) => self.handle_msg(msg), - None => break, - } - } - event = self.connection_events.next(), if !self.connection_events.is_empty() => { - if event.map(|e| e.became_direct).unwrap_or(false) { - self.metrics.connection_became_direct.inc(); - } - } - } - } - debug!("rtt-actor finished"); - } - - /// Handle actor messages. - fn handle_msg(&mut self, msg: RttMessage) { - match msg { - RttMessage::NewConnection { - connection, - conn_type_changes, - endpoint_id, - } => { - self.handle_new_connection(connection, conn_type_changes, endpoint_id); - } - } - } - - /// Handles the new connection message. - fn handle_new_connection( - &mut self, - connection: quinn::WeakConnectionHandle, - conn_type_changes: n0_watcher::Stream>, - endpoint_id: EndpointId, - ) { - self.connection_events.push(MappedStream { - stream: conn_type_changes, - connection, - endpoint_id, - was_direct_before: false, - }); - self.metrics.connection_handshake_success.inc(); - } -} diff --git a/iroh/src/magicsock.rs b/iroh/src/magicsock.rs index f87abdf2c3f..9ac643050fc 100644 --- a/iroh/src/magicsock.rs +++ b/iroh/src/magicsock.rs @@ -20,16 +20,13 @@ use std::{ fmt::Display, io, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, - pin::Pin, sync::{ Arc, Mutex, RwLock, - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, }, - task::{Context, Poll}, }; use bytes::Bytes; -use data_encoding::HEXLOWER; use iroh_base::{EndpointAddr, EndpointId, PublicKey, RelayUrl, SecretKey, TransportAddr}; use iroh_relay::{RelayConfig, RelayMap}; use n0_future::{ @@ -41,58 +38,71 @@ use nested_enum_utils::common_fields; use netwatch::netmon; #[cfg(not(wasm_browser))] use netwatch::{UdpSocket, ip::LocalAddresses}; -use quinn::{AsyncUdpSocket, ServerConfig}; +use quinn::ServerConfig; use rand::Rng; -use smallvec::SmallVec; use snafu::{ResultExt, Snafu}; -use tokio::sync::{Mutex as AsyncMutex, mpsc}; +use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot}; use tokio_util::sync::CancellationToken; -use tracing::{ - Instrument, Level, debug, error, event, info, info_span, instrument, trace, trace_span, warn, -}; -use transports::LocalAddrsWatch; +use tracing::{Instrument, Level, debug, event, info, info_span, instrument, trace, warn}; +use transports::{LocalAddrsWatch, MagicTransport}; use url::Url; #[cfg(not(wasm_browser))] use self::transports::IpTransport; use self::{ - endpoint_map::{EndpointMap, PingAction, PingRole, SendPing}, + endpoint_map::{EndpointMap, EndpointStateMessage}, metrics::Metrics as MagicsockMetrics, - transports::{RelayActorConfig, RelayTransport, Transports, UdpSender}, + transports::{RelayActorConfig, RelayTransport, Transports, TransportsSender}, }; #[cfg(not(wasm_browser))] use crate::dns::DnsResolver; #[cfg(any(test, feature = "test-utils"))] use crate::endpoint::PathSelection; #[cfg(not(wasm_browser))] -use crate::net_report::{IpMappedAddr, QuicConfig}; +use crate::net_report::QuicConfig; use crate::{ defaults::timeouts::NET_REPORT_TIMEOUT, - disco::{self, SendAddr, TransactionId}, + disco::{self, SendAddr}, discovery::{ConcurrentDiscovery, Discovery, EndpointData, UserData}, key::{DecryptionError, SharedSecret, public_ed_box, secret_ed_box}, - magicsock::endpoint_map::RemoteInfo, metrics::EndpointMetrics, - net_report::{self, IfStateDetails, IpMappedAddresses, Report}, + net_report::{self, IfStateDetails, Report}, }; -mod endpoint_map; mod metrics; +pub(crate) mod endpoint_map; +pub(crate) mod mapped_addrs; pub(crate) mod transports; -pub use endpoint_map::Source; +use mapped_addrs::{EndpointIdMappedAddr, MappedAddr}; pub use self::{ - endpoint_map::{ConnectionType, ControlMsg, DirectAddrInfo}, + endpoint_map::{ConnectionType, PathInfo}, metrics::Metrics, }; -/// How long we consider a QAD-derived endpoint valid for. UDP NAT mappings typically -/// expire at 30 seconds, so this is a few seconds shy of that. -const ENDPOINTS_FRESH_ENOUGH_DURATION: Duration = Duration::from_secs(27); +// TODO: Use this +// /// How long we consider a QAD-derived endpoint valid for. UDP NAT mappings typically +// /// expire at 30 seconds, so this is a few seconds shy of that. +// const ENDPOINTS_FRESH_ENOUGH_DURATION: Duration = Duration::from_secs(27); + +/// The duration in which we send keep-alives. +/// +/// If a path is idle for this long, a PING frame will be sent to keep the connection +/// alive. +pub(crate) const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); -const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); +/// The maximum time a path can stay idle before being closed. +/// +/// This is [`HEARTBEAT_INTERVAL`] + 1.5s. This gives us a chance to send a PING frame and +/// some retries. +pub(crate) const PATH_MAX_IDLE_TIMEOUT: Duration = Duration::from_millis(6500); + +/// Maximum number of concurrent QUIC multipath paths per connection. +/// +/// Pretty arbitrary and high right now. +pub(crate) const MAX_MULTIPATH_PATHS: u32 = 16; /// Contains options for `MagicSock::listen`. #[derive(derive_more::Debug)] @@ -187,9 +197,8 @@ pub(crate) struct MagicSock { /// If the last net_report report, reports IPv6 to be available. ipv6_reported: Arc, /// Tracks the networkmap endpoint entity for each endpoint discovery key. - endpoint_map: EndpointMap, - /// Tracks the mapped IP addresses - ip_mapped_addrs: IpMappedAddresses, + pub(crate) endpoint_map: EndpointMap, + /// Local addresses local_addrs_watch: LocalAddrsWatch, /// Currently bound IP addresses of all sockets @@ -263,6 +272,42 @@ impl MagicSock { self.local_addrs_watch.clone().get() } + /// Registers the connection in the [`NodeStateActor`]. + /// + /// The actor is responsible for holepunching and opening additional paths to this + /// connection. + /// + /// [`NodeStateActor`]: crate::magicsock::node_map::node_state::NodeStateActor + pub(crate) fn register_connection( + &self, + remote: EndpointId, + conn: &quinn::Connection, + paths_info: n0_watcher::Watchable>, + ) { + // TODO: Spawning tasks like this is obviously bad. But it is solvable: + // - This is only called from inside Connection::new. + // - Connection::new is called from: + // - impl Future for IncomingFuture + // - impl Future for Connecting + // - Connecting::into_0rtt() + // + // The first two can keep returning Pending until this message is also sent. It'll + // require storing the pinned future but it'll work. + // + // The last one is trickier. But we can make that function async. Or more likely + // we'll end up changing Connecting::into_0rtt() to return a ZrttConnection. Then + // have a ZrttConnection::into_connection() function which can be async and actually + // send this. Before the handshake has completed we don't have anything useful to + // do with this connection inside of the NodeStateActor anyway. + let weak_handle = conn.weak_handle(); + let node_state = self.endpoint_map.endpoint_state_actor(remote); + let msg = EndpointStateMessage::AddConnection(weak_handle, paths_info); + + tokio::task::spawn(async move { + node_state.send(msg).await.ok(); + }); + } + #[cfg(not(wasm_browser))] fn ip_bind_addrs(&self) -> &[SocketAddr] { &self.ip_bind_addrs @@ -275,21 +320,13 @@ impl MagicSock { } /// Returns `true` if we have at least one candidate address where we can send packets to. - pub(crate) fn has_send_address(&self, endpoint_key: PublicKey) -> bool { - self.remote_info(endpoint_key) - .map(|info| info.has_send_address()) - .unwrap_or(false) - } - - /// Return the [`RemoteInfo`]s of all endpoints in the endpoint map. - #[cfg(test)] - pub(crate) fn list_remote_infos(&self) -> Vec { - self.endpoint_map.list_remote_infos(Instant::now()) - } - - /// Return the [`RemoteInfo`] for a single endpoint in the endpoint map. - pub(crate) fn remote_info(&self, endpoint_id: EndpointId) -> Option { - self.endpoint_map.remote_info(endpoint_id) + pub(crate) async fn has_send_address(&self, eid: EndpointId) -> bool { + let actor = self.endpoint_map.endpoint_state_actor(eid); + let (tx, rx) = oneshot::channel(); + if actor.send(EndpointStateMessage::CanSend(tx)).await.is_err() { + return false; + } + rx.await.unwrap_or(false) } pub(crate) async fn insert_relay( @@ -377,33 +414,41 @@ impl MagicSock { /// Returns a [`n0_watcher::Direct`] that reports the [`ConnectionType`] we have to the /// given `endpoint_id`. /// - /// This gets us a copy of the [`n0_watcher::Direct`] for the [`Watchable`] with a [`ConnectionType`] - /// that the `EndpointMap` stores for each `endpoint_id`'s endpoint. + /// This gets us a copy of the [`n0_watcher::Direct`] for the [`Watchable`] with a + /// [`ConnectionType`] that the `EndpointMap` stores for each `endpoint_id`'s endpoint. /// /// # Errors /// /// Will return `None` if there is no address information known about the /// given `endpoint_id`. - pub(crate) fn conn_type( - &self, - endpoint_id: EndpointId, - ) -> Option> { - self.endpoint_map.conn_type(endpoint_id) + pub(crate) fn conn_type(&self, eid: EndpointId) -> Option> { + self.endpoint_map.conn_type(eid) } - pub(crate) fn latency(&self, endpoint_id: EndpointId) -> Option { - self.endpoint_map.latency(endpoint_id) + // TODO: Build better info to expose to the user about remote nodes. We probably want + // to expose this as part of path information instead. + pub(crate) async fn latency(&self, eid: EndpointId) -> Option { + let endpoint_state = self.endpoint_map.endpoint_state_actor(eid); + let (tx, rx) = oneshot::channel(); + endpoint_state + .send(EndpointStateMessage::Latency(tx)) + .await + .ok(); + rx.await.unwrap_or_default() } /// Returns the socket address which can be used by the QUIC layer to dial this endpoint. - pub(crate) fn get_mapping_addr(&self, endpoint_id: EndpointId) -> Option { - self.endpoint_map - .get_quic_mapped_addr_for_endpoint_key(endpoint_id) + pub(crate) fn get_endpoint_mapped_addr(&self, eid: EndpointId) -> EndpointIdMappedAddr { + self.endpoint_map.endpoint_mapped_addr(eid) } - /// Add addresses for an endpoint to the magic socket's addresbook. + /// Add potential addresses for a node to the [`EndpointStateActor`]. + /// + /// This is used to add possible paths that the remote node might be reachable on. They + /// will be used when there is no active connection to the node to attempt to establish + /// a connection. #[instrument(skip_all)] - pub(crate) fn add_endpoint_addr( + pub(crate) async fn add_endpoint_addr( &self, mut addr: EndpointAddr, source: endpoint_map::Source, @@ -416,9 +461,10 @@ impl MagicSock { } } if !addr.is_empty() { - let have_ipv6 = self.ipv6_reported.load(Ordering::Relaxed); + // Add addr to the internal EndpointMap self.endpoint_map - .add_endpoint_addr(addr, source, have_ipv6, &self.metrics.magicsock); + .add_endpoint_addr(addr.clone(), source) + .await; Ok(()) } else if pruned != 0 { Err(EmptyPrunedSnafu { pruned }.build()) @@ -434,8 +480,6 @@ impl MagicSock { pub(super) fn store_direct_addresses(&self, addrs: BTreeSet) { let updated = self.direct_addrs.update(addrs); if updated { - self.endpoint_map - .on_direct_addr_discovered(self.direct_addrs.sockaddrs().collect()); self.publish_my_addr(); } } @@ -499,97 +543,14 @@ impl MagicSock { } } - /// Searches the `endpoint_map` to determine the current transports to be used. - #[instrument(skip_all)] - fn prepare_send( - &self, - udp_sender: &UdpSender, - transmit: &quinn_udp::Transmit, - ) -> io::Result> { - self.metrics - .magicsock - .send_data - .inc_by(transmit.contents.len() as _); - - if self.is_closed() { - self.metrics - .magicsock - .send_data_network_down - .inc_by(transmit.contents.len() as _); - return Err(io::Error::new( - io::ErrorKind::NotConnected, - "connection closed", - )); - } - - let mut active_paths = SmallVec::<[_; 3]>::new(); - - match MappedAddr::from(transmit.destination) { - MappedAddr::None(dest) => { - error!(%dest, "Cannot convert to a mapped address."); - } - MappedAddr::EndpointId(dest) => { - trace!( - dst = %dest, - src = ?transmit.src_ip, - len = %transmit.contents.len(), - "sending", - ); - - // Get the endpoint's relay address and best direct address, as well - // as any pings that need to be sent for hole-punching purposes. - match self.endpoint_map.get_send_addrs( - dest, - self.ipv6_reported.load(Ordering::Relaxed), - &self.metrics.magicsock, - ) { - Some((endpoint_id, udp_addr, relay_url, ping_actions)) => { - if !ping_actions.is_empty() { - self.try_send_ping_actions(udp_sender, ping_actions).ok(); - } - if let Some(addr) = udp_addr { - active_paths.push(transports::Addr::from(addr)); - } - if let Some(url) = relay_url { - active_paths.push(transports::Addr::Relay(url, endpoint_id)); - } - } - None => { - error!(%dest, "no EndpointState for mapped address"); - } - } - } - #[cfg(not(wasm_browser))] - MappedAddr::Ip(dest) => { - trace!( - dst = %dest, - src = ?transmit.src_ip, - len = %transmit.contents.len(), - "sending", - ); - - // Check if this is a known IpMappedAddr, and if so, send over UDP - // Get the socket addr - match self.ip_mapped_addrs.get_ip_addr(&dest) { - Some(addr) => { - active_paths.push(transports::Addr::from(addr)); - } - None => { - error!(%dest, "unknown mapped address"); - } - } - } - } - - Ok(active_paths) - } - - /// Process datagrams received from UDP sockets. + /// Process datagrams received from all the transports. /// /// All the `bufs` and `metas` should have initialized packets in them. /// - /// This fixes up the datagrams to use the correct [`EndpointIdMappedAddr`] and extracts DISCO - /// packets, processing them inside the magic socket. + /// This fixes up the datagrams to use the correct [`MultipathMappedAddr`] and extracts + /// DISCO packets, processing them inside the magic socket. + /// + /// [`MultipathMappedAddr`]: mapped_addrs::MultipathMappedAddr fn process_datagrams( &self, bufs: &mut [io::IoSliceMut<'_>], @@ -651,11 +612,11 @@ impl MagicSock { // relies on quinn::EndpointConfig::grease_quic_bit being set to `false`, // which we do in Endpoint::bind. if let Some((sender, sealed_box)) = disco::source_and_box(datagram) { - trace!(src = ?source_addr, len = %quinn_meta.stride, "UDP recv: disco packet"); + trace!(src = ?source_addr, len = datagram.len(), "UDP recv: DISCO packet"); self.handle_disco_message(sender, sealed_box, source_addr); datagram[0] = 0u8; } else { - trace!(src = ?source_addr, len = %quinn_meta.stride, "UDP recv: quic packet"); + trace!(src = ?source_addr, len = datagram.len(), "UDP recv: QUIC packet"); match source_addr { transports::Addr::Ip(SocketAddr::V4(..)) => { self.metrics @@ -689,54 +650,15 @@ impl MagicSock { panic!("cannot use IP based addressing in the browser"); } #[cfg(not(wasm_browser))] - transports::Addr::Ip(addr) => { - // UDP - - // Update the EndpointMap and remap RecvMeta to the EndpointIdMappedAddr. - match self.endpoint_map.receive_udp(*addr) { - None => { - // Check if this address is mapped to an IpMappedAddr - if let Some(ip_mapped_addr) = - self.ip_mapped_addrs.get_mapped_addr(addr) - { - trace!( - src = %addr, - count = %quic_datagram_count, - len = quinn_meta.len, - "UDP recv QUIC address discovery packets", - ); - quic_packets_total += quic_datagram_count; - quinn_meta.addr = ip_mapped_addr.private_socket_addr(); - } else { - warn!( - src = %addr, - count = %quic_datagram_count, - len = quinn_meta.len, - "UDP recv quic packets: no endpoint state found, skipping", - ); - // If we have no endpoint state for the from addr, set len to 0 to make - // quinn skip the buf completely. - quinn_meta.len = 0; - } - } - Some((endpoint_id, quic_mapped_addr)) => { - trace!( - src = %addr, - endpoint = %endpoint_id.fmt_short(), - count = %quic_datagram_count, - len = quinn_meta.len, - "UDP recv quic packets", - ); - quic_packets_total += quic_datagram_count; - quinn_meta.addr = quic_mapped_addr.private_socket_addr(); - } - } + transports::Addr::Ip(_addr) => { + quic_packets_total += quic_datagram_count; } - transports::Addr::Relay(src_url, src_endpoint) => { - // Relay - let quic_mapped_addr = - self.endpoint_map.receive_relay(src_url, *src_endpoint); - quinn_meta.addr = quic_mapped_addr.private_socket_addr(); + transports::Addr::Relay(src_url, src_node) => { + let mapped_addr = self + .endpoint_map + .relay_mapped_addrs + .get(&(src_url.clone(), *src_node)); + quinn_meta.addr = mapped_addr.private_socket_addr(); } } } else { @@ -760,7 +682,6 @@ impl MagicSock { /// Handles a discovery message. #[instrument("disco_in", skip_all, fields(endpoint = %sender.fmt_short(), ?src))] fn handle_disco_message(&self, sender: PublicKey, sealed_box: &[u8], src: &transports::Addr) { - trace!("handle_disco_message start"); if self.is_closed() { return; } @@ -804,202 +725,28 @@ impl MagicSock { self.metrics.magicsock.recv_disco_udp.inc(); } - let span = trace_span!("handle_disco", ?dm); - let _guard = span.enter(); - trace!("receive disco message"); + trace!(?dm, "receive disco message"); match dm { disco::Message::Ping(ping) => { self.metrics.magicsock.recv_disco_ping.inc(); - self.handle_ping(ping, sender, src); + self.endpoint_map.handle_ping(ping, sender, src.clone()); } disco::Message::Pong(pong) => { self.metrics.magicsock.recv_disco_pong.inc(); - self.endpoint_map - .handle_pong(sender, src, pong, &self.metrics.magicsock); + self.endpoint_map.handle_pong(pong, sender, src.clone()); } disco::Message::CallMeMaybe(cm) => { self.metrics.magicsock.recv_disco_call_me_maybe.inc(); - match src { - transports::Addr::Relay(url, _) => { - event!( - target: "iroh::_events::call-me-maybe::recv", - Level::DEBUG, - remote_endpoint = %sender.fmt_short(), - via = ?url, - their_addrs = ?cm.my_numbers, - ); - } - _ => { - warn!("call-me-maybe packets should only come via relay"); - return; - } - } - let ping_actions = - self.endpoint_map - .handle_call_me_maybe(sender, cm, &self.metrics.magicsock); - for action in ping_actions { - match action { - PingAction::SendCallMeMaybe { .. } => { - warn!("Unexpected CallMeMaybe as response of handling a CallMeMaybe"); - } - PingAction::SendPing(ping) => { - self.send_ping_queued(ping); - } - } - } - } - } - trace!("disco message handled"); - } - - /// Handle a ping message. - fn handle_ping(&self, dm: disco::Ping, sender: EndpointId, src: &transports::Addr) { - // Insert the ping into the endpoint map, and return whether a ping with this tx_id was already - // received. - let addr: SendAddr = src.clone().into(); - let handled = self - .endpoint_map - .handle_ping(sender, addr.clone(), dm.tx_id); - match handled.role { - PingRole::Duplicate => { - debug!(?src, tx = %HEXLOWER.encode(&dm.tx_id), "received ping: path already confirmed, skip"); - return; - } - PingRole::LikelyHeartbeat => {} - PingRole::NewPath => { - debug!(?src, tx = %HEXLOWER.encode(&dm.tx_id), "received ping: new path"); - } - PingRole::Activate => { - debug!(?src, tx = %HEXLOWER.encode(&dm.tx_id), "received ping: path active"); - } - } - - // Send a pong. - debug!(tx = %HEXLOWER.encode(&dm.tx_id), %addr, dstkey = %sender.fmt_short(), - "sending pong"); - let pong = disco::Message::Pong(disco::Pong { - tx_id: dm.tx_id, - ping_observed_addr: addr.clone(), - }); - event!( - target: "iroh::_events::pong::sent", - Level::DEBUG, - remote_endpoint = %sender.fmt_short(), - dst = ?addr, - txn = ?dm.tx_id, - ); - - if !self.disco.try_send(addr.clone(), sender, pong) { - warn!(%addr, "failed to queue pong"); - } - - if let Some(ping) = handled.needs_ping_back { - debug!( - %addr, - dstkey = %sender.fmt_short(), - "sending direct ping back", - ); - self.send_ping_queued(ping); - } - } - - fn send_ping_queued(&self, ping: SendPing) { - let SendPing { - id, - dst, - dst_endpoint, - tx_id, - purpose, - } = ping; - let msg = disco::Message::Ping(disco::Ping { - tx_id, - endpoint_key: self.public_key, - }); - let sent = self.disco.try_send(dst.clone(), dst_endpoint, msg); - if sent { - let msg_sender = self.actor_sender.clone(); - trace!(%dst, tx = %HEXLOWER.encode(&tx_id), ?purpose, "ping sent (queued)"); - self.endpoint_map - .notify_ping_sent(id, dst, tx_id, purpose, msg_sender); - } else { - warn!(dst = ?dst, tx = %HEXLOWER.encode(&tx_id), ?purpose, "failed to send ping: queues full"); - } - } - - /// Send the given ping actions out. - async fn send_ping_actions(&self, sender: &UdpSender, msgs: Vec) -> io::Result<()> { - for msg in msgs { - // Abort sending as soon as we know we are shutting down. - if self.is_closing() || self.is_closed() { - return Ok(()); - } - match msg { - PingAction::SendCallMeMaybe { - relay_url, - dst_endpoint, - } => { - // Sends the call-me-maybe DISCO message, queuing if addresses are too stale. - // - // To send the call-me-maybe message, we need to know our current direct addresses. If - // this information is too stale, the call-me-maybe is queued while a net_report run is - // scheduled. Once this run finishes, the call-me-maybe will be sent. - match self.direct_addrs.fresh_enough() { - Ok(()) => { - let msg = disco::Message::CallMeMaybe( - self.direct_addrs.to_call_me_maybe_message(), - ); - if !self.disco.try_send( - SendAddr::Relay(relay_url.clone()), - dst_endpoint, - msg.clone(), - ) { - warn!(dstkey = %dst_endpoint.fmt_short(), %relay_url, "relay channel full, dropping call-me-maybe"); - } else { - debug!(dstkey = %dst_endpoint.fmt_short(), %relay_url, "call-me-maybe sent"); - } - } - Err(last_refresh_ago) => { - debug!( - ?last_refresh_ago, - "want call-me-maybe but direct addrs stale; queuing after restun", - ); - self.actor_sender - .try_send(ActorMessage::ScheduleDirectAddrUpdate( - UpdateReason::RefreshForPeering, - Some((dst_endpoint, relay_url)), - )) - .ok(); - } - } - } - PingAction::SendPing(SendPing { - id, - dst, - dst_endpoint, - tx_id, - purpose, - }) => { - let msg = disco::Message::Ping(disco::Ping { - tx_id, - endpoint_key: self.public_key, - }); - - self.send_disco_message(sender, dst.clone(), dst_endpoint, msg) - .await?; - debug!(%dst, tx = %HEXLOWER.encode(&tx_id), ?purpose, "ping sent"); - let msg_sender = self.actor_sender.clone(); - self.endpoint_map - .notify_ping_sent(id, dst, tx_id, purpose, msg_sender); - } + self.endpoint_map + .handle_call_me_maybe(cm, sender, src.clone()); } } - Ok(()) } /// Sends out a disco message. async fn send_disco_message( &self, - sender: &UdpSender, + sender: &TransportsSender, dst: SendAddr, dst_key: PublicKey, msg: disco::Message, @@ -1017,7 +764,7 @@ impl MagicSock { )); } - let pkt = self.disco.encode_and_seal(self.public_key, dst_key, &msg); + let pkt = self.disco.encode_and_seal(dst_key, &msg); let transmit = transports::Transmit { contents: &pkt, @@ -1039,118 +786,6 @@ impl MagicSock { } } } - /// Tries to send out the given ping actions out. - fn try_send_ping_actions(&self, sender: &UdpSender, msgs: Vec) -> io::Result<()> { - for msg in msgs { - // Abort sending as soon as we know we are shutting down. - if self.is_closing() || self.is_closed() { - return Ok(()); - } - match msg { - PingAction::SendCallMeMaybe { - relay_url, - dst_endpoint, - } => { - // Sends the call-me-maybe DISCO message, queuing if addresses are too stale. - // - // To send the call-me-maybe message, we need to know our current direct addresses. If - // this information is too stale, the call-me-maybe is queued while a net_report run is - // scheduled. Once this run finishes, the call-me-maybe will be sent. - match self.direct_addrs.fresh_enough() { - Ok(()) => { - let msg = disco::Message::CallMeMaybe( - self.direct_addrs.to_call_me_maybe_message(), - ); - if !self.disco.try_send( - SendAddr::Relay(relay_url.clone()), - dst_endpoint, - msg.clone(), - ) { - warn!(dstkey = %dst_endpoint.fmt_short(), %relay_url, "relay channel full, dropping call-me-maybe"); - } else { - debug!(dstkey = %dst_endpoint.fmt_short(), %relay_url, "call-me-maybe sent"); - } - } - Err(last_refresh_ago) => { - debug!( - ?last_refresh_ago, - "want call-me-maybe but direct addrs stale; queuing after restun", - ); - self.actor_sender - .try_send(ActorMessage::ScheduleDirectAddrUpdate( - UpdateReason::RefreshForPeering, - Some((dst_endpoint, relay_url)), - )) - .ok(); - } - } - } - PingAction::SendPing(SendPing { - id, - dst, - dst_endpoint, - tx_id, - purpose, - }) => { - let msg = disco::Message::Ping(disco::Ping { - tx_id, - endpoint_key: self.public_key, - }); - - self.try_send_disco_message(sender, dst.clone(), dst_endpoint, msg)?; - debug!(%dst, tx = %HEXLOWER.encode(&tx_id), ?purpose, "ping sent"); - let msg_sender = self.actor_sender.clone(); - self.endpoint_map - .notify_ping_sent(id, dst, tx_id, purpose, msg_sender); - } - } - } - Ok(()) - } - - /// Tries to send out a disco message. - fn try_send_disco_message( - &self, - sender: &UdpSender, - dst: SendAddr, - dst_key: PublicKey, - msg: disco::Message, - ) -> io::Result<()> { - let dst = match dst { - SendAddr::Udp(addr) => transports::Addr::Ip(addr), - SendAddr::Relay(url) => transports::Addr::Relay(url, dst_key), - }; - - trace!(?dst, %msg, "send disco message (UDP)"); - if self.is_closed() { - return Err(io::Error::new( - io::ErrorKind::NotConnected, - "connection closed", - )); - } - - let pkt = self.disco.encode_and_seal(self.public_key, dst_key, &msg); - - let transmit = transports::Transmit { - contents: &pkt, - ecn: None, - segment_size: None, - }; - - let dst2 = dst.clone(); - match sender.inner_try_send(&dst2, None, &transmit) { - Ok(()) => { - trace!(?dst, %msg, "sent disco message"); - self.metrics.magicsock.sent_disco_udp.inc(); - disco_message_sent(&msg, &self.metrics.magicsock); - Ok(()) - } - Err(err) => { - warn!(?dst, ?msg, ?err, "failed to send disco message"); - Err(err) - } - } - } /// Publishes our address to a discovery service, if configured. /// @@ -1181,32 +816,6 @@ impl MagicSock { } } -#[derive(Clone, Debug)] -enum MappedAddr { - EndpointId(EndpointIdMappedAddr), - #[cfg(not(wasm_browser))] - Ip(IpMappedAddr), - None(SocketAddr), -} - -impl From for MappedAddr { - fn from(value: SocketAddr) -> Self { - match value.ip() { - IpAddr::V4(_) => MappedAddr::None(value), - IpAddr::V6(addr) => { - if let Ok(endpoint_id_mapped_addr) = EndpointIdMappedAddr::try_from(addr) { - return MappedAddr::EndpointId(endpoint_id_mapped_addr); - } - #[cfg(not(wasm_browser))] - if let Ok(ip_mapped_addr) = IpMappedAddr::try_from(addr) { - return MappedAddr::Ip(ip_mapped_addr); - } - MappedAddr::None(value) - } - } - } -} - /// Manages currently running direct addr discovery, aka net_report runs. /// /// Invariants: @@ -1231,7 +840,6 @@ enum UpdateReason { /// Initial state #[default] None, - RefreshForPeering, Periodic, PortmapUpdated, LinkChangeMajor, @@ -1390,23 +998,10 @@ impl Handle { let (ip_transports, port_mapper) = bind_ip(addr_v4, addr_v6, &metrics).context(BindSocketsSnafu)?; - let ip_mapped_addrs = IpMappedAddresses::default(); - let (actor_sender, actor_receiver) = mpsc::channel(256); - let ipv6_reported = false; - - // load the endpoint data - let endpoint_map = EndpointMap::load_from_vec( - Vec::new(), - #[cfg(any(test, feature = "test-utils"))] - path_selection, - ipv6_reported, - &metrics.magicsock, - ); - let my_relay = Watchable::new(None); - let ipv6_reported = Arc::new(AtomicBool::new(ipv6_reported)); + let ipv6_reported = Arc::new(AtomicBool::new(false)); let relay_transport = RelayTransport::new(RelayActorConfig { my_relay: my_relay.clone(), @@ -1421,7 +1016,6 @@ impl Handle { }); let relay_transports = vec![relay_transport]; - let secret_encryption_key = secret_ed_box(&secret_key); #[cfg(not(wasm_browser))] let ipv6 = ip_transports.iter().any(|t| t.bind_addr().is_ipv6()); @@ -1430,7 +1024,21 @@ impl Handle { #[cfg(wasm_browser)] let transports = Transports::new(relay_transports); - let (disco, disco_receiver) = DiscoState::new(secret_encryption_key); + let direct_addrs = DiscoveredDirectAddrs::default(); + let (disco, disco_receiver) = DiscoState::new(&secret_key); + + let endpoint_map = { + let sender = transports.create_sender(); + EndpointMap::new( + secret_key.public(), + #[cfg(any(test, feature = "test-utils"))] + path_selection, + metrics.magicsock.clone(), + sender, + direct_addrs.addrs.watch(), + disco.clone(), + ) + }; let msock = Arc::new(MagicSock { public_key: secret_key.public(), @@ -1440,11 +1048,10 @@ impl Handle { actor_sender: actor_sender.clone(), ipv6_reported, endpoint_map, - ip_mapped_addrs: ip_mapped_addrs.clone(), discovery, relay_map: relay_map.clone(), discovery_user_data: RwLock::new(discovery_user_data), - direct_addrs: DiscoveredDirectAddrs::default(), + direct_addrs, net_report: Watchable::new((None, UpdateReason::None)), #[cfg(not(wasm_browser))] dns_resolver: dns_resolver.clone(), @@ -1462,17 +1069,14 @@ impl Handle { // the packet if grease_quic_bit is set to false. endpoint_config.grease_quic_bit(false); - let sender = transports.create_sender(msock.clone()); + let sender = transports.create_sender(); let local_addrs_watch = transports.local_addrs_watch(); let network_change_sender = transports.create_network_change_sender(); let endpoint = quinn::Endpoint::new_with_abstract_socket( endpoint_config, Some(server_config), - Box::new(MagicUdpSocket { - socket: msock.clone(), - transports, - }), + Box::new(MagicTransport::new(msock.clone(), transports)), #[cfg(not(wasm_browser))] Arc::new(quinn::TokioRuntime), #[cfg(wasm_browser)] @@ -1512,7 +1116,6 @@ impl Handle { #[cfg(not(wasm_browser))] dns_resolver, #[cfg(not(wasm_browser))] - Some(ip_mapped_addrs), relay_map.clone(), net_report_config, metrics.net_report.clone(), @@ -1573,9 +1176,11 @@ impl Handle { /// Closes the connection. /// - /// Only the first close does anything. Any later closes return nil. - /// Polling the socket ([`AsyncUdpSocket::poll_recv`]) will return [`Poll::Pending`] - /// indefinitely after this call. + /// Only the first close does anything. Any later closes return nil. Polling the socket + /// ([`quinn::AsyncUdpSocket::poll_recv`]) will return [`Poll::Pending`] indefinitely + /// after this call. + /// + /// [`Poll::Pending`]: std::task::Poll::Pending #[instrument(skip_all)] pub(crate) async fn close(&self) { trace!(me = ?self.public_key, "magicsock closing..."); @@ -1643,25 +1248,30 @@ fn default_quic_client_config() -> rustls::ClientConfig { .with_no_client_auth() } -#[derive(Debug)] +#[derive(Debug, Clone)] struct DiscoState { + /// The EndpointId/PublikeKey of this node. + this_id: EndpointId, /// Encryption key for this endpoint. - secret_encryption_key: crypto_box::SecretKey, + secret_encryption_key: Arc, /// The state for an active DiscoKey. - secrets: Mutex>, + secrets: Arc>>, /// Disco (ping) queue sender: mpsc::Sender<(SendAddr, PublicKey, disco::Message)>, } impl DiscoState { fn new( - secret_encryption_key: crypto_box::SecretKey, + secret_key: &SecretKey, ) -> (Self, mpsc::Receiver<(SendAddr, PublicKey, disco::Message)>) { + let this_id = secret_key.public(); + let secret_encryption_key = secret_ed_box(secret_key); let (disco_sender, disco_receiver) = mpsc::channel(256); ( Self { - secret_encryption_key, + this_id, + secret_encryption_key: Arc::new(secret_encryption_key), secrets: Default::default(), sender: disco_sender, }, @@ -1669,28 +1279,23 @@ impl DiscoState { ) } - fn try_send(&self, dst: SendAddr, endpoint_id: PublicKey, msg: disco::Message) -> bool { - self.sender.try_send((dst, endpoint_id, msg)).is_ok() + fn try_send(&self, dst: SendAddr, dst_key: PublicKey, msg: disco::Message) -> bool { + self.sender.try_send((dst, dst_key, msg)).is_ok() } - fn encode_and_seal( - &self, - this_endpoint_id: EndpointId, - other_endpoint_id: EndpointId, - msg: &disco::Message, - ) -> Bytes { + fn encode_and_seal(&self, other_key: PublicKey, msg: &disco::Message) -> Bytes { let mut seal = msg.as_bytes(); - self.get_secret(other_endpoint_id, |secret| secret.seal(&mut seal)); - disco::encode_message(&this_endpoint_id, seal).into() + self.get_secret(other_key, |secret| secret.seal(&mut seal)); + disco::encode_message(&self.this_id, seal).into() } fn unseal_and_decode( &self, - endpoint_id: PublicKey, + endpoint_key: PublicKey, sealed_box: &[u8], ) -> Result { let mut sealed_box = sealed_box.to_vec(); - self.get_secret(endpoint_id, |secret| secret.open(&mut sealed_box)) + self.get_secret(endpoint_key, |secret| secret.open(&mut sealed_box)) .context(OpenSnafu)?; disco::Message::from_bytes(&sealed_box).context(ParseSnafu) } @@ -1730,70 +1335,9 @@ enum DiscoBoxError { } #[derive(Debug)] -struct MagicUdpSocket { - socket: Arc, - transports: Transports, -} - -impl AsyncUdpSocket for MagicUdpSocket { - fn create_sender(&self) -> Pin> { - Box::pin(self.transports.create_sender(self.socket.clone())) - } - - /// NOTE: Receiving on a closed socket will return [`Poll::Pending`] indefinitely. - fn poll_recv( - &mut self, - cx: &mut Context, - bufs: &mut [io::IoSliceMut<'_>], - metas: &mut [quinn_udp::RecvMeta], - ) -> Poll> { - self.transports.poll_recv(cx, bufs, metas, &self.socket) - } - - #[cfg(not(wasm_browser))] - fn local_addr(&self) -> io::Result { - let addrs: Vec<_> = self - .transports - .local_addrs() - .into_iter() - .filter_map(|addr| { - let addr: SocketAddr = addr.into_socket_addr()?; - Some(addr) - }) - .collect(); - - if let Some(addr) = addrs.iter().find(|addr| addr.is_ipv6()) { - return Ok(*addr); - } - if let Some(SocketAddr::V4(addr)) = addrs.first() { - // Pretend to be IPv6, because our `MappedAddr`s need to be IPv6. - let ip = addr.ip().to_ipv6_mapped().into(); - return Ok(SocketAddr::new(ip, addr.port())); - } - - Err(io::Error::other("no valid address available")) - } - - #[cfg(wasm_browser)] - fn local_addr(&self) -> io::Result { - // Again, we need to pretend we're IPv6, because of our `MappedAddr`s. - Ok(SocketAddr::new(std::net::Ipv6Addr::LOCALHOST.into(), 0)) - } - - fn max_receive_segments(&self) -> usize { - self.transports.max_receive_segments() - } - - fn may_fragment(&self) -> bool { - self.transports.may_fragment() - } -} - -#[derive(Debug)] +#[allow(clippy::enum_variant_names)] enum ActorMessage { - EndpointPingExpired(usize, TransactionId), NetworkChange, - ScheduleDirectAddrUpdate(UpdateReason, Option<(EndpointId, RelayUrl)>), RelayMapChange, #[cfg(test)] ForceNetworkChange(bool), @@ -1873,14 +1417,11 @@ impl Actor { mut self, shutdown_token: CancellationToken, mut watcher: impl Watcher> + Send + Sync, - sender: UdpSender, + sender: TransportsSender, ) { // Setup network monitoring let mut current_netmon_state = self.netmon_watcher.get(); - #[cfg(not(wasm_browser))] - let mut direct_addr_heartbeat_timer = time::interval(HEARTBEAT_INTERVAL); - #[cfg(not(wasm_browser))] let mut portmap_watcher = self .direct_addr_update_state @@ -1903,11 +1444,6 @@ impl Actor { #[cfg(wasm_browser)] let portmap_watcher_changed = n0_future::future::pending(); - #[cfg(not(wasm_browser))] - let direct_addr_heartbeat_timer_tick = direct_addr_heartbeat_timer.tick(); - #[cfg(wasm_browser)] - let direct_addr_heartbeat_timer_tick = n0_future::future::pending(); - tokio::select! { _ = shutdown_token.cancelled() => { debug!("shutting down"); @@ -1990,22 +1526,6 @@ impl Actor { #[cfg(wasm_browser)] let _unused_in_browsers = change; }, - _ = direct_addr_heartbeat_timer_tick => { - #[cfg(not(wasm_browser))] - { - trace!( - "tick: direct addr heartbeat {} direct addrs", - self.msock.endpoint_map.endpoint_count(), - ); - self.msock.metrics.magicsock.actor_tick_direct_addr_heartbeat.inc(); - // TODO: this might trigger too many packets at once, pace this - - self.msock.endpoint_map.prune_inactive(); - let have_v6 = self.netmon_watcher.clone().get().have_v6; - let msgs = self.msock.endpoint_map.endpoints_stayin_alive(have_v6); - self.handle_ping_actions(&sender, msgs).await; - } - } state = self.netmon_watcher.updated() => { let Ok(state) = state else { trace!("tick: link change receiver closed"); @@ -2043,7 +1563,6 @@ impl Actor { #[cfg(not(wasm_browser))] self.msock.dns_resolver.reset().await; self.re_stun(UpdateReason::LinkChangeMajor); - self.reset_endpoint_states(); } else { self.re_stun(UpdateReason::LinkChangeMinor); } @@ -2059,36 +1578,14 @@ impl Actor { .schedule_run(why, state.into()); } - #[instrument(skip_all)] - async fn handle_ping_actions(&mut self, sender: &UdpSender, msgs: Vec) { - if let Err(err) = self.msock.send_ping_actions(sender, msgs).await { - warn!("Failed to send ping actions: {err:#}"); - } - } - /// Processes an incoming actor message. /// /// Returns `true` if it was a shutdown. async fn handle_actor_message(&mut self, msg: ActorMessage) { match msg { - ActorMessage::EndpointPingExpired(id, txid) => { - self.msock.endpoint_map.notify_ping_timeout( - id, - txid, - &self.msock.metrics.magicsock, - ); - } ActorMessage::NetworkChange => { self.network_monitor.network_change().await.ok(); } - ActorMessage::ScheduleDirectAddrUpdate(why, data) => { - if let Some((endpoint, url)) = data { - self.pending_call_me_maybes.insert(endpoint, url); - } - let state = self.netmon_watcher.get(); - self.direct_addr_update_state - .schedule_run(why, state.into()); - } ActorMessage::RelayMapChange => { self.handle_relay_map_change(); } @@ -2274,15 +1771,6 @@ impl Actor { #[cfg(not(wasm_browser))] self.update_direct_addresses(report.as_ref()); } - - /// Resets the preferred address for all endpoints. - /// This is called when connectivity changes enough that we no longer trust the old routes. - #[instrument(skip_all)] - fn reset_endpoint_states(&mut self) { - self.msock - .endpoint_map - .reset_endpoint_states(&self.msock.metrics.magicsock) - } } fn new_re_stun_timer(initial_delay: bool) -> time::Interval { @@ -2362,117 +1850,12 @@ impl DiscoveredDirectAddrs { self.addrs.get().into_iter().map(|da| da.addr) } - /// Whether the direct addr information is considered "fresh". - /// - /// If not fresh you should probably update the direct addresses before using this info. - /// - /// Returns `Ok(())` if fresh enough and `Err(elapsed)` if not fresh enough. - /// `elapsed` is the time elapsed since the direct addresses were last updated. - /// - /// If there is no direct address information `Err(Duration::ZERO)` is returned. - fn fresh_enough(&self) -> Result<(), Duration> { - match *self.updated_at.read().expect("poisoned") { - None => Err(Duration::ZERO), - Some(time) => { - let elapsed = time.elapsed(); - if elapsed <= ENDPOINTS_FRESH_ENOUGH_DURATION { - Ok(()) - } else { - Err(elapsed) - } - } - } - } - fn to_call_me_maybe_message(&self) -> disco::CallMeMaybe { let my_numbers = self.addrs.get().into_iter().map(|da| da.addr).collect(); disco::CallMeMaybe { my_numbers } } } -/// The fake address used by the QUIC layer to address an endpoint. -/// -/// You can consider this as nothing more than a lookup key for an endpoint the [`MagicSock`] knows -/// about. -/// -/// [`MagicSock`] can reach an endpoint by several real socket addresses, or maybe even via the relay -/// endpoint. The QUIC layer however needs to address an endpoint by a stable [`SocketAddr`] so -/// that normal socket APIs can function. Thus when a new endpoint is introduced to a [`MagicSock`] -/// it is given a new fake address. This is the type of that address. -/// -/// It is but a newtype. And in our QUIC-facing socket APIs like [`AsyncUdpSocket`] it -/// comes in as the inner [`Ipv6Addr`], in those interfaces we have to be careful to do -/// the conversion to this type. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub(crate) struct EndpointIdMappedAddr(Ipv6Addr); - -/// Can occur when converting a [`SocketAddr`] to an [`EndpointIdMappedAddr`] -#[derive(Debug, Snafu)] -#[snafu(display("Failed to convert"))] -pub struct EndpointIdMappedAddrError; - -/// Counter to always generate unique addresses for [`EndpointIdMappedAddr`]. -static ENDPOINT_ID_ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); - -impl EndpointIdMappedAddr { - /// The Prefix/L of our Unique Local Addresses. - const ADDR_PREFIXL: u8 = 0xfd; - /// The Global ID used in our Unique Local Addresses. - const ADDR_GLOBAL_ID: [u8; 5] = [21, 7, 10, 81, 11]; - /// The Subnet ID used in our Unique Local Addresses. - const ADDR_SUBNET: [u8; 2] = [0; 2]; - - /// The dummy port used for all [`EndpointIdMappedAddr`]s. - const ENDPOINT_ID_MAPPED_PORT: u16 = 12345; - - /// Generates a globally unique fake UDP address. - /// - /// This generates and IPv6 Unique Local Address according to RFC 4193. - pub(crate) fn generate() -> Self { - let mut addr = [0u8; 16]; - addr[0] = Self::ADDR_PREFIXL; - addr[1..6].copy_from_slice(&Self::ADDR_GLOBAL_ID); - addr[6..8].copy_from_slice(&Self::ADDR_SUBNET); - - let counter = ENDPOINT_ID_ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); - addr[8..16].copy_from_slice(&counter.to_be_bytes()); - - Self(Ipv6Addr::from(addr)) - } - - /// Returns a consistent [`SocketAddr`] for the [`EndpointIdMappedAddr`]. - /// - /// This socket address does not have a routable IP address. - /// - /// This uses a made-up port number, since the port does not play a role in looking up - /// the endpoint in the [`EndpointMap`]. This socket address is only to be used to pass into - /// Quinn. - pub(crate) fn private_socket_addr(&self) -> SocketAddr { - SocketAddr::new(IpAddr::from(self.0), Self::ENDPOINT_ID_MAPPED_PORT) - } -} - -impl TryFrom for EndpointIdMappedAddr { - type Error = EndpointIdMappedAddrError; - - fn try_from(value: Ipv6Addr) -> Result { - let octets = value.octets(); - if octets[0] == Self::ADDR_PREFIXL - && octets[1..6] == Self::ADDR_GLOBAL_ID - && octets[6..8] == Self::ADDR_SUBNET - { - return Ok(Self(value)); - } - Err(EndpointIdMappedAddrError) - } -} - -impl std::fmt::Display for EndpointIdMappedAddr { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "EndpointIdMappedAddr({})", self.0) - } -} - fn disco_message_sent(msg: &disco::Message, metrics: &MagicsockMetrics) { match msg { disco::Message::Ping(_) => { @@ -2492,6 +1875,9 @@ fn disco_message_sent(msg: &disco::Message, metrics: &MagicsockMetrics) { /// Direct addresses are UDP socket addresses on which an iroh endpoint could potentially be /// contacted. These can come from various sources depending on the network topology of the /// iroh endpoint, see [`DirectAddrType`] for the several kinds of sources. +/// +/// This is essentially a combination of our local addresses combined with any reflexive +/// transport addresses we disovered using QAD. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct DirectAddr { /// The address. @@ -2543,26 +1929,26 @@ impl Display for DirectAddrType { #[cfg(test)] mod tests { - use std::{collections::BTreeSet, net::SocketAddr, sync::Arc, time::Duration}; + use std::{sync::Arc, time::Duration}; use data_encoding::HEXLOWER; - use iroh_base::{EndpointAddr, EndpointId, PublicKey, TransportAddr}; - use n0_future::{StreamExt, time}; + use iroh_base::{EndpointAddr, EndpointId, TransportAddr}; + use n0_future::{MergeBounded, StreamExt, time}; use n0_snafu::{Result, ResultExt}; use n0_watcher::Watcher; use quinn::ServerConfig; use rand::{CryptoRng, Rng, RngCore, SeedableRng}; - use tokio::task::JoinSet; use tokio_util::task::AbortOnDropHandle; use tracing::{Instrument, error, info, info_span, instrument}; use tracing_test::traced_test; - use super::{EndpointIdMappedAddr, Options}; + use super::{EndpointIdMappedAddr, Options, endpoint_map::Source, mapped_addrs::MappedAddr}; use crate::{ Endpoint, RelayMap, RelayMode, SecretKey, + discovery::static_provider::StaticProvider, dns::DnsResolver, - endpoint::{PathSelection, Source}, - magicsock::{Handle, MagicSock, endpoint_map}, + endpoint::PathSelection, + magicsock::{Handle, MagicSock}, tls::{self, DEFAULT_MAX_TLS_TICKETS}, }; @@ -2598,137 +1984,10 @@ mod tests { server_config } - impl MagicSock { - #[track_caller] - pub fn add_test_addr(&self, endpoint_addr: EndpointAddr) { - self.add_endpoint_addr( - endpoint_addr, - Source::NamedApp { - name: "test".into(), - }, - ) - .unwrap() - } - } - - /// Magicsock plus wrappers for sending packets - #[derive(Clone)] - struct MagicStack { - secret_key: SecretKey, - endpoint: Endpoint, - } - - impl MagicStack { - async fn new(rng: &mut R, relay_mode: RelayMode) -> Self { - let secret_key = SecretKey::generate(rng); - - let mut transport_config = quinn::TransportConfig::default(); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - - let endpoint = Endpoint::empty_builder(relay_mode) - .secret_key(secret_key.clone()) - .transport_config(transport_config) - .alpns(vec![ALPN.to_vec()]) - .bind() - .await - .unwrap(); - - Self { - secret_key, - endpoint, - } - } - - fn tracked_endpoints(&self) -> Vec { - self.endpoint - .magic_sock() - .list_remote_infos() - .into_iter() - .map(|ep| ep.endpoint_id) - .collect() - } - - fn public(&self) -> PublicKey { - self.secret_key.public() - } - } - - /// Monitors endpoint changes and plumbs things together. - /// - /// This is a way of connecting endpoints without a relay server. Whenever the local - /// endpoints of a magic endpoint change this address is added to the other magic - /// sockets. This function will await until the endpoints are connected the first time - /// before returning. - /// - /// When the returned drop guard is dropped, the tasks doing this updating are stopped. - #[instrument(skip_all)] - async fn mesh_stacks(stacks: Vec) -> Result> { - /// Registers endpoint addresses of an endpoint to all other endpoints. - fn update_direct_addrs( - stacks: &[MagicStack], - my_idx: usize, - new_addrs: BTreeSet, - ) { - let me = &stacks[my_idx]; - for (i, m) in stacks.iter().enumerate() { - if i == my_idx { - continue; - } - - let addr = EndpointAddr { - id: me.public(), - addrs: new_addrs.iter().copied().map(TransportAddr::Ip).collect(), - }; - m.endpoint.magic_sock().add_test_addr(addr); - } - } - - // For each endpoint, start a task which monitors its local endpoints and registers them - // with the other endpoints as local endpoints become known. - let mut tasks = JoinSet::new(); - for (my_idx, m) in stacks.iter().enumerate() { - let m = m.clone(); - let stacks = stacks.clone(); - tasks.spawn(async move { - let me = m.endpoint.id().fmt_short(); - let mut stream = m.endpoint.watch_addr().stream(); - while let Some(addr) = stream.next().await { - info!(%me, "conn{} endpoints update: {:?}", my_idx + 1, addr.ip_addrs().collect::>()); - update_direct_addrs(&stacks, my_idx, addr.ip_addrs().copied().collect()); - } - }); - } - - // Wait for all endpoints to be registered with each other. - time::timeout(Duration::from_secs(10), async move { - let all_endpoint_ids: Vec<_> = stacks.iter().map(|ms| ms.endpoint.id()).collect(); - loop { - let mut ready = Vec::with_capacity(stacks.len()); - for ms in stacks.iter() { - let endpoints = ms.tracked_endpoints(); - let my_endpoint_id = ms.endpoint.id(); - let all_endpoints_meshed = all_endpoint_ids - .iter() - .filter(|endpoint_id| **endpoint_id != my_endpoint_id) - .all(|endpoint_id| endpoints.contains(endpoint_id)); - ready.push(all_endpoints_meshed); - } - if ready.iter().all(|meshed| *meshed) { - break; - } - time::sleep(Duration::from_millis(200)).await; - } - }) - .await - .context("timeout")?; - info!("all endpoints meshed"); - Ok(tasks) - } - - #[instrument(skip_all, fields(me = %ep.endpoint.id().fmt_short()))] - async fn echo_receiver(ep: MagicStack, loss: ExpectedLoss) -> Result { + #[instrument(skip_all, fields(me = %ep.id().fmt_short()))] + async fn echo_receiver(ep: Endpoint, loss: ExpectedLoss) -> Result { info!("accepting conn"); - let conn = ep.endpoint.accept().await.expect("no conn"); + let conn = ep.accept().await.expect("no conn"); info!("connecting"); let conn = conn.await.context("connecting")?; @@ -2754,30 +2013,32 @@ mod tests { info!("stats: {:#?}", stats); // TODO: ensure panics in this function are reported ok if matches!(loss, ExpectedLoss::AlmostNone) { - assert!( - stats.path.lost_packets < 10, - "[receiver] should not loose many packets", - ); + for (id, path) in &stats.paths { + assert!( + path.lost_packets < 10, + "[receiver] path {id:?} should not loose many packets", + ); + } } info!("close"); conn.close(0u32.into(), b"done"); info!("wait idle"); - ep.endpoint.endpoint().wait_idle().await; + ep.endpoint().wait_idle().await; Ok(()) } - #[instrument(skip_all, fields(me = %ep.endpoint.id().fmt_short()))] + #[instrument(skip_all, fields(me = %ep.id().fmt_short()))] async fn echo_sender( - ep: MagicStack, - dest_id: PublicKey, + ep: Endpoint, + dest_id: EndpointId, msg: &[u8], loss: ExpectedLoss, ) -> Result { info!("connecting to {}", dest_id.fmt_short()); let dest = EndpointAddr::new(dest_id); - let conn = ep.endpoint.connect(dest, ALPN).await?; + let conn = ep.connect(dest, ALPN).await?; info!("opening bi"); let (mut send_bi, mut recv_bi) = conn.open_bi().await.context("open bi")?; @@ -2805,16 +2066,18 @@ mod tests { let stats = conn.stats(); info!("stats: {:#?}", stats); if matches!(loss, ExpectedLoss::AlmostNone) { - assert!( - stats.path.lost_packets < 10, - "[sender] should not loose many packets", - ); + for (id, path) in &stats.paths { + assert!( + path.lost_packets < 10, + "[sender] path {id:?} should not loose many packets", + ); + } } info!("close"); conn.close(0u32.into(), b"done"); info!("wait idle"); - ep.endpoint.endpoint().wait_idle().await; + ep.endpoint().wait_idle().await; Ok(()) } @@ -2826,13 +2089,13 @@ mod tests { /// Runs a roundtrip between the [`echo_sender`] and [`echo_receiver`]. async fn run_roundtrip( - sender: MagicStack, - receiver: MagicStack, + sender: Endpoint, + receiver: Endpoint, payload: &[u8], loss: ExpectedLoss, ) { - let send_endpoint_id = sender.endpoint.id(); - let recv_endpoint_id = receiver.endpoint.id(); + let send_endpoint_id = sender.id(); + let recv_endpoint_id = receiver.id(); info!("\nroundtrip: {send_endpoint_id:#} -> {recv_endpoint_id:#}"); let receiver_task = tokio::spawn(echo_receiver(receiver, loss)); @@ -2864,14 +2127,48 @@ mod tests { } } + /// Returns a pair of endpoints with a shared [`StaticDiscovery`]. + /// + /// The endpoints do not use a relay server but can connect to each other via local + /// addresses. Dialing by [`NodeId`] is possible, and the addresses get updated even if + /// the endpoints rebind. + async fn endpoint_pair() -> (AbortOnDropHandle<()>, Endpoint, Endpoint) { + let discovery = StaticProvider::new(); + let ep1 = Endpoint::builder() + .relay_mode(RelayMode::Disabled) + .alpns(vec![ALPN.to_vec()]) + .discovery(discovery.clone()) + .bind() + .await + .unwrap(); + let ep2 = Endpoint::builder() + .relay_mode(RelayMode::Disabled) + .alpns(vec![ALPN.to_vec()]) + .discovery(discovery.clone()) + .bind() + .await + .unwrap(); + discovery.add_endpoint_info(ep1.addr()); + discovery.add_endpoint_info(ep2.addr()); + + let ep1_addr_stream = ep1.watch_addr().stream(); + let ep2_addr_stream = ep2.watch_addr().stream(); + let mut addr_stream = MergeBounded::from_iter([ep1_addr_stream, ep2_addr_stream]); + let task = tokio::spawn(async move { + loop { + while let Some(addr) = addr_stream.next().await { + discovery.add_endpoint_info(addr); + } + } + }); + + (AbortOnDropHandle::new(task), ep1, ep2) + } + #[tokio::test(flavor = "multi_thread")] #[traced_test] async fn test_two_devices_roundtrip_quinn_magic() -> Result { - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let m1 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - let m2 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + let (_guard, m1, m2) = endpoint_pair().await; for i in 0..5 { info!("\n-- round {i}"); @@ -2892,6 +2189,7 @@ mod tests { info!("\n-- larger data"); let mut data = vec![0u8; 10 * 1024]; + let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); rng.fill_bytes(&mut data); run_roundtrip(m1.clone(), m2.clone(), &data, ExpectedLoss::AlmostNone).await; run_roundtrip(m2.clone(), m1.clone(), &data, ExpectedLoss::AlmostNone).await; @@ -2903,18 +2201,14 @@ mod tests { #[tokio::test] #[traced_test] async fn test_regression_network_change_rebind_wakes_connection_driver() -> n0_snafu::Result { - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let m1 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - let m2 = MagicStack::new(&mut rng, RelayMode::Disabled).await; + let (_guard, m1, m2) = endpoint_pair().await; println!("Net change"); - m1.endpoint.magic_sock().force_network_change(true).await; + m1.magic_sock().force_network_change(true).await; tokio::time::sleep(Duration::from_secs(1)).await; // wait for socket rebinding - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; - let _handle = AbortOnDropHandle::new(tokio::spawn({ - let endpoint = m2.endpoint.clone(); + let endpoint = m2.clone(); async move { while let Some(incoming) = endpoint.accept().await { println!("Incoming first conn!"); @@ -2927,7 +2221,7 @@ mod tests { })); println!("first conn!"); - let conn = m1.endpoint.connect(m2.endpoint.addr(), ALPN).await?; + let conn = m1.connect(m2.addr(), ALPN).await?; println!("Closing first conn"); conn.close(0u32.into(), b"bye lolz"); conn.closed().await; @@ -2951,10 +2245,7 @@ mod tests { /// with (simulated) network changes. async fn test_two_devices_roundtrip_network_change_impl() -> Result { let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let m1 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - let m2 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + let (_guard, m1, m2) = endpoint_pair().await; let offset = |rng: &mut rand_chacha::ChaCha8Rng| { let delay = rng.random_range(10..=500); @@ -2969,7 +2260,7 @@ mod tests { let task = tokio::spawn(async move { loop { println!("[m1] network change"); - m1.endpoint.magic_sock().force_network_change(true).await; + m1.magic_sock().force_network_change(true).await; time::sleep(offset(&mut rng)).await; } }); @@ -2997,7 +2288,7 @@ mod tests { let task = tokio::spawn(async move { loop { println!("[m2] network change"); - m2.endpoint.magic_sock().force_network_change(true).await; + m2.magic_sock().force_network_change(true).await; time::sleep(offset(&mut rng)).await; } }); @@ -3025,9 +2316,9 @@ mod tests { let mut rng = rng.clone(); let task = tokio::spawn(async move { println!("-- [m1] network change"); - m1.endpoint.magic_sock().force_network_change(true).await; + m1.magic_sock().force_network_change(true).await; println!("-- [m2] network change"); - m2.endpoint.magic_sock().force_network_change(true).await; + m2.magic_sock().force_network_change(true).await; time::sleep(offset(&mut rng)).await; }); AbortOnDropHandle::new(task) @@ -3052,20 +2343,16 @@ mod tests { #[tokio::test(flavor = "multi_thread")] #[traced_test] async fn test_two_devices_setup_teardown() -> Result { - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); for i in 0..10 { println!("-- round {i}"); println!("setting up magic stack"); - let m1 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - let m2 = MagicStack::new(&mut rng, RelayMode::Disabled).await; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + let (_guard, m1, m2) = endpoint_pair().await; println!("closing endpoints"); - let msock1 = m1.endpoint.magic_sock(); - let msock2 = m2.endpoint.magic_sock(); - m1.endpoint.close().await; - m2.endpoint.close().await; + let msock1 = m1.magic_sock(); + let msock2 = m2.magic_sock(); + m1.close().await; + m2.close().await; assert!(msock1.msock.is_closed()); assert!(msock2.msock.is_closed()); @@ -3253,8 +2540,9 @@ mod tests { name: "test".into(), }, ) + .await .unwrap(); - let addr = msock_1.get_mapping_addr(endpoint_id_2).unwrap(); + let addr = msock_1.get_endpoint_mapped_addr(endpoint_id_2); let res = tokio::time::timeout( Duration::from_secs(10), magicsock_connect( @@ -3312,18 +2600,19 @@ mod tests { let _accept_task = AbortOnDropHandle::new(accept_task); // Add an empty entry in the EndpointMap of ep_1 - msock_1.endpoint_map.add_endpoint_addr( - EndpointAddr { - id: endpoint_id_2, - addrs: Default::default(), - }, - Source::NamedApp { - name: "test".into(), - }, - true, - &msock_1.metrics.magicsock, - ); - let addr_2 = msock_1.get_mapping_addr(endpoint_id_2).unwrap(); + msock_1 + .endpoint_map + .add_endpoint_addr( + EndpointAddr { + id: endpoint_id_2, + addrs: Default::default(), + }, + Source::NamedApp { + name: "test".into(), + }, + ) + .await; + let addr_2 = msock_1.get_endpoint_mapped_addr(endpoint_id_2); // Set a low max_idle_timeout so quinn gives up on this quickly and our test does // not take forever. You need to check the log output to verify this is really @@ -3349,23 +2638,23 @@ mod tests { info!("first connect timed out as expected"); // Provide correct addressing information - let addrs = msock_2 - .ip_addrs() - .get() - .into_iter() - .map(|x| TransportAddr::Ip(x.addr)) - .collect(); - msock_1.endpoint_map.add_endpoint_addr( - EndpointAddr { - id: endpoint_id_2, - addrs, - }, - Source::NamedApp { - name: "test".into(), - }, - true, - &msock_1.metrics.magicsock, - ); + msock_1 + .endpoint_map + .add_endpoint_addr( + EndpointAddr { + id: endpoint_id_2, + addrs: msock_2 + .ip_addrs() + .get() + .into_iter() + .map(|x| TransportAddr::Ip(x.addr)) + .collect(), + }, + Source::NamedApp { + name: "test".into(), + }, + ) + .await; // We can now connect tokio::time::timeout(Duration::from_secs(10), async move { @@ -3391,70 +2680,4 @@ mod tests { // TODO: could remove the addresses again, send, add it back and see it recover. // But we don't have that much private access to the EndpointMap. This will do for now. } - - #[tokio::test] - async fn test_add_endpoint_addr() -> Result { - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let stack = MagicStack::new(&mut rng, RelayMode::Default).await; - - assert_eq!(stack.endpoint.magic_sock().endpoint_map.endpoint_count(), 0); - - // Empty - let empty_addr = EndpointAddr::new(SecretKey::generate(&mut rng).public()); - - let err = stack - .endpoint - .magic_sock() - .add_endpoint_addr(empty_addr, endpoint_map::Source::App) - .unwrap_err(); - assert!( - err.to_string() - .to_lowercase() - .contains("empty addressing info") - ); - - // relay url only - let addr = EndpointAddr { - id: SecretKey::generate(&mut rng).public(), - addrs: [TransportAddr::Relay("http://my-relay.com".parse().unwrap())] - .into_iter() - .collect(), - }; - stack - .endpoint - .magic_sock() - .add_endpoint_addr(addr, endpoint_map::Source::App)?; - assert_eq!(stack.endpoint.magic_sock().endpoint_map.endpoint_count(), 1); - - // addrs only - let addr = EndpointAddr { - id: SecretKey::generate(&mut rng).public(), - addrs: [TransportAddr::Ip("127.0.0.1:1234".parse().unwrap())] - .into_iter() - .collect(), - }; - stack - .endpoint - .magic_sock() - .add_endpoint_addr(addr, endpoint_map::Source::App)?; - assert_eq!(stack.endpoint.magic_sock().endpoint_map.endpoint_count(), 2); - - // both - let addr = EndpointAddr { - id: SecretKey::generate(&mut rng).public(), - addrs: [ - TransportAddr::Relay("http://my-relay.com".parse().unwrap()), - TransportAddr::Ip("127.0.0.1:1234".parse().unwrap()), - ] - .into_iter() - .collect(), - }; - stack - .endpoint - .magic_sock() - .add_endpoint_addr(addr, endpoint_map::Source::App)?; - assert_eq!(stack.endpoint.magic_sock().endpoint_map.endpoint_count(), 3); - - Ok(()) - } } diff --git a/iroh/src/magicsock/endpoint_map.rs b/iroh/src/magicsock/endpoint_map.rs index e23da332321..411d2a89e42 100644 --- a/iroh/src/magicsock/endpoint_map.rs +++ b/iroh/src/magicsock/endpoint_map.rs @@ -1,77 +1,66 @@ use std::{ - collections::{BTreeSet, HashMap, hash_map::Entry}, + collections::{BTreeSet, HashMap}, hash::Hash, net::{IpAddr, SocketAddr}, - sync::Mutex, - time::Duration, + sync::{Arc, Mutex}, }; -use iroh_base::{EndpointAddr, EndpointId, PublicKey, RelayUrl}; -use n0_future::time::Instant; +use iroh_base::{EndpointAddr, EndpointId, RelayUrl, TransportAddr}; +use n0_future::task::AbortOnDropHandle; use serde::{Deserialize, Serialize}; -use tracing::{debug, info, instrument, trace, warn}; +use tokio::sync::mpsc; +use tracing::{Instrument, error, info_span, trace, warn}; -use self::endpoint_state::{EndpointState, Options, PingHandled}; -use super::{ActorMessage, EndpointIdMappedAddr, metrics::Metrics, transports}; -use crate::disco::{CallMeMaybe, Pong, SendAddr, TransactionId}; +#[cfg(any(test, feature = "test-utils"))] +use super::transports::TransportsSender; +#[cfg(not(any(test, feature = "test-utils")))] +use super::transports::TransportsSender; +use super::{ + DirectAddr, DiscoState, MagicsockMetrics, + mapped_addrs::{AddrMap, EndpointIdMappedAddr, MultipathMappedAddr, RelayMappedAddr}, + transports::{self, OwnedTransmit}, +}; +use crate::disco::{self}; #[cfg(any(test, feature = "test-utils"))] use crate::endpoint::PathSelection; mod endpoint_state; mod path_state; -mod path_validity; -mod udp_paths; -pub use endpoint_state::{ConnectionType, ControlMsg, DirectAddrInfo}; -pub(super) use endpoint_state::{DiscoPingPurpose, PingAction, PingRole, RemoteInfo, SendPing}; +pub(super) use endpoint_state::EndpointStateMessage; +pub use endpoint_state::{ConnectionType, PathInfo}; +use endpoint_state::{EndpointStateActor, EndpointStateHandle}; -/// Number of endpoints that are inactive for which we keep info about. This limit is enforced -/// periodically via [`EndpointMap::prune_inactive`]. -const MAX_INACTIVE_ENDPOINTS: usize = 30; +// TODO: use this +// /// Number of endpoints that are inactive for which we keep info about. This limit is enforced +// /// periodically via [`NodeMap::prune_inactive`]. +// const MAX_INACTIVE_NODES: usize = 30; /// Map of the [`EndpointState`] information for all the known endpoints. -/// -/// The endpoints can be looked up by: -/// -/// - The endpoint's ID in this map, only useful if you know the ID from an insert or lookup. -/// This is static and never changes. -/// -/// - The [`EndpointIdMappedAddr`] which internally identifies the endpoint to the QUIC stack. This -/// is static and never changes. -/// -/// - The endpoints's public key, aka `PublicKey` or "endpoint_key". This is static and never changes, -/// however an endpoint could be added when this is not yet known. -/// -/// - A public socket address on which they are reachable on the internet, known as ip-port. -/// These come and go as the endpoint moves around on the internet -/// -/// An index of endpointInfos by endpoint key, EndpointIdMappedAddr, and discovered ip:port endpoints. -#[derive(Debug, Default)] -pub(super) struct EndpointMap { +#[derive(Debug)] +pub(crate) struct EndpointMap { + /// The endpoint ID of the local endpoint. + local_endpoint_id: EndpointId, inner: Mutex, + /// The mapping between [`EndpointId`]s and [`EndpointIdMappedAddr`]s. + pub(super) endpoint_mapped_addrs: AddrMap, + /// The mapping between endpoints via a relay and their [`RelayMappedAddr`]s. + pub(super) relay_mapped_addrs: AddrMap<(RelayUrl, EndpointId), RelayMappedAddr>, } -#[derive(Default, Debug)] +#[derive(Debug)] pub(super) struct EndpointMapInner { - by_endpoint_key: HashMap, - by_ip_port: HashMap, - by_quic_mapped_addr: HashMap, - by_id: HashMap, - next_id: usize, + metrics: Arc, + /// Handle to an actor that can send over the transports. + transports_handle: TransportsSenderHandle, + local_addrs: n0_watcher::Direct>, + disco: DiscoState, #[cfg(any(test, feature = "test-utils"))] path_selection: PathSelection, -} - -/// Identifier to look up a [`EndpointState`] in the [`EndpointMap`]. -/// -/// You can look up entries in [`EndpointMap`] with various keys, depending on the context you -/// have for the endpoint. These are all the keys the [`EndpointMap`] can use. -#[derive(Debug, Clone)] -enum EndpointStateKey { - Idx(usize), - EndpointId(EndpointId), - EndpointIdMappedAddr(EndpointIdMappedAddr), - IpPort(IpPort), + /// The [`EndpointStateActor`] for each remote endpoint. + /// + /// [`EndpointStateActor`]: endpoint_state::EndpointStateActor + endpoint_states: HashMap, } /// The origin or *source* through which an address associated with a remote endpoint @@ -83,14 +72,6 @@ enum EndpointStateKey { /// A [`Source`] helps track how and where an address was learned. Multiple /// sources can be associated with a single address, if we have discovered this /// address through multiple means. -/// -/// Each time a [`EndpointAddr`] is added to the endpoint map a [`Source`] must be supplied to indicate -/// how the address was obtained. -/// -/// A [`Source`] can describe a variety of places that an address or endpoint was -/// discovered, such as a configured discovery service, the network itself -/// (if another endpoint has reached out to us), or as a user supplied [`EndpointAddr`]. - #[derive(Serialize, Deserialize, strum::Display, Debug, Clone, Eq, PartialEq, Hash)] #[strum(serialize_all = "kebab-case")] pub enum Source { @@ -114,201 +95,84 @@ pub enum Source { /// The name of the application that added the endpoint name: String, }, + /// The address was advertised by a call-me-maybe DISCO message. + CallMeMaybe, + /// We received a ping on the path. + Ping, + /// We established a connection on this address. + /// + /// Currently this means the path was in uses as [`PathId::ZERO`] when the a connection + /// was added to the [`NodeStateActor`]. + /// + /// [`PathId::ZERO`]: quinn_proto::PathId::ZERO + /// [`NodeStateActor`]: self::node_state::NodeStateActor + Connection, } impl EndpointMap { - /// Create a new [`EndpointMap`] from a list of [`EndpointAddr`]s. - pub(super) fn load_from_vec( - endpoints: Vec, + /// Creates a new [`EndpointMap`]. + pub(super) fn new( + local_endpoint_id: EndpointId, #[cfg(any(test, feature = "test-utils"))] path_selection: PathSelection, - have_ipv6: bool, - metrics: &Metrics, + metrics: Arc, + sender: TransportsSender, + local_addrs: n0_watcher::Direct>, + disco: DiscoState, ) -> Self { - Self::from_inner(EndpointMapInner::load_from_vec( - endpoints, - #[cfg(any(test, feature = "test-utils"))] - path_selection, - have_ipv6, - metrics, - )) - } + #[cfg(not(any(test, feature = "test-utils")))] + let inner = EndpointMapInner::new(metrics, sender, local_addrs, disco); + + #[cfg(any(test, feature = "test-utils"))] + let inner = { + let mut inner = EndpointMapInner::new(metrics, sender, local_addrs, disco); + inner.path_selection = path_selection; + inner + }; - fn from_inner(inner: EndpointMapInner) -> Self { Self { + local_endpoint_id, inner: Mutex::new(inner), + endpoint_mapped_addrs: Default::default(), + relay_mapped_addrs: Default::default(), } } - /// Add the contact information for an endpoint. - pub(super) fn add_endpoint_addr( - &self, - endpoint_addr: EndpointAddr, - source: Source, - have_v6: bool, - metrics: &Metrics, - ) { - self.inner.lock().expect("poisoned").add_endpoint_addr( - endpoint_addr, - source, - have_v6, - metrics, - ) - } - - /// Number of endpoints currently listed. - pub(super) fn endpoint_count(&self) -> usize { - self.inner.lock().expect("poisoned").endpoint_count() - } - - #[cfg(not(wasm_browser))] - pub(super) fn receive_udp( - &self, - udp_addr: SocketAddr, - ) -> Option<(PublicKey, EndpointIdMappedAddr)> { - self.inner.lock().expect("poisoned").receive_udp(udp_addr) - } - - pub(super) fn receive_relay( - &self, - relay_url: &RelayUrl, - src: EndpointId, - ) -> EndpointIdMappedAddr { - self.inner - .lock() - .expect("poisoned") - .receive_relay(relay_url, src) - } - - pub(super) fn notify_ping_sent( - &self, - id: usize, - dst: SendAddr, - tx_id: TransactionId, - purpose: DiscoPingPurpose, - msg_sender: tokio::sync::mpsc::Sender, - ) { - if let Some(ep) = self - .inner - .lock() - .expect("poisoned") - .get_mut(EndpointStateKey::Idx(id)) - { - ep.ping_sent(dst, tx_id, purpose, msg_sender); - } - } - - pub(super) fn notify_ping_timeout(&self, id: usize, tx_id: TransactionId, metrics: &Metrics) { - if let Some(ep) = self - .inner - .lock() - .expect("poisoned") - .get_mut(EndpointStateKey::Idx(id)) - { - ep.ping_timeout(tx_id, Instant::now(), metrics); + /// Adds addresses where a node might be contactable. + pub(super) async fn add_endpoint_addr(&self, endpoint_addr: EndpointAddr, source: Source) { + for url in endpoint_addr.relay_urls() { + // Ensure we have a RelayMappedAddress. + self.relay_mapped_addrs + .get(&(url.clone(), endpoint_addr.id)); } - } - - pub(super) fn get_quic_mapped_addr_for_endpoint_key( - &self, - endpoint_key: EndpointId, - ) -> Option { - self.inner - .lock() - .expect("poisoned") - .get(EndpointStateKey::EndpointId(endpoint_key)) - .map(|ep| *ep.quic_mapped_addr()) - } - - /// Insert a received ping into the endpoint map, and return whether a ping with this tx_id was already - /// received. - pub(super) fn handle_ping( - &self, - sender: PublicKey, - src: SendAddr, - tx_id: TransactionId, - ) -> PingHandled { - self.inner - .lock() - .expect("poisoned") - .handle_ping(sender, src, tx_id) - } + let actor = self.endpoint_state_actor(endpoint_addr.id); - pub(super) fn handle_pong( - &self, - sender: PublicKey, - src: &transports::Addr, - pong: Pong, - metrics: &Metrics, - ) { - self.inner - .lock() - .expect("poisoned") - .handle_pong(sender, src, pong, metrics) - } - - #[must_use = "actions must be handled"] - pub(super) fn handle_call_me_maybe( - &self, - sender: PublicKey, - cm: CallMeMaybe, - metrics: &Metrics, - ) -> Vec { - self.inner - .lock() - .expect("poisoned") - .handle_call_me_maybe(sender, cm, metrics) + // This only fails if the sender is closed. That means the NodeStateActor has + // stopped, which only happens during shutdown. + actor + .send(EndpointStateMessage::AddEndpointAddr(endpoint_addr, source)) + .await + .ok(); } - #[allow(clippy::type_complexity)] - pub(super) fn get_send_addrs( - &self, - addr: EndpointIdMappedAddr, - have_ipv6: bool, - metrics: &Metrics, - ) -> Option<( - PublicKey, - Option, - Option, - Vec, - )> { - let mut inner = self.inner.lock().expect("poisoned"); - let ep = inner.get_mut(EndpointStateKey::EndpointIdMappedAddr(addr))?; - let public_key = *ep.public_key(); - trace!(dest = %addr, endpoint_id = %public_key.fmt_short(), "dst mapped to EndpointId"); - let (udp_addr, relay_url, ping_actions) = ep.get_send_addrs(have_ipv6, metrics); - Some((public_key, udp_addr, relay_url, ping_actions)) + pub(super) fn endpoint_mapped_addr(&self, eid: EndpointId) -> EndpointIdMappedAddr { + self.endpoint_mapped_addrs.get(&eid) } - pub(super) fn reset_endpoint_states(&self, metrics: &Metrics) { - let now = Instant::now(); - let mut inner = self.inner.lock().expect("poisoned"); - for (_, ep) in inner.endpoint_states_mut() { - ep.note_connectivity_change(now, metrics); + /// Converts a mapped address as we use them inside Quinn. + pub(crate) fn transport_addr_from_mapped(&self, mapped: SocketAddr) -> Option { + match MultipathMappedAddr::from(mapped) { + MultipathMappedAddr::Mixed(_) => None, + MultipathMappedAddr::Relay(addr) => match self.relay_mapped_addrs.lookup(&addr) { + Some((url, _)) => Some(TransportAddr::Relay(url)), + None => { + error!("Unknown RelayMappedAddr"); + None + } + }, + MultipathMappedAddr::Ip(addr) => Some(TransportAddr::Ip(addr)), } } - pub(super) fn endpoints_stayin_alive(&self, have_ipv6: bool) -> Vec { - let mut inner = self.inner.lock().expect("poisoned"); - inner - .endpoint_states_mut() - .flat_map(|(_idx, endpoint_state)| endpoint_state.stayin_alive(have_ipv6)) - .collect() - } - - /// Returns the [`RemoteInfo`]s for each endpoint in the endpoint map. - #[cfg(test)] - pub(super) fn list_remote_infos(&self, now: Instant) -> Vec { - // NOTE: calls to this method will often call `into_iter` (or similar methods). Note that - // we can't avoid `collect` here since it would hold a lock for an indefinite time. Even if - // we were to find this acceptable, dealing with the lifetimes of the mutex's guard and the - // internal iterator will be a hassle, if possible at all. - self.inner - .lock() - .expect("poisoned") - .remote_infos_iter(now) - .collect() - } - /// Returns a [`n0_watcher::Direct`] for given endpoint's [`ConnectionType`]. /// /// # Errors @@ -322,206 +186,107 @@ impl EndpointMap { self.inner.lock().expect("poisoned").conn_type(endpoint_id) } - pub(super) fn latency(&self, endpoint_id: EndpointId) -> Option { - self.inner.lock().expect("poisoned").latency(endpoint_id) - } - - /// Get the [`RemoteInfo`]s for the endpoint identified by [`EndpointId`]. - pub(super) fn remote_info(&self, endpoint_id: EndpointId) -> Option { - self.inner - .lock() - .expect("poisoned") - .remote_info(endpoint_id) - } - - /// Prunes endpoints without recent activity so that at most [`MAX_INACTIVE_ENDPOINTS`] are kept. - pub(super) fn prune_inactive(&self) { - self.inner.lock().expect("poisoned").prune_inactive(); - } - - pub(crate) fn on_direct_addr_discovered(&self, discovered: BTreeSet) { - self.inner - .lock() - .expect("poisoned") - .on_direct_addr_discovered(discovered, Instant::now()); - } -} - -impl EndpointMapInner { - /// Create a new [`EndpointMap`] from a list of [`EndpointAddr`]s. - fn load_from_vec( - endpoints: Vec, - #[cfg(any(test, feature = "test-utils"))] path_selection: PathSelection, - have_ipv6: bool, - metrics: &Metrics, - ) -> Self { - let mut me = Self { - #[cfg(any(test, feature = "test-utils"))] - path_selection, - ..Default::default() - }; - for endpoint_addr in endpoints { - me.add_endpoint_addr(endpoint_addr, Source::Saved, have_ipv6, metrics); + /// Returns the sender for the [`EndpointStateActor`]. + /// + /// If needed a new actor is started on demand. + /// + /// [`EndpointStateActor`]: endpoint_state::EndpointStateActor + pub(super) fn endpoint_state_actor( + &self, + eid: EndpointId, + ) -> mpsc::Sender { + let mut inner = self.inner.lock().expect("poisoned"); + match inner.endpoint_states.get(&eid) { + Some(handle) => handle.sender.clone(), + None => { + // Create a new EndpointStateActor and insert it into the endpoint map. + let sender = inner.transports_handle.inbox.clone(); + let local_addrs = inner.local_addrs.clone(); + let disco = inner.disco.clone(); + let metrics = inner.metrics.clone(); + let actor = EndpointStateActor::new( + eid, + self.local_endpoint_id, + sender, + local_addrs, + disco, + self.relay_mapped_addrs.clone(), + metrics, + ); + let handle = actor.start(); + let sender = handle.sender.clone(); + inner.endpoint_states.insert(eid, handle); + + // Ensure there is a EndpointMappedAddr for this EndpointId. + self.endpoint_mapped_addrs.get(&eid); + sender + } } - me } - /// Add the contact information for an endpoint. - #[instrument(skip_all, fields(endpoint = %endpoint_addr.id.fmt_short()))] - fn add_endpoint_addr( - &mut self, - endpoint_addr: EndpointAddr, - source: Source, - have_ipv6: bool, - metrics: &Metrics, - ) { - let source0 = source.clone(); - let endpoint_id = endpoint_addr.id; - let relay_url = endpoint_addr.relay_urls().next().cloned(); - #[cfg(any(test, feature = "test-utils"))] - let path_selection = self.path_selection; - let endpoint_state = - self.get_or_insert_with(EndpointStateKey::EndpointId(endpoint_id), || Options { - endpoint_id, - relay_url, - active: false, - source, - #[cfg(any(test, feature = "test-utils"))] - path_selection, - }); - endpoint_state.update_from_endpoint_addr( - endpoint_addr.relay_urls().next(), - endpoint_addr.ip_addrs().copied(), - source0, - have_ipv6, - metrics, - ); - let id = endpoint_state.id(); - for addr in endpoint_addr.ip_addrs() { - self.set_endpoint_state_for_ip_port(*addr, id); + pub(super) fn handle_ping(&self, msg: disco::Ping, sender: EndpointId, src: transports::Addr) { + if msg.endpoint_key != sender { + warn!("DISCO Ping EndpointId mismatch, ignoring ping"); + return; } - } - - /// Prunes direct addresses from endpoints that claim to share an address we know points to us. - pub(super) fn on_direct_addr_discovered( - &mut self, - discovered: BTreeSet, - now: Instant, - ) { - for addr in discovered { - self.remove_by_ipp(addr.into(), now, "matches our local addr") + let endpoint_state = self.endpoint_state_actor(sender); + if let Err(err) = endpoint_state.try_send(EndpointStateMessage::PingReceived(msg, src)) { + // TODO: This is really, really bad and will drop pings under load. But + // DISCO pings are going away with QUIC-NAT-TRAVERSAL so I don't care. + warn!("DISCO Ping dropped: {err:#}"); } } - /// Removes a direct address from an endpoint. - fn remove_by_ipp(&mut self, ipp: IpPort, now: Instant, why: &'static str) { - if let Some(id) = self.by_ip_port.remove(&ipp) { - if let Entry::Occupied(mut entry) = self.by_id.entry(id) { - let endpoint = entry.get_mut(); - endpoint.remove_direct_addr(&ipp, now, why); - if endpoint.ip_addrs().count() == 0 { - let endpoint_id = endpoint.public_key(); - let mapped_addr = endpoint.quic_mapped_addr(); - self.by_endpoint_key.remove(endpoint_id); - self.by_quic_mapped_addr.remove(mapped_addr); - debug!(endpoint_id=%endpoint_id.fmt_short(), why, "removing endpoint"); - entry.remove(); - } - } + pub(super) fn handle_pong(&self, msg: disco::Pong, sender: EndpointId, src: transports::Addr) { + let actor = self.endpoint_state_actor(sender); + if let Err(err) = actor.try_send(EndpointStateMessage::PongReceived(msg, src)) { + // TODO: This is really, really bad and will drop pongs under load. But + // DISCO pongs are going away with QUIC-NAT-TRAVERSAL so I don't care. + warn!("DISCO Pong dropped: {err:#}"); } } - fn get_id(&self, id: EndpointStateKey) -> Option { - match id { - EndpointStateKey::Idx(id) => Some(id), - EndpointStateKey::EndpointId(endpoint_key) => { - self.by_endpoint_key.get(&endpoint_key).copied() - } - EndpointStateKey::EndpointIdMappedAddr(addr) => { - self.by_quic_mapped_addr.get(&addr).copied() - } - EndpointStateKey::IpPort(ipp) => self.by_ip_port.get(&ipp).copied(), + pub(super) fn handle_call_me_maybe( + &self, + msg: disco::CallMeMaybe, + sender: EndpointId, + src: transports::Addr, + ) { + if !src.is_relay() { + warn!("DISCO CallMeMaybe packets should only come via relay"); + return; } - } - - fn get_mut(&mut self, id: EndpointStateKey) -> Option<&mut EndpointState> { - self.get_id(id).and_then(|id| self.by_id.get_mut(&id)) - } - - fn get(&self, id: EndpointStateKey) -> Option<&EndpointState> { - self.get_id(id).and_then(|id| self.by_id.get(&id)) - } - - fn get_or_insert_with( - &mut self, - id: EndpointStateKey, - f: impl FnOnce() -> Options, - ) -> &mut EndpointState { - let id = self.get_id(id); - match id { - None => self.insert_endpoint(f()), - Some(id) => self.by_id.get_mut(&id).expect("is not empty"), + let actor = self.endpoint_state_actor(sender); + if let Err(err) = actor.try_send(EndpointStateMessage::CallMeMaybeReceived(msg)) { + // TODO: This is bad and will drop call-me-maybe's under load. But + // DISCO CallMeMaybe going away with QUIC-NAT-TRAVERSAL so I don't care. + warn!("DISCO CallMeMaybe dropped: {err:#}"); } } +} - /// Number of endpoints currently listed. - fn endpoint_count(&self) -> usize { - self.by_id.len() - } - - /// Marks the endpoint we believe to be at `ipp` as recently used. - #[cfg(not(wasm_browser))] - fn receive_udp(&mut self, udp_addr: SocketAddr) -> Option<(EndpointId, EndpointIdMappedAddr)> { - let ip_port: IpPort = udp_addr.into(); - let Some(endpoint_state) = self.get_mut(EndpointStateKey::IpPort(ip_port)) else { - trace!(src=%udp_addr, "receive_udp: no endpoint_state found for addr, ignore"); - return None; - }; - endpoint_state.receive_udp(ip_port, Instant::now()); - Some(( - *endpoint_state.public_key(), - *endpoint_state.quic_mapped_addr(), - )) - } - - #[instrument(skip_all, fields(src = %src.fmt_short()))] - fn receive_relay(&mut self, relay_url: &RelayUrl, src: EndpointId) -> EndpointIdMappedAddr { - #[cfg(any(test, feature = "test-utils"))] - let path_selection = self.path_selection; - let endpoint_state = self.get_or_insert_with(EndpointStateKey::EndpointId(src), || { - trace!("packets from unknown endpoint, insert into endpoint map"); - Options { - endpoint_id: src, - relay_url: Some(relay_url.clone()), - active: true, - source: Source::Relay, - #[cfg(any(test, feature = "test-utils"))] - path_selection, - } - }); - endpoint_state.receive_relay(relay_url, src, Instant::now()); - *endpoint_state.quic_mapped_addr() - } - - #[cfg(test)] - fn endpoint_states(&self) -> impl Iterator { - self.by_id.iter() - } - - fn endpoint_states_mut(&mut self) -> impl Iterator { - self.by_id.iter_mut() - } - - /// Get the [`RemoteInfo`]s for all endpoints. - #[cfg(test)] - fn remote_infos_iter(&self, now: Instant) -> impl Iterator + '_ { - self.endpoint_states().map(move |(_, ep)| ep.info(now)) +impl EndpointMapInner { + fn new( + metrics: Arc, + sender: TransportsSender, + local_addrs: n0_watcher::Direct>, + disco: DiscoState, + ) -> Self { + let transports_handle = Self::start_transports_sender(sender); + Self { + metrics, + transports_handle, + local_addrs, + disco, + #[cfg(any(test, feature = "test-utils"))] + path_selection: Default::default(), + endpoint_states: Default::default(), + } } - /// Get the [`RemoteInfo`]s for each endpoint. - fn remote_info(&self, endpoint_id: EndpointId) -> Option { - self.get(EndpointStateKey::EndpointId(endpoint_id)) - .map(|ep| ep.info(Instant::now())) + fn start_transports_sender(sender: TransportsSender) -> TransportsSenderHandle { + let actor = TransportsSenderActor::new(sender); + actor.start() } /// Returns a stream of [`ConnectionType`]. @@ -533,186 +298,8 @@ impl EndpointMapInner { /// /// Will return `None` if there is not an entry in the [`EndpointMap`] for /// the `public_key` - fn conn_type(&self, endpoint_id: EndpointId) -> Option> { - self.get(EndpointStateKey::EndpointId(endpoint_id)) - .map(|ep| ep.conn_type()) - } - - fn latency(&self, endpoint_id: EndpointId) -> Option { - self.get(EndpointStateKey::EndpointId(endpoint_id)) - .and_then(|ep| ep.latency()) - } - - fn handle_pong( - &mut self, - sender: EndpointId, - src: &transports::Addr, - pong: Pong, - metrics: &Metrics, - ) { - if let Some(ns) = self.get_mut(EndpointStateKey::EndpointId(sender)).as_mut() { - let insert = ns.handle_pong(&pong, src.clone().into(), metrics); - if let Some((src, key)) = insert { - self.set_endpoint_key_for_ip_port(src, &key); - } - trace!(?insert, "received pong") - } else { - warn!("received pong: endpoint unknown, ignore") - } - } - - #[must_use = "actions must be handled"] - fn handle_call_me_maybe( - &mut self, - sender: EndpointId, - cm: CallMeMaybe, - metrics: &Metrics, - ) -> Vec { - let ns_id = EndpointStateKey::EndpointId(sender); - if let Some(id) = self.get_id(ns_id.clone()) { - for number in &cm.my_numbers { - // ensure the new addrs are known - self.set_endpoint_state_for_ip_port(*number, id); - } - } - match self.get_mut(ns_id) { - None => { - debug!("received call-me-maybe: ignore, endpoint is unknown"); - metrics.recv_disco_call_me_maybe_bad_disco.inc(); - vec![] - } - Some(ns) => { - debug!(endpoints = ?cm.my_numbers, "received call-me-maybe"); - - ns.handle_call_me_maybe(cm, metrics) - } - } - } - - fn handle_ping( - &mut self, - sender: EndpointId, - src: SendAddr, - tx_id: TransactionId, - ) -> PingHandled { - #[cfg(any(test, feature = "test-utils"))] - let path_selection = self.path_selection; - let endpoint_state = self.get_or_insert_with(EndpointStateKey::EndpointId(sender), || { - debug!("received ping: endpoint unknown, add to endpoint map"); - let source = if src.is_relay() { - Source::Relay - } else { - Source::Udp - }; - Options { - endpoint_id: sender, - relay_url: src.relay_url(), - active: true, - source, - #[cfg(any(test, feature = "test-utils"))] - path_selection, - } - }); - - let handled = endpoint_state.handle_ping(src.clone(), tx_id); - if let SendAddr::Udp(ref addr) = src { - if matches!(handled.role, PingRole::NewPath) { - self.set_endpoint_key_for_ip_port(*addr, &sender); - } - } - handled - } - - /// Inserts a new endpoint into the [`EndpointMap`]. - fn insert_endpoint(&mut self, options: Options) -> &mut EndpointState { - info!( - endpoint = %options.endpoint_id.fmt_short(), - relay_url = ?options.relay_url, - source = %options.source, - "inserting new endpoint in EndpointMap", - ); - let id = self.next_id; - self.next_id = self.next_id.wrapping_add(1); - let endpoint_state = EndpointState::new(id, options); - - // update indices - self.by_quic_mapped_addr - .insert(*endpoint_state.quic_mapped_addr(), id); - self.by_endpoint_key - .insert(*endpoint_state.public_key(), id); - - self.by_id.insert(id, endpoint_state); - self.by_id.get_mut(&id).expect("just inserted") - } - - /// Makes future endpoint lookups by ipp return the same endpoint as a lookup by nk. - /// - /// This should only be called with a fully verified mapping of ipp to - /// nk, because calling this function defines the endpoint we hand to - /// WireGuard for packets received from ipp. - fn set_endpoint_key_for_ip_port(&mut self, ipp: impl Into, nk: &PublicKey) { - let ipp = ipp.into(); - if let Some(id) = self.by_ip_port.get(&ipp) { - if !self.by_endpoint_key.contains_key(nk) { - self.by_endpoint_key.insert(*nk, *id); - } - self.by_ip_port.remove(&ipp); - } - if let Some(id) = self.by_endpoint_key.get(nk) { - trace!("insert ip -> id: {:?} -> {}", ipp, id); - self.by_ip_port.insert(ipp, *id); - } - } - - fn set_endpoint_state_for_ip_port(&mut self, ipp: impl Into, id: usize) { - let ipp = ipp.into(); - trace!(?ipp, ?id, "set endpoint for ip:port"); - self.by_ip_port.insert(ipp, id); - } - - /// Prunes endpoints without recent activity so that at most [`MAX_INACTIVE_ENDPOINTS`] are kept. - fn prune_inactive(&mut self) { - let now = Instant::now(); - let mut prune_candidates: Vec<_> = self - .by_id - .values() - .filter(|endpoint| !endpoint.is_active(&now)) - .map(|endpoint| (*endpoint.public_key(), endpoint.last_used())) - .collect(); - - let prune_count = prune_candidates - .len() - .saturating_sub(MAX_INACTIVE_ENDPOINTS); - if prune_count == 0 { - // within limits - return; - } - - prune_candidates.sort_unstable_by_key(|(_pk, last_used)| *last_used); - prune_candidates.truncate(prune_count); - for (public_key, last_used) in prune_candidates.into_iter() { - let endpoint = public_key.fmt_short(); - match last_used.map(|instant| instant.elapsed()) { - Some(last_used) => trace!(%endpoint, ?last_used, "pruning inactive"), - None => trace!(%endpoint, last_used=%"never", "pruning inactive"), - } - - let Some(id) = self.by_endpoint_key.remove(&public_key) else { - debug_assert!(false, "missing by_endpoint_key entry for pk in by_id"); - continue; - }; - - let Some(ep) = self.by_id.remove(&id) else { - debug_assert!(false, "missing by_id entry for id in by_endpoint_key"); - continue; - }; - - for ip_port in ep.ip_addrs() { - self.by_ip_port.remove(&ip_port); - } - - self.by_quic_mapped_addr.remove(ep.quic_mapped_addr()); - } + fn conn_type(&self, _eid: EndpointId) -> Option> { + todo!(); } } @@ -753,204 +340,227 @@ impl IpPort { } } -#[cfg(test)] -mod tests { - use std::net::Ipv4Addr; +/// An actor that can send datagrams onto iroh transports. +/// +/// The [`NodeStateActor`]s want to be able to send datagrams. Because we can not create +/// [`TransportsSender`]s on demand we must share one for the entire [`NodeMap`], which +/// lives in this actor. +/// +/// [`NodeStateActor`]: node_state::NodeStateActor +#[derive(Debug)] +struct TransportsSenderActor { + sender: TransportsSender, +} - use iroh_base::{SecretKey, TransportAddr}; - use rand::SeedableRng; - use tracing_test::traced_test; +impl TransportsSenderActor { + fn new(sender: TransportsSender) -> Self { + Self { sender } + } + + fn start(self) -> TransportsSenderHandle { + // This actor gets an inbox size of exactly 1. This is the same as if they had the + // underlying sender directly: either you can send or not, or you await until you + // can. No need to introduce extra buffering. + let (tx, rx) = mpsc::channel(1); - use super::{endpoint_state::MAX_INACTIVE_DIRECT_ADDRESSES, *}; - - impl EndpointMap { - #[track_caller] - fn add_test_addr(&self, endpoint_addr: EndpointAddr) { - self.add_endpoint_addr( - endpoint_addr, - Source::NamedApp { - name: "test".into(), - }, - true, - &Default::default(), - ) + let task = tokio::spawn( + async move { + self.run(rx).await; + } + .instrument(info_span!("TransportsSenderActor")), + ); + TransportsSenderHandle { + inbox: tx, + _task: AbortOnDropHandle::new(task), } } - /// Test persisting and loading of known endpoints. - #[tokio::test] - #[traced_test] - async fn restore_from_vec() { - let endpoint_map = EndpointMap::default(); - - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let endpoint_a = SecretKey::generate(&mut rng).public(); - let endpoint_b = SecretKey::generate(&mut rng).public(); - let endpoint_c = SecretKey::generate(&mut rng).public(); - let endpoint_d = SecretKey::generate(&mut rng).public(); - - let relay_x: RelayUrl = "https://my-relay-1.com".parse().unwrap(); - let relay_y: RelayUrl = "https://my-relay-2.com".parse().unwrap(); - - let ip_addresses_a = [TransportAddr::Ip(addr(4000)), TransportAddr::Ip(addr(4001))]; - let ip_addresses_c = [TransportAddr::Ip(addr(5000))]; - - let addrs_a = std::iter::once(TransportAddr::Relay(relay_x)).chain(ip_addresses_a); - let endpoint_addr_a = EndpointAddr::new(endpoint_a).with_addrs(addrs_a); - let endpoint_addr_b = EndpointAddr::new(endpoint_b).with_relay_url(relay_y); - let endpoint_addr_c = EndpointAddr::new(endpoint_c).with_addrs(ip_addresses_c); - let endpoint_addr_d = EndpointAddr::new(endpoint_d); - - endpoint_map.add_test_addr(endpoint_addr_a); - endpoint_map.add_test_addr(endpoint_addr_b); - endpoint_map.add_test_addr(endpoint_addr_c); - endpoint_map.add_test_addr(endpoint_addr_d); - - let mut addrs: Vec = endpoint_map - .list_remote_infos(Instant::now()) - .into_iter() - .filter_map(|info| { - let addr: EndpointAddr = info.into(); - if addr.is_empty() { - return None; - } - Some(addr) - }) - .collect(); - let loaded_endpoint_map = EndpointMap::load_from_vec( - addrs.clone(), - PathSelection::default(), - true, - &Default::default(), - ); + async fn run(self, mut inbox: mpsc::Receiver) { + use TransportsSenderMessage::SendDatagram; - let mut loaded: Vec = loaded_endpoint_map - .list_remote_infos(Instant::now()) - .into_iter() - .filter_map(|info| { - let addr: EndpointAddr = info.into(); - if addr.is_empty() { - return None; + while let Some(SendDatagram(dst, owned_transmit)) = inbox.recv().await { + let transmit = transports::Transmit { + ecn: owned_transmit.ecn, + contents: owned_transmit.contents.as_ref(), + segment_size: owned_transmit.segment_size, + }; + let len = transmit.contents.len(); + match self.sender.send(&dst, None, &transmit).await { + Ok(()) => {} + Err(err) => { + trace!(?dst, %len, "transmit failed to send: {err:#}"); } - Some(addr) - }) - .collect(); + }; + } + trace!("actor terminating"); + } +} - loaded.sort_unstable(); - addrs.sort_unstable(); +#[derive(Debug)] +struct TransportsSenderHandle { + inbox: mpsc::Sender, + _task: AbortOnDropHandle<()>, +} - // compare the endpoint maps via their known endpoints - assert_eq!(addrs, loaded); - } +#[derive(Debug)] +enum TransportsSenderMessage { + SendDatagram(transports::Addr, OwnedTransmit), +} - fn addr(port: u16) -> SocketAddr { - (std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), port).into() +impl From<(transports::Addr, OwnedTransmit)> for TransportsSenderMessage { + fn from(source: (transports::Addr, OwnedTransmit)) -> Self { + Self::SendDatagram(source.0, source.1) } +} - #[test] - #[traced_test] - fn test_prune_direct_addresses() { - let endpoint_map = EndpointMap::default(); - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - let public_key = SecretKey::generate(&mut rng).public(); - let id = endpoint_map - .inner - .lock() - .unwrap() - .insert_endpoint(Options { - endpoint_id: public_key, - relay_url: None, - active: false, - source: Source::NamedApp { - name: "test".into(), - }, - path_selection: PathSelection::default(), - }) - .id(); - - const LOCALHOST: IpAddr = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); - - // add [`MAX_INACTIVE_DIRECT_ADDRESSES`] active direct addresses and double - // [`MAX_INACTIVE_DIRECT_ADDRESSES`] that are inactive - - info!("Adding active addresses"); - for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES { - let addr = SocketAddr::new(LOCALHOST, 5000 + i as u16); - let endpoint_addr = EndpointAddr::new(public_key).with_ip_addr(addr); - // add address - endpoint_map.add_test_addr(endpoint_addr); - // make it active - endpoint_map.inner.lock().unwrap().receive_udp(addr); - } - - info!("Adding offline/inactive addresses"); - for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES * 2 { - let addr = SocketAddr::new(LOCALHOST, 6000 + i as u16); - let endpoint_addr = EndpointAddr::new(public_key).with_ip_addr(addr); - endpoint_map.add_test_addr(endpoint_addr); - } +#[cfg(test)] +mod tests { - let mut endpoint_map_inner = endpoint_map.inner.lock().unwrap(); - let endpoint = endpoint_map_inner.by_id.get_mut(&id).unwrap(); + use tracing_test::traced_test; - info!("Adding alive addresses"); - for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES { - let addr = SendAddr::Udp(SocketAddr::new(LOCALHOST, 7000 + i as u16)); - let txid = TransactionId::from([i as u8; 12]); - // Note that this already invokes .prune_direct_addresses() because these are - // new UDP paths. - endpoint.handle_ping(addr, txid); - } + // use super::*; - info!("Pruning addresses"); - endpoint.prune_direct_addresses(Instant::now()); + // impl NodeMap { + // async fn add_test_addr(&self, node_addr: NodeAddr) { + // self.add_node_addr( + // node_addr, + // Source::NamedApp { + // name: "test".into(), + // }, + // ) + // .await; + // } + // } - // Half the offline addresses should have been pruned. All the active and alive - // addresses should have been kept. - assert_eq!( - endpoint.ip_addrs().count(), - MAX_INACTIVE_DIRECT_ADDRESSES * 3 - ); + // fn addr(port: u16) -> SocketAddr { + // (std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), port).into() + // } - // We should have both offline and alive addresses which are not active. - assert_eq!( - endpoint - .ip_addr_states() - .filter(|(_addr, state)| !state.is_active()) - .count(), - MAX_INACTIVE_DIRECT_ADDRESSES * 2 - ) + #[tokio::test] + #[traced_test] + async fn test_prune_direct_addresses() { + panic!("support this again"); + // let transports = Transports::new(Vec::new(), Vec::new(), Arc::new(1200.into())); + // let direct_addrs = DiscoveredDirectAddrs::default(); + // let secret_key = SecretKey::generate(&mut rand::rngs::OsRng); + // let (disco, _) = DiscoState::new(&secret_key); + // let node_map = NodeMap::new( + // secret_key.public(), + // Default::default(), + // transports.create_sender(), + // direct_addrs.addrs.watch(), + // disco, + // ); + // let public_key = SecretKey::generate(rand::thread_rng()).public(); + // let id = node_map + // .inner + // .lock() + // .unwrap() + // .insert_node(Options { + // node_id: public_key, + // relay_url: None, + // active: false, + // source: Source::NamedApp { + // name: "test".into(), + // }, + // path_selection: PathSelection::default(), + // }) + // .id(); + + // const LOCALHOST: IpAddr = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + + // // add [`MAX_INACTIVE_DIRECT_ADDRESSES`] active direct addresses and double + // // [`MAX_INACTIVE_DIRECT_ADDRESSES`] that are inactive + + // info!("Adding active addresses"); + // for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES { + // let addr = SocketAddr::new(LOCALHOST, 5000 + i as u16); + // let node_addr = NodeAddr::new(public_key).with_direct_addresses([addr]); + // // add address + // node_map.add_test_addr(node_addr).await; + // // make it active + // node_map.inner.lock().unwrap().receive_udp(addr); + // } + + // info!("Adding offline/inactive addresses"); + // for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES * 2 { + // let addr = SocketAddr::new(LOCALHOST, 6000 + i as u16); + // let node_addr = NodeAddr::new(public_key).with_direct_addresses([addr]); + // node_map.add_test_addr(node_addr).await; + // } + + // let mut node_map_inner = node_map.inner.lock().unwrap(); + // let endpoint = node_map_inner.by_id.get_mut(&id).unwrap(); + + // info!("Adding alive addresses"); + // for i in 0..MAX_INACTIVE_DIRECT_ADDRESSES { + // let addr = SendAddr::Udp(SocketAddr::new(LOCALHOST, 7000 + i as u16)); + // let txid = stun_rs::TransactionId::from([i as u8; 12]); + // // Note that this already invokes .prune_direct_addresses() because these are + // // new UDP paths. + // // endpoint.handle_ping(addr, txid); + // } + + // info!("Pruning addresses"); + // endpoint.prune_direct_addresses(Instant::now()); + + // // Half the offline addresses should have been pruned. All the active and alive + // // addresses should have been kept. + // assert_eq!( + // endpoint.direct_addresses().count(), + // MAX_INACTIVE_DIRECT_ADDRESSES * 3 + // ); + + // // We should have both offline and alive addresses which are not active. + // assert_eq!( + // endpoint + // .direct_address_states() + // .filter(|(_addr, state)| !state.is_active()) + // .count(), + // MAX_INACTIVE_DIRECT_ADDRESSES * 2 + // ) } - #[test] - fn test_prune_inactive() { - let endpoint_map = EndpointMap::default(); - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - - // add one active endpoint and more than MAX_INACTIVE_ENDPOINTS inactive endpoints - let active_endpoint = SecretKey::generate(&mut rng).public(); - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 167); - endpoint_map.add_test_addr(EndpointAddr::new(active_endpoint).with_ip_addr(addr)); - endpoint_map - .inner - .lock() - .unwrap() - .receive_udp(addr) - .expect("registered"); - - for _ in 0..MAX_INACTIVE_ENDPOINTS + 1 { - let endpoint = SecretKey::generate(&mut rng).public(); - endpoint_map.add_test_addr(EndpointAddr::new(endpoint)); - } - - assert_eq!(endpoint_map.endpoint_count(), MAX_INACTIVE_ENDPOINTS + 2); - endpoint_map.prune_inactive(); - assert_eq!(endpoint_map.endpoint_count(), MAX_INACTIVE_ENDPOINTS + 1); - endpoint_map - .inner - .lock() - .unwrap() - .get(EndpointStateKey::EndpointId(active_endpoint)) - .expect("should not be pruned"); + #[tokio::test] + async fn test_prune_inactive() { + panic!("support this again"); + // let transports = Transports::new(Vec::new(), Vec::new(), Arc::new(1200.into())); + // let direct_addrs = DiscoveredDirectAddrs::default(); + // let secret_key = SecretKey::generate(&mut rand::rngs::OsRng); + // let (disco, _) = DiscoState::new(&secret_key); + // let node_map = NodeMap::new( + // secret_key.public(), + // Default::default(), + // transports.create_sender(), + // direct_addrs.addrs.watch(), + // disco, + // ); + // // add one active node and more than MAX_INACTIVE_NODES inactive nodes + // let active_node = SecretKey::generate(rand::thread_rng()).public(); + // let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 167); + // node_map + // .add_test_addr(NodeAddr::new(active_node).with_direct_addresses([addr])) + // .await; + // node_map + // .inner + // .lock() + // .unwrap() + // .receive_udp(addr) + // .expect("registered"); + + // for _ in 0..MAX_INACTIVE_NODES + 1 { + // let node = SecretKey::generate(rand::thread_rng()).public(); + // node_map.add_test_addr(NodeAddr::new(node)).await; + // } + + // assert_eq!(node_map.node_count(), MAX_INACTIVE_NODES + 2); + // node_map.prune_inactive(); + // assert_eq!(node_map.node_count(), MAX_INACTIVE_NODES + 1); + // node_map + // .inner + // .lock() + // .unwrap() + // .get(NodeStateKey::NodeId(active_node)) + // .expect("should not be pruned"); } } diff --git a/iroh/src/magicsock/endpoint_map/endpoint_state.rs b/iroh/src/magicsock/endpoint_map/endpoint_state.rs index e4b1c2a2209..6071362d311 100644 --- a/iroh/src/magicsock/endpoint_map/endpoint_state.rs +++ b/iroh/src/magicsock/endpoint_map/endpoint_state.rs @@ -1,1426 +1,1011 @@ use std::{ - collections::{BTreeSet, HashMap, btree_map::Entry}, - hash::Hash, - net::{IpAddr, SocketAddr}, - sync::atomic::AtomicBool, + collections::{BTreeSet, HashMap}, + net::SocketAddr, + pin::Pin, + sync::Arc, }; -use data_encoding::HEXLOWER; -use iroh_base::{EndpointAddr, EndpointId, PublicKey, RelayUrl, TransportAddr}; +use iroh_base::{EndpointAddr, EndpointId, RelayUrl, TransportAddr}; use n0_future::{ - task::{self, AbortOnDropHandle}, - time::{self, Duration, Instant}, + MergeUnbounded, Stream, StreamExt, + task::AbortOnDropHandle, + time::{Duration, Instant}, }; -use n0_watcher::Watchable; +use n0_watcher::{Watchable, Watcher}; +use quinn::WeakConnectionHandle; +use quinn_proto::{PathEvent, PathId, PathStatus}; +use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc; -use tracing::{Level, debug, event, info, instrument, trace, warn}; - -use super::{ - IpPort, Source, - path_state::{PathState, summarize_endpoint_paths}, - udp_paths::{EndpointUdpPaths, UdpSendAddr}, -}; -#[cfg(any(test, feature = "test-utils"))] -use crate::endpoint::PathSelection; +use snafu::{ResultExt, Whatever}; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::{BroadcastStream, errors::BroadcastStreamRecvError}; +use tracing::{Instrument, Level, debug, error, event, info_span, instrument, trace, warn}; + +use super::{Source, TransportsSenderMessage, path_state::PathState}; +// TODO: Use this +// #[cfg(any(test, feature = "test-utils"))] +// use crate::endpoint::PathSelection; use crate::{ - disco::{self, SendAddr, TransactionId}, + disco::{self}, + endpoint::DirectAddr, magicsock::{ - ActorMessage, EndpointIdMappedAddr, HEARTBEAT_INTERVAL, MagicsockMetrics, - endpoint_map::path_validity::PathValidity, + DiscoState, HEARTBEAT_INTERVAL, MagicsockMetrics, PATH_MAX_IDLE_TIMEOUT, + mapped_addrs::{AddrMap, MappedAddr, MultipathMappedAddr, RelayMappedAddr}, + transports::{self, OwnedTransmit}, }, + util::MaybeFuture, }; -/// Number of addresses that are not active that we keep around per endpoint. +// TODO: use this +// /// Number of addresses that are not active that we keep around per endpoint. +// /// +// /// See [`EndpointState::prune_direct_addresses`]. +// pub(super) const MAX_INACTIVE_DIRECT_ADDRESSES: usize = 20; + +// TODO: use this +// /// How long since an endpoint path was last alive before it might be pruned. +// const LAST_ALIVE_PRUNE_DURATION: Duration = Duration::from_secs(120); + +// TODO: use this +// /// The latency at or under which we don't try to upgrade to a better path. +// const GOOD_ENOUGH_LATENCY: Duration = Duration::from_millis(5); + +// TODO: use this +// /// How long since the last activity we try to keep an established endpoint peering alive. +// /// +// /// It's also the idle time at which we stop doing QAD queries to keep NAT mappings alive. +// pub(super) const SESSION_ACTIVE_TIMEOUT: Duration = Duration::from_secs(45); + +// TODO: use this +// /// How often we try to upgrade to a better path. +// /// +// /// Even if we have some non-relay route that works. +// const UPGRADE_INTERVAL: Duration = Duration::from_secs(60); + +/// The value which we close paths. +// TODO: Quinn should just do this. Also, I made this value up. +const APPLICATION_ABANDON_PATH: u8 = 30; + +/// A stream of events from all paths for all connections. /// -/// See [`EndpointState::prune_direct_addresses`]. -pub(super) const MAX_INACTIVE_DIRECT_ADDRESSES: usize = 20; - -/// How long since an endpoint path was last alive before it might be pruned. -const LAST_ALIVE_PRUNE_DURATION: Duration = Duration::from_secs(120); - -/// How long we wait for a pong reply before assuming it's never coming. -const PING_TIMEOUT_DURATION: Duration = Duration::from_secs(5); - -/// The latency at or under which we don't try to upgrade to a better path. -const GOOD_ENOUGH_LATENCY: Duration = Duration::from_millis(5); - -/// How long since the last activity we try to keep an established endpoint peering alive. -/// It's also the idle time at which we stop doing QAD queries to keep NAT mappings alive. -pub(super) const SESSION_ACTIVE_TIMEOUT: Duration = Duration::from_secs(45); - -/// How often we try to upgrade to a better patheven if we have some non-relay route that works. -const UPGRADE_INTERVAL: Duration = Duration::from_secs(60); - -/// How long until we send a stayin alive ping -const STAYIN_ALIVE_MIN_ELAPSED: Duration = Duration::from_secs(2); - -#[derive(Debug)] -pub(in crate::magicsock) enum PingAction { - SendCallMeMaybe { - relay_url: RelayUrl, - dst_endpoint: EndpointId, - }, - SendPing(SendPing), -} - -#[derive(Debug)] -pub(in crate::magicsock) struct SendPing { - pub id: usize, - pub dst: SendAddr, - pub dst_endpoint: EndpointId, - pub tx_id: TransactionId, - pub purpose: DiscoPingPurpose, -} - -/// Indicating an [`EndpointState`] has handled a ping. -#[derive(Debug)] -pub struct PingHandled { - /// What this ping did to the [`EndpointState`]. - pub role: PingRole, - /// Whether the sender path should also be pinged. - /// - /// This is the case if an [`EndpointState`] does not yet have a direct path, i.e. it has no - /// best_addr. In this case we want to ping right back to open the direct path in this - /// direction as well. - pub needs_ping_back: Option, -} - -#[derive(Debug)] -pub enum PingRole { - Duplicate, - NewPath, - LikelyHeartbeat, - Activate, -} - -/// An iroh endpoint, which we can have connections with. +/// The connection is identified using [`ConnId`]. The event `Err` variant happens when the +/// actor has lagged processing the events, which is rather critical for us. +type PathEvents = MergeUnbounded< + Pin< + Box)> + Send + Sync>, + >, +>; + +/// The state we need to know about a single remote endpoint. /// -/// The whole point of the magicsock is that we can have multiple **paths** to a particular -/// endpoint. One of these paths is via the endpoint's home relay endpoint but as we establish a -/// connection we'll hopefully discover more direct paths. -#[derive(Debug)] -pub(super) struct EndpointState { - /// The ID used as index in the [`EndpointMap`]. - /// - /// [`EndpointMap`]: super::EndpointMap - id: usize, - /// The UDP address used on the QUIC-layer to address this endpoint. - quic_mapped_addr: EndpointIdMappedAddr, - /// The global identifier for this endpoint. +/// This actor manages all connections to the remote endpoint. It will trigger holepunching +/// and select the best path etc. +pub(super) struct EndpointStateActor { + /// The endpoint ID of the remote endpoint. endpoint_id: EndpointId, - /// The last time we pinged all endpoints. - last_full_ping: Option, - /// The url of relay endpoint that we can relay over to communicate. + /// The endpoint ID of the local endpoint. + local_endpoint_id: EndpointId, + + // Hooks into the rest of the MagicSocket. + // + /// Metrics. + metrics: Arc, + /// Allowing us to directly send datagrams. /// - /// The fallback/bootstrap path, if non-zero (non-zero for well-behaved clients). - relay_url: Option<(RelayUrl, PathState)>, - udp_paths: EndpointUdpPaths, - sent_pings: HashMap, - /// Last time this endpoint was used. + /// Used for handling [`EndpointStateMessage::SendDatagram`] messages. + transports_sender: mpsc::Sender, + /// Our local addresses. /// - /// An endpoint is marked as in use when sending datagrams to them, or when having received - /// datagrams from it. Regardless of whether the datagrams are payload or DISCO, and whether - /// they go via UDP or the relay. + /// These are our local addresses and any reflexive transport addresses. + local_addrs: n0_watcher::Direct>, + /// Shared state to allow to encrypt DISCO messages to peers. + disco: DiscoState, + /// The mapping between endpoints via a relay and their [`RelayMappedAddr`]s. + relay_mapped_addrs: AddrMap<(RelayUrl, EndpointId), RelayMappedAddr>, + + // Internal state - Quinn Connections we are managing. + // + /// All connections we have to this remote endpoint. + connections: FxHashMap, + /// Events emitted by Quinn about path changes, for all paths, all connections. + path_events: PathEvents, + + // Internal state - Holepunching and path state. + // + /// All possible paths we are aware of. /// - /// Note that sending datagrams to an endpoint does not mean the endpoint receives them. - last_used: Option, - /// Last time we sent a call-me-maybe. + /// These paths might be entirely impossible to use, since they are added by discovery + /// mechanisms. The are only potentially usable. + paths: FxHashMap, + /// Information about the last holepunching attempt. + last_holepunch: Option, + /// The path we currently consider the preferred path to the remote endpoint. /// - /// When we do not have a direct connection and we try to send some data, we will try to - /// do a full ping + call-me-maybe. Usually each side only needs to send one - /// call-me-maybe to the other for holes to be punched in both directions however. So - /// we only try and send one per [`HEARTBEAT_INTERVAL`]. Each [`HEARTBEAT_INTERVAL`] - /// the [`EndpointState::stayin_alive`] function is called, which will trigger new - /// call-me-maybe messages as backup. - last_call_me_maybe: Option, - /// The type of connection we have to the endpoint, either direct, relay, mixed, or none. - conn_type: Watchable, - /// Whether the conn_type was ever observed to be `Direct` at some point. + /// **We expect this path to work.** If we become aware this path is broken then it is + /// set back to `None`. Having a selected path does not mean we may not be able to get + /// a better path: e.g. when the selected path is a relay path we still need to trigger + /// holepunching regularly. /// - /// Used for metric reporting. - has_been_direct: AtomicBool, - /// Configuration for what path selection to use - #[cfg(any(test, feature = "test-utils"))] - path_selection: PathSelection, -} - -/// Options for creating a new [`EndpointState`]. -#[derive(Debug)] -pub(super) struct Options { - pub(super) endpoint_id: EndpointId, - pub(super) relay_url: Option, - /// Is this endpoint currently active (sending data)? - pub(super) active: bool, - pub(super) source: super::Source, - #[cfg(any(test, feature = "test-utils"))] - pub(super) path_selection: PathSelection, + /// We only select a path once the path is functional in Quinn. + selected_path: Option, + /// Time at which we should schedule the next holepunch attempt. + scheduled_holepunch: Option, } -impl EndpointState { - pub(super) fn new(id: usize, options: Options) -> Self { - let quic_mapped_addr = EndpointIdMappedAddr::generate(); - - // TODO(frando): I don't think we need to track the `num_relay_conns_added` - // metric here. We do so in `Self::addr_for_send`. - // if options.relay_url.is_some() { - // // we potentially have a relay connection to the endpoint - // inc!(MagicsockMetrics, num_relay_conns_added); - // } - - let now = Instant::now(); - - EndpointState { - id, - quic_mapped_addr, - endpoint_id: options.endpoint_id, - last_full_ping: None, - relay_url: options.relay_url.map(|url| { - ( - url.clone(), - PathState::new( - options.endpoint_id, - SendAddr::Relay(url), - options.source, - now, - ), - ) - }), - udp_paths: EndpointUdpPaths::new(), - sent_pings: HashMap::new(), - last_used: options.active.then(Instant::now), - last_call_me_maybe: None, - conn_type: Watchable::new(ConnectionType::None), - has_been_direct: AtomicBool::new(false), - #[cfg(any(test, feature = "test-utils"))] - path_selection: options.path_selection, +impl EndpointStateActor { + pub(super) fn new( + endpoint_id: EndpointId, + local_endpoint_id: EndpointId, + transports_sender: mpsc::Sender, + local_addrs: n0_watcher::Direct>, + disco: DiscoState, + relay_mapped_addrs: AddrMap<(RelayUrl, EndpointId), RelayMappedAddr>, + metrics: Arc, + ) -> Self { + Self { + endpoint_id, + local_endpoint_id, + metrics, + transports_sender, + local_addrs, + relay_mapped_addrs, + disco, + connections: FxHashMap::default(), + path_events: Default::default(), + paths: FxHashMap::default(), + last_holepunch: None, + selected_path: None, + scheduled_holepunch: None, } } - pub(super) fn public_key(&self) -> &PublicKey { - &self.endpoint_id - } - - pub(super) fn quic_mapped_addr(&self) -> &EndpointIdMappedAddr { - &self.quic_mapped_addr - } - - pub(super) fn id(&self) -> usize { - self.id - } - - pub(super) fn conn_type(&self) -> n0_watcher::Direct { - self.conn_type.watch() - } - - pub(super) fn latency(&self) -> Option { - match self.conn_type.get() { - ConnectionType::Direct(addr) => self - .udp_paths - .paths() - .get(&addr.into()) - .and_then(|state| state.latency()), - ConnectionType::Relay(ref url) => self - .relay_url - .as_ref() - .filter(|(relay_url, _)| relay_url == url) - .and_then(|(_, state)| state.latency()), - ConnectionType::Mixed(addr, ref url) => { - let addr_latency = self - .udp_paths - .paths() - .get(&addr.into()) - .and_then(|state| state.latency()); - let relay_latency = self - .relay_url - .as_ref() - .filter(|(relay_url, _)| relay_url == url) - .and_then(|(_, state)| state.latency()); - addr_latency.min(relay_latency) + pub(super) fn start(mut self) -> EndpointStateHandle { + let (tx, rx) = mpsc::channel(16); + let me = self.local_endpoint_id; + let endpoint_id = self.endpoint_id; + + // Ideally we'd use the endpoint span as parent. We'd have to plug that span into + // here somehow. Instead we have no parent and explicitly set the me attribute. If + // we don't explicitly set a span we get the spans from whatever call happens to + // first create the actor, which is often very confusing as it then keeps those + // spans for all logging of the actor. + let task = tokio::spawn( + async move { + if let Err(err) = self.run(rx).await { + error!("actor failed: {err:#}"); + } } - ConnectionType::None => None, - } - } - - /// Returns info about this endpoint. - pub(super) fn info(&self, now: Instant) -> RemoteInfo { - let conn_type = self.conn_type.get(); - let latency = self.latency(); - - let addrs = self - .udp_paths - .paths() - .iter() - .map(|(addr, path_state)| DirectAddrInfo { - addr: SocketAddr::from(*addr), - latency: path_state.validity.latency(), - last_control: path_state.last_control_msg(now), - last_payload: path_state - .last_payload_msg - .as_ref() - .map(|instant| now.duration_since(*instant)), - last_alive: path_state - .last_alive() - .map(|instant| now.duration_since(instant)), - sources: path_state - .sources - .iter() - .map(|(source, instant)| (source.clone(), now.duration_since(*instant))) - .collect(), - }) - .collect(); - - RemoteInfo { - endpoint_id: self.endpoint_id, - relay_url: self.relay_url.clone().map(|r| r.into()), - addrs, - conn_type, - latency, - last_used: self.last_used.map(|instant| now.duration_since(instant)), + .instrument(info_span!( + parent: None, + "EndpointStateActor", + me = %me.fmt_short(), + remote = %endpoint_id.fmt_short(), + )), + ); + EndpointStateHandle { + sender: tx, + _task: AbortOnDropHandle::new(task), } } - /// Returns the relay url of this endpoint - pub(super) fn relay_url(&self) -> Option { - self.relay_url.as_ref().map(|(url, _state)| url.clone()) - } - - /// Returns the address(es) that should be used for sending the next packet. + /// Runs the main loop of the actor. /// - /// This may return to send on one, both or no paths. - fn addr_for_send( - &self, - have_ipv6: bool, - metrics: &MagicsockMetrics, - ) -> (Option, Option) { - #[cfg(any(test, feature = "test-utils"))] - if self.path_selection == PathSelection::RelayOnly { - debug!( - "in `RelayOnly` mode, giving the relay address as the only viable address for this endpoint" - ); - return (None, self.relay_url()); - } - let (best_addr, relay_url) = match self.udp_paths.send_addr(have_ipv6) { - UdpSendAddr::Valid(addr) => { - // If we have a valid address we use it. - trace!(%addr, ?have_ipv6, "UdpSendAddr is valid, use it"); - (Some(*addr), None) - } - UdpSendAddr::Outdated(addr) => { - // If the address is outdated we use it, but send via relay at the same time. - // We also send disco pings so that it will become valid again if it still - // works (i.e. we don't need to holepunch again). - trace!(%addr, ?have_ipv6, "UdpSendAddr is outdated, use it together with relay"); - (Some(*addr), self.relay_url()) - } - UdpSendAddr::Unconfirmed(addr) => { - trace!(%addr, ?have_ipv6, "UdpSendAddr is unconfirmed, use it together with relay"); - (Some(*addr), self.relay_url()) - } - UdpSendAddr::None => { - trace!(?have_ipv6, "No UdpSendAddr, use relay"); - (None, self.relay_url()) - } - }; - let typ = match (best_addr, relay_url.clone()) { - (Some(best_addr), Some(relay_url)) => ConnectionType::Mixed(best_addr, relay_url), - (Some(best_addr), None) => ConnectionType::Direct(best_addr), - (None, Some(relay_url)) => ConnectionType::Relay(relay_url), - (None, None) => ConnectionType::None, - }; - if matches!(&typ, ConnectionType::Direct(_)) { - let before = self - .has_been_direct - .swap(true, std::sync::atomic::Ordering::Relaxed); - if !before { - metrics.endpoints_contacted_directly.inc(); - } - } - if let Ok(prev_typ) = self.conn_type.set(typ.clone()) { - // The connection type has changed. - event!( - target: "iroh::_events::conn_type::changed", - Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - conn_type = ?typ, - ); - info!(%typ, "new connection type"); - - // Update some metrics - match (prev_typ, typ) { - (ConnectionType::Relay(_), ConnectionType::Direct(_)) - | (ConnectionType::Mixed(_, _), ConnectionType::Direct(_)) => { - metrics.num_direct_conns_added.inc(); - metrics.num_relay_conns_removed.inc(); - } - (ConnectionType::Direct(_), ConnectionType::Relay(_)) - | (ConnectionType::Direct(_), ConnectionType::Mixed(_, _)) => { - metrics.num_direct_conns_removed.inc(); - metrics.num_relay_conns_added.inc(); - } - (ConnectionType::None, ConnectionType::Direct(_)) => { - metrics.num_direct_conns_added.inc(); + /// Note that the actor uses async handlers for tasks from the main loop. The actor is + /// not processing items from the inbox while waiting on any async calls. So some + /// dicipline is needed to not turn pending for a long time. + async fn run( + &mut self, + mut inbox: mpsc::Receiver, + ) -> Result<(), Whatever> { + trace!("actor started"); + loop { + let scheduled_hp = match self.scheduled_holepunch { + Some(when) => MaybeFuture::Some(tokio::time::sleep_until(when)), + None => MaybeFuture::None, + }; + let mut scheduled_hp = std::pin::pin!(scheduled_hp); + tokio::select! { + biased; + msg = inbox.recv() => { + match msg { + Some(msg) => self.handle_message(msg).await?, + None => break, + } } - (ConnectionType::Direct(_), ConnectionType::None) => { - metrics.num_direct_conns_removed.inc(); + Some((id, evt)) = self.path_events.next() => { + self.handle_path_event(id, evt); } - (ConnectionType::None, ConnectionType::Relay(_)) - | (ConnectionType::None, ConnectionType::Mixed(_, _)) => { - metrics.num_relay_conns_added.inc(); + _ = self.local_addrs.updated() => { + trace!("local addrs updated, triggering holepunching"); + self.trigger_holepunching().await; } - (ConnectionType::Relay(_), ConnectionType::None) - | (ConnectionType::Mixed(_, _), ConnectionType::None) => { - metrics.num_relay_conns_removed.inc(); + _ = &mut scheduled_hp => { + trace!("triggering scheduled holepunching"); + self.scheduled_holepunch = None; + self.trigger_holepunching().await; } - _ => (), } } - (best_addr, relay_url) - } - - /// Removes a direct address for this endpoint. - /// - /// If this is also the best address, it will be cleared as well. - pub(super) fn remove_direct_addr(&mut self, ip_port: &IpPort, now: Instant, why: &'static str) { - let Some(state) = self.udp_paths.access_mut(now).paths().remove(ip_port) else { - return; - }; - - match state.last_alive().map(|instant| instant.elapsed()) { - Some(last_alive) => debug!(%ip_port, ?last_alive, why, "pruning address"), - None => debug!(%ip_port, last_seen=%"never", why, "pruning address"), - } + trace!("actor terminating"); + Ok(()) } - /// Whether we need to send another call-me-maybe to the endpoint. + /// Handles an actor message. /// - /// Basically we need to send a call-me-maybe if we need to find a better path. Maybe - /// we only have a relay path, or our path is expired. - /// - /// When a call-me-maybe message is sent we also need to send pings to all known paths - /// of the endpoint. The [`EndpointState::send_call_me_maybe`] function takes care of this. - #[cfg(not(wasm_browser))] - #[instrument("want_call_me_maybe", skip_all)] - fn want_call_me_maybe(&self, now: &Instant, have_ipv6: bool) -> bool { - trace!("full ping: wanted?"); - let Some(last_full_ping) = self.last_full_ping else { - debug!("no previous full ping: need full ping"); - return true; - }; - match &self.udp_paths.send_addr(have_ipv6) { - UdpSendAddr::None | UdpSendAddr::Unconfirmed(_) => { - debug!("best addr not set: need full ping"); - true + /// Error returns are fatal and kill the actor. + #[instrument(skip(self))] + async fn handle_message(&mut self, msg: EndpointStateMessage) -> Result<(), Whatever> { + // trace!("handling message"); + match msg { + EndpointStateMessage::SendDatagram(transmit) => { + self.handle_msg_send_datagram(transmit).await?; } - UdpSendAddr::Outdated(_) => { - debug!("best addr expired: need full ping"); - true + EndpointStateMessage::AddConnection(handle, paths_info) => { + self.handle_msg_add_connection(handle, paths_info).await; } - UdpSendAddr::Valid(addr) => { - let latency = self - .udp_paths - .paths() - .get(&(*addr).into()) - .expect("send path not tracked?") - .latency() - .expect("send_addr marked valid incorrectly"); - if latency > GOOD_ENOUGH_LATENCY && *now - last_full_ping >= UPGRADE_INTERVAL { - debug!( - "full ping interval expired and latency is only {}ms: need full ping", - latency.as_millis() - ); - true - } else { - trace!(?now, "best_addr valid: not needed"); - false - } + EndpointStateMessage::AddEndpointAddr(addr, source) => { + self.handle_msg_add_endpoint_addr(addr, source); + } + EndpointStateMessage::CallMeMaybeReceived(msg) => { + self.handle_msg_call_me_maybe_received(msg).await; + } + EndpointStateMessage::PingReceived(ping, src) => { + self.handle_msg_ping_received(ping, src).await; + } + EndpointStateMessage::PongReceived(pong, src) => { + self.handle_msg_pong_received(pong, src); + } + EndpointStateMessage::CanSend(tx) => { + self.handle_msg_can_send(tx); + } + EndpointStateMessage::Latency(tx) => { + self.handle_msg_latency(tx); } } + Ok(()) } - #[cfg(wasm_browser)] - fn want_call_me_maybe(&self, _now: &Instant, _have_ipv6: bool) -> bool { - trace!("full ping: skipped in browser"); - false + /// Handles [`EndpointStateMessage::SendDatagram`]. + /// + /// Error returns are fatal and kill the actor. + async fn handle_msg_send_datagram(&mut self, transmit: OwnedTransmit) -> Result<(), Whatever> { + if let Some(ref addr) = self.selected_path { + trace!(?addr, "sending datagram to selected path"); + self.transports_sender + .send((addr.clone(), transmit).into()) + .await + .whatever_context("TransportSenderActor stopped")?; + } else { + trace!( + paths = ?self.paths.keys().collect::>(), + "sending datagram to all known paths", + ); + for addr in self.paths.keys() { + self.transports_sender + .send((addr.clone(), transmit.clone()).into()) + .await + .whatever_context("TransportSenerActor stopped")?; + } + // This message is received *before* a connection is added. So we do + // not yet have a connection to holepunch. Instead we trigger + // holepunching when AddConnection is received. + } + Ok(()) } - /// Cleanup the expired ping for the passed in txid. - #[instrument("disco", skip_all, fields(endpoint = %self.endpoint_id.fmt_short()))] - pub(super) fn ping_timeout( + /// Handles [`EndpointStateMessage::AddConnection`]. + /// + /// Error returns are fatal and kill the actor. + async fn handle_msg_add_connection( &mut self, - txid: TransactionId, - now: Instant, - metrics: &MagicsockMetrics, + handle: WeakConnectionHandle, + paths_info: Watchable>, ) { - if let Some(sp) = self.sent_pings.remove(&txid) { - debug!(tx = %HEXLOWER.encode(&txid), addr = %sp.to, "pong not received in timeout"); - match sp.to { - SendAddr::Udp(addr) => { - if let Some(path_state) = - self.udp_paths.access_mut(now).paths().get_mut(&addr.into()) + if let Some(conn) = handle.upgrade() { + // Remove any conflicting stable_ids from the local state. + let conn_id = ConnId(conn.stable_id()); + self.connections.remove(&conn_id); + + // This is a good time to clean up connections. + self.cleanup_connections(); + + // Store the connection and hook up paths events stream. + let events = BroadcastStream::new(conn.path_events()); + let stream = events.map(move |evt| (conn_id, evt)); + self.path_events.push(Box::pin(stream)); + self.connections.insert( + conn_id, + ConnectionState { + handle: handle.clone(), + pub_path_info: paths_info, + paths: Default::default(), + open_paths: Default::default(), + path_ids: Default::default(), + }, + ); + + // Store PathId(0), set path_status and select best path, check if holepunching + // is needed. + if let Some(conn) = handle.upgrade() { + if let Some(path) = conn.path(PathId::ZERO) { + if let Some(path_remote) = path + .remote_address() + .map_or(None, |remote| Some(MultipathMappedAddr::from(remote))) + .and_then(|mmaddr| mmaddr.to_transport_addr(&self.relay_mapped_addrs)) { - path_state.last_ping = None; - let consider_alive = path_state - .last_alive() - .map(|last_alive| last_alive.elapsed() <= PING_TIMEOUT_DURATION) - .unwrap_or(false); - if !consider_alive { - // If there was no sign of life from this path during the time - // which we should have received the pong, clear best addr and - // pong. Both are used to select this path again, but we know - // it's not a usable path now. - path_state.validity = PathValidity::empty(); - metrics.path_ping_failures.inc(); - - path_state.validity.record_metrics(metrics); - metrics.path_marked_outdated.inc(); - } - } - } - SendAddr::Relay(ref url) => { - if let Some((home_relay, relay_state)) = self.relay_url.as_mut() { - if home_relay == url { - // lost connectivity via relay - relay_state.last_ping = None; - } + trace!(?path_remote, "added new connection"); + let status = match path_remote { + transports::Addr::Ip(_) => PathStatus::Available, + transports::Addr::Relay(_, _) => PathStatus::Backup, + }; + path.set_status(status).ok(); + let conn_state = + self.connections.get_mut(&conn_id).expect("inserted above"); + conn_state.add_open_path(path_remote.clone(), PathId::ZERO); + self.paths + .entry(path_remote) + .or_default() + .sources + .insert(Source::Connection, Instant::now()); + self.select_path(); } } + // TODO: Make sure we are adding the relay path if we're on a direct + // path. + self.trigger_holepunching().await; } } } - #[must_use = "pings must be handled"] - fn start_ping(&self, dst: SendAddr, purpose: DiscoPingPurpose) -> Option { - #[cfg(any(test, feature = "test-utils"))] - if self.path_selection == PathSelection::RelayOnly && !dst.is_relay() { - // don't attempt any hole punching in relay only mode - warn!("in `RelayOnly` mode, ignoring request to start a hole punching attempt."); - return None; + /// Handles [`EndpointStateMessage::AddEndpointAddr`]. + fn handle_msg_add_endpoint_addr(&mut self, addr: EndpointAddr, source: Source) { + for sockaddr in addr.ip_addrs() { + let addr = transports::Addr::from(sockaddr); + self.paths + .entry(addr) + .or_default() + .sources + .insert(source.clone(), Instant::now()); } - #[cfg(wasm_browser)] - if !dst.is_relay() { - return None; // Similar to `RelayOnly` mode, we don't send UDP pings for hole-punching. + for relay_url in addr.relay_urls() { + let addr = transports::Addr::from((relay_url.clone(), self.endpoint_id)); + self.paths + .entry(addr) + .or_default() + .sources + .insert(source.clone(), Instant::now()); } + trace!("added addressing information"); + } - let tx_id = TransactionId::default(); - trace!(tx = %HEXLOWER.encode(&tx_id), %dst, ?purpose, - dst = %self.endpoint_id.fmt_short(), "start ping"); + /// Handles [`EndpointStateMessage::CallMeMaybeReceived`]. + async fn handle_msg_call_me_maybe_received(&mut self, msg: disco::CallMeMaybe) { event!( - target: "iroh::_events::ping::sent", + target: "iroh::_events::call_me_maybe::recv", Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - ?dst, - txn = ?tx_id, - ?purpose, + remote = %self.endpoint_id.fmt_short(), + addrs = ?msg.my_numbers, ); - Some(SendPing { - id: self.id, - dst, - dst_endpoint: self.endpoint_id, - tx_id, - purpose, - }) - } + let now = Instant::now(); + for addr in msg.my_numbers { + let dst = transports::Addr::Ip(addr); + let ping = disco::Ping::new(self.local_endpoint_id); - /// Record the fact that a ping has been sent out. - pub(super) fn ping_sent( - &mut self, - to: SendAddr, - tx_id: TransactionId, - purpose: DiscoPingPurpose, - sender: mpsc::Sender, - ) { - trace!(%to, tx = %HEXLOWER.encode(&tx_id), ?purpose, "record ping sent"); + let path = self.paths.entry(dst.clone()).or_default(); + path.sources.insert(Source::CallMeMaybe, now); + path.ping_sent = Some(ping.tx_id); - let now = Instant::now(); - let mut path_found = false; - match to { - SendAddr::Udp(addr) => { - if let Some(st) = self.udp_paths.access_mut(now).paths().get_mut(&addr.into()) { - st.last_ping.replace(now); - st.validity.record_ping_sent(); - path_found = true - } - } - SendAddr::Relay(ref url) => { - if let Some((home_relay, relay_state)) = self.relay_url.as_mut() { - if home_relay == url { - relay_state.last_ping.replace(now); - path_found = true - } - } - } - } - if !path_found { - // Shouldn't happen. But don't ping an endpoint that's not active for us. - warn!(%to, ?purpose, "unexpected attempt to ping no longer live path"); - return; + event!( + target: "iroh::_events::ping::sent", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + ?dst, + ); + self.send_disco_message(dst, disco::Message::Ping(ping)) + .await; } + } - let id = self.id; - let _expiry_task = AbortOnDropHandle::new(task::spawn(async move { - time::sleep(PING_TIMEOUT_DURATION).await; - sender - .send(ActorMessage::EndpointPingExpired(id, tx_id)) - .await - .ok(); - })); - self.sent_pings.insert( - tx_id, - SentPing { - to, - at: now, - purpose, - _expiry_task, - }, + /// Handles [`EndpointStateMessage::PingReceived`]. + async fn handle_msg_ping_received(&mut self, ping: disco::Ping, src: transports::Addr) { + let transports::Addr::Ip(addr) = src else { + warn!("received ping via relay transport, ignored"); + return; + }; + event!( + target: "iroh::_events::ping::recv", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + ?src, + txn = ?ping.tx_id, + ); + let pong = disco::Pong { + tx_id: ping.tx_id, + ping_observed_addr: addr.into(), + }; + event!( + target: "iroh::_events::pong::sent", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + dst = ?src, + txn = ?pong.tx_id, ); + self.send_disco_message(src.clone(), disco::Message::Pong(pong)) + .await; + + let path = self.paths.entry(src).or_default(); + path.sources.insert(Source::Ping, Instant::now()); + + trace!("ping received, triggering holepunching"); + self.trigger_holepunching().await; } - /// Send a DISCO call-me-maybe message to the peer. - /// - /// This takes care of sending the needed pings beforehand. This ensures that we open - /// our firewall's port so that when the receiver sends us DISCO pings in response to - /// our call-me-maybe they will reach us and the other side establishes a direct - /// connection upon our subsequent pong response. - /// - /// For [`SendCallMeMaybe::IfNoRecent`], **no** paths will be pinged if there already - /// was a recent call-me-maybe sent. - /// - /// The caller is responsible for sending the messages. - #[must_use = "actions must be handled"] - fn send_call_me_maybe(&mut self, now: Instant, always: SendCallMeMaybe) -> Vec { - match always { - SendCallMeMaybe::Always => (), - SendCallMeMaybe::IfNoRecent => { - let had_recent_call_me_maybe = self - .last_call_me_maybe - .map(|when| when.elapsed() < HEARTBEAT_INTERVAL) - .unwrap_or(false); - if had_recent_call_me_maybe { - trace!("skipping call-me-maybe, still recent"); - return Vec::new(); - } - } - } - // We send pings regardless of whether we have a RelayUrl. If we were given any - // direct address paths to contact but no RelayUrl, we still need to send a DISCO - // ping to the direct address paths so that the other endpoint will learn about us and - // accepts the connection. - let mut msgs = self.send_pings(now); - - if let Some(url) = self.relay_url() { - debug!(%url, "queue call-me-maybe"); - msgs.push(PingAction::SendCallMeMaybe { - relay_url: url, - dst_endpoint: self.endpoint_id, - }); - self.last_call_me_maybe = Some(now); - } else { - debug!("can not send call-me-maybe, no relay URL"); + /// Handles [`EndpointStateMessage::PongReceived`]. + fn handle_msg_pong_received(&mut self, pong: disco::Pong, src: transports::Addr) { + let Some(state) = self.paths.get(&src) else { + warn!(path = ?src, ?self.paths, "ignoring DISCO Pong for unknown path"); + return; + }; + if state.ping_sent != Some(pong.tx_id) { + debug!(path = ?src, ?state.ping_sent, pong_tx = ?pong.tx_id, + "ignoring unknown DISCO Pong for path"); + return; } + event!( + target: "iroh::_events::pong::recv", + Level::DEBUG, + remote_endpoint = %self.endpoint_id.fmt_short(), + ?src, + txn = ?pong.tx_id, + ); - msgs + self.open_path(&src); } - /// Send DISCO Pings to all the paths of this endpoint. - /// - /// Any paths to the endpoint which have not been recently pinged will be sent a disco - /// ping. - /// - /// The caller is responsible for sending the messages. - #[must_use = "actions must be handled"] - fn send_pings(&mut self, now: Instant) -> Vec { - // We allocate +1 in case the caller wants to add a call-me-maybe message. - let mut ping_msgs = Vec::with_capacity(self.udp_paths.paths().len() + 1); - - if let Some((url, state)) = self.relay_url.as_ref() { - if state.needs_ping(&now) { - debug!(%url, "relay path needs ping"); - if let Some(msg) = - self.start_ping(SendAddr::Relay(url.clone()), DiscoPingPurpose::Discovery) + /// Handles [`EndpointStateMessage::CanSend`]. + fn handle_msg_can_send(&self, tx: oneshot::Sender) { + let can_send = !self.paths.is_empty(); + tx.send(can_send).ok(); + } + + /// Handles [`EndpointStateMessage::Latency`]. + fn handle_msg_latency(&self, tx: oneshot::Sender>) { + let rtt = self.selected_path.as_ref().and_then(|addr| { + for conn_state in self.connections.values() { + let Some(path_id) = conn_state.path_ids.get(addr) else { + continue; + }; + if !conn_state.open_paths.contains_key(path_id) { + continue; + } + if let Some(stats) = conn_state + .handle + .upgrade() + .and_then(|conn| conn.stats().paths.get(path_id).copied()) { - ping_msgs.push(PingAction::SendPing(msg)) + return Some(stats.rtt); } } + None + }); + tx.send(rtt).ok(); + } + + /// Triggers holepunching to the remote endpoint. + /// + /// This will manage the entire process of holepunching with the remote endpoint. + /// + /// - If there already is a direct connection, nothing happens. + /// - If there is no relay address known, nothing happens. + /// - If there was a recent attempt, it will schedule holepunching instead. + /// - Unless there are new addresses to try. + /// - The scheduled attempt will only run if holepunching has not yet succeeded by + /// then. + /// - DISCO pings will be sent to addresses recently advertised in a call-me-maybe + /// message. + /// - A DISCO call-me-maybe message advertising our own addresses will be sent. + /// + /// If a next trigger needs to be scheduled the delay until when to call this again is + /// returned. + async fn trigger_holepunching(&mut self) { + const HOLEPUNCH_ATTEMPTS_INTERVAL: Duration = Duration::from_secs(5); + + if self.connections.is_empty() { + trace!("not holepunching: no connections"); + return; } - #[cfg(any(test, feature = "test-utils"))] - if self.path_selection == PathSelection::RelayOnly { - warn!("in `RelayOnly` mode, ignoring request to respond to a hole punching attempt."); - return ping_msgs; + if self + .selected_path + .as_ref() + .map(|addr| addr.is_ip()) + .unwrap_or_default() + { + // TODO: We should ping this path to make sure it still works. Because we now + // know things could be broken. + trace!("not holepunching: already have a direct connection"); + // TODO: If the latency is kind of bad we should retry holepunching at times. + return; } - self.prune_direct_addresses(now); - let mut ping_dsts = String::from("["); - self.udp_paths - .paths() + let remote_addrs: BTreeSet = self.remote_hp_addrs(); + let local_addrs: BTreeSet = self + .local_addrs + .get() .iter() - .filter_map(|(ipp, state)| state.needs_ping(&now).then_some(*ipp)) - .filter_map(|ipp| { - self.start_ping(SendAddr::Udp(ipp.into()), DiscoPingPurpose::Discovery) + .map(|daddr| daddr.addr) + .collect(); + let new_addrs = self + .last_holepunch + .as_ref() + .map(|last_hp| { + // Addrs are allowed to disappear, but if there are new ones we need to + // holepunch again. + trace!(?last_hp, ?local_addrs, ?remote_addrs, "addrs to holepunch?"); + !remote_addrs.is_subset(&last_hp.remote_addrs) + || !local_addrs.is_subset(&last_hp.local_addrs) }) - .for_each(|msg| { - use std::fmt::Write; - write!(&mut ping_dsts, " {} ", msg.dst).ok(); - ping_msgs.push(PingAction::SendPing(msg)); - }); - ping_dsts.push(']'); - debug!( - %ping_dsts, - dst = %self.endpoint_id.fmt_short(), - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "sending pings to endpoint", - ); - self.last_full_ping.replace(now); - ping_msgs - } - - pub(super) fn update_from_endpoint_addr( - &mut self, - new_relay_url: Option<&RelayUrl>, - new_addrs: impl Iterator, - source: super::Source, - have_ipv6: bool, - metrics: &MagicsockMetrics, - ) { - if matches!( - self.udp_paths.send_addr(have_ipv6), - UdpSendAddr::None | UdpSendAddr::Unconfirmed(_) - ) { - // we do not have a direct connection, so changing the relay information may - // have an effect on our connection status - if self.relay_url.is_none() && new_relay_url.is_some() { - // we did not have a relay connection before, but now we do - metrics.num_relay_conns_added.inc(); - } else if self.relay_url.is_some() && new_relay_url.is_none() { - // we had a relay connection before but do not have one now - metrics.num_relay_conns_removed.inc(); + .unwrap_or(true); + if !new_addrs { + if let Some(ref last_hp) = self.last_holepunch { + let next_hp = last_hp.when + HOLEPUNCH_ATTEMPTS_INTERVAL; + let now = Instant::now(); + if next_hp > now { + trace!(scheduled_in = ?(next_hp - now), "not holepunching: no new addresses"); + self.scheduled_holepunch = Some(next_hp); + return; + } } } - let now = Instant::now(); - - if new_relay_url.is_some() && new_relay_url != self.relay_url().as_ref() { - debug!( - "Changing relay endpoint from {:?} to {:?}", - self.relay_url, new_relay_url - ); - self.relay_url = new_relay_url.map(|url| { - ( - url.clone(), - PathState::new(self.endpoint_id, url.clone().into(), source.clone(), now), - ) - }); - } - - let mut access = self.udp_paths.access_mut(now); - let mut new_addrs_list = Vec::new(); - for addr in new_addrs { - access - .paths() - .entry(addr.into()) - .and_modify(|path_state| { - path_state.add_source(source.clone(), now); - }) - .or_insert_with(|| { - PathState::new(self.endpoint_id, SendAddr::from(addr), source.clone(), now) - }); - new_addrs_list.push(addr); - } - drop(access); - let paths = summarize_endpoint_paths(self.udp_paths.paths()); - debug!(new = ?new_addrs_list , %paths, "added new direct paths for endpoint"); + self.do_holepunching().await; } - /// Handle a received Disco Ping. - /// - /// - Ensures the paths the ping was received on is a known path for this endpoint. - /// - /// - If there is no best_addr for this endpoint yet, sends a ping itself to try and - /// establish one. - /// - /// This is called once we've already verified that we got a valid discovery message - /// from `self` via ep. - pub(super) fn handle_ping(&mut self, path: SendAddr, tx_id: TransactionId) -> PingHandled { - let now = Instant::now(); + /// Returns the remote addresses to holepunch against. + fn remote_hp_addrs(&self) -> BTreeSet { + const CALL_ME_MAYBE_VALIDITY: Duration = Duration::from_secs(30); - let role = match path { - SendAddr::Udp(addr) => { - match self.udp_paths.access_mut(now).paths().entry(addr.into()) { - Entry::Occupied(mut occupied) => occupied.get_mut().handle_ping(tx_id, now), - Entry::Vacant(vacant) => { - info!(%addr, "new direct addr for endpoint"); - vacant.insert(PathState::with_ping( - self.endpoint_id, - path.clone(), - tx_id, - Source::Udp, - now, - )); - PingRole::NewPath - } - } - } - SendAddr::Relay(ref url) => { - match self.relay_url.as_mut() { - Some((home_url, _state)) if home_url != url => { - // either the endpoint changed relays or we didn't have a relay address for the - // endpoint. In both cases, trust the new confirmed url - info!(%url, "new relay addr for endpoint"); - self.relay_url = Some(( - url.clone(), - PathState::with_ping( - self.endpoint_id, - path.clone(), - tx_id, - Source::Relay, - now, - ), - )); - PingRole::NewPath - } - Some((_home_url, state)) => state.handle_ping(tx_id, now), - None => { - info!(%url, "new relay addr for endpoint"); - self.relay_url = Some(( - url.clone(), - PathState::with_ping( - self.endpoint_id, - path.clone(), - tx_id, - Source::Relay, - now, - ), - )); - PingRole::NewPath - } + self.paths + .iter() + .filter_map(|(addr, state)| match addr { + transports::Addr::Ip(socket_addr) => Some((socket_addr, state)), + transports::Addr::Relay(_, _) => None, + }) + .filter_map(|(addr, state)| { + if state + .sources + .get(&Source::CallMeMaybe) + .map(|when| when.elapsed() <= CALL_ME_MAYBE_VALIDITY) + .unwrap_or_default() + || state + .sources + .get(&Source::Ping) + .map(|when| when.elapsed() <= CALL_ME_MAYBE_VALIDITY) + .unwrap_or_default() + { + Some(*addr) + } else { + None } - } - }; - event!( - target: "iroh::_events::ping::recv", - Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - src = ?path, - txn = ?tx_id, - ?role, - ); - - if matches!(path, SendAddr::Udp(_)) && matches!(role, PingRole::NewPath) { - self.prune_direct_addresses(now); - } - - // if the endpoint does not yet have a best_addr - let needs_ping_back = if matches!(path, SendAddr::Udp(_)) - && matches!( - self.udp_paths.send_addr(true), - UdpSendAddr::None | UdpSendAddr::Unconfirmed(_) | UdpSendAddr::Outdated(_) - ) { - // We also need to send a ping to make this path available to us as well. This - // is always sent together with a pong. So in the worst case the pong gets lost - // and this ping does not. In that case we ping-pong until both sides have - // received at least one pong. Once both sides have received one pong they both - // have a best_addr and this ping will stop being sent. - self.start_ping(path, DiscoPingPurpose::PingBack) - } else { - None - }; - - debug!( - ?role, - needs_ping_back = ?needs_ping_back.is_some(), - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "endpoint handled ping", - ); - PingHandled { - role, - needs_ping_back, - } + }) + .collect() } - /// Prune inactive paths. + /// Unconditionally perform holepunching. /// - /// This trims the list of inactive paths for an endpoint. At most - /// [`MAX_INACTIVE_DIRECT_ADDRESSES`] are kept. - pub(super) fn prune_direct_addresses(&mut self, now: Instant) { - // prune candidates are addresses that are not active - let mut prune_candidates: Vec<_> = self - .udp_paths - .paths() + /// - DISCO pings will be sent to addresses recently advertised in a call-me-maybe + /// message. + /// - A DISCO call-me-maybe message advertising our own addresses will be sent. + #[instrument(skip_all)] + async fn do_holepunching(&mut self) { + let Some(relay_addr) = self + .paths .iter() - .filter(|(_ip_port, state)| !state.is_active()) - .map(|(ip_port, state)| (*ip_port, state.last_alive())) - .filter(|(_ipp, last_alive)| match last_alive { - Some(last_seen) => last_seen.elapsed() > LAST_ALIVE_PRUNE_DURATION, - None => true, + .filter_map(|(addr, _)| match addr { + transports::Addr::Ip(_) => None, + transports::Addr::Relay(_, _) => Some(addr), }) - .collect(); - let prune_count = prune_candidates - .len() - .saturating_sub(MAX_INACTIVE_DIRECT_ADDRESSES); - if prune_count == 0 { - // nothing to do, within limits - debug!( - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "prune addresses: {prune_count} pruned", - ); + .next() + .cloned() + else { + warn!("holepunching requested but have no relay address"); return; - } - - // sort leaving the worst addresses first (never contacted) and better ones (most recently - // used ones) last - prune_candidates.sort_unstable_by_key(|(_ip_port, last_alive)| *last_alive); - prune_candidates.truncate(prune_count); - for (ip_port, _last_alive) in prune_candidates.into_iter() { - self.remove_direct_addr(&ip_port, now, "inactive"); - } - debug!( - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "prune addresses: {prune_count} pruned", - ); - } + }; + let remote_addrs = self.remote_hp_addrs(); - /// Called when connectivity changes enough that we should question our earlier - /// assumptions about which paths work. - #[instrument("disco", skip_all, fields(endpoint = %self.endpoint_id.fmt_short()))] - pub(super) fn note_connectivity_change(&mut self, now: Instant, metrics: &MagicsockMetrics) { - let mut guard = self.udp_paths.access_mut(now); - for es in guard.paths().values_mut() { - es.validity.record_metrics(metrics); - es.clear(); + // Send DISCO Ping messages to all CallMeMaybe-advertised paths. + for dst in remote_addrs.iter() { + let msg = disco::Ping::new(self.local_endpoint_id); + event!( + target: "iroh::_events::ping::sent", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + ?dst, + txn = ?msg.tx_id, + ); + let addr = transports::Addr::Ip(*dst); + self.paths.entry(addr.clone()).or_default().ping_sent = Some(msg.tx_id); + self.send_disco_message(addr, disco::Message::Ping(msg)) + .await; } - } - /// Handles a Pong message (a reply to an earlier ping). - /// - /// It reports the address and key that should be inserted for the endpoint if any. - #[instrument(skip(self, metrics))] - pub(super) fn handle_pong( - &mut self, - m: &disco::Pong, - src: SendAddr, - metrics: &MagicsockMetrics, - ) -> Option<(SocketAddr, PublicKey)> { + // Send the DISCO CallMeMaybe message over the relay. + let my_numbers: Vec = self + .local_addrs + .get() + .iter() + .map(|daddr| daddr.addr) + .collect(); + let local_addrs: BTreeSet = my_numbers.iter().copied().collect(); + let msg = disco::CallMeMaybe { my_numbers }; event!( - target: "iroh::_events::pong::recv", + target: "iroh::_events::call_me_maybe::sent", Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - ?src, - txn = ?m.tx_id, + remote = %self.endpoint_id.fmt_short(), + dst = ?relay_addr, + my_numbers = ?msg.my_numbers, ); - let is_relay = src.is_relay(); - match self.sent_pings.remove(&m.tx_id) { - None => { - // This is not a pong for a ping we sent. In reality however we probably - // did send this ping but it has timed-out by the time we receive this pong - // so we removed the state already. - debug!(tx = %HEXLOWER.encode(&m.tx_id), "received unknown pong (did it timeout?)"); - None - } - Some(sp) => { - let mut endpoint_map_insert = None; - - let now = Instant::now(); - let latency = now - sp.at; - - debug!( - tx = %HEXLOWER.encode(&m.tx_id), - src = %src, - reported_ping_src = %m.ping_observed_addr, - ping_dst = %sp.to, - is_relay = %src.is_relay(), - latency = %latency.as_millis(), - "received pong", - ); - - match src { - SendAddr::Udp(addr) => { - match self.udp_paths.access_mut(now).paths().get_mut(&addr.into()) { - None => { - warn!("ignoring pong: no state for src addr"); - // This is no longer an endpoint we care about. - return endpoint_map_insert; - } - Some(st) => { - endpoint_map_insert = Some((addr, self.endpoint_id)); - st.add_pong_reply( - PongReply { - latency, - pong_at: now, - from: src, - pong_src: m.ping_observed_addr.clone(), - }, - metrics, - ); - } - } - debug!( - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "handled pong", - ); - } - SendAddr::Relay(ref url) => match self.relay_url.as_mut() { - Some((home_url, state)) if home_url == url => { - state.add_pong_reply( - PongReply { - latency, - pong_at: now, - from: src, - pong_src: m.ping_observed_addr.clone(), - }, - metrics, - ); - } - other => { - // if we are here then we sent this ping, but the url changed - // waiting for the response. It was either set to None or changed to - // another relay. This should either never happen or be extremely - // unlikely. Log and ignore for now - warn!( - stored=?other, - received=?url, - "ignoring pong via relay for different relay from last one", - ); - } - }, - } + self.send_disco_message(relay_addr, disco::Message::CallMeMaybe(msg)) + .await; - // Promote this pong response to our current best address if it's lower latency. - // TODO(bradfitz): decide how latency vs. preference order affects decision - if let SendAddr::Udp(_to) = sp.to { - debug_assert!(!is_relay, "mismatching relay & udp"); - } + self.last_holepunch = Some(HolepunchAttempt { + when: Instant::now(), + local_addrs, + remote_addrs, + }); + } - endpoint_map_insert + /// Sends a DISCO message to the remote endpoint this actor manages. + #[instrument(skip(self), fields(remote = %self.endpoint_id.fmt_short()))] + async fn send_disco_message(&self, dst: transports::Addr, msg: disco::Message) { + let pkt = self.disco.encode_and_seal(self.endpoint_id, &msg); + let transmit = transports::OwnedTransmit { + ecn: None, + contents: pkt, + segment_size: None, + }; + let counter = match dst { + transports::Addr::Ip(_) => &self.metrics.send_disco_udp, + transports::Addr::Relay(_, _) => &self.metrics.send_disco_relay, + }; + match self.transports_sender.send((dst, transmit).into()).await { + Ok(()) => { + trace!("sent"); + counter.inc(); + } + Err(err) => { + warn!("failed to send disco message: {err:#}"); } } } - /// Handles a DISCO CallMeMaybe discovery message. + /// Open the path on all connections. /// - /// The contract for use of this message is that the endpoint has already pinged to us via - /// UDP, so their stateful firewall should be open. Now we can Ping back and make it - /// through. - /// - /// However if the remote side has no direct path information to us, they would not have - /// had any [`IpPort`]s to send pings to and our pings might end up blocked. But at - /// least open the firewalls on our side, giving the other side another change of making - /// it through when it pings in response. - pub(super) fn handle_call_me_maybe( - &mut self, - m: disco::CallMeMaybe, - metrics: &MagicsockMetrics, - ) -> Vec { - let now = Instant::now(); - let mut call_me_maybe_ipps = BTreeSet::new(); - - let mut guard = self.udp_paths.access_mut(now); + /// This goes through all the connections for which we are the client, and makes sure + /// the path exists, or opens it. + #[instrument(level = "warn", skip(self))] + fn open_path(&mut self, open_addr: &transports::Addr) { + let path_status = match open_addr { + transports::Addr::Ip(_) => PathStatus::Available, + transports::Addr::Relay(_, _) => PathStatus::Backup, + }; + let quic_addr = match &open_addr { + transports::Addr::Ip(socket_addr) => *socket_addr, + transports::Addr::Relay(relay_url, eid) => self + .relay_mapped_addrs + .get(&(relay_url.clone(), *eid)) + .private_socket_addr(), + }; - for peer_sockaddr in &m.my_numbers { - if let IpAddr::V6(ip) = peer_sockaddr.ip() { - if netwatch::ip::is_unicast_link_local(ip) { - // We send these out, but ignore them for now. - // TODO: teach the ping code to ping on all interfaces for these. - continue; - } + for (conn_id, conn_state) in self.connections.iter_mut() { + if conn_state.path_ids.contains_key(open_addr) { + continue; } - let ipp = IpPort::from(*peer_sockaddr); - call_me_maybe_ipps.insert(ipp); - guard - .paths() - .entry(ipp) - .or_insert_with(|| { - PathState::new( - self.endpoint_id, - SendAddr::from(*peer_sockaddr), - Source::Relay, - now, - ) - }) - .call_me_maybe_time - .replace(now); - } - - // Zero out all the last_ping times to force send_pings to send new ones, even if - // it's been less than 5 seconds ago. Also clear pongs for direct addresses not - // included in the updated set. - for (ipp, st) in guard.paths().iter_mut() { - st.last_ping = None; - if !call_me_maybe_ipps.contains(ipp) { - // TODO: This seems like a weird way to signal that the endpoint no longer - // thinks it has this IpPort as an available path. - if !st.validity.is_empty() { - debug!(path=?ipp ,"clearing recent pong"); - st.validity.record_metrics(metrics); - st.validity = PathValidity::empty(); + let Some(conn) = conn_state.handle.upgrade() else { + continue; + }; + if conn.side().is_server() { + continue; + } + let fut = conn.open_path_ensure(quic_addr, path_status); + match fut.path_id() { + Some(path_id) => { + trace!(?conn_id, ?path_id, "opening new path"); + conn_state.add_path(open_addr.clone(), path_id); + } + None => { + let ret = now_or_never(fut); + warn!(?ret, "Opening path failed"); } } } - if guard.has_best_addr_changed() { - // Clear the last call-me-maybe send time so we will send one again. - self.last_call_me_maybe = None; - } - debug!( - paths = %summarize_endpoint_paths(self.udp_paths.paths()), - "updated endpoint paths from call-me-maybe", - ); - self.send_pings(now) } - /// Marks this endpoint as having received a UDP payload message. - #[cfg(not(wasm_browser))] - pub(super) fn receive_udp(&mut self, addr: IpPort, now: Instant) { - let mut guard = self.udp_paths.access_mut(now); - let Some(state) = guard.paths().get_mut(&addr) else { - debug_assert!( - false, - "endpoint map inconsistency by_ip_port <-> direct addr" - ); + #[instrument(skip(self))] + fn handle_path_event( + &mut self, + conn_id: ConnId, + event: Result, + ) { + let Ok(event) = event else { + warn!("missed a PathEvent, EndpointStateActor lagging"); + // TODO: Is it possible to recover using the sync APIs to figure out what the + // state of the connection and it's paths are? return; }; - state.receive_payload(now); - self.last_used = Some(now); - } + let Some(conn_state) = self.connections.get_mut(&conn_id) else { + trace!("event for removed connection"); + return; + }; + let Some(conn) = conn_state.handle.upgrade() else { + trace!("event for closed connection"); + return; + }; + trace!("path event"); + match event { + PathEvent::Opened { id: path_id } => { + let Some(path) = conn.path(path_id) else { + trace!("path open event for unknown path"); + return; + }; + // TODO: We configure this as defaults when we setup the endpoint, do we + // really need to duplicate this? + path.set_keep_alive_interval(Some(HEARTBEAT_INTERVAL)).ok(); + path.set_max_idle_timeout(Some(PATH_MAX_IDLE_TIMEOUT)).ok(); + + if let Some(path_remote) = path + .remote_address() + .map_or(None, |remote| Some(MultipathMappedAddr::from(remote))) + .and_then(|mmaddr| mmaddr.to_transport_addr(&self.relay_mapped_addrs)) + { + event!( + target: "iroh::_events::path::open", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + ?path_remote, + ?conn_id, + ?path_id, + ); + conn_state.add_open_path(path_remote.clone(), path_id); + self.paths + .entry(path_remote.clone()) + .or_default() + .sources + .insert(Source::Connection, Instant::now()); + } - pub(super) fn receive_relay(&mut self, url: &RelayUrl, src: EndpointId, now: Instant) { - match self.relay_url.as_mut() { - Some((current_home, state)) if current_home == url => { - // We received on the expected url. update state. - state.receive_payload(now); + self.select_path(); } - Some((_current_home, _state)) => { - // we have a different url. we only update on ping, not on receive_relay. + PathEvent::Abandoned { id, path_stats } => { + trace!(?path_stats, "path abandoned"); + // This is the last event for this path. + conn_state.remove_path(&id); } - None => { - self.relay_url = Some(( - url.clone(), - PathState::with_last_payload( - src, - SendAddr::from(url.clone()), - Source::Relay, - now, - ), - )); + PathEvent::Closed { id, .. } | PathEvent::LocallyClosed { id, .. } => { + let Some(path_remote) = conn_state.paths.get(&id).cloned() else { + debug!("path not in path_id_map"); + return; + }; + event!( + target: "iroh::_events::path::closed", + Level::DEBUG, + remote = %self.endpoint_id.fmt_short(), + ?path_remote, + ?conn_id, + path_id = ?id, + ); + conn_state.remove_open_path(&id); + + // If one connection closes this path, close it on all connections. + for (conn_id, conn_state) in self.connections.iter_mut() { + let Some(path_id) = conn_state.path_ids.get(&path_remote) else { + continue; + }; + let Some(conn) = conn_state.handle.upgrade() else { + continue; + }; + if let Some(path) = conn.path(*path_id) { + trace!(?path_remote, ?conn_id, ?path_id, "closing path"); + if let Err(err) = path.close(APPLICATION_ABANDON_PATH.into()) { + trace!( + ?path_remote, + ?conn_id, + ?path_id, + "path close failed: {err:#}" + ); + } + } + } + } + PathEvent::RemoteStatus { .. } | PathEvent::ObservedAddr { .. } => { + // Nothing to do for these events. } - } - self.last_used = Some(now); - } - - pub(super) fn last_ping(&self, addr: &SendAddr) -> Option { - match addr { - SendAddr::Udp(addr) => self - .udp_paths - .paths() - .get(&(*addr).into()) - .and_then(|ep| ep.last_ping), - SendAddr::Relay(url) => self - .relay_url - .as_ref() - .filter(|(home_url, _state)| home_url == url) - .and_then(|(_home_url, state)| state.last_ping), } } - /// Checks if this `Endpoint` is currently actively being used. - pub(super) fn is_active(&self, now: &Instant) -> bool { - match self.last_used { - Some(last_active) => now.duration_since(last_active) <= SESSION_ACTIVE_TIMEOUT, - None => false, - } + /// Clean up connections which no longer exist. + // TODO: Call this on a schedule. + fn cleanup_connections(&mut self) { + self.connections.retain(|_, c| c.handle.upgrade().is_some()); } - /// Send a heartbeat to the endpoint to keep the connection alive, or trigger a full ping - /// if necessary. - #[instrument("stayin_alive", skip_all, fields(endpoint = %self.endpoint_id.fmt_short()))] - pub(super) fn stayin_alive(&mut self, have_ipv6: bool) -> Vec { - trace!("stayin_alive"); - let now = Instant::now(); - if !self.is_active(&now) { - trace!("skipping stayin alive: session is inactive"); - return Vec::new(); - } - - // If we do not have an optimal addr, send pings to all known places. - if self.want_call_me_maybe(&now, have_ipv6) { - debug!("sending a call-me-maybe"); - return self.send_call_me_maybe(now, SendCallMeMaybe::Always); - } - - // Send heartbeat ping to keep the current addr going as long as we need it. - if let Some(udp_addr) = self.udp_paths.send_addr(have_ipv6).get_addr() { - let elapsed = self.last_ping(&SendAddr::Udp(udp_addr)).map(|l| now - l); - // Send a ping if the last ping is older than 2 seconds. - let needs_ping = match elapsed { - Some(e) => e >= STAYIN_ALIVE_MIN_ELAPSED, - None => false, + /// Selects the path with the lowest RTT, prefers direct paths. + /// + /// If there are direct paths, this selects the direct path with the lowest RTT. If + /// there are only relay paths, the relay path with the lowest RTT is chosen. + /// + /// The selected path is added to any connections which do not yet have it. Any unused + /// direct paths are closed for all connections. + #[instrument(skip_all)] + fn select_path(&mut self) { + // Find the lowest RTT across all connections for each open path. The long way, so + // we get to log *all* RTTs. + let mut all_path_rtts: FxHashMap> = FxHashMap::default(); + for conn_state in self.connections.values() { + let Some(conn) = conn_state.handle.upgrade() else { + continue; }; - - if needs_ping { - debug!( - dst = %udp_addr, - since_last_ping=?elapsed, - "send stayin alive ping", - ); - if let Some(msg) = - self.start_ping(SendAddr::Udp(udp_addr), DiscoPingPurpose::StayinAlive) - { - return vec![PingAction::SendPing(msg)]; + let stats = conn.stats(); + for (path_id, stats) in stats.paths { + if let Some(addr) = conn_state.open_paths.get(&path_id) { + all_path_rtts + .entry(addr.clone()) + .or_default() + .push(stats.rtt); } } } - - Vec::new() - } - - /// Returns the addresses on which a payload should be sent right now. - /// - /// This is in the hot path of `.poll_send()`. - // TODO(matheus23): Make this take &self. That's not quite possible yet due to `send_call_me_maybe` - // eventually calling `prune_direct_addresses` (which needs &mut self) - #[instrument("get_send_addrs", skip_all, fields(endpoint = %self.endpoint_id.fmt_short()))] - pub(crate) fn get_send_addrs( - &mut self, - have_ipv6: bool, - metrics: &MagicsockMetrics, - ) -> (Option, Option, Vec) { - let now = Instant::now(); - let prev = self.last_used.replace(now); - if prev.is_none() { - // this is the first time we are trying to connect to this endpoint - metrics.endpoints_contacted.inc(); - } - let (udp_addr, relay_url) = self.addr_for_send(have_ipv6, metrics); - - let ping_msgs = if self.want_call_me_maybe(&now, have_ipv6) { - self.send_call_me_maybe(now, SendCallMeMaybe::IfNoRecent) - } else { - Vec::new() - }; - trace!( - ?udp_addr, - ?relay_url, - pings = %ping_msgs.len(), - "found send address", - ); - (udp_addr, relay_url, ping_msgs) - } - - /// Get the IP addresses for this endpoint. - pub(super) fn ip_addrs(&self) -> impl Iterator + '_ { - self.udp_paths.paths().keys().copied() - } - - #[cfg(test)] - pub(super) fn ip_addr_states(&self) -> impl Iterator + '_ { - self.udp_paths.paths().iter() - } - - pub(super) fn last_used(&self) -> Option { - self.last_used - } -} - -impl From for EndpointAddr { - fn from(info: RemoteInfo) -> Self { - let mut addrs = info - .addrs + trace!(?all_path_rtts, "dumping all path RTTs"); + let path_rtts: FxHashMap = all_path_rtts .into_iter() - .map(|info| TransportAddr::Ip(info.addr)) - .collect::>(); - - if let Some(url) = info.relay_url { - addrs.insert(TransportAddr::Relay(url.into())); - } + .filter_map(|(addr, rtts)| rtts.into_iter().min().map(|rtt| (addr, rtt))) + .collect(); - EndpointAddr { - id: info.endpoint_id, - addrs, + // Find the fastest direct or relay path. + const IPV6_RTT_ADVANTAGE: Duration = Duration::from_millis(3); + let direct_path = path_rtts + .iter() + .filter(|(addr, _rtt)| addr.is_ip()) + .map(|(addr, rtt)| { + if addr.is_ipv4() { + (*rtt + IPV6_RTT_ADVANTAGE, addr) + } else { + (*rtt, addr) + } + }) + .min(); + let selected_path = direct_path.or_else(|| { + // Find the fasted relay path. + path_rtts + .iter() + .filter(|(addr, _rtt)| addr.is_relay()) + .map(|(addr, rtt)| (*rtt, addr)) + .min() + }); + if let Some((rtt, addr)) = selected_path { + let prev = self.selected_path.replace(addr.clone()); + if prev.as_ref() != Some(addr) { + debug!(?addr, ?rtt, ?prev, "selected new path"); + } + self.open_path(addr); + self.close_redundant_paths(addr); } } -} - -/// Whether to send a call-me-maybe message after sending pings to all known paths. -/// -/// `IfNoRecent` will only send a call-me-maybe if no previous one was sent in the last -/// [`HEARTBEAT_INTERVAL`]. -#[derive(Debug)] -enum SendCallMeMaybe { - Always, - IfNoRecent, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(super) struct PongReply { - pub(super) latency: Duration, - /// When we received the pong. - pub(super) pong_at: Instant, - /// The pong's src (usually same as endpoint map key). - pub(super) from: SendAddr, - /// What they reported they heard. - pub(super) pong_src: SendAddr, -} - -#[derive(Debug)] -pub(super) struct SentPing { - pub(super) to: SendAddr, - pub(super) at: Instant, - #[allow(dead_code)] - pub(super) purpose: DiscoPingPurpose, - pub(super) _expiry_task: AbortOnDropHandle<()>, -} -/// The reason why a discovery ping message was sent. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DiscoPingPurpose { - /// The purpose of a ping was to see if a path was valid. - Discovery, - /// Ping to ensure the current route is still valid. - StayinAlive, - /// When a ping was received and no direct connection exists yet. + /// Closes any direct paths not selected. /// - /// When a ping was received we suspect a direct connection is possible. If we do not - /// yet have one that triggers a ping, indicated with this reason. - PingBack, -} - -/// The type of control message we have received. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, derive_more::Display)] -pub enum ControlMsg { - /// We received a Ping from the endpoint. - #[display("ping←")] - Ping, - /// We received a Pong from the endpoint. - #[display("pong←")] - Pong, - /// We received a CallMeMaybe. - #[display("call me")] - CallMeMaybe, + /// Makes sure not to close the last direct path. Relay paths are never closed + /// currently, because we only have one relay path at this time. + // TODO: Need to handle this on a timer as well probably. In .select_path() we open new + // paths and immediately call this. But the new paths are probably not yet open on + // all connections. + fn close_redundant_paths(&mut self, selected_path: &transports::Addr) { + debug_assert_eq!(self.selected_path.as_ref(), Some(selected_path)); + + for (conn_id, conn_state) in self.connections.iter() { + for (path_id, path_remote) in conn_state + .open_paths + .iter() + .filter(|(_, addr)| addr.is_ip()) + .filter(|(_, addr)| *addr != selected_path) + { + if conn_state.open_paths.values().filter(|a| a.is_ip()).count() <= 1 { + continue; // Do not close the last direct path. + } + if let Some(path) = conn_state + .handle + .upgrade() + .and_then(|conn| conn.path(*path_id)) + { + trace!(?path_remote, ?conn_id, ?path_id, "closing direct path"); + match path.close(APPLICATION_ABANDON_PATH.into()) { + Err(quinn_proto::ClosePathError::LastOpenPath) => { + error!("could not close last open path"); + } + Err(quinn_proto::ClosePathError::ClosedPath) => { + // We already closed this. + } + Ok(_fut) => { + // We will handle the event in Self::handle_path_events. + } + } + } + } + } + } } -/// Information about a *direct address*. -/// -/// The *direct addresses* of an iroh endpoint are those that could be used by other endpoints to -/// establish direct connectivity, depending on the network situation. Due to NAT configurations, -/// for example, not all direct addresses of an endpoint are usable by all peers. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct DirectAddrInfo { - /// The UDP address reported by the remote endpoint. - pub addr: SocketAddr, - /// The latency to the remote endpoint over this network path. - /// - /// If there has never been any connectivity via this address no latency will be known. - pub latency: Option, - /// Last control message received by this endpoint about this address. - /// - /// This contains the elapsed duration since the control message was received and the - /// kind of control message received at that time. Only the most recent control message - /// is returned. - /// - /// Note that [`ControlMsg::CallMeMaybe`] is received via a relay path, while - /// [`ControlMsg::Ping`] and [`ControlMsg::Pong`] are received on the path to - /// [`DirectAddrInfo::addr`] itself and thus convey very different information. - pub last_control: Option<(Duration, ControlMsg)>, - /// Elapsed time since the last payload message was received on this network path. +/// Messages to send to the [`EndpointStateActor`]. +#[derive(derive_more::Debug)] +pub(crate) enum EndpointStateMessage { + /// Sends a datagram to all known paths. /// - /// This indicates how long ago a QUIC datagram was received from the remote endpoint sent - /// from this [`DirectAddrInfo::addr`]. It indicates the network path was in use to - /// transport payload data. - pub last_payload: Option, - /// Elapsed time since this network path was known to exist. + /// Used to send QUIC Initial packets. If there is no working direct path this will + /// trigger holepunching. /// - /// A network path is considered to exist only because the remote endpoint advertised it. - /// It may not mean the path is usable. However, if there was any communication with - /// the remote endpoint over this network path it also means the path exists. + /// This is not acceptable to use on the normal send path, as it is an async send + /// operation with a bunch more copying. So it should only be used for sending QUIC + /// Initial packets. + #[debug("SendDatagram(..)")] + SendDatagram(OwnedTransmit), + /// Adds an active connection to this remote endpoint. /// - /// The elapsed time since *any* confirmation of the path's existence was received is - /// returned. If the remote endpoint moved networks and no longer has this path, this could - /// be a long duration. - pub last_alive: Option, - /// A [`HashMap`] of [`Source`]s to [`Duration`]s. + /// The connection will now be managed by this actor. Holepunching will happen when + /// needed, any new paths discovered via holepunching will be added. And closed paths + /// will be removed etc. + #[debug("AddConnection(..)")] + AddConnection( + WeakConnectionHandle, + Watchable>, + ), + /// Adds a [`EndpointAddr`] with locations where the endpoint might be reachable. + AddEndpointAddr(EndpointAddr, Source), + /// Process a received DISCO CallMeMaybe message. + CallMeMaybeReceived(disco::CallMeMaybe), + /// Process a received DISCO Ping message. + #[debug("PingReceived({:?}, src: {_1:?})", _0.tx_id)] + PingReceived(disco::Ping, transports::Addr), + /// Process a received DISCO Pong message. + #[debug("PongReceived({:?}, src: {_1:?})", _0.tx_id)] + PongReceived(disco::Pong, transports::Addr), + /// Asks if there is any possible path that could be used. /// - /// The [`Duration`] indicates the elapsed time since this source last - /// recorded this address. + /// This does not mean there is any guarantee that the remote endpoint is reachable. + #[debug("CanSend(..)")] + CanSend(oneshot::Sender), + /// Returns the current latency to the remote endpoint. /// - /// The [`Duration`] will always indicate the most recent time the source - /// recorded this address. - pub sources: HashMap, + /// TODO: This is more of a placeholder message currently. Check MagicSock::latency. + #[debug("Latency(..)")] + Latency(oneshot::Sender>), } -/// Information about the network path to a remote endpoint via a relay server. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct RelayUrlInfo { - /// The relay URL. - pub relay_url: RelayUrl, - /// Elapsed time since this relay path last received payload or control data. - pub last_alive: Option, - /// Latency to the remote endpoint over this relayed network path. - pub latency: Option, -} - -impl From<(RelayUrl, PathState)> for RelayUrlInfo { - fn from(value: (RelayUrl, PathState)) -> Self { - RelayUrlInfo { - relay_url: value.0, - last_alive: value.1.last_alive().map(|i| i.elapsed()), - latency: value.1.latency(), - } - } -} - -impl From for RelayUrl { - fn from(value: RelayUrlInfo) -> Self { - value.relay_url - } +/// A handle to a [`EndpointStateActor`]. +/// +/// Dropping this will stop the actor. +#[derive(Debug)] +pub(super) struct EndpointStateHandle { + pub(super) sender: mpsc::Sender, + _task: AbortOnDropHandle<()>, } -/// Details about a remote iroh endpoint which is known to this endpoint. -/// -/// Having details of an endpoint does not mean it can be connected to, nor that it has ever been -/// connected to in the past. There are various reasons an endpoint might be known: it could have -/// been manually added via [`Endpoint::add_endpoint_addr`], it could have been added by some -/// discovery mechanism, the endpoint could have contacted this endpoint, etc. -/// -/// [`Endpoint::add_endpoint_addr`]: crate::endpoint::Endpoint::add_endpoint_addr -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub(crate) struct RemoteInfo { - /// The globally unique identifier for this endpoint. - pub endpoint_id: EndpointId, - /// Relay server information, if available. - pub relay_url: Option, - /// The addresses at which this endpoint might be reachable. +/// Information about a holepunch attempt. +#[derive(Debug)] +struct HolepunchAttempt { + when: Instant, + /// The set of local addresses which could take part in holepunching. /// - /// Some of these addresses might only be valid for networks we are not part of, but the remote - /// endpoint might be a part of. - pub addrs: Vec, - /// The type of connection we have to the endpoint, either direct or over relay. - pub conn_type: ConnectionType, - /// The latency of the current network path to the remote endpoint. - pub latency: Option, - /// Time elapsed time since last we have sent to or received from the endpoint. + /// This does not mean every address here participated in the holepunching. E.g. we + /// could have tried only a sub-set of the addresses because a previous attempt already + /// covered part of the range. /// - /// This is the duration since *any* data (payload or control messages) was sent or receive - /// from the remote endpoint. Note that sending to the remote endpoint does not imply - /// the remote endpoint received anything. - pub last_used: Option, -} - -impl RemoteInfo { - /// Get the duration since the last activity we received from this endpoint - /// on any of its direct addresses. - pub(crate) fn last_received(&self) -> Option { - self.addrs - .iter() - .filter_map(|addr| addr.last_control.map(|x| x.0).min(addr.last_payload)) - .min() - } - - /// Whether there is a possible known network path to the remote endpoint. + /// We do not store this as a [`DirectAddr`] because this is checked for equality and we + /// do not want to compare the sources of these addresses. + local_addrs: BTreeSet, + /// The set of remote addresses which could take part in holepunching. /// - /// Note that this does not provide any guarantees of whether any network path is - /// usable. - pub(crate) fn has_send_address(&self) -> bool { - self.relay_url.is_some() || !self.addrs.is_empty() - } + /// Like `local_addrs` we may not have used them. + remote_addrs: BTreeSet, } /// The type of connection we have to the endpoint. @@ -1444,306 +1029,110 @@ pub enum ConnectionType { None, } -#[cfg(test)] -mod tests { - use std::{collections::BTreeMap, net::Ipv4Addr}; - - use iroh_base::SecretKey; - use rand::SeedableRng; - - use super::*; - use crate::magicsock::endpoint_map::{EndpointMap, EndpointMapInner}; - - #[test] - fn test_remote_infos() { - let now = Instant::now(); - let elapsed = Duration::from_secs(3); - let later = now + elapsed; - let send_addr: RelayUrl = "https://my-relay.com".parse().unwrap(); - let pong_src = SendAddr::Udp("0.0.0.0:1".parse().unwrap()); - let latency = Duration::from_millis(50); - - let relay_and_state = |endpoint_id: EndpointId, url: RelayUrl| { - let relay_state = PathState::with_pong_reply( - endpoint_id, - PongReply { - latency, - pong_at: now, - from: SendAddr::Relay(send_addr.clone()), - pong_src: pong_src.clone(), - }, - ); - Some((url, relay_state)) - }; - - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); +/// Newtype to track Connections. +/// +/// The wrapped value is the [`Connection::stable_id`] value, and is thus only valid for +/// active connections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct ConnId(usize); - // endpoint with a `best_addr` that has a latency but no relay - let (a_endpoint, a_socket_addr) = { - let key = SecretKey::generate(&mut rng); - let endpoint_id = key.public(); - let ip_port = IpPort { - ip: Ipv4Addr::UNSPECIFIED.into(), - port: 10, - }; - let endpoint_state = BTreeMap::from([( - ip_port, - PathState::with_pong_reply( - endpoint_id, - PongReply { - latency, - pong_at: now, - from: SendAddr::Udp(ip_port.into()), - pong_src: pong_src.clone(), - }, - ), - )]); - ( - EndpointState { - id: 0, - quic_mapped_addr: EndpointIdMappedAddr::generate(), - endpoint_id: key.public(), - last_full_ping: None, - relay_url: None, - udp_paths: EndpointUdpPaths::from_parts( - endpoint_state, - UdpSendAddr::Valid(ip_port.into()), - ), - sent_pings: HashMap::new(), - last_used: Some(now), - last_call_me_maybe: None, - conn_type: Watchable::new(ConnectionType::Direct(ip_port.into())), - has_been_direct: AtomicBool::new(true), - #[cfg(any(test, feature = "test-utils"))] - path_selection: PathSelection::default(), - }, - ip_port.into(), - ) - }; - // endpoint w/ no best addr but a relay w/ latency - let b_endpoint = { - // let socket_addr = "0.0.0.0:9".parse().unwrap(); - let key = SecretKey::generate(&mut rng); - EndpointState { - id: 1, - quic_mapped_addr: EndpointIdMappedAddr::generate(), - endpoint_id: key.public(), - last_full_ping: None, - relay_url: relay_and_state(key.public(), send_addr.clone()), - udp_paths: EndpointUdpPaths::new(), - sent_pings: HashMap::new(), - last_used: Some(now), - last_call_me_maybe: None, - conn_type: Watchable::new(ConnectionType::Relay(send_addr.clone())), - has_been_direct: AtomicBool::new(false), - #[cfg(any(test, feature = "test-utils"))] - path_selection: PathSelection::default(), - } - }; +/// State about one connection. +#[derive(Debug)] +struct ConnectionState { + /// Weak handle to the connection. + handle: WeakConnectionHandle, + /// The information we publish to users about the paths used in this connection. + pub_path_info: Watchable>, + /// The paths that exist on this connection. + /// + /// This could be in any state, e.g. while still validating the path or already closed + /// but not yet fully removed from the connection. This exists as long as Quinn knows + /// about the [`PathId`]. + paths: FxHashMap, + /// The open paths on this connection, a subset of [`Self::paths`]. + open_paths: FxHashMap, + /// Reverse map of [`Self::paths]. + path_ids: FxHashMap, +} - // endpoint w/ no best addr but a relay w/ no latency - let c_endpoint = { - // let socket_addr = "0.0.0.0:8".parse().unwrap(); - let key = SecretKey::generate(&mut rng); - EndpointState { - id: 2, - quic_mapped_addr: EndpointIdMappedAddr::generate(), - endpoint_id: key.public(), - last_full_ping: None, - relay_url: Some(( - send_addr.clone(), - PathState::new( - key.public(), - SendAddr::from(send_addr.clone()), - Source::App, - now, - ), - )), - udp_paths: EndpointUdpPaths::new(), - sent_pings: HashMap::new(), - last_used: Some(now), - last_call_me_maybe: None, - conn_type: Watchable::new(ConnectionType::Relay(send_addr.clone())), - has_been_direct: AtomicBool::new(false), - #[cfg(any(test, feature = "test-utils"))] - path_selection: PathSelection::default(), - } - }; +impl ConnectionState { + /// Tracks a path for the connection. + fn add_path(&mut self, remote: transports::Addr, path_id: PathId) { + self.paths.insert(path_id, remote.clone()); + self.path_ids.insert(remote, path_id); + } - // endpoint w/ expired best addr and relay w/ latency - let (d_endpoint, d_socket_addr) = { - let socket_addr: SocketAddr = "0.0.0.0:7".parse().unwrap(); - let key = SecretKey::generate(&mut rng); - let endpoint_id = key.public(); - let endpoint_state = BTreeMap::from([( - IpPort::from(socket_addr), - PathState::with_pong_reply( - endpoint_id, - PongReply { - latency, - pong_at: now, - from: SendAddr::Udp(socket_addr), - pong_src: pong_src.clone(), - }, - ), - )]); - ( - EndpointState { - id: 3, - quic_mapped_addr: EndpointIdMappedAddr::generate(), - endpoint_id: key.public(), - last_full_ping: None, - relay_url: relay_and_state(key.public(), send_addr.clone()), - udp_paths: EndpointUdpPaths::from_parts( - endpoint_state, - UdpSendAddr::Outdated(socket_addr), - ), - sent_pings: HashMap::new(), - last_used: Some(now), - last_call_me_maybe: None, - conn_type: Watchable::new(ConnectionType::Mixed( - socket_addr, - send_addr.clone(), - )), - has_been_direct: AtomicBool::new(false), - #[cfg(any(test, feature = "test-utils"))] - path_selection: PathSelection::default(), - }, - socket_addr, - ) - }; + /// Tracks an open path for the connection. + fn add_open_path(&mut self, remote: transports::Addr, path_id: PathId) { + self.paths.insert(path_id, remote.clone()); + self.open_paths.insert(path_id, remote.clone()); + self.path_ids.insert(remote, path_id); - let mut expect = Vec::from([ - RemoteInfo { - endpoint_id: a_endpoint.endpoint_id, - relay_url: None, - addrs: Vec::from([DirectAddrInfo { - addr: a_socket_addr, - latency: Some(latency), - last_control: Some((elapsed, ControlMsg::Pong)), - last_payload: None, - last_alive: Some(elapsed), - sources: HashMap::new(), - }]), - conn_type: ConnectionType::Direct(a_socket_addr), - latency: Some(latency), - last_used: Some(elapsed), - }, - RemoteInfo { - endpoint_id: b_endpoint.endpoint_id, - relay_url: Some(RelayUrlInfo { - relay_url: b_endpoint.relay_url.as_ref().unwrap().0.clone(), - last_alive: None, - latency: Some(latency), - }), - addrs: Vec::new(), - conn_type: ConnectionType::Relay(send_addr.clone()), - latency: Some(latency), - last_used: Some(elapsed), - }, - RemoteInfo { - endpoint_id: c_endpoint.endpoint_id, - relay_url: Some(RelayUrlInfo { - relay_url: c_endpoint.relay_url.as_ref().unwrap().0.clone(), - last_alive: None, - latency: None, - }), - addrs: Vec::new(), - conn_type: ConnectionType::Relay(send_addr.clone()), - latency: None, - last_used: Some(elapsed), - }, - RemoteInfo { - endpoint_id: d_endpoint.endpoint_id, - relay_url: Some(RelayUrlInfo { - relay_url: d_endpoint.relay_url.as_ref().unwrap().0.clone(), - last_alive: None, - latency: Some(latency), - }), - addrs: Vec::from([DirectAddrInfo { - addr: d_socket_addr, - latency: Some(latency), - last_control: Some((elapsed, ControlMsg::Pong)), - last_payload: None, - last_alive: Some(elapsed), - sources: HashMap::new(), - }]), - conn_type: ConnectionType::Mixed(d_socket_addr, send_addr.clone()), - latency: Some(Duration::from_millis(50)), - last_used: Some(elapsed), - }, - ]); - - let endpoint_map = EndpointMap::from_inner(EndpointMapInner { - by_endpoint_key: HashMap::from([ - (a_endpoint.endpoint_id, a_endpoint.id), - (b_endpoint.endpoint_id, b_endpoint.id), - (c_endpoint.endpoint_id, c_endpoint.id), - (d_endpoint.endpoint_id, d_endpoint.id), - ]), - by_ip_port: HashMap::from([ - (a_socket_addr.into(), a_endpoint.id), - (d_socket_addr.into(), d_endpoint.id), - ]), - by_quic_mapped_addr: HashMap::from([ - (a_endpoint.quic_mapped_addr, a_endpoint.id), - (b_endpoint.quic_mapped_addr, b_endpoint.id), - (c_endpoint.quic_mapped_addr, c_endpoint.id), - (d_endpoint.quic_mapped_addr, d_endpoint.id), - ]), - by_id: HashMap::from([ - (a_endpoint.id, a_endpoint), - (b_endpoint.id, b_endpoint), - (c_endpoint.id, c_endpoint), - (d_endpoint.id, d_endpoint), - ]), - next_id: 5, - path_selection: PathSelection::default(), - }); - let mut got = endpoint_map.list_remote_infos(later); - got.sort_by_key(|p| p.endpoint_id); - expect.sort_by_key(|p| p.endpoint_id); - remove_non_deterministic_fields(&mut got); - assert_eq!(expect, got); + self.update_pub_path_info(); } - fn remove_non_deterministic_fields(infos: &mut [RemoteInfo]) { - for info in infos.iter_mut() { - if info.relay_url.is_some() { - info.relay_url.as_mut().unwrap().last_alive = None; - } + /// Completely removes a path from this connection. + fn remove_path(&mut self, path_id: &PathId) { + if let Some(addr) = self.paths.remove(path_id) { + self.path_ids.remove(&addr); } + self.open_paths.remove(path_id); } - #[test] - fn test_prune_direct_addresses() { - // When we handle a call-me-maybe with more than MAX_INACTIVE_DIRECT_ADDRESSES we do - // not want to prune them right away but send pings to all of them. - let mut rng = rand_chacha::ChaCha8Rng::seed_from_u64(0u64); - - let key = SecretKey::generate(&mut rng); - let opts = Options { - endpoint_id: key.public(), - relay_url: None, - active: true, - source: crate::magicsock::Source::NamedApp { - name: "test".into(), - }, - path_selection: PathSelection::default(), - }; - let mut ep = EndpointState::new(0, opts); + /// Removes the path from the open paths. + fn remove_open_path(&mut self, path_id: &PathId) { + self.open_paths.remove(path_id); - let my_numbers_count: u16 = (MAX_INACTIVE_DIRECT_ADDRESSES + 5).try_into().unwrap(); - let my_numbers = (0u16..my_numbers_count) - .map(|i| SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1000 + i)) - .collect(); - let call_me_maybe = disco::CallMeMaybe { my_numbers }; + self.update_pub_path_info(); + } - let metrics = MagicsockMetrics::default(); - let ping_messages = ep.handle_call_me_maybe(call_me_maybe, &metrics); + /// Sets the new [`PathInfo`] structs for the public [`Connection`]. + fn update_pub_path_info(&self) { + let new = self + .open_paths + .iter() + .map(|(path_id, remote)| { + let remote = TransportAddr::from(remote.clone()); + ( + remote.clone(), + PathInfo { + remote: remote.clone(), + path_id: *path_id, + }, + ) + }) + .collect::>(); + + self.pub_path_info.set(new).ok(); + } +} + +/// Information about a network path used by a [`Connection`]. +/// +/// [`Connection`]: crate::endpoint::Connection +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PathInfo { + /// The remote transport address used by this network path. + pub remote: TransportAddr, + /// The internal path identifier for the [`Connection`] + /// + /// This is unique for the lifetime of the connection. Can be used to look up the path + /// statistics in the [`ConnectionStats::paths`], returned by [`Connection::stats`]. + /// + /// [`Connection`]: crate::endpoint::Connection + /// [`Connection::stats`]: crate::endpoint::Connection::stats + /// [`ConnectionStats::paths`]: crate::endpoint::ConnectionStats::paths + // TODO: Decide if exposing this is a good idea. Maybe we should just hide this + // entirely try to provide Self::stats(). But that would mean this needs to have a + // WeakConnectionHandle. + pub path_id: PathId, +} - // We have no relay server and no previous direct addresses, so we should get the same - // number of pings as direct addresses in the call-me-maybe. - assert_eq!(ping_messages.len(), my_numbers_count as usize); +/// Poll a future once, like n0_future::future::poll_once but sync. +fn now_or_never>(fut: F) -> Option { + let fut = std::pin::pin!(fut); + match fut.poll(&mut std::task::Context::from_waker(std::task::Waker::noop())) { + std::task::Poll::Ready(res) => Some(res), + std::task::Poll::Pending => None, } } diff --git a/iroh/src/magicsock/endpoint_map/path_state.rs b/iroh/src/magicsock/endpoint_map/path_state.rs index d78b67179f3..44fdef65b93 100644 --- a/iroh/src/magicsock/endpoint_map/path_state.rs +++ b/iroh/src/magicsock/endpoint_map/path_state.rs @@ -1,335 +1,25 @@ //! The state kept for each network path to a remote endpoint. -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; -use iroh_base::EndpointId; -use n0_future::time::{Duration, Instant}; -use tracing::{Level, debug, event}; +use n0_future::time::Instant; -use super::{ - IpPort, PingRole, Source, - endpoint_state::{ControlMsg, PongReply, SESSION_ACTIVE_TIMEOUT}, -}; -use crate::{ - disco::{SendAddr, TransactionId}, - magicsock::{ - HEARTBEAT_INTERVAL, Metrics as MagicsockMetrics, - endpoint_map::path_validity::{self, PathValidity}, - }, -}; +use super::Source; +use crate::disco::TransactionId; -/// The minimum time between pings to an endpoint. +/// The state of a single path to the remote endpoint. /// -/// Except in the case of CallMeMaybe frames resetting the counter, as the first pings -/// likely didn't through the firewall. -const DISCO_PING_INTERVAL: Duration = Duration::from_secs(5); - -/// State about a particular path to another [`EndpointState`]. -/// -/// This state is used for both the relay path and any direct UDP paths. +/// Each path is identified by the destination [`transports::Addr`] and they are stored in +/// the [`NodeStateActor::paths`] map. /// -/// [`EndpointState`]: super::endpoint_state::EndpointState -#[derive(Debug, Clone)] +/// [`NodeStateActor::paths`]: super::node_state::NodeStateActor +#[derive(Debug, Default)] pub(super) struct PathState { - /// The endpoint for which this path exists. - endpoint_id: EndpointId, - /// The path this applies for. - path: SendAddr, - /// The last (outgoing) ping time. - pub(super) last_ping: Option, - - /// If non-zero, means that this was an endpoint that we learned about at runtime (from an - /// incoming ping). If so, we keep the time updated and use it to discard old candidates. - // NOTE: tx_id Originally added in tailscale due to . - last_got_ping: Option<(Instant, TransactionId)>, - - /// The time this endpoint was last advertised via a call-me-maybe DISCO message. - pub(super) call_me_maybe_time: Option, - - /// Tracks whether this path is valid. + /// How we learned about this path, and when. /// - /// Also stores the latest [`PongReply`], if there is one. - /// - /// See [`PathValidity`] docs. - pub(super) validity: PathValidity, - /// When the last payload data was **received** via this path. - /// - /// This excludes DISCO messages. - pub(super) last_payload_msg: Option, - /// Sources is a map of [`Source`]s to [`Instant`]s, keeping track of all the ways we have - /// learned about this path - /// - /// We keep track of only the latest [`Instant`] for each [`Source`], keeping the size of - /// the map of sources down to one entry per type of source. + /// We keep track of only the latest [`Instant`] for each [`Source`], keeping the size + /// of the map of sources down to one entry per type of source. pub(super) sources: HashMap, -} - -impl PathState { - pub(super) fn new( - endpoint_id: EndpointId, - path: SendAddr, - source: Source, - now: Instant, - ) -> Self { - let mut sources = HashMap::new(); - sources.insert(source, now); - Self { - endpoint_id, - path, - last_ping: None, - last_got_ping: None, - call_me_maybe_time: None, - validity: PathValidity::empty(), - last_payload_msg: None, - sources, - } - } - - pub(super) fn with_last_payload( - endpoint_id: EndpointId, - path: SendAddr, - source: Source, - now: Instant, - ) -> Self { - let mut sources = HashMap::new(); - sources.insert(source, now); - PathState { - endpoint_id, - path, - last_ping: None, - last_got_ping: None, - call_me_maybe_time: None, - validity: PathValidity::empty(), - last_payload_msg: Some(now), - sources, - } - } - - pub(super) fn with_ping( - endpoint_id: EndpointId, - path: SendAddr, - tx_id: TransactionId, - source: Source, - now: Instant, - ) -> Self { - let mut new = PathState::new(endpoint_id, path, source, now); - new.handle_ping(tx_id, now); - new - } - - pub(super) fn add_pong_reply(&mut self, r: PongReply, metrics: &MagicsockMetrics) { - if let SendAddr::Udp(ref path) = self.path { - if self.validity.is_empty() { - event!( - target: "iroh::_events::holepunched", - Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - path = ?path, - direction = "outgoing", - ); - } - } - - self.validity.update_pong(r.pong_at, r.latency); - - self.validity.record_metrics(metrics); - } - - pub(super) fn receive_payload(&mut self, now: Instant) { - self.last_payload_msg = Some(now); - self.validity - .receive_payload(now, path_validity::Source::QuicPayload); - } - - #[cfg(test)] - pub(super) fn with_pong_reply(endpoint_id: EndpointId, r: PongReply) -> Self { - PathState { - endpoint_id, - path: r.from.clone(), - last_ping: None, - last_got_ping: None, - call_me_maybe_time: None, - validity: PathValidity::new(r.pong_at, r.latency), - last_payload_msg: None, - sources: HashMap::new(), - } - } - - /// Check whether this path is considered active. - /// - /// Active means the path has received payload messages within the last - /// [`SESSION_ACTIVE_TIMEOUT`]. - /// - /// Note that a path might be alive but not active if it's contactable but not in - /// use. - pub(super) fn is_active(&self) -> bool { - self.last_payload_msg - .as_ref() - .map(|instant| instant.elapsed() <= SESSION_ACTIVE_TIMEOUT) - .unwrap_or(false) - } - - /// Returns the instant the last incoming ping was received. - pub(super) fn last_incoming_ping(&self) -> Option<&Instant> { - self.last_got_ping.as_ref().map(|(time, _tx_id)| time) - } - - /// Reports the last instant this path was considered alive. - /// - /// Alive means the path is considered in use by the remote endpoint. Either because we - /// received a payload message, a DISCO message (ping, pong) or it was advertised in a - /// call-me-maybe message. - /// - /// This is the most recent instant between: - /// - when last pong was received. - /// - when this path was last advertised in a received CallMeMaybe message. - /// - When the last payload transmission occurred. - /// - when the last ping from them was received. - pub(super) fn last_alive(&self) -> Option { - self.validity - .latest_pong() - .into_iter() - .chain(self.last_payload_msg) - .chain(self.call_me_maybe_time) - .chain(self.last_incoming_ping().cloned()) - .max() - } - - /// The last control or DISCO message **about** this path. - /// - /// This is the most recent instant among: - /// - when last pong was received. - /// - when this path was last advertised in a received CallMeMaybe message. - /// - when the last ping from them was received. - /// - /// Returns the time elapsed since the last control message, and the type of control message. - pub(super) fn last_control_msg(&self, now: Instant) -> Option<(Duration, ControlMsg)> { - // get every control message and assign it its kind - let last_pong = self - .validity - .latest_pong() - .map(|pong_at| (pong_at, ControlMsg::Pong)); - let last_call_me_maybe = self - .call_me_maybe_time - .as_ref() - .map(|call_me| (*call_me, ControlMsg::CallMeMaybe)); - let last_ping = self - .last_incoming_ping() - .map(|ping| (*ping, ControlMsg::Ping)); - - last_pong - .into_iter() - .chain(last_call_me_maybe) - .chain(last_ping) - .max_by_key(|(instant, _kind)| *instant) - .map(|(instant, kind)| (now.duration_since(instant), kind)) - } - - /// Returns the latency from the most recent pong, if available. - pub(super) fn latency(&self) -> Option { - self.validity.latency() - } - - pub(super) fn needs_ping(&self, now: &Instant) -> bool { - match self.last_ping { - None => true, - Some(last_ping) => { - let elapsed = now.duration_since(last_ping); - - // TODO: remove! - // This logs "ping is too new" for each send whenever the endpoint does *not* need - // a ping. Pretty sure this is not a useful log, but maybe there was a reason? - // if !needs_ping { - // debug!("ping is too new: {}ms", elapsed.as_millis()); - // } - elapsed > DISCO_PING_INTERVAL - } - } - } - - pub(super) fn handle_ping(&mut self, tx_id: TransactionId, now: Instant) -> PingRole { - if Some(&tx_id) == self.last_got_ping.as_ref().map(|(_t, tx_id)| tx_id) { - PingRole::Duplicate - } else { - let prev = self.last_got_ping.replace((now, tx_id)); - let heartbeat_deadline = HEARTBEAT_INTERVAL + (HEARTBEAT_INTERVAL / 2); - match prev { - Some((prev_time, _tx)) if now.duration_since(prev_time) <= heartbeat_deadline => { - PingRole::LikelyHeartbeat - } - Some((prev_time, _tx)) => { - debug!( - elapsed = ?now.duration_since(prev_time), - "heartbeat missed, reactivating", - ); - PingRole::Activate - } - None => { - if let SendAddr::Udp(ref addr) = self.path { - event!( - target: "iroh::_events::holepunched", - Level::DEBUG, - remote_endpoint = %self.endpoint_id.fmt_short(), - path = ?addr, - direction = "incoming", - ); - } - PingRole::Activate - } - } - } - } - - pub(super) fn add_source(&mut self, source: Source, now: Instant) { - self.sources.insert(source, now); - } - - pub(super) fn clear(&mut self) { - self.last_ping = None; - self.last_got_ping = None; - self.call_me_maybe_time = None; - self.validity = PathValidity::empty(); - } - - fn summary(&self, mut w: impl std::fmt::Write) -> std::fmt::Result { - write!(w, "{{ ")?; - if self.is_active() { - write!(w, "active ")?; - } - if let Some(pong_at) = self.validity.latest_pong() { - write!(w, "pong-received({:?} ago) ", pong_at.elapsed())?; - } - if let Some(when) = self.last_incoming_ping() { - write!(w, "ping-received({:?} ago) ", when.elapsed())?; - } - if let Some(ref when) = self.last_ping { - write!(w, "ping-sent({:?} ago) ", when.elapsed())?; - } - if let Some(last_source) = self.sources.iter().max_by_key(|&(_, instant)| instant) { - write!( - w, - "last-source: {}({:?} ago)", - last_source.0, - last_source.1.elapsed() - )?; - } - write!(w, "}}") - } -} - -// TODO: Make an `EndpointPaths` struct and do things nicely. -pub(super) fn summarize_endpoint_paths(paths: &BTreeMap) -> String { - use std::fmt::Write; - - let mut w = String::new(); - write!(&mut w, "[").ok(); - for (i, (ipp, state)) in paths.iter().enumerate() { - if i > 0 { - write!(&mut w, ", ").ok(); - } - write!(&mut w, "{ipp}").ok(); - state.summary(&mut w).ok(); - } - write!(&mut w, "]").ok(); - w + /// The last ping sent on this path. + pub(super) ping_sent: Option, } diff --git a/iroh/src/magicsock/endpoint_map/path_validity.rs b/iroh/src/magicsock/endpoint_map/path_validity.rs deleted file mode 100644 index 3a33993c864..00000000000 --- a/iroh/src/magicsock/endpoint_map/path_validity.rs +++ /dev/null @@ -1,398 +0,0 @@ -use n0_future::time::{Duration, Instant}; - -use crate::magicsock::Metrics as MagicsockMetrics; - -/// How long we trust a UDP address as the exclusive path (i.e. without also sending via the relay). -/// -/// Trust for a UDP address begins when we receive a DISCO UDP pong on that address. -/// It is then further extended by this duration every time we receive QUIC payload data while it's -/// currently trusted. -/// -/// If trust goes away, it can be brought back with another valid DISCO UDP pong. -const TRUST_UDP_ADDR_DURATION: Duration = Duration::from_millis(6500); - -/// Tracks a path's validity. -/// -/// A path is valid: -/// - For [`Source::trust_duration`] after a successful [`PongReply`]. -/// - For [`Source::trust_duration`] longer starting at the most recent -/// received application payload *while the path was valid*. -/// -/// [`PongReply`]: super::endpoint_state::PongReply -#[derive(Debug, Clone, Default)] -pub(super) struct PathValidity(Option); - -#[derive(Debug, Clone)] -struct Inner { - latest_pong: Instant, - latency: Duration, - trust_until: Instant, - congestion_metrics: CongestionMetrics, -} - -/// Congestion tracking for a UDP path. -#[derive(Debug, Default, Clone)] -struct CongestionMetrics { - /// Rolling window of recent latency measurements (stores up to 8 samples). - latency_samples: [Option; 8], - /// Index for next sample insertion (circular buffer). - sample_index: usize, - /// Total pings sent on this path. - pings_sent: u32, - /// Total pongs received on this path. - pongs_received: u32, -} - -impl CongestionMetrics { - fn add_latency_sample(&mut self, latency: Duration) { - self.latency_samples[self.sample_index] = Some(latency); - self.sample_index = (self.sample_index + 1) % self.latency_samples.len(); - self.pongs_received = self.pongs_received.saturating_add(1); - } - - fn record_ping_sent(&mut self) { - self.pings_sent = self.pings_sent.saturating_add(1); - } - - /// Calculate packet loss rate (0.0 to 1.0). - fn packet_loss_rate(&self) -> f64 { - if self.pings_sent == 0 { - return 0.0; - } - let lost = self.pings_sent.saturating_sub(self.pongs_received); - lost as f64 / self.pings_sent as f64 - } - - /// Calculate RTT variance as a congestion indicator. - /// Higher variance suggests congestion or unstable path. - fn rtt_variance(&self) -> Option { - let samples: Vec = self.latency_samples.iter().filter_map(|&s| s).collect(); - - if samples.len() < 2 { - return None; - } - - let mean = samples.iter().sum::() / samples.len() as u32; - let variance: f64 = samples - .iter() - .map(|&s| { - let diff = s.as_secs_f64() - mean.as_secs_f64(); - diff * diff - }) - .sum::() - / samples.len() as f64; - - Some(Duration::from_secs_f64(variance.sqrt())) - } - - /// Calculate average latency from recent samples. - #[cfg(test)] - fn avg_latency(&self) -> Option { - let samples: Vec = self.latency_samples.iter().filter_map(|&s| s).collect(); - - if samples.is_empty() { - return None; - } - - Some(samples.iter().sum::() / samples.len() as u32) - } - - /// Path quality score (0.0 = worst, 1.0 = best). - /// Factors in packet loss and RTT variance. - fn quality_score(&self) -> f64 { - let packet_loss = self.packet_loss_rate(); - - // Defensive: packet_loss should never exceed 1.0, but clamp just in case - if packet_loss > 1.0 { - tracing::warn!( - packet_loss, - pings_sent = self.pings_sent, - pongs_received = self.pongs_received, - "packet loss rate exceeded 1.0 - possible bug in tracking" - ); - } - let loss_penalty = (1.0 - packet_loss).clamp(0.0, 1.0); - - // Penalize high RTT variance - let variance_penalty = match self.rtt_variance() { - Some(var) if var.as_millis() > 50 => 0.7, - Some(var) if var.as_millis() > 20 => 0.85, - Some(_) => 1.0, - None => 1.0, - }; - - loss_penalty * variance_penalty - } -} - -#[derive(Debug)] -pub(super) enum Source { - ReceivedPong, - QuicPayload, -} - -impl Source { - fn trust_duration(&self) -> Duration { - match self { - Source::ReceivedPong => TRUST_UDP_ADDR_DURATION, - Source::QuicPayload => TRUST_UDP_ADDR_DURATION, - } - } -} - -impl PathValidity { - pub(super) fn new(pong_at: Instant, latency: Duration) -> Self { - let mut metrics = CongestionMetrics::default(); - // Account for the ping that must have been sent to receive this pong - metrics.record_ping_sent(); - metrics.add_latency_sample(latency); - Self(Some(Inner { - trust_until: pong_at + Source::ReceivedPong.trust_duration(), - latest_pong: pong_at, - latency, - congestion_metrics: metrics, - })) - } - - /// Update with a new pong, preserving congestion history. - pub(super) fn update_pong(&mut self, pong_at: Instant, latency: Duration) { - match &mut self.0 { - Some(inner) => { - inner.trust_until = pong_at + Source::ReceivedPong.trust_duration(); - inner.latest_pong = pong_at; - inner.latency = latency; - inner.congestion_metrics.add_latency_sample(latency); - } - None => { - *self = Self::new(pong_at, latency); - } - } - } - - pub(super) fn empty() -> Self { - Self(None) - } - - pub(super) fn is_empty(&self) -> bool { - self.0.is_none() - } - - pub(super) fn is_valid(&self, now: Instant) -> bool { - let Some(state) = self.0.as_ref() else { - return false; - }; - - state.is_valid(now) - } - - pub(super) fn latency_if_valid(&self, now: Instant) -> Option { - let state = self.0.as_ref()?; - state.is_valid(now).then_some(state.latency) - } - - pub(super) fn is_outdated(&self, now: Instant) -> bool { - let Some(state) = self.0.as_ref() else { - return false; - }; - - // We *used* to be valid, but are now outdated. - // This happens when we had a DISCO pong but didn't receive - // any payload data or further pongs for at least TRUST_UDP_ADDR_DURATION - state.is_outdated(now) - } - - pub(super) fn latency_if_outdated(&self, now: Instant) -> Option { - let state = self.0.as_ref()?; - state.is_outdated(now).then_some(state.latency) - } - - /// Reconfirms path validity, if a payload was received while the - /// path was valid. - pub(super) fn receive_payload(&mut self, now: Instant, source: Source) { - let Some(state) = self.0.as_mut() else { - return; - }; - - if state.is_valid(now) { - state.trust_until = now + source.trust_duration(); - } - } - - pub(super) fn latency(&self) -> Option { - Some(self.0.as_ref()?.latency) - } - - pub(super) fn latest_pong(&self) -> Option { - Some(self.0.as_ref()?.latest_pong) - } - - /// Record that a ping was sent on this path. - pub(super) fn record_ping_sent(&mut self) { - if let Some(state) = self.0.as_mut() { - state.congestion_metrics.record_ping_sent(); - } - } - - /// Get the path quality score (0.0 = worst, 1.0 = best). - #[cfg(test)] - pub(super) fn quality_score(&self) -> f64 { - self.0 - .as_ref() - .map(|state| state.congestion_metrics.quality_score()) - .unwrap_or(0.0) - } - - /// Get packet loss rate for this path. - #[cfg(test)] - pub(super) fn packet_loss_rate(&self) -> f64 { - self.0 - .as_ref() - .map(|state| state.congestion_metrics.packet_loss_rate()) - .unwrap_or(0.0) - } - - /// Get RTT variance as congestion indicator. - #[cfg(test)] - pub(super) fn rtt_variance(&self) -> Option { - self.0 - .as_ref() - .and_then(|state| state.congestion_metrics.rtt_variance()) - } - - /// Get average latency from recent samples. - #[cfg(test)] - pub(super) fn avg_latency(&self) -> Option { - self.0 - .as_ref() - .and_then(|state| state.congestion_metrics.avg_latency()) - } - - /// Record congestion metrics to the metrics system. - /// Should be called periodically or on significant events. - pub(super) fn record_metrics(&self, metrics: &MagicsockMetrics) { - let Some(state) = self.0.as_ref() else { - return; - }; - - let loss_rate = state.congestion_metrics.packet_loss_rate(); - metrics.path_packet_loss_rate.observe(loss_rate); - - if let Some(variance) = state.congestion_metrics.rtt_variance() { - metrics - .path_rtt_variance_ms - .observe(variance.as_millis() as f64); - } - - let quality = state.congestion_metrics.quality_score(); - metrics.path_quality_score.observe(quality); - } -} - -impl Inner { - fn is_valid(&self, now: Instant) -> bool { - self.latest_pong <= now && now < self.trust_until - } - - fn is_outdated(&self, now: Instant) -> bool { - self.latest_pong <= now && self.trust_until <= now - } -} - -#[cfg(test)] -mod tests { - use n0_future::time::{Duration, Instant}; - - use super::{PathValidity, Source, TRUST_UDP_ADDR_DURATION}; - - #[tokio::test(start_paused = true)] - async fn test_basic_path_validity_lifetime() { - let mut validity = PathValidity(None); - assert!(!validity.is_valid(Instant::now())); - assert!(!validity.is_outdated(Instant::now())); - - validity = PathValidity::new(Instant::now(), Duration::from_millis(20)); - assert!(validity.is_valid(Instant::now())); - assert!(!validity.is_outdated(Instant::now())); - - tokio::time::advance(TRUST_UDP_ADDR_DURATION / 2).await; - assert!(validity.is_valid(Instant::now())); - assert!(!validity.is_outdated(Instant::now())); - - validity.receive_payload(Instant::now(), Source::QuicPayload); - assert!(validity.is_valid(Instant::now())); - assert!(!validity.is_outdated(Instant::now())); - - tokio::time::advance(TRUST_UDP_ADDR_DURATION / 2).await; - assert!(validity.is_valid(Instant::now())); - assert!(!validity.is_outdated(Instant::now())); - - tokio::time::advance(TRUST_UDP_ADDR_DURATION / 2).await; - assert!(!validity.is_valid(Instant::now())); - assert!(validity.is_outdated(Instant::now())); - } - #[tokio::test] - async fn test_congestion_metrics() { - let mut validity = PathValidity::new(Instant::now(), Duration::from_millis(10)); - // new() initializes with pings_sent=1, pongs_received=1 - - // Record some additional ping sends - validity.record_ping_sent(); - validity.record_ping_sent(); - validity.record_ping_sent(); - // Now: pings_sent=4, pongs_received=1 - - validity.update_pong(Instant::now(), Duration::from_millis(15)); - // Now: pings_sent=4, pongs_received=2 - - // Packet loss should be (4-2)/4 = 0.5 - let loss_rate = validity.packet_loss_rate(); - assert!((loss_rate - 0.5).abs() < 0.01); - - // Quality score should be reduced due to packet loss - let quality = validity.quality_score(); - assert!(quality < 1.0); - assert!(quality > 0.45); // Should still be relatively good (1.0 - 0.5 = 0.5) - } - - #[tokio::test] - async fn test_congestion_rtt_variance() { - let mut validity = PathValidity::new(Instant::now(), Duration::from_millis(10)); - - // Add varying latencies - validity.update_pong(Instant::now(), Duration::from_millis(10)); - validity.update_pong(Instant::now(), Duration::from_millis(50)); - validity.update_pong(Instant::now(), Duration::from_millis(20)); - validity.update_pong(Instant::now(), Duration::from_millis(40)); - - // Should have variance - let variance = validity.rtt_variance(); - assert!(variance.is_some()); - assert!(variance.unwrap().as_millis() > 0); - - // Average latency should be around 30ms - let avg = validity.avg_latency(); - assert!(avg.is_some()); - let avg_ms = avg.unwrap().as_millis(); - assert!((25..=35).contains(&avg_ms)); - } - - #[tokio::test] - async fn test_quality_score_with_high_variance() { - let mut validity = PathValidity::new(Instant::now(), Duration::from_millis(10)); - - // Add highly varying latencies (simulating congestion) - for i in 0..8 { - let latency = if i % 2 == 0 { - Duration::from_millis(10) - } else { - Duration::from_millis(100) - }; - validity.update_pong(Instant::now(), latency); - validity.record_ping_sent(); - } - - // Quality should be penalized due to high variance - let quality = validity.quality_score(); - assert!(quality < 0.9); // Should be penalized - } -} diff --git a/iroh/src/magicsock/endpoint_map/udp_paths.rs b/iroh/src/magicsock/endpoint_map/udp_paths.rs deleted file mode 100644 index a378ed0bdf2..00000000000 --- a/iroh/src/magicsock/endpoint_map/udp_paths.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! Path state for UDP addresses of a single peer endpoint. -//! -//! This started as simply moving the [`EndpointState`]'s `direct_addresses` and `best_addr` -//! into one place together. The aim is for external places to not directly interact with -//! the inside and instead only notifies this struct of state changes to each path. -//! -//! [`EndpointState`]: super::endpoint_state::EndpointState -use std::{collections::BTreeMap, net::SocketAddr}; - -use n0_future::time::Instant; -use tracing::{Level, event}; - -use super::{IpPort, path_state::PathState}; - -/// The address on which to send datagrams over UDP. -/// -/// The [`MagicSock`] sends packets to zero or one UDP address, depending on the known paths -/// to the remote endpoint. This conveys the UDP address to send on from the [`EndpointUdpPaths`] -/// to the [`EndpointState`]. -/// -/// [`EndpointUdpPaths`] contains all the UDP path states, while [`EndpointState`] has to decide the -/// bigger picture including the relay server. -/// -/// See [`EndpointUdpPaths::send_addr`]. -/// -/// [`MagicSock`]: crate::magicsock::MagicSock -/// [`EndpointState`]: super::endpoint_state::EndpointState -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] -pub(super) enum UdpSendAddr { - /// The UDP address can be relied on to deliver data to the remote endpoint. - /// - /// This means this path is usable with a reasonable latency and can be fully trusted to - /// transport payload data to the remote endpoint. - Valid(SocketAddr), - /// The UDP address is highly likely to work, but has not been used for a while. - /// - /// The path should be usable but has not carried DISCO or payload data for a little too - /// long. It is best to also use a backup, i.e. relay, path if possible. - Outdated(SocketAddr), - /// The UDP address is not known to work, but it might. - /// - /// We know this UDP address belongs to the remote endpoint, but we do not know if the path - /// already works or may need holepunching before it will start to work. It might even - /// never work. It is still useful to send to this together with backup path, - /// i.e. relay, in case the path works: if the path does not need holepunching it might - /// be much faster. And if there is no relay path at all it might be the only way to - /// establish a connection. - Unconfirmed(SocketAddr), - /// No known UDP path exists to the remote endpoint. - #[default] - None, -} - -impl UdpSendAddr { - pub fn get_addr(&self) -> Option { - match self { - UdpSendAddr::Valid(addr) - | UdpSendAddr::Outdated(addr) - | UdpSendAddr::Unconfirmed(addr) => Some(*addr), - UdpSendAddr::None => None, - } - } -} - -/// The UDP paths for a single endpoint. -/// -/// Paths are identified by the [`IpPort`] of their UDP address. -/// -/// Initially this collects two structs directly from the [`EndpointState`] into one place, -/// leaving the APIs and astractions the same. The goal is that this slowly migrates -/// directly interacting with this data into only receiving [`PathState`] updates. This -/// will consolidate the logic of direct path selection and make this simpler to reason -/// about. However doing that all at once is too large a refactor. -/// -/// [`EndpointState`]: super::endpoint_state::EndpointState -#[derive(Debug, Default)] -pub(super) struct EndpointUdpPaths { - /// The state for each of this endpoint's direct paths. - paths: BTreeMap, - /// The current address we use to send on. - /// - /// This is *almost* the same as going through `paths` and finding - /// the best one, except that this is - /// 1. Not updated in `send_addr`, but instead when there's changes to `paths`, so that `send_addr` can take `&self`. - /// 2. Slightly sticky: It only changes when - /// - the current send addr is not a validated path anymore or - /// - we received a pong with lower latency. - best: UdpSendAddr, - /// The current best address to send on from all IPv4 addresses we have available. - /// - /// Follows the same logic as `best` above, but doesn't include any IPv6 addresses. - best_ipv4: UdpSendAddr, -} - -pub(super) struct MutAccess<'a> { - now: Instant, - inner: &'a mut EndpointUdpPaths, -} - -impl<'a> MutAccess<'a> { - pub fn paths(&mut self) -> &mut BTreeMap { - &mut self.inner.paths - } - - pub fn has_best_addr_changed(self) -> bool { - let changed = self.inner.update_to_best_addr(self.now); - std::mem::forget(self); // don't run drop - changed - } -} - -impl Drop for MutAccess<'_> { - fn drop(&mut self) { - self.inner.update_to_best_addr(self.now); - } -} - -impl EndpointUdpPaths { - pub(super) fn new() -> Self { - Default::default() - } - - #[cfg(test)] - pub(super) fn from_parts(paths: BTreeMap, best: UdpSendAddr) -> Self { - Self { - paths, - best_ipv4: best, // we only use ipv4 addrs in tests - best, - } - } - - /// Returns the current UDP address to send on. - pub(super) fn send_addr(&self, have_ipv6: bool) -> &UdpSendAddr { - if !have_ipv6 { - // If it's a valid address, it doesn't matter if our interface scan determined that we - // "probably" don't have IPv6, because we clearly were able to send and receive a ping/pong over IPv6. - if matches!(&self.best, UdpSendAddr::Valid(_)) { - return &self.best; - } - return &self.best_ipv4; - } - &self.best - } - - /// Returns a guard for accessing the inner paths mutably. - /// - /// This guard ensures that [`Self::send_addr`] will be updated on drop. - pub(super) fn access_mut(&mut self, now: Instant) -> MutAccess<'_> { - MutAccess { now, inner: self } - } - - /// Returns immutable access to the inner paths. - pub(super) fn paths(&self) -> &BTreeMap { - &self.paths - } - - /// Changes the current best address(es) to ones chosen as described in [`Self::best_addr`] docs. - /// - /// Returns whether one of the best addresses had to change. - /// - /// This should be called any time that `paths` is modified. - fn update_to_best_addr(&mut self, now: Instant) -> bool { - let best_ipv4 = self.best_addr(false, now); - let best = self.best_addr(true, now); - let mut changed = false; - if best_ipv4 != self.best_ipv4 { - event!( - target: "iroh::_events::udp::best_ipv4", - Level::DEBUG, - ?best_ipv4, - ); - changed = true; - } - if best != self.best { - event!( - target: "iroh::_events::udp::best", - Level::DEBUG, - ?best, - ); - changed = true; - } - self.best_ipv4 = best_ipv4; - self.best = best; - changed - } - - /// Returns the current best address of all available paths, ignoring - /// the currently chosen best address. - /// - /// We try to find the lowest latency [`UdpSendAddr::Valid`], if one exists, otherwise - /// we try to find the lowest latency [`UdpSendAddr::Outdated`], if one exists, otherwise - /// we return essentially an arbitrary [`UdpSendAddr::Unconfirmed`]. - /// - /// If we don't have any addresses, returns [`UdpSendAddr::None`]. - /// - /// If `have_ipv6` is false, we only search among ipv4 candidates. - fn best_addr(&self, have_ipv6: bool, now: Instant) -> UdpSendAddr { - let Some((ipp, path)) = self - .paths - .iter() - .filter(|(ipp, _)| have_ipv6 || ipp.ip.is_ipv4()) - .max_by_key(|(ipp, path)| { - // We find the best by sorting on a key of type (Option>, Option>, bool) - // where the first is set to Some(ReverseOrd(latency)) iff path.is_valid(now) and - // the second is set to Some(ReverseOrd(latency)) if path.is_outdated(now) and - // the third is set to whether the ipp is ipv6. - // This makes max_by_key sort for the lowest valid latency first, then sort for - // the lowest outdated latency second, and if latencies are equal, it'll sort IPv6 paths first. - let is_ipv6 = ipp.ip.is_ipv6(); - if let Some(latency) = path.validity.latency_if_valid(now) { - (Some(ReverseOrd(latency)), None, is_ipv6) - } else if let Some(latency) = path.validity.latency_if_outdated(now) { - (None, Some(ReverseOrd(latency)), is_ipv6) - } else { - (None, None, is_ipv6) - } - }) - else { - return UdpSendAddr::None; - }; - - if path.validity.is_valid(now) { - UdpSendAddr::Valid((*ipp).into()) - } else if path.validity.is_outdated(now) { - UdpSendAddr::Outdated((*ipp).into()) - } else { - UdpSendAddr::Unconfirmed((*ipp).into()) - } - } -} - -/// Implements the reverse [`Ord`] implementation for the wrapped type. -/// -/// Literally calls [`std::cmp::Ordering::reverse`] on the inner value's -/// ordering. -#[derive(PartialEq, Eq)] -struct ReverseOrd(N); - -impl Ord for ReverseOrd { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.0.cmp(&other.0).reverse() - } -} - -impl PartialOrd for ReverseOrd { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} diff --git a/iroh/src/magicsock/mapped_addrs.rs b/iroh/src/magicsock/mapped_addrs.rs new file mode 100644 index 00000000000..0f8b473bc28 --- /dev/null +++ b/iroh/src/magicsock/mapped_addrs.rs @@ -0,0 +1,309 @@ +//! The various mapped addresses we use. + +//! We use non-IP transports to carry datagrams. Yet Quinn needs to address those +//! transports using IPv6 addresses. These defines mappings of several IPv6 Unique Local +//! Address ranges we use to keep track of the various "fake" address types we use. + +use std::{ + fmt, + hash::Hash, + net::{IpAddr, Ipv6Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, +}; + +use iroh_base::{EndpointId, RelayUrl}; +use rustc_hash::FxHashMap; +use snafu::Snafu; +use tracing::{error, trace}; + +use super::transports; + +/// The Prefix/L of all Unique Local Addresses. +const ADDR_PREFIXL: u8 = 0xfd; + +/// The Global ID used in n0's Unique Local Addresses. +const ADDR_GLOBAL_ID: [u8; 5] = [21, 7, 10, 81, 11]; + +/// The Subnet ID for [`RelayMappedAddr]. +const RELAY_MAPPED_SUBNET: [u8; 2] = [0, 1]; + +/// The Subnet ID for [`EndpointIdMappedAddr`]. +const ENDPOINT_ID_SUBNET: [u8; 2] = [0; 2]; + +/// The dummy port used for all mapped addresses. +/// +/// We map each entity, usually an [`EndpointId`], to an IPv6 address. But socket addresses +/// involve ports, so we use a dummy fixed port when creating socket addresses. +const MAPPED_PORT: u16 = 12345; + +/// Counter to always generate unique addresses for [`RelayMappedAddr`]. +static RELAY_ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); + +/// Counter to always generate unique addresses for [`EndpointIdMappedAddr`]. +static ENDPOINT_ID_ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); + +/// Generic mapped address. +/// +/// Allows implementing [`AddrMap`]. +pub(crate) trait MappedAddr { + /// Generates a new mapped address in the IPv6 Unique Local Address space. + fn generate() -> Self; + + /// Returns a consistent [`SocketAddr`] for the mapped addr. + /// + /// This socket address does not have a routable IP address. It uses a fake but + /// consistent port number, since the port does not play a role in the addressing. This + /// socket address is only to be used to pass into Quinn. + fn private_socket_addr(&self) -> SocketAddr; +} + +/// An enum encompassing all the mapped and unmapped addresses. +/// +/// This can consistently convert a socket address as we use them in Quinn and return a real +/// socket address or a mapped address. Note that this does not mean that the mapped +/// address exists, only that it is semantically a valid mapped address. +#[derive(Clone, Debug)] +pub(crate) enum MultipathMappedAddr { + /// An address for a [`EndpointId`], via one or more paths. + Mixed(EndpointIdMappedAddr), + /// An address for a particular [`EndpointId`] via a particular relay. + Relay(RelayMappedAddr), + /// An IP based transport address. + #[cfg(not(wasm_browser))] + Ip(SocketAddr), +} + +impl From for MultipathMappedAddr { + fn from(value: SocketAddr) -> Self { + match value.ip() { + IpAddr::V4(_) => Self::Ip(value), + IpAddr::V6(addr) => { + if let Ok(addr) = EndpointIdMappedAddr::try_from(addr) { + return Self::Mixed(addr); + } + if let Ok(addr) = RelayMappedAddr::try_from(addr) { + return Self::Relay(addr); + } + #[cfg(not(wasm_browser))] + Self::Ip(value) + } + } + } +} + +impl MultipathMappedAddr { + pub(super) fn to_transport_addr( + &self, + relay_mapped_addrs: &AddrMap<(RelayUrl, EndpointId), RelayMappedAddr>, + ) -> Option { + match self { + Self::Mixed(_) => { + error!("Mixed addr has no transports::Addr"); + None + } + Self::Relay(mapped) => match relay_mapped_addrs.lookup(mapped) { + Some(parts) => Some(transports::Addr::from(parts)), + None => { + error!("Unknown RelayMappedAddr"); + None + } + }, + Self::Ip(addr) => Some(transports::Addr::from(*addr)), + } + } +} + +/// An address used to address a endpoint on any or all paths. +/// +/// This is only used for initially connecting to a remote endpoint. We instruct Quinn to +/// send to this address, and duplicate all packets for this address to send on all paths we +/// might want to send the initial on: +/// +/// - If this the first connection to the remote endpoint we don't know which path will work +/// and send to all of them. +/// +/// - If there already is an active connection to this endpoint we now which path to use. +/// +/// It is but a newtype around an IPv6 Unique Local Addr. And in our QUIC-facing socket +/// APIs like [`quinn::AsyncUdpSocket`] it comes in as the inner [`Ipv6Addr`], in those +/// interfaces we have to be careful to do the conversion to this type. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct EndpointIdMappedAddr(Ipv6Addr); + +impl MappedAddr for EndpointIdMappedAddr { + /// Generates a globally unique fake UDP address. + /// + /// This generates and IPv6 Unique Local Address according to RFC 4193. + fn generate() -> Self { + let mut addr = [0u8; 16]; + addr[0] = ADDR_PREFIXL; + addr[1..6].copy_from_slice(&ADDR_GLOBAL_ID); + addr[6..8].copy_from_slice(&ENDPOINT_ID_SUBNET); + + let counter = ENDPOINT_ID_ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); + addr[8..16].copy_from_slice(&counter.to_be_bytes()); + + Self(Ipv6Addr::from(addr)) + } + + /// Returns a consistent [`SocketAddr`] for the [`EndpointIdMappedAddr`]. + /// + /// This socket address does not have a routable IP address. + /// + /// This uses a made-up port number, since the port does not play a role in the + /// addressing. This socket address is only to be used to pass into Quinn. + fn private_socket_addr(&self) -> SocketAddr { + SocketAddr::new(IpAddr::from(self.0), MAPPED_PORT) + } +} + +impl std::fmt::Display for EndpointIdMappedAddr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "EndpointIdMappedAddr({})", self.0) + } +} + +impl TryFrom for EndpointIdMappedAddr { + type Error = EndpointIdMappedAddrError; + + fn try_from(value: Ipv6Addr) -> Result { + let octets = value.octets(); + if octets[0] == ADDR_PREFIXL + && octets[1..6] == ADDR_GLOBAL_ID + && octets[6..8] == ENDPOINT_ID_SUBNET + { + return Ok(Self(value)); + } + Err(EndpointIdMappedAddrError) + } +} + +/// Can occur when converting a [`SocketAddr`] to an [`EndpointIdMappedAddr`] +#[derive(Debug, Snafu)] +#[snafu(display("Failed to convert"))] +pub(crate) struct EndpointIdMappedAddrError; + +/// An Ipv6 ULA address, identifying a relay path for a [`EndpointId`]. +/// +/// Since iroh endpoint are reachable via a relay server we have a network path indicated by +/// the `(EndpointId, RelayUrl)`. However Quinn can only handle socket addresses, so we use +/// IPv6 addresses in a private IPv6 Unique Local Address range, which map to a unique +/// `(EndointId, RelayUrl)` pair. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub(crate) struct RelayMappedAddr(Ipv6Addr); + +impl MappedAddr for RelayMappedAddr { + /// Generates a globally unique fake UDP address. + /// + /// This generates a new IPv6 address in the Unique Local Address range (RFC 4193) + /// which is recognised by iroh as an IP mapped address. + fn generate() -> Self { + let mut addr = [0u8; 16]; + addr[0] = ADDR_PREFIXL; + addr[1..6].copy_from_slice(&ADDR_GLOBAL_ID); + addr[6..8].copy_from_slice(&RELAY_MAPPED_SUBNET); + + let counter = RELAY_ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); + addr[8..16].copy_from_slice(&counter.to_be_bytes()); + + Self(Ipv6Addr::from(addr)) + } + + /// Returns a consistent [`SocketAddr`] for the [`RelayMappedAddr`]. + /// + /// This does not have a routable IP address. + /// + /// This uses a made-up, but fixed port number. The [`RelayAddrMap`] creates a unique + /// [`RelayMappedAddr`] for each `(EndpointId, RelayUrl)` pair and thus does not use the + /// port to map back to the original [`SocketAddr`]. + fn private_socket_addr(&self) -> SocketAddr { + SocketAddr::new(IpAddr::from(self.0), MAPPED_PORT) + } +} + +impl TryFrom for RelayMappedAddr { + type Error = RelayMappedAddrError; + + fn try_from(value: Ipv6Addr) -> std::result::Result { + let octets = value.octets(); + if octets[0] == ADDR_PREFIXL + && octets[1..6] == ADDR_GLOBAL_ID + && octets[6..8] == RELAY_MAPPED_SUBNET + { + return Ok(Self(value)); + } + Err(RelayMappedAddrError) + } +} + +/// Can occur when converting a [`SocketAddr`] to an [`RelayMappedAddr`] +#[derive(Debug, Snafu)] +#[snafu(display("Failed to convert"))] +pub(crate) struct RelayMappedAddrError; + +impl std::fmt::Display for RelayMappedAddr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "RelayMappedAddr({})", self.0) + } +} + +/// A bi-directional map between a key and a [`MappedAddr`]. +#[derive(Debug, Clone)] +pub(super) struct AddrMap { + inner: Arc>>, +} + +// Manual impl because derive ends up requiring T: Default. +impl Default for AddrMap { + fn default() -> Self { + Self { + inner: Default::default(), + } + } +} + +impl AddrMap +where + K: Eq + Hash + Clone + fmt::Debug, + V: MappedAddr + Eq + Hash + Copy + fmt::Debug, +{ + /// Returns the [`MappedAddr`], generating one if needed. + pub(super) fn get(&self, key: &K) -> V { + let mut inner = self.inner.lock().expect("poisoned"); + match inner.addrs.get(key) { + Some(addr) => *addr, + None => { + let addr = V::generate(); + inner.addrs.insert(key.clone(), addr); + inner.lookup.insert(addr, key.clone()); + trace!(?addr, ?key, "generated new addr"); + addr + } + } + } + + /// Performs the reverse lookup. + pub(super) fn lookup(&self, addr: &V) -> Option { + let inner = self.inner.lock().expect("poisoned"); + inner.lookup.get(addr).cloned() + } +} + +#[derive(Debug)] +struct AddrMapInner { + addrs: FxHashMap, + lookup: FxHashMap, +} + +// Manual impl because derive ends up requiring T: Default. +impl Default for AddrMapInner { + fn default() -> Self { + Self { + addrs: Default::default(), + lookup: Default::default(), + } + } +} diff --git a/iroh/src/magicsock/transports.rs b/iroh/src/magicsock/transports.rs index 02632d72e77..677bced59ce 100644 --- a/iroh/src/magicsock/transports.rs +++ b/iroh/src/magicsock/transports.rs @@ -1,4 +1,5 @@ use std::{ + fmt, io::{self, IoSliceMut}, net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}, pin::Pin, @@ -6,11 +7,14 @@ use std::{ task::{Context, Poll}, }; -use iroh_base::{EndpointId, RelayUrl}; +use bytes::Bytes; +use iroh_base::{EndpointId, RelayUrl, TransportAddr}; use n0_watcher::Watcher; use relay::{RelayNetworkChangeSender, RelaySender}; -use smallvec::SmallVec; -use tracing::{error, trace, warn}; +use tracing::{debug, error, instrument, trace, warn}; + +use super::{MagicSock, endpoint_map::EndpointStateMessage, mapped_addrs::MultipathMappedAddr}; +use crate::net_report::Report; #[cfg(not(wasm_browser))] mod ip; @@ -21,8 +25,6 @@ pub(crate) use self::ip::IpTransport; #[cfg(not(wasm_browser))] use self::ip::{IpNetworkChangeSender, IpSender}; pub(crate) use self::relay::{RelayActorConfig, RelayTransport}; -use super::MagicSock; -use crate::net_report::Report; /// Manages the different underlying data transports that the magicsock /// can support. @@ -222,16 +224,15 @@ impl Transports { false } - pub(crate) fn create_sender(&self, msock: Arc) -> UdpSender { + pub(crate) fn create_sender(&self) -> TransportsSender { #[cfg(not(wasm_browser))] let ip = self.ip.iter().map(|t| t.create_sender()).collect(); let relay = self.relay.iter().map(|t| t.create_sender()).collect(); let max_transmit_segments = self.max_transmit_segments(); - UdpSender { + TransportsSender { #[cfg(not(wasm_browser))] ip, - msock, relay, max_transmit_segments, } @@ -304,12 +305,42 @@ pub(crate) struct Transmit<'a> { pub(crate) segment_size: Option, } -#[derive(Debug, Clone, PartialEq, Eq)] +/// An outgoing packet that can be sent across channels. +#[derive(Debug, Clone)] +pub(crate) struct OwnedTransmit { + pub(crate) ecn: Option, + pub(crate) contents: Bytes, + pub(crate) segment_size: Option, +} + +impl From<&quinn_udp::Transmit<'_>> for OwnedTransmit { + fn from(source: &quinn_udp::Transmit<'_>) -> Self { + Self { + ecn: source.ecn, + contents: Bytes::copy_from_slice(source.contents), + segment_size: source.segment_size, + } + } +} + +/// Transports address. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub(crate) enum Addr { + /// An IP address, should always be stored in its canonical form. Ip(SocketAddr), + /// A relay address. Relay(RelayUrl, EndpointId), } +impl fmt::Debug for Addr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Addr::Ip(addr) => write!(f, "Ip({addr})"), + Addr::Relay(url, node_id) => write!(f, "Relay({url}, {})", node_id.fmt_short()), + } + } +} + impl Default for Addr { fn default() -> Self { Self::Ip(SocketAddr::V6(SocketAddrV6::new( @@ -323,7 +354,23 @@ impl Default for Addr { impl From for Addr { fn from(value: SocketAddr) -> Self { - Self::Ip(value) + match value { + SocketAddr::V4(_) => Self::Ip(value), + SocketAddr::V6(addr) => { + Self::Ip(SocketAddr::new(addr.ip().to_canonical(), addr.port())) + } + } + } +} + +impl From<&SocketAddr> for Addr { + fn from(value: &SocketAddr) -> Self { + match value { + SocketAddr::V4(_) => Self::Ip(*value), + SocketAddr::V6(addr) => { + Self::Ip(SocketAddr::new(addr.ip().to_canonical(), addr.port())) + } + } } } @@ -333,11 +380,31 @@ impl From<(RelayUrl, EndpointId)> for Addr { } } +impl From for TransportAddr { + fn from(value: Addr) -> Self { + match value { + Addr::Ip(addr) => TransportAddr::Ip(addr), + Addr::Relay(url, _) => TransportAddr::Relay(url), + } + } +} + impl Addr { pub(crate) fn is_relay(&self) -> bool { matches!(self, Self::Relay(..)) } + pub(crate) fn is_ip(&self) -> bool { + matches!(self, Self::Ip(_)) + } + + pub(crate) fn is_ipv4(&self) -> bool { + match self { + Addr::Ip(socket_addr) => socket_addr.is_ipv4(), + Addr::Relay(_, _) => false, + } + } + /// Returns `None` if not an `Ip`. pub(crate) fn into_socket_addr(self) -> Option { match self { @@ -347,26 +414,25 @@ impl Addr { } } +/// A sender that sends to all our transports. #[derive(Debug)] -pub(crate) struct UdpSender { - msock: Arc, // :( +pub(crate) struct TransportsSender { #[cfg(not(wasm_browser))] ip: Vec, relay: Vec, max_transmit_segments: usize, } -impl UdpSender { +impl TransportsSender { + #[instrument(skip(self, transmit), fields(len = transmit.contents.len()))] pub(crate) async fn send( &self, - destination: &Addr, + dst: &Addr, src: Option, transmit: &Transmit<'_>, ) -> io::Result<()> { - trace!(?destination, "sending"); - let mut any_match = false; - match destination { + match dst { #[cfg(wasm_browser)] Addr::Ip(..) => return Err(io::Error::other("IP is unsupported in browser")), #[cfg(not(wasm_browser))] @@ -376,6 +442,7 @@ impl UdpSender { any_match = true; match sender.send(*addr, src, transmit).await { Ok(()) => { + trace!("sent"); return Ok(()); } Err(err) => { @@ -391,6 +458,7 @@ impl UdpSender { any_match = true; match sender.send(url.clone(), *endpoint_id, transmit).await { Ok(()) => { + trace!("sent"); return Ok(()); } Err(err) => { @@ -408,16 +476,15 @@ impl UdpSender { } } + #[instrument(name = "poll_send", skip(self, cx, transmit), fields(len = transmit.contents.len()))] pub(crate) fn inner_poll_send( mut self: Pin<&mut Self>, cx: &mut std::task::Context, - destination: &Addr, + dst: &Addr, src: Option, transmit: &Transmit<'_>, ) -> Poll> { - trace!(?destination, "sending"); - - match destination { + match dst { #[cfg(wasm_browser)] Addr::Ip(..) => { return Poll::Ready(Err(io::Error::other("IP is unsupported in browser"))); @@ -428,7 +495,13 @@ impl UdpSender { if sender.is_valid_send_addr(addr) { match Pin::new(sender).poll_send(cx, *addr, src, transmit) { Poll::Pending => {} - Poll::Ready(res) => return Poll::Ready(res), + Poll::Ready(res) => { + match &res { + Ok(()) => trace!("sent"), + Err(err) => trace!("send failed: {err:#}"), + } + return Poll::Ready(res); + } } } } @@ -438,7 +511,13 @@ impl UdpSender { if sender.is_valid_send_addr(url, endpoint_id) { match sender.poll_send(cx, url.clone(), *endpoint_id, transmit) { Poll::Pending => {} - Poll::Ready(res) => return Poll::Ready(res), + Poll::Ready(res) => { + match &res { + Ok(()) => trace!("sent"), + Err(err) => trace!("send failed: {err:#}"), + } + return Poll::Ready(res); + } } } } @@ -446,149 +525,211 @@ impl UdpSender { } Poll::Pending } +} - /// Best effort sending - pub(crate) fn inner_try_send( - &self, - destination: &Addr, - src: Option, - transmit: &Transmit<'_>, - ) -> io::Result<()> { - trace!(?destination, "sending, best effort"); +/// A [`Transports`] that works with [`MultipathMappedAddr`]s and their IPv6 representation. +/// +/// The [`MultipathMappedAddr`]s have an IPv6 representation that Quinn uses. This struct +/// knows about these and maps them back to the transport [`Addr`]s used by the wrapped +/// [`Transports`]. +#[derive(Debug)] +pub(crate) struct MagicTransport { + msock: Arc, + transports: Transports, +} - match destination { - #[cfg(wasm_browser)] - Addr::Ip(..) => return Err(io::Error::other("IP is unsupported in browser")), - #[cfg(not(wasm_browser))] - Addr::Ip(addr) => { - for transport in &self.ip { - if transport.is_valid_send_addr(addr) { - match transport.try_send(*addr, src, transmit) { - Ok(()) => return Ok(()), - Err(_err) => { - continue; - } - } - } - } - } - Addr::Relay(url, endpoint_id) => { - for transport in &self.relay { - if transport.is_valid_send_addr(url, endpoint_id) { - match transport.try_send(url.clone(), *endpoint_id, transmit) { - Ok(()) => return Ok(()), - Err(_err) => { - continue; - } - } - } - } - } - } - Err(io::Error::new( - io::ErrorKind::WouldBlock, - "no transport ready", - )) +impl MagicTransport { + pub(crate) fn new(msock: Arc, transports: Transports) -> Self { + Self { msock, transports } } } -impl quinn::UdpSender for UdpSender { - fn poll_send( - mut self: Pin<&mut Self>, - transmit: &quinn_udp::Transmit, +impl quinn::AsyncUdpSocket for MagicTransport { + fn create_sender(&self) -> Pin> { + Box::pin(MagicSender { + msock: self.msock.clone(), + sender: self.transports.create_sender(), + }) + } + + fn poll_recv( + &mut self, cx: &mut Context, - ) -> Poll> { - let active_paths = self.msock.prepare_send(&self, transmit)?; - - if active_paths.is_empty() { - // Returning Ok here means we let QUIC timeout. - // Returning an error would immediately fail a connection. - // The philosophy of quinn-udp is that a UDP connection could - // come back at any time or missing should be transient so chooses to let - // these kind of errors time out. See test_try_send_no_send_addr to try - // this out. - error!("no paths available for endpoint, voiding transmit"); - return Poll::Ready(Ok(())); - } + bufs: &mut [IoSliceMut<'_>], + meta: &mut [quinn_udp::RecvMeta], + ) -> Poll> { + self.transports.poll_recv(cx, bufs, meta, &self.msock) + } - let mut results = SmallVec::<[_; 3]>::new(); + #[cfg(not(wasm_browser))] + fn local_addr(&self) -> io::Result { + let addrs: Vec<_> = self + .transports + .local_addrs() + .into_iter() + .filter_map(|addr| { + let addr: SocketAddr = addr.into_socket_addr()?; + Some(addr) + }) + .collect(); - trace!(?active_paths, "attempting to send"); + if let Some(addr) = addrs.iter().find(|addr| addr.is_ipv6()) { + return Ok(*addr); + } + if let Some(SocketAddr::V4(addr)) = addrs.first() { + // Pretend to be IPv6, because our `MappedAddr`s need to be IPv6. + let ip = addr.ip().to_ipv6_mapped().into(); + return Ok(SocketAddr::new(ip, addr.port())); + } - for destination in active_paths { - let src = transmit.src_ip; - let transmit = Transmit { - ecn: transmit.ecn, - contents: transmit.contents, - segment_size: transmit.segment_size, - }; + Err(io::Error::other("no valid address available")) + } - let res = self - .as_mut() - .inner_poll_send(cx, &destination, src, &transmit); - match res { - Poll::Ready(Ok(())) => { - trace!(dst = ?destination, "sent transmit"); - } - Poll::Ready(Err(ref err)) => { - warn!(dst = ?destination, "failed to send: {err:#}"); - } - Poll::Pending => {} - } - results.push(res); - } + #[cfg(wasm_browser)] + fn local_addr(&self) -> io::Result { + // Again, we need to pretend we're IPv6, because of our `MappedAddr`s. + Ok(SocketAddr::new(std::net::Ipv6Addr::LOCALHOST.into(), 0)) + } - if results.iter().all(|p| matches!(p, Poll::Pending)) { - // Handle backpressure. - return Poll::Pending; - } - Poll::Ready(Ok(())) + fn max_receive_segments(&self) -> usize { + self.transports.max_receive_segments() } - fn max_transmit_segments(&self) -> usize { - self.max_transmit_segments - } - - fn try_send(self: Pin<&mut Self>, transmit: &quinn_udp::Transmit) -> io::Result<()> { - let active_paths = self.msock.prepare_send(&self, transmit)?; - if active_paths.is_empty() { - // Returning Ok here means we let QUIC timeout. - // Returning an error would immediately fail a connection. - // The philosophy of quinn-udp is that a UDP connection could - // come back at any time or missing should be transient so chooses to let - // these kind of errors time out. See test_try_send_no_send_addr to try - // this out. - error!("no paths available for endpoint, voiding transmit"); - return Ok(()); - } + fn may_fragment(&self) -> bool { + self.transports.may_fragment() + } +} - let mut results = SmallVec::<[_; 3]>::new(); +/// A sender for [`MagicTransport`]. +/// +/// This is special in that it handles [`MultipathMappedAddr::Mixed`] by delegating to the +/// [`MagicSock`] which expands it back to one or more [`Addr`]s and sends it +/// using the underlying [`Transports`]. +// TODO: Can I just send the TransportsSender along in the NodeStateMessage::SendDatagram +// message?? That way you don't have to hook up the sender into the NodeMap! +#[derive(Debug)] +#[pin_project::pin_project] +pub(crate) struct MagicSender { + msock: Arc, + #[pin] + sender: TransportsSender, +} - trace!(?active_paths, "attempting to send"); +impl MagicSender { + /// Extracts the right [`Addr`] from the [`quinn_udp::Transmit`]. + /// + /// Because Quinn does only know about IP transports we map other transports to private + /// IPv6 Unique Local Address ranges. This extracts the transport addresses out of the + /// transmit's destination. + fn mapped_addr(&self, transmit: &quinn_udp::Transmit) -> io::Result { + if self.msock.is_closed() { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "connection closed", + )); + } - for destination in active_paths { - let src = transmit.src_ip; - let transmit = Transmit { - ecn: transmit.ecn, - contents: transmit.contents, - segment_size: transmit.segment_size, - }; + Ok(MultipathMappedAddr::from(transmit.destination)) + } +} - let res = self.inner_try_send(&destination, src, &transmit); - match res { - Ok(()) => { - trace!(dst = ?destination, "sent transmit"); +impl quinn::UdpSender for MagicSender { + fn poll_send( + self: Pin<&mut Self>, + quinn_transmit: &quinn_udp::Transmit, + cx: &mut Context, + ) -> Poll> { + // On errors this methods prefers returning Ok(()) to Quinn. Returning an error + // should only happen if the error is permanent and fatal and it will never be + // possible to send anything again. Doing so kills the Quinn EndpointDriver. Most + // send errors are intermittent errors, returning Ok(()) in those cases will mean + // Quinn eventually considers the packets that had send errors as lost and will try + // and re-send them. + let mapped_addr = self.mapped_addr(quinn_transmit)?; + + let transport_addr = match mapped_addr { + MultipathMappedAddr::Mixed(mapped_addr) => { + let Some(node_id) = self + .msock + .endpoint_map + .endpoint_mapped_addrs + .lookup(&mapped_addr) + else { + error!(dst = ?mapped_addr, "unknown NodeIdMappedAddr, dropped transmit"); + return Poll::Ready(Ok(())); + }; + + // Note we drop the src_ip set in the Quinn Transmit. This is only the + // Initial packet we are sending, so we do not yet have an src address we + // need to respond from. + if let Some(src_ip) = quinn_transmit.src_ip { + warn!(dst = ?mapped_addr, ?src_ip, dst_node = %node_id.fmt_short(), + "oops, flub didn't think this would happen"); } - Err(ref err) => { - warn!(dst = ?destination, "failed to send: {err:#}"); + + let sender = self.msock.endpoint_map.endpoint_state_actor(node_id); + let transmit = OwnedTransmit::from(quinn_transmit); + return match sender.try_send(EndpointStateMessage::SendDatagram(transmit)) { + Ok(()) => { + trace!(dst = ?mapped_addr, dst_node = %node_id.fmt_short(), "sent transmit"); + Poll::Ready(Ok(())) + } + Err(err) => { + // We do not want to block the next send which might be on a + // different transport. Instead we let Quinn handle this as + // a lost datagram. + // TODO: Revisit this: we might want to do something better. + debug!(dst = ?mapped_addr, dst_node = %node_id.fmt_short(), + "NodeStateActor inbox {err:#}, dropped transmit"); + Poll::Ready(Ok(())) + } + }; + } + MultipathMappedAddr::Relay(relay_mapped_addr) => { + match self + .msock + .endpoint_map + .relay_mapped_addrs + .lookup(&relay_mapped_addr) + { + Some((relay_url, node_id)) => Addr::Relay(relay_url, node_id), + None => { + error!("unknown RelayMappedAddr, dropped transmit"); + return Poll::Ready(Ok(())); + } } } - results.push(res); + MultipathMappedAddr::Ip(socket_addr) => Addr::Ip(socket_addr), + }; + + let transmit = Transmit { + ecn: quinn_transmit.ecn, + contents: quinn_transmit.contents, + segment_size: quinn_transmit.segment_size, + }; + let this = self.project(); + + match this + .sender + .inner_poll_send(cx, &transport_addr, quinn_transmit.src_ip, &transmit) + { + Poll::Ready(Ok(())) => Poll::Ready(Ok(())), + Poll::Ready(Err(ref err)) => { + warn!("dropped transmit: {err:#}"); + Poll::Ready(Ok(())) + } + Poll::Pending => { + // We do not want to block the next send which might be on a + // different transport. Instead we let Quinn handle this as a lost + // datagram. + // TODO: Revisit this: we might want to do something better. + trace!("transport pending, dropped transmit"); + Poll::Ready(Ok(())) + } } + } - if results.iter().all(|p| p.is_err()) { - return Err(io::Error::other("all failed")); - } - Ok(()) + fn max_transmit_segments(&self) -> usize { + self.sender.max_transmit_segments } } diff --git a/iroh/src/magicsock/transports/ip.rs b/iroh/src/magicsock/transports/ip.rs index 95e4093435c..a1c2fabad16 100644 --- a/iroh/src/magicsock/transports/ip.rs +++ b/iroh/src/magicsock/transports/ip.rs @@ -51,8 +51,21 @@ impl IpTransport { match self.socket.poll_recv_quinn(cx, bufs, metas) { Poll::Pending => Poll::Pending, Poll::Ready(Ok(n)) => { - for (addr, el) in source_addrs.iter_mut().zip(metas.iter()).take(n) { - *addr = el.addr.into(); + for (source_addr, meta) in source_addrs.iter_mut().zip(metas.iter_mut()).take(n) { + if meta.addr.is_ipv4() { + // The AsyncUdpSocket is an AF_INET6 socket and needs to show this + // as coming from an IPv4-mapped IPv6 addresses, since Quinn will + // use those when sending on an INET6 socket. + let v6_ip = match meta.addr.ip() { + IpAddr::V4(ipv4_addr) => ipv4_addr.to_ipv6_mapped(), + IpAddr::V6(ipv6_addr) => ipv6_addr, + }; + meta.addr = SocketAddr::new(v6_ip.into(), meta.addr.port()); + } + // The transport addresses are internal to iroh and we always want those + // to remain the canonical address. + *source_addr = + SocketAddr::new(meta.addr.ip().to_canonical(), meta.addr.port()).into(); } Poll::Ready(Ok(n)) } @@ -143,29 +156,37 @@ impl IpSender { } } + /// Creates a canonical socket address. + /// + /// We may be asked to send IPv4-mapped IPv6 addresses. But our sockets are configured + /// to only send their actual family. So we need to map those back to the canonical + /// addresses. + #[inline] + fn canonical_addr(addr: SocketAddr) -> SocketAddr { + SocketAddr::new(addr.ip().to_canonical(), addr.port()) + } + pub(super) async fn send( &self, - destination: SocketAddr, + dst: SocketAddr, src: Option, transmit: &Transmit<'_>, ) -> io::Result<()> { - trace!("sending to {}", destination); let total_bytes = transmit.contents.len() as u64; let res = self .sender .send(&quinn_udp::Transmit { - destination: Self::canonical_addr(destination), + destination: Self::canonical_addr(dst), ecn: transmit.ecn, contents: transmit.contents, segment_size: transmit.segment_size, src_ip: src, }) .await; - trace!("send res: {:?}", res); match res { Ok(res) => { - match destination { + match dst { SocketAddr::V4(_) => { self.metrics.send_ipv4.inc_by(total_bytes); } @@ -179,28 +200,17 @@ impl IpSender { } } - /// Creates a canonical socket address. - /// - /// We may be asked to send IPv4-mapped IPv6 addresses. But our sockets are configured - /// to only send their actual family. So we need to map those back to the canonical - /// addresses. - #[inline] - fn canonical_addr(addr: SocketAddr) -> SocketAddr { - SocketAddr::new(addr.ip().to_canonical(), addr.port()) - } - pub(super) fn poll_send( mut self: Pin<&mut Self>, cx: &mut std::task::Context, - destination: SocketAddr, + dst: SocketAddr, src: Option, transmit: &Transmit<'_>, ) -> Poll> { - trace!("sending to {}", destination); let total_bytes = transmit.contents.len() as u64; let res = Pin::new(&mut self.sender).poll_send( &quinn_udp::Transmit { - destination: Self::canonical_addr(destination), + destination: Self::canonical_addr(dst), ecn: transmit.ecn, contents: transmit.contents, segment_size: transmit.segment_size, @@ -208,11 +218,10 @@ impl IpSender { }, cx, ); - trace!("send res: {:?}", res); match res { Poll::Ready(Ok(res)) => { - match destination { + match dst { SocketAddr::V4(_) => { self.metrics.send_ipv4.inc_by(total_bytes); } @@ -226,37 +235,4 @@ impl IpSender { Poll::Pending => Poll::Pending, } } - - pub(super) fn try_send( - &self, - destination: SocketAddr, - src: Option, - transmit: &Transmit<'_>, - ) -> io::Result<()> { - trace!("sending to {}", destination); - let total_bytes = transmit.contents.len() as u64; - let res = self.sender.try_send(&quinn_udp::Transmit { - destination, - ecn: transmit.ecn, - contents: transmit.contents, - segment_size: transmit.segment_size, - src_ip: src, - }); - trace!("send res: {:?}", res); - - match res { - Ok(res) => { - match destination { - SocketAddr::V4(_) => { - self.metrics.send_ipv4.inc_by(total_bytes); - } - SocketAddr::V6(_) => { - self.metrics.send_ipv6.inc_by(total_bytes); - } - } - Ok(res) - } - Err(err) => Err(err), - } - } } diff --git a/iroh/src/magicsock/transports/relay.rs b/iroh/src/magicsock/transports/relay.rs index b1f3f043fb1..35114eb5ae0 100644 --- a/iroh/src/magicsock/transports/relay.rs +++ b/iroh/src/magicsock/transports/relay.rs @@ -14,7 +14,7 @@ use n0_future::{ use n0_watcher::{Watchable, Watcher as _}; use tokio::sync::mpsc; use tokio_util::sync::PollSender; -use tracing::{Instrument, error, info_span, trace, warn}; +use tracing::{Instrument, error, info_span, warn}; use super::{Addr, Transmit}; @@ -147,7 +147,7 @@ impl RelayTransport { .segment_size .map_or(dm.datagrams.contents.len(), |s| u16::from(s) as usize); meta_out.ecn = None; - meta_out.dst_ip = None; // TODO: insert the relay url for this relay + meta_out.dst_ip = None; *addr = (dm.url, dm.src).into(); num_msgs += 1; @@ -259,25 +259,15 @@ impl RelaySender { datagrams: contents, }; - let dest_endpoint = item.remote_endpoint; - let dest_url = item.url.clone(); let Some(sender) = self.sender.get_ref() else { return Err(io::Error::other("channel closed")); }; match sender.send(item).await { - Ok(_) => { - trace!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message queued"); - Ok(()) - } - Err(mpsc::error::SendError(_)) => { - error!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message dropped, channel to actor is closed"); - Err(io::Error::new( - io::ErrorKind::ConnectionReset, - "channel to actor is closed", - )) - } + Ok(_) => Ok(()), + Err(mpsc::error::SendError(_)) => Err(io::Error::new( + io::ErrorKind::ConnectionReset, + "channel to actor is closed", + )), } } @@ -290,81 +280,24 @@ impl RelaySender { ) -> Poll> { match ready!(self.sender.poll_reserve(cx)) { Ok(()) => { - trace!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message queued"); - let contents = datagrams_from_transmit(transmit); let item = RelaySendItem { remote_endpoint: dest_endpoint, url: dest_url.clone(), datagrams: contents, }; - let dest_endpoint = item.remote_endpoint; - let dest_url = item.url.clone(); - match self.sender.send_item(item) { Ok(()) => Poll::Ready(Ok(())), - Err(_err) => { - error!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message dropped, channel to actor is closed"); - Poll::Ready(Err(io::Error::new( - io::ErrorKind::ConnectionReset, - "channel to actor is closed", - ))) - } + Err(_err) => Poll::Ready(Err(io::Error::new( + io::ErrorKind::ConnectionReset, + "channel to actor is closed", + ))), } } - Err(_err) => { - error!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message dropped, channel to actor is closed"); - Poll::Ready(Err(io::Error::new( - io::ErrorKind::ConnectionReset, - "channel to actor is closed", - ))) - } - } - } - - pub(super) fn try_send( - &self, - dest_url: RelayUrl, - dest_endpoint: EndpointId, - transmit: &Transmit<'_>, - ) -> io::Result<()> { - let contents = datagrams_from_transmit(transmit); - - let item = RelaySendItem { - remote_endpoint: dest_endpoint, - url: dest_url.clone(), - datagrams: contents, - }; - - let dest_endpoint = item.remote_endpoint; - let dest_url = item.url.clone(); - - let Some(sender) = self.sender.get_ref() else { - return Err(io::Error::other("channel closed")); - }; - - match sender.try_send(item) { - Ok(_) => { - trace!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message queued"); - Ok(()) - } - Err(mpsc::error::TrySendError::Closed(_)) => { - error!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message dropped, channel to actor is closed"); - Err(io::Error::new( - io::ErrorKind::ConnectionReset, - "channel to actor is closed", - )) - } - Err(mpsc::error::TrySendError::Full(_)) => { - warn!(endpoint = %dest_endpoint.fmt_short(), relay_url = %dest_url, - "send relay: message dropped, channel to actor is full"); - Err(io::Error::new(io::ErrorKind::WouldBlock, "channel full")) - } + Err(_err) => Poll::Ready(Err(io::Error::new( + io::ErrorKind::ConnectionReset, + "channel to actor is closed", + ))), } } } diff --git a/iroh/src/magicsock/transports/relay/actor.rs b/iroh/src/magicsock/transports/relay/actor.rs index 8ad69779c9a..7461593f61b 100644 --- a/iroh/src/magicsock/transports/relay/actor.rs +++ b/iroh/src/magicsock/transports/relay/actor.rs @@ -406,7 +406,7 @@ impl ActiveRelayActor { /// Returns `None` if the actor needs to shut down. Returns `Some(Ok(client))` when the /// connection is established, and `Some(Err(err))` if dialing the relay failed. async fn run_dialing(&mut self) -> Option> { - debug!("Actor loop: connecting to relay."); + trace!("Actor loop: connecting to relay."); // We regularly flush the relay_datagrams_send queue so it is not full of stale // packets while reconnecting. Those datagrams are dropped and the QUIC congestion diff --git a/iroh/src/net_report.rs b/iroh/src/net_report.rs index 0ba629544b2..920e397e518 100644 --- a/iroh/src/net_report.rs +++ b/iroh/src/net_report.rs @@ -48,7 +48,6 @@ use self::reportgen::QadProbeReport; use self::reportgen::{ProbeFinished, ProbeReport}; mod defaults; -mod ip_mapped_addrs; mod metrics; mod probes; mod report; @@ -75,8 +74,6 @@ pub(crate) mod portmapper { } } -pub(crate) use ip_mapped_addrs::{IpMappedAddr, IpMappedAddresses}; - pub(crate) use self::reportgen::IfStateDetails; #[cfg(not(wasm_browser))] use self::reportgen::SocketState; @@ -207,7 +204,6 @@ impl Client { /// Creates a new net_report client. pub(crate) fn new( #[cfg(not(wasm_browser))] dns_resolver: DnsResolver, - #[cfg(not(wasm_browser))] ip_mapped_addrs: Option, relay_map: RelayMap, opts: Options, metrics: Arc, @@ -225,7 +221,6 @@ impl Client { let socket_state = SocketState { quic_client, dns_resolver, - ip_mapped_addrs, }; Client { @@ -388,8 +383,6 @@ impl Client { ) -> Vec { use tracing::{Instrument, warn_span}; - debug!("spawning QAD probes"); - let Some(ref quic_client) = self.socket_state.quic_client else { return Vec::new(); }; @@ -432,6 +425,8 @@ impl Client { return reports; } + trace!("spawning QAD probes"); + // TODO: randomize choice? const MAX_RELAYS: usize = 5; @@ -444,7 +439,6 @@ impl Client { for relay in relays.into_iter().take(MAX_RELAYS) { if if_state.have_v4 && needs_v4_probe { debug!(?relay.url, "v4 QAD probe"); - let ip_mapped_addrs = self.socket_state.ip_mapped_addrs.clone(); let relay = relay.clone(); let dns_resolver = self.socket_state.dns_resolver.clone(); let quic_client = quic_client.clone(); @@ -454,15 +448,13 @@ impl Client { .child_token() .run_until_cancelled_owned(time::timeout( PROBES_TIMEOUT, - run_probe_v4(ip_mapped_addrs, relay, quic_client, dns_resolver), + run_probe_v4(relay, quic_client, dns_resolver), )) - .instrument(warn_span!("QAD-IPv4", %relay_url)), + .instrument(warn_span!("QADv4", %relay_url)), ); } - if if_state.have_v6 && needs_v6_probe { debug!(?relay.url, "v6 QAD probe"); - let ip_mapped_addrs = self.socket_state.ip_mapped_addrs.clone(); let relay = relay.clone(); let dns_resolver = self.socket_state.dns_resolver.clone(); let quic_client = quic_client.clone(); @@ -472,9 +464,9 @@ impl Client { .child_token() .run_until_cancelled_owned(time::timeout( PROBES_TIMEOUT, - run_probe_v6(ip_mapped_addrs, relay, quic_client, dns_resolver), + run_probe_v6(relay, quic_client, dns_resolver), )) - .instrument(warn_span!("QAD-IPv6", %relay_url)), + .instrument(warn_span!("QADv6", %relay_url)), ); } } @@ -489,6 +481,7 @@ impl Client { loop { // We early-abort the tasks once we have at least `enough_relays` reports, // and at least one ipv4 and one ipv6 report completed (if they were started, see comment above). + if reports.len() >= enough_relays && !ipv4_pending && !ipv6_pending { debug!("enough probes: {}", reports.len()); cancel_v4.cancel(); @@ -500,12 +493,14 @@ impl Client { biased; val = v4_buf.join_next(), if !v4_buf.is_empty() => { + let span = warn_span!("QADv4"); + let _guard = span.enter(); ipv4_pending = false; match val { Some(Ok(Some(Ok(res)))) => { match res { Ok((r, conn)) => { - debug!(?r, "got v4 QAD conn"); + debug!(?r, "probe report"); let url = r.relay.clone(); reports.push(ProbeReport::QadIpv4(r)); if self.qad_conns.v4.is_none() { @@ -515,32 +510,34 @@ impl Client { } } Err(err) => { - debug!("probe v4 failed: {err:?}"); + debug!("probe failed: {err:#}"); } } } Some(Err(err)) => { if err.is_panic() { - panic!("probe v4 panicked: {err:?}"); + panic!("probe panicked: {err:#}"); } - warn!("probe v4 failed: {err:?}"); + warn!("probe failed: {err:#}"); } Some(Ok(None)) => { - debug!("probe v4 canceled"); + debug!("probe canceled"); } Some(Ok(Some(Err(time::Elapsed { .. })))) => { - debug!("probe v4 timed out"); + debug!("probe timed out"); } None => {} } } val = v6_buf.join_next(), if !v6_buf.is_empty() => { + let span = warn_span!("QADv6"); + let _guard = span.enter(); ipv6_pending = false; match val { Some(Ok(Some(Ok(res)))) => { match res { Ok((r, conn)) => { - debug!(?r, "got v6 QAD conn"); + debug!(?r, "probe report"); let url = r.relay.clone(); reports.push(ProbeReport::QadIpv6(r)); if self.qad_conns.v6.is_none() { @@ -550,21 +547,21 @@ impl Client { } } Err(err) => { - debug!("probe v6 failed: {err:?}"); + debug!("probe failed: {err:#}"); } } } Some(Err(err)) => { if err.is_panic() { - panic!("probe v6 panicked: {err:?}"); + panic!("probe panicked: {err:#}"); } - warn!("probe v6 failed: {err:?}"); + warn!("probe failed: {err:#}"); } Some(Ok(None)) => { - debug!("probe v6 canceled"); + debug!("probe canceled"); } Some(Ok(Some(Err(time::Elapsed { .. })))) => { - debug!("probe v6 timed out"); + debug!("probe timed out"); } None => {} } @@ -745,20 +742,17 @@ impl Client { #[cfg(not(wasm_browser))] async fn run_probe_v4( - ip_mapped_addrs: Option, relay: Arc, quic_client: QuicClient, dns_resolver: DnsResolver, ) -> n0_snafu::Result<(QadProbeReport, QadConn)> { use n0_snafu::ResultExt; - let relay_addr_orig = reportgen::get_relay_addr_ipv4(&dns_resolver, &relay).await?; - let relay_addr = - reportgen::maybe_to_mapped_addr(ip_mapped_addrs.as_ref(), relay_addr_orig.into()); + let relay_addr = reportgen::get_relay_addr_ipv4(&dns_resolver, &relay).await?; - debug!(?relay_addr_orig, ?relay_addr, "relay addr v4"); + trace!(?relay_addr, "resolved relay server address"); let host = relay.url.host_str().context("missing host url")?; - let conn = quic_client.create_conn(relay_addr, host).await?; + let conn = quic_client.create_conn(relay_addr.into(), host).await?; let mut receiver = conn.observed_external_addr(); // wait for an addr @@ -785,7 +779,6 @@ async fn run_probe_v4( // that is ivp6 then the address is an [IPv4-Mapped IPv6 Addresses](https://doc.rust-lang.org/beta/std/net/struct.Ipv6Addr.html#ipv4-mapped-ipv6-addresses) let val = val.map(|val| SocketAddr::new(val.ip().to_canonical(), val.port())); let latency = conn.rtt(); - trace!(?val, ?relay_addr, ?latency, "got addr V4"); observer .set(val.map(|addr| QadProbeReport { relay: endpoint.clone(), @@ -813,19 +806,16 @@ async fn run_probe_v4( #[cfg(not(wasm_browser))] async fn run_probe_v6( - ip_mapped_addrs: Option, relay: Arc, quic_client: QuicClient, dns_resolver: DnsResolver, ) -> n0_snafu::Result<(QadProbeReport, QadConn)> { use n0_snafu::ResultExt; - let relay_addr_orig = reportgen::get_relay_addr_ipv6(&dns_resolver, &relay).await?; - let relay_addr = - reportgen::maybe_to_mapped_addr(ip_mapped_addrs.as_ref(), relay_addr_orig.into()); + let relay_addr = reportgen::get_relay_addr_ipv6(&dns_resolver, &relay).await?; - debug!(?relay_addr_orig, ?relay_addr, "relay addr v6"); + trace!(?relay_addr, "resolved relay server address"); let host = relay.url.host_str().context("missing host url")?; - let conn = quic_client.create_conn(relay_addr, host).await?; + let conn = quic_client.create_conn(relay_addr.into(), host).await?; let mut receiver = conn.observed_external_addr(); // wait for an addr @@ -852,7 +842,6 @@ async fn run_probe_v6( // that is ivp6 then the address is an [IPv4-Mapped IPv6 Addresses](https://doc.rust-lang.org/beta/std/net/struct.Ipv6Addr.html#ipv4-mapped-ipv6-addresses) let val = val.map(|val| SocketAddr::new(val.ip().to_canonical(), val.port())); let latency = conn.rtt(); - trace!(?val, ?relay_addr, ?latency, "got addr V6"); observer .set(val.map(|addr| QadProbeReport { relay: endpoint.clone(), @@ -948,7 +937,6 @@ mod tests { .insecure_skip_relay_cert_verify(true); let mut client = Client::new( resolver.clone(), - None, relay_map.clone(), opts.clone(), Default::default(), @@ -1149,8 +1137,7 @@ mod tests { println!("test: {}", tt.name); let relay_map = RelayMap::empty(); let opts = Options::default(); - let mut client = - Client::new(resolver.clone(), None, relay_map, opts, Default::default()); + let mut client = Client::new(resolver.clone(), relay_map, opts, Default::default()); for s in &mut tt.steps { // trigger the timer tokio::time::advance(Duration::from_secs(s.after)).await; diff --git a/iroh/src/net_report/ip_mapped_addrs.rs b/iroh/src/net_report/ip_mapped_addrs.rs deleted file mode 100644 index 2a8927edd92..00000000000 --- a/iroh/src/net_report/ip_mapped_addrs.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::{ - collections::BTreeMap, - net::{IpAddr, Ipv6Addr, SocketAddr}, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, -}; - -use snafu::Snafu; - -/// Can occur when converting a [`SocketAddr`] to an [`IpMappedAddr`] -#[derive(Debug, Snafu)] -#[snafu(display("Failed to convert"))] -pub struct IpMappedAddrError; - -/// A map fake Ipv6 address with an actual IP address. -/// -/// It is essentially a lookup key for an IP that iroh's magicsocket knows about. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] -pub(crate) struct IpMappedAddr(Ipv6Addr); - -/// Counter to always generate unique addresses for [`IpMappedAddr`]. -static IP_ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); - -impl IpMappedAddr { - /// The Prefix/L of our Unique Local Addresses. - const ADDR_PREFIXL: u8 = 0xfd; - /// The Global ID used in our Unique Local Addresses. - const ADDR_GLOBAL_ID: [u8; 5] = [21, 7, 10, 81, 11]; - /// The Subnet ID used in our Unique Local Addresses. - const ADDR_SUBNET: [u8; 2] = [0, 1]; - - /// The dummy port used for all mapped addresses. - const MAPPED_ADDR_PORT: u16 = 12345; - - /// Generates a globally unique fake UDP address. - /// - /// This generates a new IPv6 address in the Unique Local Address range (RFC 4193) - /// which is recognised by iroh as an IP mapped address. - pub(super) fn generate() -> Self { - let mut addr = [0u8; 16]; - addr[0] = Self::ADDR_PREFIXL; - addr[1..6].copy_from_slice(&Self::ADDR_GLOBAL_ID); - addr[6..8].copy_from_slice(&Self::ADDR_SUBNET); - - let counter = IP_ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); - addr[8..16].copy_from_slice(&counter.to_be_bytes()); - - Self(Ipv6Addr::from(addr)) - } - - /// Returns a consistent [`SocketAddr`] for the [`IpMappedAddr`]. - /// - /// This does not have a routable IP address. - /// - /// This uses a made-up, but fixed port number. The [IpMappedAddresses`] map this is - /// made for creates a unique [`IpMappedAddr`] for each IP+port and thus does not use - /// the port to map back to the original [`SocketAddr`]. - pub(crate) fn private_socket_addr(&self) -> SocketAddr { - SocketAddr::new(IpAddr::from(self.0), Self::MAPPED_ADDR_PORT) - } -} - -impl TryFrom for IpMappedAddr { - type Error = IpMappedAddrError; - - fn try_from(value: Ipv6Addr) -> std::result::Result { - let octets = value.octets(); - if octets[0] == Self::ADDR_PREFIXL - && octets[1..6] == Self::ADDR_GLOBAL_ID - && octets[6..8] == Self::ADDR_SUBNET - { - return Ok(Self(value)); - } - Err(IpMappedAddrError) - } -} - -impl std::fmt::Display for IpMappedAddr { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "IpMappedAddr({})", self.0) - } -} - -/// A Map of [`IpMappedAddresses`] to [`SocketAddr`]. -// TODO(ramfox): before this is ready to be used beyond QAD, we should add -// mechanisms for keeping track of "aliveness" and pruning address, as we do -// with the `EndpointMap` -#[derive(Debug, Clone, Default)] -pub(crate) struct IpMappedAddresses(Arc>); - -#[derive(Debug, Default)] -pub(super) struct Inner { - by_mapped_addr: BTreeMap, - /// Because [`std::net::SocketAddrV6`] contains extra fields besides the IP - /// address and port (ie, flow_info and scope_id), the a [`std::net::SocketAddrV6`] - /// with the same IP addr and port might Hash to something different. - /// So to get a hashable key for the map, we are using `(IpAddr, u6)`. - by_ip_port: BTreeMap<(IpAddr, u16), IpMappedAddr>, -} - -impl IpMappedAddresses { - /// Adds a [`SocketAddr`] to the map and returns the generated [`IpMappedAddr`]. - /// - /// If this [`SocketAddr`] already exists in the map, it returns its - /// associated [`IpMappedAddr`]. - /// - /// Otherwise a new [`IpMappedAddr`] is generated for it and returned. - pub(super) fn get_or_register(&self, socket_addr: SocketAddr) -> IpMappedAddr { - let ip_port = (socket_addr.ip(), socket_addr.port()); - let mut inner = self.0.lock().expect("poisoned"); - if let Some(mapped_addr) = inner.by_ip_port.get(&ip_port) { - return *mapped_addr; - } - let ip_mapped_addr = IpMappedAddr::generate(); - inner.by_mapped_addr.insert(ip_mapped_addr, socket_addr); - inner.by_ip_port.insert(ip_port, ip_mapped_addr); - ip_mapped_addr - } - - /// Returns the [`IpMappedAddr`] for the given [`SocketAddr`]. - pub(crate) fn get_mapped_addr(&self, socket_addr: &SocketAddr) -> Option { - let ip_port = (socket_addr.ip(), socket_addr.port()); - let inner = self.0.lock().expect("poisoned"); - inner.by_ip_port.get(&ip_port).copied() - } - - /// Returns the [`SocketAddr`] for the given [`IpMappedAddr`]. - pub(crate) fn get_ip_addr(&self, mapped_addr: &IpMappedAddr) -> Option { - let inner = self.0.lock().expect("poisoned"); - inner.by_mapped_addr.get(mapped_addr).copied() - } -} diff --git a/iroh/src/net_report/report.rs b/iroh/src/net_report/report.rs index dca3047a384..004a140dc66 100644 --- a/iroh/src/net_report/report.rs +++ b/iroh/src/net_report/report.rs @@ -7,7 +7,7 @@ use std::{ use iroh_base::RelayUrl; use serde::{Deserialize, Serialize}; -use tracing::warn; +use tracing::{trace, warn}; use super::{ProbeReport, probes::Probe}; @@ -82,7 +82,6 @@ impl Report { self.udp_v4 = true; - tracing::debug!(?self.global_v4, ?self.mapping_varies_by_dest_ipv4, %ipp,"got"); if let Some(global) = self.global_v4 { if global == ipp { if self.mapping_varies_by_dest_ipv4.is_none() { @@ -95,6 +94,7 @@ impl Report { } else { self.global_v4 = Some(ipp); } + trace!(?self.global_v4, ?self.mapping_varies_by_dest_ipv4, %ipp, "stored report"); } #[cfg(not(wasm_browser))] ProbeReport::QadIpv6(report) => { @@ -109,7 +109,6 @@ impl Report { }; self.udp_v6 = true; - tracing::debug!(?self.global_v6, ?self.mapping_varies_by_dest_ipv6, %ipp,"got"); if let Some(global) = self.global_v6 { if global == ipp { if self.mapping_varies_by_dest_ipv6.is_none() { @@ -122,6 +121,7 @@ impl Report { } else { self.global_v6 = Some(ipp); } + trace!(?self.global_v6, ?self.mapping_varies_by_dest_ipv6, %ipp, "stored report"); } } } diff --git a/iroh/src/net_report/reportgen.rs b/iroh/src/net_report/reportgen.rs index 10ec6d19ea5..87455237728 100644 --- a/iroh/src/net_report/reportgen.rs +++ b/iroh/src/net_report/reportgen.rs @@ -47,6 +47,8 @@ use tokio_util::sync::CancellationToken; use tracing::{Instrument, debug, error, trace, warn, warn_span}; use url::Host; +#[cfg(not(wasm_browser))] +use super::defaults::timeouts::DNS_TIMEOUT; #[cfg(wasm_browser)] use super::portmapper; // We stub the library use super::{ @@ -54,8 +56,6 @@ use super::{ probes::{Probe, ProbePlan}, }; #[cfg(not(wasm_browser))] -use super::{defaults::timeouts::DNS_TIMEOUT, ip_mapped_addrs::IpMappedAddresses}; -#[cfg(not(wasm_browser))] use crate::discovery::dns::DNS_STAGGERING_MS; use crate::{ net_report::defaults::timeouts::{ @@ -110,8 +110,6 @@ pub(crate) struct SocketState { pub(crate) quic_client: Option, /// The DNS resolver to use for probes that need to resolve DNS records. pub(crate) dns_resolver: DnsResolver, - /// Optional [`IpMappedAddresses`] used to enable QAD in iroh - pub(crate) ip_mapped_addrs: Option, } impl Client { @@ -200,7 +198,7 @@ pub(super) enum ProbeFinished { impl Actor { async fn run(self) { match time::timeout(OVERALL_REPORT_TIMEOUT, self.run_inner()).await { - Ok(()) => debug!("reportgen actor finished"), + Ok(()) => trace!("reportgen actor finished"), Err(time::Elapsed { .. }) => { warn!("reportgen timed out"); } @@ -219,7 +217,7 @@ impl Actor { /// - Updates the report, cancels unneeded futures. /// - Sends the report to the net_report actor. async fn run_inner(self) { - debug!("reportstate actor starting"); + trace!("reportgen actor starting"); let mut probes = JoinSet::default(); @@ -350,7 +348,7 @@ impl Actor { if_state: IfStateDetails, probes: &mut JoinSet, ) -> CancellationToken { - debug!(?if_state, "local interface details"); + trace!(?if_state, "local interface details"); let plan = match self.last_report { Some(ref report) => { ProbePlan::with_last_report(&self.relay_map, report, &self.protocols) @@ -523,17 +521,6 @@ impl Probe { } } -#[cfg(not(wasm_browser))] -pub(super) fn maybe_to_mapped_addr( - ip_mapped_addrs: Option<&IpMappedAddresses>, - addr: SocketAddr, -) -> SocketAddr { - if let Some(ip_mapped_addrs) = ip_mapped_addrs { - return ip_mapped_addrs.get_or_register(addr).private_socket_addr(); - } - addr -} - #[cfg(not(wasm_browser))] #[derive(Debug, Snafu)] #[snafu(module)] @@ -652,8 +639,11 @@ fn get_quic_port(relay: &RelayConfig) -> Option { pub enum GetRelayAddrError { #[snafu(display("No valid hostname in the relay URL"))] InvalidHostname, - #[snafu(display("No suitable relay address found"))] - NoAddrFound, + #[snafu(display("No suitable relay address found for {url} ({addr_type})"))] + NoAddrFound { + url: RelayUrl, + addr_type: &'static str, + }, #[snafu(display("DNS lookup failed"))] DnsLookup { source: StaggeredError }, #[snafu(display("Relay is not suitable"))] @@ -706,12 +696,22 @@ async fn relay_lookup_ipv4_staggered( IpAddr::V4(ip) => SocketAddrV4::new(ip, port), IpAddr::V6(_) => unreachable!("bad DNS lookup: {:?}", addr), }) - .ok_or(get_relay_addr_error::NoAddrFoundSnafu.build()), + .ok_or( + get_relay_addr_error::NoAddrFoundSnafu { + url: relay.url.clone(), + addr_type: "A", + } + .build(), + ), Err(err) => Err(get_relay_addr_error::DnsLookupSnafu.into_error(err)), } } Some(url::Host::Ipv4(addr)) => Ok(SocketAddrV4::new(addr, port)), - Some(url::Host::Ipv6(_addr)) => Err(get_relay_addr_error::NoAddrFoundSnafu.build()), + Some(url::Host::Ipv6(_addr)) => Err(get_relay_addr_error::NoAddrFoundSnafu { + url: relay.url.clone(), + addr_type: "A", + } + .build()), None => Err(get_relay_addr_error::InvalidHostnameSnafu.build()), } } @@ -738,11 +738,21 @@ async fn relay_lookup_ipv6_staggered( IpAddr::V4(_) => unreachable!("bad DNS lookup: {:?}", addr), IpAddr::V6(ip) => SocketAddrV6::new(ip, port, 0, 0), }) - .ok_or(get_relay_addr_error::NoAddrFoundSnafu.build()), + .ok_or( + get_relay_addr_error::NoAddrFoundSnafu { + url: relay.url.clone(), + addr_type: "AAAA", + } + .build(), + ), Err(err) => Err(get_relay_addr_error::DnsLookupSnafu.into_error(err)), } } - Some(url::Host::Ipv4(_addr)) => Err(get_relay_addr_error::NoAddrFoundSnafu.build()), + Some(url::Host::Ipv4(_addr)) => Err(get_relay_addr_error::NoAddrFoundSnafu { + url: relay.url.clone(), + addr_type: "AAAA", + } + .build()), Some(url::Host::Ipv6(addr)) => Ok(SocketAddrV6::new(addr, port, 0, 0)), None => Err(get_relay_addr_error::InvalidHostnameSnafu.build()), } @@ -876,7 +886,7 @@ mod tests { let quic_client = iroh_relay::quic::QuicClient::new(ep.clone(), client_config); let dns_resolver = DnsResolver::default(); - let (report, conn) = super::super::run_probe_v4(None, relay, quic_client, dns_resolver) + let (report, conn) = super::super::run_probe_v4(relay, quic_client, dns_resolver) .await .unwrap();