Added some ratelimiting middleware

Cargo.lock

@@ -263,6 +263,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -317,6 +326,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.3.3"
@@ -346,6 +368,7 @@ dependencies = [
  "axum",
  "axum-client-ip",
  "clap",
+ "governor",
  "idna 0.3.0",
  "lazy_static",
  "maxminddb",
@@ -418,19 +441,46 @@ dependencies = [
 ]
 
 [[package]]
-name = "futures-channel"
+name = "futures"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
 
 [[package]]
 name = "futures-io"
@@ -439,19 +489,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
 
 [[package]]
-name = "futures-task"
+name = "futures-macro"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
+checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
+
+[[package]]
+name = "futures-task"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
+
+[[package]]
+name = "futures-timer"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
 
 [[package]]
 name = "futures-util"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
+checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
 dependencies = [
+ "futures-channel",
  "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
  "futures-task",
+ "memchr",
  "pin-project-lite",
  "pin-utils",
  "slab",
@@ -475,7 +553,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -502,6 +580,24 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "governor"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5"
+dependencies = [
+ "cfg-if",
+ "dashmap",
+ "futures",
+ "futures-timer",
+ "no-std-compat",
+ "nonzero_ext",
+ "parking_lot",
+ "quanta",
+ "rand",
+ "smallvec",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -831,6 +927,15 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "mach"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "match_cfg"
 version = "0.1.0"
@@ -881,10 +986,16 @@ checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.42.0",
 ]
 
+[[package]]
+name = "no-std-compat"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
+
 [[package]]
 name = "nom8"
 version = "0.2.0"
@@ -900,6 +1011,12 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7"
 
+[[package]]
+name = "nonzero_ext"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@@ -1133,6 +1250,22 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "mach",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.10.2+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -1178,6 +1311,15 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "raw-cpuid"
+version = "10.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c307f7aacdbab3f0adee67d52739a1d71112cc068d6fab169ddeb18e48877fad"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -1823,6 +1965,12 @@ dependencies = [
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -1883,6 +2031,16 @@ version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
 
+[[package]]
+name = "web-sys"
+version = "0.3.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "widestring"
 version = "0.5.1"

Cargo.toml

@@ -10,6 +10,7 @@ authors = ["Slatian <baschdel@disroot.org>"]
 axum = { version = "0.6", features = ["macros", "headers"] }
 axum-client-ip = "0.4"
 clap = { version = "4", features = ["derive"] }
+governor = "0.5"
 idna = "0.3"
 lazy_static = "1.4.0"
 regex = "1.7"

README.md

@@ -58,7 +58,8 @@ For a public service you should use a reverse proxy like Caddy, apache2 or nginx
 
 ### Denail of Service
 
-`echoip-slatecave` currently doesn't have any protection mechanisms against overuse or a full (D)DOS, make sure you know how to to use your filewall (i.e. [nftables](https://nftables.org)) or you have fail2ban set up.
+`echoip-slatecave` has some simle ratelimiting built in (see the `[ratelimit]` section in the configuration file) this should help you with too frequest automated requests causung high load.
+The default configuration is pretty liberal so that the average human probably won't notice the rate limit, but a misbehavin bot will be limited to one request every 3 seconds after 15 requests.
 
 ## License
 

echoip_config.toml

@@ -3,12 +3,15 @@
 listen_on = "127.0.0.1:3000"
 
 # What header your reverse proxy sets that contains the real ip-address
-# Possible Values: Every Variation of SecureClientIpSource in the axum_client_ip package
+# Possible Values: Every Variation of SecureClientIpSource in the axum_client_ip crate
 # https://docs.rs/axum-client-ip/latest/axum_client_ip/enum.SecureClientIpSource.html
+# Note: This one is also used for rate limiting
+# Related: https://adam-p.ca/blog/2022/03/x-forwarded-for/
 ip_header = "RightmostXForwardedFor"
 # When you don't want to use a proxy server:
 #ip_header = "ConnectInfo"
 
+
 # Allow querying of private range ips
 # enable if you want to use this service
 # on your internal network for some reason
@@ -45,3 +48,17 @@ template_location = "templates"
 # Prefixes of user agents that should get a text reponse by default
 text_user_agents = ["curl/"]
 
+[ratelimit]
+# Configure a Quota for the Rate limiter
+
+# Please note that this depends on the ip_header being
+# configured correctly!
+
+# How many requests per minute are allowed
+# (How fast the leaky bucket drains)
+per_minute = 20
+# How many requests may come in at once
+# (How much capacity the leaky bucket has)
+burst = 15
+
+#Note: The ratelimit is implemented using the governor crate

echoip_test.toml

@@ -45,3 +45,19 @@ template_location = "templates"
 
 # Prefixes of user agents that should get a text reponse by default
 text_user_agents = ["curl/"]
+
+[ratelimit]
+# Configure a Quota for the Rate limiter
+
+# Please note that this depends on the ip_header being
+# configured correctly!
+
+# How many requests per minute are allowed
+# (How fast the leaky bucket drains)
+per_minute = 20
+# How many requests may come in at once
+# (How much capacity the leaky bucket has)
+burst = 15
+
+#Note: The ratelimit is implemented using the governor crate
+

src/config.rs

@@ -1,5 +1,6 @@
 use axum_client_ip::SecureClientIpSource;
 use std::net::SocketAddr;
+use std::num::NonZeroU32;
 
 #[derive(serde::Deserialize, Default, Clone)]
 pub struct EchoIpServiceConfig {
@@ -7,6 +8,7 @@ pub struct EchoIpServiceConfig {
 	pub dns: DnsConfig,
 	pub geoip: GeoIpConfig,
 	pub template: TemplateConfig,
+	pub ratelimit: RatelimitConfig,
 }
 
 #[derive(serde::Deserialize, Clone)]
@@ -38,6 +40,12 @@ pub struct TemplateConfig {
 	pub text_user_agents: Vec<String>,
 }
 
+#[derive(serde::Deserialize, Clone)]
+pub struct RatelimitConfig {
+	pub per_minute: NonZeroU32,
+	pub burst: NonZeroU32,
+}
+
 impl Default for ServerConfig {
 	fn default() -> Self {
 		ServerConfig {
@@ -76,3 +84,12 @@ impl Default for TemplateConfig {
 		}
 	}
 }
+
+impl Default for RatelimitConfig {
+	fn default() -> Self {
+		RatelimitConfig {
+			per_minute: NonZeroU32::new(20).unwrap(),
+			burst: NonZeroU32::new(15).unwrap(),
+		}
+	}
+}

src/main.rs

@@ -33,6 +33,7 @@ use std::path::Path;
 mod config;
 mod geoip;
 mod ipinfo;
+mod ratelimit;
 mod simple_dns;
 mod templating_engine;
 mod idna;
@@ -85,6 +86,7 @@ pub struct DigResult {
 	partial_lookup: bool,
 }
 
+
 struct ServiceSharedState {
 	templating_engine: templating_engine::Engine,
 	dns_resolver:      TokioAsyncResolver,
@@ -110,7 +112,6 @@ fn match_domain_hidden_list(domain: &String, hidden_list: &Vec<String>) -> bool
 	let name = domain.trim_end_matches(".");
 	for suffix in hidden_list {
 		if name.ends_with(suffix) {
-			println!("Blocked {name} …");
 			return true;
 		}
 	}
@@ -185,6 +186,9 @@ async fn main() {
 		template_config: template_extra_config,
 	};
 
+	// Initalize Rate Limiter
+
+
 	// Initalize GeoIP Database
 
 	let mut asn_db = geoip::MMDBCarrier {
@@ -243,9 +247,12 @@ async fn main() {
     	.with_state(shared_state)
     	.layer(
 			ServiceBuilder::new()
-	    		.layer(ip_header.into_extension())
+				.layer(ip_header.into_extension())
-	    		.layer(Extension(config))
+				.layer(ratelimit::build_rate_limiting_state(
-    			.layer(middleware::from_fn(format_and_language_middleware))
+					config.ratelimit.per_minute, config.ratelimit.burst))
+				.layer(middleware::from_fn(ratelimit::rate_limit_middleware))
+				.layer(Extension(config))
+				.layer(middleware::from_fn(format_and_language_middleware))
     	)
     	;
 
@@ -257,6 +264,7 @@ async fn main() {
         .unwrap();
 }
 
+
 async fn format_and_language_middleware<B>(
 	Query(query): Query<SettingsQuery>,
 	Extension(config): Extension<config::EchoIpServiceConfig>,

src/ratelimit.rs

@@ -0,0 +1,71 @@
+use axum_client_ip::SecureClientIp;
+use axum::{
+	extract::Extension,
+	http::{
+		Request,
+		StatusCode,
+	},
+	middleware::Next,
+	response::{
+		IntoResponse,
+		Response,
+	},
+};
+use governor::{
+	clock::DefaultClock,
+	Quota,
+	RateLimiter,
+	state::keyed::DefaultKeyedStateStore,
+};
+
+use std::net::IpAddr;
+use std::num::NonZeroU32;
+use std::sync::Arc;
+
+pub type SimpleRateLimiter<Key> =
+	RateLimiter<Key, DefaultKeyedStateStore<Key>, DefaultClock>;
+
+pub fn build_rate_limiting_state(
+	requests_per_minute: NonZeroU32,
+	request_burst_capacity: NonZeroU32,
+) -> Extension<Arc<SimpleRateLimiter<IpAddr>>> {
+
+	let quota = Quota::per_minute(requests_per_minute)
+		.allow_burst(request_burst_capacity);
+	
+	let arc_limiter : Arc<SimpleRateLimiter<IpAddr>> = Arc::new(
+		RateLimiter::keyed(quota)
+	);
+	
+	Extension(arc_limiter)
+}
+
+pub async fn rate_limit_middleware<B>(
+	SecureClientIp(address): SecureClientIp,
+	Extension(arc_limiter): Extension<Arc<SimpleRateLimiter<IpAddr>>>,
+	req: Request<B>,
+	next: Next<B>
+) -> Response {
+	let limiter = Arc::clone(&arc_limiter);
+
+	match limiter.check_key(&address) {
+		Ok(_) => {
+			//Little hack to prevent too many cleanups in cases of very high load
+			if limiter.check_key(&IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED)).is_ok() {
+				let oldlen = limiter.len();
+				if oldlen > 100 {
+					println!("Doing limiter cleanup ...");
+					limiter.retain_recent();
+					limiter.shrink_to_fit();
+					println!("Old limiter store size: {oldlen} New limiter store size: {}", limiter.len());
+				}
+			}
+			next.run(req).await
+		},
+		Err(_) => (
+			StatusCode::TOO_MANY_REQUESTS,
+			"You make too many requests! Please slow down a bit."
+		).into_response(),
+	}
+}
+