Add robots meta tag to templates to make it less likely for a crawler to get stuck

Cargo.toml

@@ -7,15 +7,14 @@ authors = ["Slatian <baschdel@disroot.org>"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-lib-humus = { version="0.2", features=["axum-view+cookie"], git="https://codeberg.org/slatian/lib-humus.git" }
+lib-humus = { version="0.2", features=["axum-view+cookie"] }
 
 axum = { version = "0.7", features = ["macros"] }
 axum-extra = { version = "0.9", features = ["cookie", "typed-header"] }
-axum-client-ip = "0.5"
-clap = { version = "4", features = ["derive"] }
+axum-client-ip = "0.6"
+clap = { version = "4.5", features = ["derive"] }
 governor = "0.6"
-idna = "0.4"
-lazy_static = "1.4.0"
+idna = "1.0"
 parking_lot = "0.12"
 regex = "1.10"
 serde = { version = "1", features = ["derive","rc"] }
@@ -26,6 +25,6 @@ tower = "0.4"
 tower-http = { version = "0.5", features = ["fs"] }
 hickory-proto = "0.24"
 hickory-resolver =  { version = "0.24", features = ["dns-over-rustls","dns-over-https","dns-over-quic","native-certs"] }
-maxminddb = "0.23"
+maxminddb = "0.24"
 mime = "0.3"
-http = "1.0"
+http = "1.1"

README.md

@@ -1,9 +1,20 @@
 # echoip-slatecave
 
-This is a service inspired by ifconfig.co, but built from scratch with a more useful feature set (in rust !!1!).
+This is a service inspired by ifconfig.co, but built from scratch with a more useful feature set. Currently live on [echoip.slatecave.net](https://echoip.slatecave.net).
 
 It is Licensed under the AGPL-v3 license.
 
+## Maintainence Mode
+
+This project is in maintanance mode.
+
+This means the following will happen:
+* Dependency updates
+* Bugfixes
+* Small quality of life improvements
+
+But no active feature development by Slatian.
+
 ## Building
 
 Simply run `cargo build` after cloning. The binary should be called `target/debug/echoip-slatecave`.
@@ -63,10 +74,16 @@ In addition to that the following fields are accessible from inside the template
 
 The templates are covered by the AGPL as well, please share them with your users if you modified them.
 
-### GeoLite2 database
+### Geolocation databases
 
 For geolocation to work you need a MaxMind format database, for full functionality you need the GeoLite2-ASN and GeoLite2-City databses. Unfortunately you have to sign up with [MaxMind](https://maxmind.com) to obtain them. Once you have a license key there is a helper script in [contrib/maxmind-download.sh](contrib/maxmind-download.sh) that helps you with keeping the databse updated.
 
+As an alternative to MaxMind there is also [DB-IP who offer their free databases without a login](https://db-ip.com/db/lite.php). You want the City and ASN databases in mmdb format.
+
+**Don't forget to set the atttribution in the template configuration appropriately.**
+
+See the file `templates/extra.toml`.
+
 Since v1.0 echoip-slatecave reloads the databses when it rececieves a `USR1` signal.
 
 ## Security Considerations

src/config/dns.rs

@@ -86,7 +86,7 @@ impl Into<Protocol> for DnsProtocol {
 }
 
 impl DnsResolverConfig {
-	pub fn to_trust_resolver_config(
+	pub fn to_hickory_resolver_config(
 		&self
 	) -> HickoryResolverConfig {
 		let mut resolver = HickoryResolverConfig::new();

src/config/mod.rs

@@ -6,7 +6,7 @@ use std::num::NonZeroU32;
 
 mod dns;
 
-pub use crate::config::dns::{DnsConfig, DnsProtocol, DnsResolverConfig};
+pub use crate::config::dns::{DnsConfig, DnsResolverConfig};
 
 #[derive(Deserialize, Default, Clone)]
 pub struct EchoIpServiceConfig {
@@ -23,7 +23,6 @@ pub struct ServerConfig {
 	pub ip_header: SecureClientIpSource,
 
 	pub allow_private_ip_lookup: bool,
-	pub static_location: Option<String>,
 }
 
 
@@ -53,7 +52,6 @@ impl Default for ServerConfig {
 			listen_on: "127.0.0.1:3000".parse().unwrap(),
 			ip_header: SecureClientIpSource::ConnectInfo,
 			allow_private_ip_lookup: false,
-			static_location: None,
 		}
 	}
 }

src/main.rs

@@ -17,7 +17,6 @@ use axum_client_ip::SecureClientIp;
 use axum_extra::headers;
 use axum_extra::TypedHeader;
 use clap::Parser;
-use lazy_static::lazy_static;
 use regex::Regex;
 use serde::{Deserialize,Serialize};
 use tower::ServiceBuilder;
@@ -33,7 +32,9 @@ use tokio::task;
 
 use std::collections::HashMap;
 use std::net::IpAddr;
+use std::net::SocketAddr;
 use std::sync::Arc;
+use std::sync::LazyLock;
 
 use lib_humus::TemplateEngineLoader;
 use lib_humus::read_toml_from_file;
@@ -67,6 +68,7 @@ pub struct SettingsQuery {
 	format: Option<ResponseFormat>,
 	lang:   Option<String>,
 	dns:	Option<String>,
+	dns_self_lookup: Option<bool>,
 }
 
 #[derive(Deserialize, Serialize, Clone)]
@@ -82,6 +84,7 @@ pub struct IpResult {
 	location: Option<LocationResult>,
 	ip_info:  AddressInfo,
 	used_dns_resolver: Option<Arc<str>>,
+	reverse_dns_disabled_for_privacy: bool,
 }
 
 // We need this one to hide the partial lookup field when irelevant
@@ -119,7 +122,7 @@ struct CliArgs {
 	#[arg(short, long)]
 	config: Option<String>,
 	#[arg(short, long)]
-	listen_on: Option<String>,
+	listen_on: Option<SocketAddr>,
 	#[arg(short, long)]
 	template_location: Option<String>,
 	#[arg(short,long)]
@@ -227,7 +230,7 @@ async fn main() {
 	for (key, resolver_config) in &config.dns.resolver {
 		println!("Initalizing {} resolver ...", key);
 		let resolver = TokioAsyncResolver::tokio(
-			resolver_config.to_trust_resolver_config(),
+			resolver_config.to_hickory_resolver_config(),
 			Default::default()
 		);
 		dns_resolver_map.insert(key.clone(), resolver);
@@ -241,7 +244,7 @@ async fn main() {
 		}
 	}
 	
-	let listen_on = config.server.listen_on;
+	let listen_on = cli_args.listen_on.unwrap_or(config.server.listen_on);
 	let ip_header = config.server.ip_header.clone();
 
 	// Initialize shared state
@@ -357,6 +360,7 @@ async fn settings_query_middleware(
 		lang: query.lang.unwrap_or("en".to_string()),
 		available_dns_resolvers: derived_config.dns_resolver_selectables,
 		dns_resolver_id: dns_resolver_id,
+		dns_disable_self_lookup: !query.dns_self_lookup.unwrap_or(false),
 	});
 	next.run(req).await
 }
@@ -400,7 +404,7 @@ async fn handle_default_route(
 	State(arc_state): State<Arc<ServiceSharedState>>,
 	Extension(settings): Extension<QuerySettings>,
 	user_agent_header: Option<TypedHeader<headers::UserAgent>>,
-	SecureClientIp(address): SecureClientIp
+	SecureClientIp(client_ip): SecureClientIp
 ) -> Response {
 	
 	let state = Arc::clone(&arc_state);
@@ -411,12 +415,20 @@ async fn handle_default_route(
 				search_query,
 				false,
 				settings,
-				state
+				state,
+				&client_ip
 			).await;
 		}
 	}
 
-	let result = get_ip_result(&address, &settings.lang, &"default".into(), &state).await;
+	let result = get_ip_result(
+		&client_ip,
+		&settings.lang,
+		&settings.dns_resolver_id,
+		settings.dns_disable_self_lookup,
+		&client_ip,
+		&state,
+	).await;
 
 	let user_agent: Option<String> = match user_agent_header {
 		Some(TypedHeader(user_agent)) => Some(user_agent.to_string()),
@@ -432,22 +444,20 @@ async fn handle_default_route(
 	)
 }
 
+static ASN_REGEX: LazyLock<Regex> = LazyLock::new(|| { Regex::new(r"^[Aa][Ss][Nn]?\s*(\d{1,7})$").unwrap() });
+static VIA_REGEX: LazyLock<Regex> = LazyLock::new(|| { Regex::new(r"[Vv][Ii][Aa]\s+(\S+)").unwrap() });
 
 async fn handle_search_request(
 	search_query: String,
 	this_should_have_been_an_ip: bool,
 	settings: QuerySettings,
 	arc_state: Arc<ServiceSharedState>,
+	client_ip: &IpAddr,
 ) -> Response {
 
 	let mut search_query = search_query.trim().to_string();
 	let mut settings = settings;
 
-	lazy_static!{
-		static ref ASN_REGEX: Regex = Regex::new(r"^[Aa][Ss][Nn]?\s*(\d{1,7})$").unwrap();
-		static ref VIA_REGEX: Regex = Regex::new(r"[Vv][Ii][Aa]\s+(\S+)").unwrap();
-	}
-
 	//If someone asked for an asn, give an asn answer
 	if let Some(asn_cap) = ASN_REGEX.captures(&search_query) {
 		if let Some(asn) = asn_cap.get(1).map_or(None, |m| m.as_str().parse::<u32>().ok()) {
@@ -474,7 +484,7 @@ async fn handle_search_request(
 
 	// Try to interpret as an IP-Address
 	if let Ok(address) = search_query.parse() {
-		return handle_ip_request(address, settings, arc_state).await;
+		return handle_ip_request(address, settings, arc_state, client_ip).await;
 	}
 
 	// Fall back to treating it as a hostname
@@ -522,11 +532,12 @@ async fn handle_ip_route_with_path(
 	Extension(settings): Extension<QuerySettings>,
 	State(arc_state): State<Arc<ServiceSharedState>>,
 	extract::Path(query): extract::Path<String>,
+	SecureClientIp(client_ip): SecureClientIp
 ) -> Response {
 	if let Ok(address) = query.parse() {
-		return handle_ip_request(address, settings, arc_state).await
+		return handle_ip_request(address, settings, arc_state, &client_ip).await
 	} else {
-		return handle_search_request(query, true, settings, arc_state).await;
+		return handle_search_request(query, true, settings, arc_state, &client_ip).await;
 	}
 }
 
@@ -534,6 +545,7 @@ async fn handle_ip_request(
 	address: IpAddr,
 	settings: QuerySettings,
 	arc_state: Arc<ServiceSharedState>,
+	client_ip: &IpAddr,
 ) -> Response {
 
 	let state = Arc::clone(&arc_state);
@@ -541,6 +553,8 @@ async fn handle_ip_request(
 		&address,
 		&settings.lang,
 		&settings.dns_resolver_id,
+		settings.dns_disable_self_lookup,
+		client_ip,
 		&state).await;
 
 	state.templating_engine.render_view(
@@ -553,9 +567,19 @@ async fn get_ip_result(
 	address: &IpAddr,
 	lang: &String,
 	dns_resolver_name: &Arc<str>,
+	dns_disable_self_lookup: bool,
+	client_ip: &IpAddr,
 	state: &ServiceSharedState,
 ) -> IpResult {
 
+	let mut reverse_dns_disabled_for_privacy = false;
+	
+	if state.config.dns.allow_reverse_lookup {
+		if address == client_ip && dns_disable_self_lookup {
+			reverse_dns_disabled_for_privacy = true;
+		}
+	}
+		
 	let ip_info = AddressInfo::new(&address);
 
 	if !(ip_info.scope == AddressScope::Global || ip_info.scope == AddressScope::Shared) || ip_info.cast != AddressCast::Unicast {
@@ -567,6 +591,7 @@ async fn get_ip_result(
 				location: None,
 				ip_info: ip_info,
 				used_dns_resolver: None,
+				reverse_dns_disabled_for_privacy: reverse_dns_disabled_for_privacy,
 			}
 		}
 	} 
@@ -574,7 +599,7 @@ async fn get_ip_result(
 	// do reverse lookup
 	let mut hostname: Option<String> = None;
 	let mut used_dns_resolver: Option<Arc<str>> = None;
-	if state.config.dns.allow_reverse_lookup {
+	if state.config.dns.allow_reverse_lookup && !reverse_dns_disabled_for_privacy {
 		if let Some(dns_resolver) = &state.dns_resolvers.get(dns_resolver_name) {
 			hostname = simple_dns::reverse_lookup(&dns_resolver, &address).await;
 			used_dns_resolver = Some(dns_resolver_name.clone());
@@ -605,6 +630,7 @@ async fn get_ip_result(
 		location: location_result,
 		ip_info: ip_info,
 		used_dns_resolver: used_dns_resolver,
+		reverse_dns_disabled_for_privacy: reverse_dns_disabled_for_privacy,
 	}
 }
 

src/settings.rs

@@ -17,6 +17,7 @@ pub struct QuerySettings {
 	pub lang: String,
 	pub available_dns_resolvers: Vec<Selectable>,
 	pub dns_resolver_id: Arc<str>,
+	pub dns_disable_self_lookup: bool,
 }
 
 #[derive(Deserialize, Serialize, Clone)]

templates/base.html

@@ -5,6 +5,9 @@
 	<meta charset="utf-8">
 	<title>{% block title %}{{ extra[view].title | default(value="…") }}{% endblock %} | {{extra.site_name|default(value="echoip")}}</title>
 	<meta content="width=device-width, initial-scale=1" name="viewport">
+	<meta name="color-scheme" content="echoip-slatecave <https://codeberg.org/slatian/service.echoip-slatecave>">
+	{% block robots_meta %}
+	{% endblock robots_meta %}
 	<!-- Open-Graph -->
 	{% block metadata %}
 	<meta name="description" property="og:description" content="{% block description %}{{ extra[view].description | default(value="One of the best echoip services") | escape_xml }}{% endblock %}" />

templates/dig.html

@@ -2,6 +2,8 @@
 {% import "helpers.html" as helper %}
 {% import "links.html" as links %}
 
+{% block robots_meta %}<meta name="robots" content="noindex,nofollow">{% endblock %}
+
 {% block title %}dig {{ data.query }}{% endblock %}
 {% block og_title %}dig {{ data.query }}{% endblock %}
 {% block h1 %}dig <code>{{ helper::breadcrumb_domain(extra=extra, name=data.query) }}</code> <small>via <a href="{{extra.base_url}}/dns_resolver/{{data.result.used_dns_resolver}}">{{data.result.used_dns_resolver}}</a></small>{% endblock %}

templates/extra.toml

@@ -21,6 +21,13 @@ display_icon = "/icon_64.png"
 v4_url="http://v4.localhost:3000/"
 v6_url="http://v6.localhost:3000/"
 
+# Geolocation Attribution for MaxMind
+#geo_attribution_html="The Geolocation and ASN information is provided by the GeoLite2 database created by <a href='https://www.maxmind.com/'>MaxMind</a>."
+
+# Geolocation Attribution for DB-IP
+#geo_attribution_html="The Geolocation and ASN information is provided by <a href='https://db-ip.com/'>DB-IP</a>."
+
+
 [404]
 # configure the 404 page, this is available for other pages too!
 # Use the template name as the section name.

templates/helpers.html

@@ -17,8 +17,8 @@
 <a href="{{ self::dig_link(extra=extra, name=name) }}">{% if prefix %}{{ prefix }} {% endif %}{% if fqdn or name=="." %}{{ name }}{% else %}{{ name | trim_end_matches(pat=".") }}{% endif %}</a>
 {% endmacro dig %}
 
-{% macro ip(extra, ip, text=false) %}
-<a href="{{ extra.base_url }}/ip/{{ ip | urlencode_strict | replace(from="%2e", to=".") | replace(from="%3a", to=":") | safe }}"><code>{% if text %}{{ text }}{% else %}{{ ip }}{% endif %}</code></a>
+{% macro ip(extra, ip, text=false, with_self_lookup=false) %}
+<a href="{{ extra.base_url }}/ip/{{ ip | urlencode_strict | replace(from="%2e", to=".") | replace(from="%3a", to=":") | safe }}{% if with_self_lookup %}?dns_self_lookup=true{% endif %}"><code>{% if text %}{{ text }}{% else %}{{ ip }}{% endif %}</code></a>
 {% endmacro dig %}
 
 {% macro breadcrumb_domain(extra, name) %}

templates/index.html

@@ -1,6 +1,8 @@
 {% extends "ip.html" %}
 {% import "helpers.html" as helper %}
 
+{% block robots_meta %}{# Allow indexing for landing page #}{% endblock %}
+
 {% block title %}Your IP: {{ data.result.address }}{% endblock %}
 {% block og_title %}What is my IP-Address?{% endblock %}
 {% block h1 %}Your IPv{% if data.result.ip_info.is_v6_address %}6{% else %}4{% endif %}: <code>{{ data.result.address }}</code>{% endblock %}

templates/ip.html

@@ -2,6 +2,8 @@
 {% import "helpers.html" as helper %}
 {% import "links.html" as links %}
 
+{% block robots_meta %}<meta name="robots" content="noindex,nofollow">{% endblock %}
+
 {% block title %}{{ data.result.address }}{% endblock %}
 {% block og_title %}Lookup {{ data.result.address }}{% endblock %}
 {% block h1 %}Lookup <code>{{ data.result.address }}</code>{% endblock %}
@@ -18,6 +20,9 @@
 		{% if r.hostname %}
 			<dt>Hostname</dt>
 			<dd>{{ helper::dig(extra=extra, name=r.hostname) }}</dd>
+		{% elif r.reverse_dns_disabled_for_privacy %}
+			<dt>Hostname</dt>
+			<dd>Lookup disabled by default: {{ helper::ip(ip=r.address, extra=extra, text="enable", with_self_lookup=true)}}</dd>
 		{% endif %}
 		{% if r.asn %}
 			<dt><abbr="Autonomous System Number">ASN</abbr></dt>
@@ -31,43 +36,44 @@
 	{% if r.location %}
 	<section>
 		<h2>Geolocation</h2>
-		<dl>
-			{{ helper::place_dl(place=r.location.continent, label="Continent") }}
-			{{ helper::place_dl(place=r.location.country, label="Country") }}
-			{% if r.location.country.iso_code | default(value="") != r.location.registered_country.iso_code | default(value="") %}
-			{{ helper::place_dl(place=r.location.registered_country, label="Registered in") }}
-			{% endif %}
-			{% if r.location.country.iso_code | default(value="") != r.location.represented_country.iso_code | default(value="")%}
-			{{ helper::place_dl(place=r.location.represented_country, label="Represents") }}
-			{% endif %}
-			{% if r.location.subdivisions %}
-			{% for sd in r.location.subdivisions %}
-			{{ helper::place_dl(place=sd, label="Subdivision", iso_code_prefix=r.location.country.iso_code|default(value="")) }}
-			{% endfor %}
-			{% endif %}
-			{{ helper::place_dl(place=r.location.city, label="City") }}
-			{% if r.location.postal_code %}
-			<dt>Postal Code</dt>
-			<dd>{{r.location.postal_code}}</dd>
-			{% endif %}
-			{% if r.location.time_zone %}
-			<dt>Timezone</dt>
-			<dd>{{r.location.time_zone}}</dd>
-			{% endif %}
-			{% if r.location.accuracy %}
-			<dt>Accuracy</dt>
-			<dd>~{{r.location.accuracy}}km</dd>
-			{% endif %}
-			{% if r.location.coordinates %}
-			<dt>Coordinates</dt>
-			<dd><a target="_blank" href="{{ links::map_link(lat=r.location.coordinates.lat, lon=r.location.coordinates.lon)}}">lat: {{r.location.coordinates.lat}}, lon: {{r.location.coordinates.lon}}</a></dd>
-			{% endif %}
-		</dl>
-		<!--We have to put that there to comply with maxminds licensing-->
-		<p><small>
-			The GeoIP and ASN information is provided by the GeoLite2 database created by
-			<a target="_blank" href="https://www.maxmind.com">MaxMind</a>.
-		</small></p>
+		{% if extra.geo_attribution_html %}
+			<dl>
+				{{ helper::place_dl(place=r.location.continent, label="Continent") }}
+				{{ helper::place_dl(place=r.location.country, label="Country") }}
+				{% if r.location.country.iso_code | default(value="") != r.location.registered_country.iso_code | default(value="") %}
+				{{ helper::place_dl(place=r.location.registered_country, label="Registered in") }}
+				{% endif %}
+				{% if r.location.country.iso_code | default(value="") != r.location.represented_country.iso_code | default(value="")%}
+				{{ helper::place_dl(place=r.location.represented_country, label="Represents") }}
+				{% endif %}
+				{% if r.location.subdivisions %}
+				{% for sd in r.location.subdivisions %}
+				{{ helper::place_dl(place=sd, label="Subdivision", iso_code_prefix=r.location.country.iso_code|default(value="")) }}
+				{% endfor %}
+				{% endif %}
+				{{ helper::place_dl(place=r.location.city, label="City") }}
+				{% if r.location.postal_code %}
+				<dt>Postal Code</dt>
+				<dd>{{r.location.postal_code}}</dd>
+				{% endif %}
+				{% if r.location.time_zone %}
+				<dt>Timezone</dt>
+				<dd>{{r.location.time_zone}}</dd>
+				{% endif %}
+				{% if r.location.accuracy %}
+				<dt>Accuracy</dt>
+				<dd>~{{r.location.accuracy}}km</dd>
+				{% endif %}
+				{% if r.location.coordinates %}
+				<dt>Coordinates</dt>
+				<dd><a target="_blank" href="{{ links::map_link(lat=r.location.coordinates.lat, lon=r.location.coordinates.lon)}}">lat: {{r.location.coordinates.lat}}, lon: {{r.location.coordinates.lon}}</a></dd>
+				{% endif %}
+			</dl>
+			<p><small>{{extra.geo_attribution_html | safe}}</small></p>
+		{% else %}
+			<p><strong style="font-size: 2em">Please configure the <code>geo_attribution_html</code> key in the template extra configuration!</strong></p>
+			<p>The geolocation information will then become visible.</p>
+		{% endif %}
 	</section>
 	{% endif %}
 	{% block extra_content %}{% endblock %}

templates/ip.txt

@@ -13,12 +13,19 @@
 * Type of Address: {{ helper::ip_info(ip_info=r.ip_info) }}
 {% if r.hostname -%}
 * Hostname: {{ r.hostname }}
+{%- elif r.reverse_dns_disabled_for_privacy %}
+* Hostname: Lookup disabled by default
 {%- endif %}
 {% if r.asn -%}
 * ASN: AS{{ r.asn.asn }}
 * AS Name: {{r.asn.name}}
 {%- endif -%}
 
+{%- if r.reverse_dns_disabled_for_privacy %}
+
+=> /ip/{{ data.result.address }}?dns_self_lookup=true Do a reverse DNS lookup
+{% endif %}
+
 {%- if r.location %}
 
 ## Geolocation
@@ -52,7 +59,7 @@ lat: {{r.location.coordinates.lat}}, lon: {{r.location.coordinates.lon}}
 => {{ links::map_link(lat=r.location.coordinates.lat, lon=r.location.coordinates.lon)}}
 {%- endif %}
 
-The GeoIP and ASN information is provided by the GeoLite2 database created by MaxMind.
+{{ extra.geo_attribution_html | default(value="Please configure the geo_attribution_html key in the template extra configuration.") | striptags }}
 {% endif -%}
 
 {%- block extra_content %}{% endblock -%}