v0.9.1: normalize X/Twitter GraphQL URLs for cache hit rate (issue #16)

User @barzamini pointed out an optimization from the Python community
(originally from seramo_ir): X/Twitter GraphQL URLs look like

  https://x.com/i/api/graphql/{hash}/{op}?variables=...&features=...&fieldToggles=...

The features and fieldToggles params change across sessions and even
within a session, busting our 50 MB response cache on every request to
the same logical query. Stripping everything after 'variables=' lets
identical logical queries collapse into one cache entry, dramatically
reducing quota usage when browsing Twitter through the relay.

Implementation:
  - src/domain_fronter.rs: new normalize_x_graphql_url() helper. Matches
    exactly the Python patch's pattern (host == 'x.com', path starts
    with /i/api/graphql/, query starts with variables=). Truncates at
    the first '&' past the '?'. Applied at the top of relay() so the
    normalized URL feeds BOTH the cache key AND the request sent to
    Apps Script — so we save on Apps Script quota too, not just on
    return-trip bytes.
  - src/config.rs: new opt-in normalize_x_graphql bool (default false).
    Off by default because strict X endpoints may reject trimmed requests;
    user should flip it on and watch for regressions.
  - src/bin/ui.rs: checkbox in the Advanced section,
    'Normalize X/Twitter GraphQL URLs', with tooltip explaining the
    trade-off and crediting the source.
  - Four new unit tests in domain_fronter::tests covering: the happy
    path trim, non-x.com hosts pass through unchanged, non-graphql x.com
    paths pass through unchanged, and idempotency. 48 tests total, all
    green.

Credit: idea by seramo_ir, Python patch at
https://gist.github.com/seramo/0ae9e5d30ac23a73d5eb3bd2710fcd67,
implementation request by @barzamini in issue #16.

Author: therealaleph

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mhrv-rs"
-version = "0.9.0"
+version = "0.9.1"
 edition = "2021"
 description = "Rust port of MasterHttpRelayVPN -- DPI bypass via Google Apps Script relay with domain fronting"
 license = "MIT"

Cargo.toml

@@ -172,7 +172,8 @@ struct FormState {
     fetch_ips_from_api: bool,
     max_ips_to_scan: usize,
     scan_batch_size:usize,
-    google_ip_validation: bool
+    google_ip_validation: bool,
+    normalize_x_graphql: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -247,7 +248,8 @@ fn load_form() -> (FormState, Option<String>) {
             fetch_ips_from_api:c.fetch_ips_from_api,
             max_ips_to_scan:c.max_ips_to_scan,
             google_ip_validation: c.google_ip_validation,
-            scan_batch_size:c.scan_batch_size
+            scan_batch_size:c.scan_batch_size,
+            normalize_x_graphql: c.normalize_x_graphql,
         }
     } else {
         FormState {
@@ -270,7 +272,8 @@ fn load_form() -> (FormState, Option<String>) {
             fetch_ips_from_api:false,
             max_ips_to_scan:100,
             google_ip_validation:true,
-            scan_batch_size:500
+            scan_batch_size:500,
+            normalize_x_graphql: false,
         }
     };
     (form, load_err)
@@ -396,7 +399,8 @@ impl FormState {
             fetch_ips_from_api:self.fetch_ips_from_api,
             max_ips_to_scan: self.max_ips_to_scan,
             google_ip_validation:self.google_ip_validation,
-            scan_batch_size:self.scan_batch_size
+            scan_batch_size:self.scan_batch_size,
+            normalize_x_graphql: self.normalize_x_graphql,
         })
     }
 }
@@ -433,6 +437,12 @@ struct ConfigWire<'a> {
     parallel_relay: u8,
     #[serde(skip_serializing_if = "Option::is_none")]
     sni_hosts: Option<Vec<&'a str>>,
+    #[serde(skip_serializing_if = "is_false")]
+    normalize_x_graphql: bool,
+}
+
+fn is_false(b: &bool) -> bool {
+    !*b
 }
 
 fn is_zero_u8(v: &u8) -> bool {
@@ -470,6 +480,7 @@ impl<'a> From<&'a Config> for ConfigWire<'a> {
                 .sni_hosts
                 .as_ref()
                 .map(|v| v.iter().map(String::as_str).collect()),
+            normalize_x_graphql: c.normalize_x_graphql,
         }
     }
 }
@@ -763,6 +774,16 @@ impl eframe::App for App {
                         ui.add_space(120.0 + 8.0);
                         ui.checkbox(&mut self.form.show_auth_key, "Show auth key");
                     });
+                    ui.horizontal(|ui| {
+                        ui.add_space(120.0 + 8.0);
+                        ui.checkbox(&mut self.form.normalize_x_graphql, "Normalize X/Twitter GraphQL URLs")
+                            .on_hover_text(
+                                "Trim the `features` / `fieldToggles` query params from x.com/i/api/graphql/… \
+                                 requests before relaying. Massively improves cache hit rate when browsing \
+                                 Twitter/X. Off by default — some endpoints may reject trimmed requests. \
+                                 Credit: seramo_ir + Persian Python community (issue #16).",
+                            );
+                    });
                 });
             });
 

src/bin/ui.rs

@@ -90,7 +90,21 @@ pub struct Config {
     pub scan_batch_size:usize,
 
     #[serde(default = "default_google_ip_validation")]
-    pub google_ip_validation: bool
+    pub google_ip_validation: bool,
+    /// When true, GET requests to `x.com/i/api/graphql/<hash>/<op>?variables=…`
+    /// have their query trimmed to just the `variables=` param before being
+    /// relayed. The `features` / `fieldToggles` params that X ships with
+    /// these requests change frequently and bust the response cache —
+    /// stripping them dramatically improves hit rate on Twitter/X browsing.
+    ///
+    /// Credit: idea from seramo_ir, originally adapted to the Python
+    /// MasterHttpRelayVPN by the Persian community
+    /// (https://gist.github.com/seramo/0ae9e5d30ac23a73d5eb3bd2710fcd67).
+    ///
+    /// Off by default — some X endpoints may reject calls that omit
+    /// features. Turn on and observe.
+    #[serde(default)]
+    pub normalize_x_graphql: bool,
 }
 
 fn default_fetch_ips_from_api() -> bool { false }

src/config.rs

@@ -77,6 +77,12 @@ pub struct DomainFronter {
     /// Fan-out factor: fire this many Apps Script instances in parallel
     /// per request and return first success. `<= 1` = off.
     parallel_relay: usize,
+    /// Enable the `normalize_x_graphql` URL rewrite (issue #16, credit
+    /// seramo_ir). When true, GETs to `x.com/i/api/graphql/<hash>/<op>`
+    /// have their query trimmed to the first `variables=` block so the
+    /// response cache isn't busted by the constantly-changing `features`
+    /// / `fieldToggles` params.
+    normalize_x_graphql: bool,
     tls_connector: TlsConnector,
     pool: Arc<Mutex<Vec<PoolEntry>>>,
     cache: Arc<ResponseCache>,
@@ -172,6 +178,7 @@ impl DomainFronter {
             http_host: "script.google.com",
             auth_key: config.auth_key.clone(),
             parallel_relay: config.parallel_relay as usize,
+            normalize_x_graphql: config.normalize_x_graphql,
             script_ids,
             script_idx: AtomicUsize::new(0),
             tls_connector,
@@ -388,6 +395,19 @@ impl DomainFronter {
         headers: &[(String, String)],
         body: &[u8],
     ) -> Vec<u8> {
+        // Optional URL rewrite for X/Twitter GraphQL (issue #16). Applied
+        // here, at the top of relay(), so it affects BOTH the cache key
+        // (so matching requests collapse into one entry) AND the URL that
+        // gets sent upstream to Apps Script (so Apps Script only has to
+        // fetch the trimmed variant, cutting quota usage).
+        let normalized;
+        let url: &str = if self.normalize_x_graphql {
+            normalized = normalize_x_graphql_url(url);
+            normalized.as_str()
+        } else {
+            url
+        };
+
         let coalescible = is_cacheable_method(method) && body.is_empty();
         let key = if coalescible { Some(cache_key(method, url)) } else { None };
         let t_start = Instant::now();
@@ -698,6 +718,58 @@ impl DomainFronter {
 /// strip Accept-Encoding: br (Apps Script can't decompress brotli).
 /// Extract the host (no scheme, no port, no path) from a URL string.
 /// Returns None for malformed / scheme-less inputs.
+/// Trim X/Twitter GraphQL URLs down to just the `variables=` query param,
+/// stripping everything from the first `&` in the query onward. See the
+/// `normalize_x_graphql` config field for the why.
+///
+/// Exact pattern mirrored from the Python community patch (issue #16):
+///
+///   host == "x.com"
+///   && path starts with "/i/api/graphql/"
+///   && query starts with "variables="
+///   → truncate at first `&` past the `?`.
+///
+/// Returns the possibly-rewritten URL. If the URL doesn't match the
+/// pattern the input is returned unchanged (as an owned String — the
+/// allocation is cheap on the slow path and keeps the caller's
+/// type-signature-juggling simple).
+fn normalize_x_graphql_url(url: &str) -> String {
+    // Split host from the rest. We accept both "x.com" and common legacy
+    // forms; the Python patch only checks x.com so we do the same to be
+    // safe about the endpoint actually accepting truncated requests.
+    let Some(rest) = url.strip_prefix("https://").or_else(|| url.strip_prefix("http://")) else {
+        return url.to_string();
+    };
+    let Some(slash) = rest.find('/') else {
+        return url.to_string();
+    };
+    let host = &rest[..slash];
+    let path_and_query = &rest[slash..];
+
+    // Strip port if present in host.
+    let host_no_port = host.split(':').next().unwrap_or(host);
+    if host_no_port != "x.com" {
+        return url.to_string();
+    }
+
+    let Some(q_idx) = path_and_query.find('?') else {
+        return url.to_string();
+    };
+    let path = &path_and_query[..q_idx];
+    let query = &path_and_query[q_idx + 1..];
+
+    if !path.starts_with("/i/api/graphql/") || !query.starts_with("variables=") {
+        return url.to_string();
+    }
+
+    let new_query = match query.find('&') {
+        Some(amp) => &query[..amp],
+        None => query,
+    };
+    let scheme = if url.starts_with("https://") { "https://" } else { "http://" };
+    format!("{}{}{}?{}", scheme, host, path, new_query)
+}
+
 fn extract_host(url: &str) -> Option<String> {
     let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url);
     let authority = after_scheme.split('/').next().unwrap_or("");
@@ -1230,6 +1302,53 @@ impl ServerCertVerifier for NoVerify {
 mod tests {
     use super::*;
 
+    #[test]
+    fn normalize_x_graphql_trims_after_variables() {
+        // Real-looking x.com GraphQL URL with variables + features +
+        // fieldToggles. Only the variables= prefix should survive.
+        let in_url = "https://x.com/i/api/graphql/abcd1234/TweetDetail?variables=%7B%22focalTweetId%22%3A%221234%22%7D&features=%7B%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%7D";
+        let out = normalize_x_graphql_url(in_url);
+        assert!(out.starts_with("https://x.com/i/api/graphql/abcd1234/TweetDetail?variables="));
+        assert!(!out.contains("features="));
+        assert!(!out.contains("fieldToggles="));
+        assert!(!out.contains('&'));
+    }
+
+    #[test]
+    fn normalize_x_graphql_leaves_non_x_hosts_alone() {
+        let cases = [
+            "https://twitter.com/i/api/graphql/x/y?variables=z&features=q",
+            "https://x.co/i/api/graphql/x/y?variables=z&features=q",
+            "https://api.x.com/i/api/graphql/x/y?variables=z&features=q",
+            "https://example.com/?variables=1&other=2",
+        ];
+        for u in cases {
+            assert_eq!(normalize_x_graphql_url(u), u, "should pass through: {}", u);
+        }
+    }
+
+    #[test]
+    fn normalize_x_graphql_leaves_non_graphql_paths_alone() {
+        let cases = [
+            "https://x.com/home",
+            "https://x.com/i/api/2/notifications/view/generic.json",
+            "https://x.com/i/api/graphql/x/y",       // no query
+            "https://x.com/i/api/graphql/x/y?features=1&variables=2", // variables not first
+        ];
+        for u in cases {
+            assert_eq!(normalize_x_graphql_url(u), u, "should pass through: {}", u);
+        }
+    }
+
+    #[test]
+    fn normalize_x_graphql_is_idempotent() {
+        let once = normalize_x_graphql_url(
+            "https://x.com/i/api/graphql/H/Op?variables=%7B%7D&features=%7B%7D",
+        );
+        let twice = normalize_x_graphql_url(&once);
+        assert_eq!(once, twice);
+    }
+
     #[test]
     fn extract_host_strips_scheme_port_path() {
         assert_eq!(extract_host("https://example.com/foo"), Some("example.com".into()));