Remove tracking tags from submitted URLs

2 years ago · ead3c4d6bb
5 changed files with 230 additions and 1 deletions
--- a/package.json
+++ b/package.json
@ -40,7 +40,10 @@
    "mobius1-selectr": "^2.4.13",
    "moment": "^2.24.0",
    "node-fetch": "^2.6.0",
+    "parse-domain": "^3.0.2",
    "prettier": "^2.0.4",
+    "psl": "^1.8.0",
+    "qs": "^6.9.4",
    "reconnecting-websocket": "^4.4.0",
    "rxjs": "^6.5.5",
    "terser": "^4.6.11",
@ -48,6 +51,7 @@
    "toastify-js": "^1.7.0",
    "tributejs": "^5.1.3",
    "twemoji": "^12.1.2",
+    "url": "^0.11.0",
    "ws": "^7.2.3"
  },
  "devDependencies": {
--- a/src/clean-url.ts
+++ b/src/clean-url.ts
@ -0,0 +1,171 @@
+import url from 'url';
+// import psl from 'psl';
+import { parseDomain, fromUrl } from 'parse-domain';
+import qs from 'qs';
+
+export function cleanURL(params) {
+  const oUrl = params.url;
+  const includeHash =
+    (typeof params.includeHash !== 'undefined' && params.includeHash) || false;
+
+  const urlObj = url.parse(oUrl, true);
+
+  delete urlObj.search;
+
+  // Remove tracking tokens.
+  delete urlObj.query.CNDID;
+  delete urlObj.query.__twitter_impression;
+  delete urlObj.query._hsenc;
+  delete urlObj.query._openstat;
+  delete urlObj.query.action_object_map;
+  delete urlObj.query.action_ref_map;
+  delete urlObj.query.action_type_map;
+  delete urlObj.query.amp;
+  delete urlObj.query.fb_action_ids;
+  delete urlObj.query.fb_action_types;
+  delete urlObj.query.fb_ref;
+  delete urlObj.query.fb_source;
+  delete urlObj.query.fbclid;
+  delete urlObj.query.ga_campaign;
+  delete urlObj.query.ga_content;
+  delete urlObj.query.ga_medium;
+  delete urlObj.query.ga_place;
+  delete urlObj.query.ga_source;
+  delete urlObj.query.ga_term;
+  delete urlObj.query.gs_l;
+  delete urlObj.query.hmb_campaign;
+  delete urlObj.query.hmb_medium;
+  delete urlObj.query.hmb_source;
+  delete urlObj.query.mbid;
+  delete urlObj.query.mc_cid;
+  delete urlObj.query.mc_eid;
+  delete urlObj.query.mkt_tok;
+  delete urlObj.query.referrer;
+  delete urlObj.query.spJobID;
+  delete urlObj.query.spMailingID;
+  delete urlObj.query.spReportId;
+  delete urlObj.query.spUserID;
+  delete urlObj.query.utm_brand;
+  delete urlObj.query.utm_campaign;
+  delete urlObj.query.utm_cid;
+  delete urlObj.query.utm_content;
+  delete urlObj.query.utm_int;
+  delete urlObj.query.utm_mailing;
+  delete urlObj.query.utm_medium;
+  delete urlObj.query.utm_name;
+  delete urlObj.query.utm_place;
+  delete urlObj.query.utm_pubreferrer;
+  delete urlObj.query.utm_reader;
+  delete urlObj.query.utm_social;
+  delete urlObj.query.utm_source;
+  delete urlObj.query.utm_swu;
+  delete urlObj.query.utm_term;
+  delete urlObj.query.utm_userid;
+  delete urlObj.query.utm_viz_id;
+  delete urlObj.query.wt_mc_o;
+  delete urlObj.query.yclid;
+  delete urlObj.query['WT.mc_id'];
+  delete urlObj.query['WT.mc_ev'];
+  delete urlObj.query['WT.srch'];
+
+  // Get domain without subdomain.
+  const result = parseDomain(fromUrl(urlObj.hostname));
+  console.log(result);
+
+  const domain: string = result.hostname.toString();
+  // const domain = parsed.domain;
+
+  // Remove domain specific tracking tokens.
+  if (domain.match(/^amazon\.(.*)$/)) {
+    delete urlObj.query._encoding;
+    delete urlObj.query.pd_rd_r;
+    delete urlObj.query.pd_rd_w;
+    delete urlObj.query.pd_rd_wg;
+    delete urlObj.query.psc;
+    delete urlObj.query.tag;
+  }
+
+  if (domain.match(/^google\.(.*)$/)) {
+    delete urlObj.query.ei;
+    delete urlObj.query.gws_rd;
+    delete urlObj.query.sei;
+    delete urlObj.query.ved;
+  }
+
+  if (domain.match(/^bing\.com$/)) {
+    delete urlObj.query.cvid;
+    delete urlObj.query.form;
+    delete urlObj.query.pq;
+    delete urlObj.query.qs;
+    delete urlObj.query.sc;
+    delete urlObj.query.sk;
+    delete urlObj.query.sp;
+  }
+
+  if (domain.match(/^youtube\.com$/)) {
+    delete urlObj.query.ab_channel;
+    delete urlObj.query.attr_tag;
+    delete urlObj.query.feature;
+    delete urlObj.query.gclid;
+    delete urlObj.query.kw;
+  }
+
+  if (domain.match(/^reddit\.com$/)) {
+    delete urlObj.query.st;
+  }
+
+  if (domain.match(/^twitter\.com$/)) {
+    delete urlObj.query.s;
+    delete urlObj.query.ref_src;
+    delete urlObj.query.ref_url;
+  }
+
+  if (domain.match(/^nytimes\.com$/)) {
+    delete urlObj.query.emc;
+    delete urlObj.query.partner;
+  }
+
+  if (domain.match(/^instagram\.com$/)) {
+    delete urlObj.query.igshid;
+  }
+
+  if (domain.match(/^amazon\.(com|ca|co\.jp|co\.uk|de|es|jp)$/)) {
+    delete urlObj.query.ie;
+    delete urlObj.query.creative;
+    delete urlObj.query.linkCode;
+    delete urlObj.query.creativeASIN;
+    delete urlObj.query.linkId;
+
+    // Check for /ref=xxx at the end of the pathname.
+    const match = urlObj.pathname.match(/\/ref=(\w*)$/);
+    if (match) {
+      const firstIndex = urlObj.pathname.indexOf(match[0]);
+      urlObj.pathname = urlObj.pathname.substr(0, firstIndex);
+    }
+  }
+
+  if (includeHash == true && typeof urlObj.hash === 'string') {
+    const hashQuery = qs.parse(urlObj.hash.substr(1));
+
+    // Remove keys from hash query.
+    delete hashQuery.Echobox;
+
+    urlObj.hash = `#${qs.stringify(hashQuery)}`;
+  }
+
+  let formatted = url.format(urlObj);
+
+  if (formatted.substr(-1) === '=' && oUrl.substr(-1) !== '=') {
+    formatted = formatted.substr(0, formatted.length - 1);
+  }
+
+  if (formatted.substr(-1) === '#' && oUrl.substr(-1) !== '#') {
+    formatted = formatted.substr(0, formatted.length - 1);
+  }
+
+  if (formatted.substr(-1) === '/' && oUrl.substr(-1) !== '/') {
+    formatted = formatted.substr(0, formatted.length - 1);
+  }
+
+  return formatted;
+}
--- a/src/components/main.tsx
+++ b/src/components/main.tsx
@ -319,6 +319,10 @@ export class Main extends Component<any, MainState> {
            <div className="my-2">
              It is currently {getMoscowTime()} in Moscow
            </div>
+            <div className="my-2">
+              Please send any thoughts, suggestions, or complaints to Virgil
+              Texas
+            </div>
            <ul class="my-2 list-inline">
              {/*
              <li className="list-inline-item badge badge-secondary">
--- a/src/components/post-form.tsx
+++ b/src/components/post-form.tsx
@ -43,6 +43,7 @@ import Tribute from 'tributejs/src/Tribute.js';
 import emojiShortName from 'emoji-short-name';
 import Selectr from 'mobius1-selectr';
 import { i18n } from '../i18next';
+import { cleanURL } from '../clean-url';

 const MAX_POST_TITLE_LENGTH = 200;

@ -419,6 +420,11 @@ export class PostForm extends Component<PostFormProps, PostFormState> {
      i.state.postForm.url = undefined;
    }

+    // remove trackers from URLs
+    const cleanedURL = cleanURL({ url: i.state.postForm.url });
+
+    i.state.postForm.url = cleanedURL;
+
    if (i.props.post) {
      WebSocketService.Instance.editPost(i.state.postForm);
    } else {
--- a/yarn.lock
+++ b/yarn.lock
@ -3289,6 +3289,11 @@ [email protected]^2.1.0:
  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
  integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=

+[email protected]^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.1.0.tgz#5ad62f685a14edb421abebc2fff8db94df67b455"
+  integrity sha512-pKnZpbgCTfH/1NLIlOduP/V+WRXzC2MOz3Qo8xmxk8C5GudJLgK5QyLVXOSWy3ParAH7Eemurl3xjv/WXYFvMA==
+
 [email protected]:
  version "1.9.0"
  resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.0.tgz#37df74e430a0e47550fe54a2defe30d8acd95f65"
@ -3444,6 +3449,13 @@ [email protected]^4.0.0, [email protected]^4.0.1:
  dependencies:
    is-extglob "^2.1.1"

+[email protected]^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/is-ip/-/is-ip-3.1.0.tgz#2ae5ddfafaf05cb8008a62093cf29734f657c5d8"
+  integrity sha512-35vd5necO7IitFPjd/YBeqwWnyDWbuLH9ZXQdMfDA8TEo7pv5X8yfrvVO3xbJbLUlERCMvf6X0hTUamQxCYJ9Q==
+  dependencies:
+    ip-regex "^4.0.0"
+
 [email protected]^2.0.2, [email protected]^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f"
@ -4905,6 +4917,15 @@ [email protected]^1.0.0:
  dependencies:
    callsites "^3.0.0"

+[email protected]^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/parse-domain/-/parse-domain-3.0.2.tgz#ed13cf18c81d2bdeb6fbe6cf21ba8ea5c8f53818"
+  integrity sha512-eR48amwZBKL6VEdnqGTNILLBfYaOyDBh256Pjbxt+pqRYvMzG68raIJ/Q8bKrAjJJeRX84JGsoOmFW7/vY9KrQ==
+  dependencies:
+    is-ip "^3.1.0"
+    node-fetch "^2.6.0"
+    punycode "^2.1.1"
+
 [email protected]^3.0.4:
  version "3.0.4"
  resolved "https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c"
@ -5160,7 +5181,7 @@ [email protected]^1.1.24:
  resolved "https://registry.yarnpkg.com/psl/-/psl-1.7.0.tgz#f1c4c47a8ef97167dea5d6bbf4816d736e884a3c"
  integrity sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==

-[email protected]^1.1.28:
+[email protected]^1.1.28, [email protected]^1.8.0:
  version "1.8.0"
  resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
  integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
@ -5173,6 +5194,11 @@ [email protected]^3.0.0:
    end-of-stream "^1.1.0"
    once "^1.3.1"

+[email protected]:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
+  integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
+
 [email protected]^1.4.1:
  version "1.4.1"
  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
@ -5188,11 +5214,21 @@ [email protected]:
  resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
  integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==

+[email protected]^6.9.4:
+  version "6.9.4"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687"
+  integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==
+
 [email protected]~6.5.2:
  version "6.5.2"
  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
  integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==

+[email protected]:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 [email protected]^3.0.0:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed"
@ -6535,6 +6571,14 @@ [email protected]^0.1.0:
  resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
  integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=

+[email protected]^0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
+  integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 [email protected]^3.1.0:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"