Browse Source

Remove tracking tags from submitted URLs

unoptim
GreatBearShark 2 years ago
parent
commit
ead3c4d6bb
  1. 4
      package.json
  2. 171
      src/clean-url.ts
  3. 4
      src/components/main.tsx
  4. 6
      src/components/post-form.tsx
  5. 46
      yarn.lock

4
package.json

@ -40,7 +40,10 @@
"mobius1-selectr": "^2.4.13",
"moment": "^2.24.0",
"node-fetch": "^2.6.0",
"parse-domain": "^3.0.2",
"prettier": "^2.0.4",
"psl": "^1.8.0",
"qs": "^6.9.4",
"reconnecting-websocket": "^4.4.0",
"rxjs": "^6.5.5",
"terser": "^4.6.11",
@ -48,6 +51,7 @@
"toastify-js": "^1.7.0",
"tributejs": "^5.1.3",
"twemoji": "^12.1.2",
"url": "^0.11.0",
"ws": "^7.2.3"
},
"devDependencies": {

171
src/clean-url.ts

@ -0,0 +1,171 @@
import url from 'url';
// import psl from 'psl';
import { parseDomain, fromUrl } from 'parse-domain';
import qs from 'qs';
export function cleanURL(params) {
const oUrl = params.url;
const includeHash =
(typeof params.includeHash !== 'undefined' && params.includeHash) || false;
const urlObj = url.parse(oUrl, true);
delete urlObj.search;
// Remove tracking tokens.
delete urlObj.query.CNDID;
delete urlObj.query.__twitter_impression;
delete urlObj.query._hsenc;
delete urlObj.query._openstat;
delete urlObj.query.action_object_map;
delete urlObj.query.action_ref_map;
delete urlObj.query.action_type_map;
delete urlObj.query.amp;
delete urlObj.query.fb_action_ids;
delete urlObj.query.fb_action_types;
delete urlObj.query.fb_ref;
delete urlObj.query.fb_source;
delete urlObj.query.fbclid;
delete urlObj.query.ga_campaign;
delete urlObj.query.ga_content;
delete urlObj.query.ga_medium;
delete urlObj.query.ga_place;
delete urlObj.query.ga_source;
delete urlObj.query.ga_term;
delete urlObj.query.gs_l;
delete urlObj.query.hmb_campaign;
delete urlObj.query.hmb_medium;
delete urlObj.query.hmb_source;
delete urlObj.query.mbid;
delete urlObj.query.mc_cid;
delete urlObj.query.mc_eid;
delete urlObj.query.mkt_tok;
delete urlObj.query.referrer;
delete urlObj.query.spJobID;
delete urlObj.query.spMailingID;
delete urlObj.query.spReportId;
delete urlObj.query.spUserID;
delete urlObj.query.utm_brand;
delete urlObj.query.utm_campaign;
delete urlObj.query.utm_cid;
delete urlObj.query.utm_content;
delete urlObj.query.utm_int;
delete urlObj.query.utm_mailing;
delete urlObj.query.utm_medium;
delete urlObj.query.utm_name;
delete urlObj.query.utm_place;
delete urlObj.query.utm_pubreferrer;
delete urlObj.query.utm_reader;
delete urlObj.query.utm_social;
delete urlObj.query.utm_source;
delete urlObj.query.utm_swu;
delete urlObj.query.utm_term;
delete urlObj.query.utm_userid;
delete urlObj.query.utm_viz_id;
delete urlObj.query.wt_mc_o;
delete urlObj.query.yclid;
delete urlObj.query['WT.mc_id'];
delete urlObj.query['WT.mc_ev'];
delete urlObj.query['WT.srch'];
// Get domain without subdomain.
const result = parseDomain(fromUrl(urlObj.hostname));
console.log(result);
const domain: string = result.hostname.toString();
// const domain = parsed.domain;
// Remove domain specific tracking tokens.
if (domain.match(/^amazon\.(.*)$/)) {
delete urlObj.query._encoding;
delete urlObj.query.pd_rd_r;
delete urlObj.query.pd_rd_w;
delete urlObj.query.pd_rd_wg;
delete urlObj.query.psc;
delete urlObj.query.tag;
}
if (domain.match(/^google\.(.*)$/)) {
delete urlObj.query.ei;
delete urlObj.query.gws_rd;
delete urlObj.query.sei;
delete urlObj.query.ved;
}
if (domain.match(/^bing\.com$/)) {
delete urlObj.query.cvid;
delete urlObj.query.form;
delete urlObj.query.pq;
delete urlObj.query.qs;
delete urlObj.query.sc;
delete urlObj.query.sk;
delete urlObj.query.sp;
}
if (domain.match(/^youtube\.com$/)) {
delete urlObj.query.ab_channel;
delete urlObj.query.attr_tag;
delete urlObj.query.feature;
delete urlObj.query.gclid;
delete urlObj.query.kw;
}
if (domain.match(/^reddit\.com$/)) {
delete urlObj.query.st;
}
if (domain.match(/^twitter\.com$/)) {
delete urlObj.query.s;
delete urlObj.query.ref_src;
delete urlObj.query.ref_url;
}
if (domain.match(/^nytimes\.com$/)) {
delete urlObj.query.emc;
delete urlObj.query.partner;
}
if (domain.match(/^instagram\.com$/)) {
delete urlObj.query.igshid;
}
if (domain.match(/^amazon\.(com|ca|co\.jp|co\.uk|de|es|jp)$/)) {
delete urlObj.query.ie;
delete urlObj.query.creative;
delete urlObj.query.linkCode;
delete urlObj.query.creativeASIN;
delete urlObj.query.linkId;
// Check for /ref=xxx at the end of the pathname.
const match = urlObj.pathname.match(/\/ref=(\w*)$/);
if (match) {
const firstIndex = urlObj.pathname.indexOf(match[0]);
urlObj.pathname = urlObj.pathname.substr(0, firstIndex);
}
}
if (includeHash == true && typeof urlObj.hash === 'string') {
const hashQuery = qs.parse(urlObj.hash.substr(1));
// Remove keys from hash query.
delete hashQuery.Echobox;
urlObj.hash = `#${qs.stringify(hashQuery)}`;
}
let formatted = url.format(urlObj);
if (formatted.substr(-1) === '=' && oUrl.substr(-1) !== '=') {
formatted = formatted.substr(0, formatted.length - 1);
}
if (formatted.substr(-1) === '#' && oUrl.substr(-1) !== '#') {
formatted = formatted.substr(0, formatted.length - 1);
}
if (formatted.substr(-1) === '/' && oUrl.substr(-1) !== '/') {
formatted = formatted.substr(0, formatted.length - 1);
}
return formatted;
}

4
src/components/main.tsx

@ -319,6 +319,10 @@ export class Main extends Component<any, MainState> {
<div className="my-2">
It is currently {getMoscowTime()} in Moscow
</div>
<div className="my-2">
Please send any thoughts, suggestions, or complaints to Virgil
Texas
</div>
<ul class="my-2 list-inline">
{/*
<li className="list-inline-item badge badge-secondary">

6
src/components/post-form.tsx

@ -43,6 +43,7 @@ import Tribute from 'tributejs/src/Tribute.js';
import emojiShortName from 'emoji-short-name';
import Selectr from 'mobius1-selectr';
import { i18n } from '../i18next';
import { cleanURL } from '../clean-url';
const MAX_POST_TITLE_LENGTH = 200;
@ -419,6 +420,11 @@ export class PostForm extends Component<PostFormProps, PostFormState> {
i.state.postForm.url = undefined;
}
// remove trackers from URLs
const cleanedURL = cleanURL({ url: i.state.postForm.url });
i.state.postForm.url = cleanedURL;
if (i.props.post) {
WebSocketService.Instance.editPost(i.state.postForm);
} else {

46
yarn.lock

@ -3289,6 +3289,11 @@ [email protected]^2.1.0:
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=
[email protected]^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.1.0.tgz#5ad62f685a14edb421abebc2fff8db94df67b455"
integrity sha512-pKnZpbgCTfH/1NLIlOduP/V+WRXzC2MOz3Qo8xmxk8C5GudJLgK5QyLVXOSWy3ParAH7Eemurl3xjv/WXYFvMA==
[email protected]:
version "1.9.0"
resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.0.tgz#37df74e430a0e47550fe54a2defe30d8acd95f65"
@ -3444,6 +3449,13 @@ [email protected]^4.0.0, [email protected]^4.0.1:
dependencies:
is-extglob "^2.1.1"
[email protected]^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/is-ip/-/is-ip-3.1.0.tgz#2ae5ddfafaf05cb8008a62093cf29734f657c5d8"
integrity sha512-35vd5necO7IitFPjd/YBeqwWnyDWbuLH9ZXQdMfDA8TEo7pv5X8yfrvVO3xbJbLUlERCMvf6X0hTUamQxCYJ9Q==
dependencies:
ip-regex "^4.0.0"
[email protected]^2.0.2, [email protected]^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f"
@ -4905,6 +4917,15 @@ [email protected]^1.0.0:
dependencies:
callsites "^3.0.0"
[email protected]^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/parse-domain/-/parse-domain-3.0.2.tgz#ed13cf18c81d2bdeb6fbe6cf21ba8ea5c8f53818"
integrity sha512-eR48amwZBKL6VEdnqGTNILLBfYaOyDBh256Pjbxt+pqRYvMzG68raIJ/Q8bKrAjJJeRX84JGsoOmFW7/vY9KrQ==
dependencies:
is-ip "^3.1.0"
node-fetch "^2.6.0"
punycode "^2.1.1"
[email protected]^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c"
@ -5160,7 +5181,7 @@ [email protected]^1.1.24:
resolved "https://registry.yarnpkg.com/psl/-/psl-1.7.0.tgz#f1c4c47a8ef97167dea5d6bbf4816d736e884a3c"
integrity sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==
[email protected]^1.1.28:
[email protected]^1.1.28, [email protected]^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
@ -5173,6 +5194,11 @@ [email protected]^3.0.0:
end-of-stream "^1.1.0"
once "^1.3.1"
[email protected]:
version "1.3.2"
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
[email protected]^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
@ -5188,11 +5214,21 @@ [email protected]:
resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==
[email protected]^6.9.4:
version "6.9.4"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687"
integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==
[email protected]~6.5.2:
version "6.5.2"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
[email protected]:
version "0.2.0"
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
[email protected]^3.0.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed"
@ -6535,6 +6571,14 @@ [email protected]^0.1.0:
resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
[email protected]^0.11.0:
version "0.11.0"
resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=
dependencies:
punycode "1.3.2"
querystring "0.2.0"
[email protected]^3.1.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"

Loading…
Cancel
Save