diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/assets/javascripts/services.js | 1702 | ||||
| -rw-r--r-- | src/config/config.json | 1220 | ||||
| -rw-r--r-- | src/instances/get_instances.py | 161 |
3 files changed, 1550 insertions, 1533 deletions
diff --git a/src/assets/javascripts/services.js b/src/assets/javascripts/services.js index d02e03f6..e67ed842 100644 --- a/src/assets/javascripts/services.js +++ b/src/assets/javascripts/services.js @@ -1,851 +1,851 @@ -window.browser = window.browser || window.chrome - -import utils from "./utils.js" - -let config, options, redirects, targets - -function init() { - return new Promise(async resolve => { - browser.storage.local.get(["options", "redirects", "targets"], r => { - options = r.options - redirects = r.redirects - targets = r.targets - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - config = JSON.parse(configData) - resolve() - }) - }) - }) -} - -init() -browser.storage.onChanged.addListener(init) - -function fetchFrontendInstanceList(service, frontend, redirects, options, config) { - let tmp = [] - if (config.services[service].frontends[frontend].instanceList) { - for (const network in config.networks) { - tmp.push(...redirects[network], ...options[frontend][network].custom) - } - } else if (config.services[service].frontends[frontend].singleInstance) tmp = config.services[service].frontends[frontend].singleInstance - return tmp -} - -function all(service, frontend, options, config, redirects) { - let instances = [] - if (!frontend) { - for (const frontend in config.services[service].frontends) { - instances.push(...fetchFrontendInstanceList(service, frontend, redirects[frontend], options, config)) - } - } else { - instances.push(...fetchFrontendInstanceList(service, frontend, redirects[frontend], options, config)) - } - return instances -} - -function regexArray(service, url, config) { - if (config.services[service].targets == "datajson") { - if (targets[service].includes(utils.protocolHost(url))) return true - } else { - const targetList = config.services[service].targets - for (const targetString in targetList) { - const target = new RegExp(targetList[targetString]) - if (target.test(url.href)) return true - } - } - return false -} - -function redirect(url, type, initiator, forceRedirection) { - if (type != "main_frame" && type != "sub_frame") return - let randomInstance - let frontend - for (const service in config.services) { - if (!forceRedirection && !options[service].enabled) continue - if (config.services[service].embeddable && type != options[service].redirectType && options[service].redirectType != "both") continue - if (!config.services[service].embeddable && type != "main_frame") continue - // let targets = new RegExp(config.services[service].targets.join("|"), "i") - - if (!regexArray(service, url, config)) continue - // if (initiator) { - // console.log(initiator.host) - // if (targets.test(initiator.host)) continue - // //if (all(service, null, options, config, redirects).includes(initiator.origin) && reverse(initiator) == url) return "BYPASSTAB" - // } - - if (Object.keys(config.services[service].frontends).length > 1) { - if (type == "sub_frame" && config.services[service].embeddable && !config.services[service].frontends[options[service].frontend].embeddable) frontend = options[service].embedFrontend - else frontend = options[service].frontend - } else frontend = Object.keys(config.services[service].frontends)[0] - - if (config.services[service].frontends[frontend].instanceList) { - let instanceList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] - if (instanceList.length === 0 && options.networkFallback) instanceList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] - if (instanceList.length === 0) return - randomInstance = utils.getRandomInstance(instanceList) - } else if (config.services[service].frontends[frontend].singleInstance) randomInstance = config.services[service].frontends[frontend].singleInstance - break - } - if (!frontend) return - - // Here is a (temperory) space for defining constants required in 2 or more switch cases. - // When possible, try have the two switch cases share all their code as done with searx and searxng. - // Do not do that when they do not share 100% of their code. - - const mapCentreRegex = /@(-?\d[0-9.]*),(-?\d[0-9.]*),(\d{1,2})[.z]/ - const dataLatLngRegex = /!3d(-?[0-9]{1,}.[0-9]{1,})!4d(-?[0-9]{1,}.[0-9]{1,})/ - const placeRegex = /\/place\/(.*)\// - function convertMapCentre() { - let [lat, lon, zoom] = [null, null, null] - if (url.pathname.match(mapCentreRegex)) { - // Set map centre if present - ;[lat, lon, zoom] = url.pathname.match(mapCentreRegex) - } else if (url.searchParams.has("center")) { - ;[lat, lon] = url.searchParams.get("center").split(",") - zoom = url.searchParams.get("zoom") ?? "17" - } - return [zoom, lon, lat] - } - - switch (frontend) { - // This is where all instance-specific code must be ran to convert the service url to one that can be understood by the frontend. - case "beatbump": - return `${randomInstance}${url.pathname}${url.search}` - .replace("/watch?v=", "/listen?id=") - .replace("/channel/", "/artist/") - .replace("/playlist?list=", "/playlist/VL") - .replace(/\/search\?q=.*/, searchQuery => searchQuery.replace("?q=", "/") + "?filter=all") - case "hyperpipe": - return `${randomInstance}${url.pathname}${url.search}`.replace(/\/search\?q=.*/, searchQuery => searchQuery.replace("?q=", "/")) - case "bibliogram": - const reservedPaths = ["u", "p", "privacy"] - if (url.pathname === "/" || reservedPaths.includes(url.pathname.split("/")[1])) return `${randomInstance}${url.pathname}${url.search}` - if (url.pathname.startsWith("/reel") || url.pathname.startsWith("/tv")) return `${randomInstance}/p${url.pathname.replace(/\/reel|\/tv/i, "")}${url.search}` - else return `${randomInstance}/u${url.pathname}${url.search}` // Likely a user profile, redirect to '/u/...' - case "lbryDesktop": - return url.href.replace(/^https?:\/{2}odysee\.com\//, "lbry://").replace(/:(?=[a-zA-Z0-9])/g, "#") - case "neuters": - if (url.pathname.startsWith("/article/") || url.pathname.startsWith("/pf/") || url.pathname.startsWith("/arc/") || url.pathname.startsWith("/resizer/")) return null - else if (url.pathname.endsWith("/")) return `${randomInstance}${url.pathname}` - else return `${randomInstance}${url.pathname}/` - case "searx": - case "searxng": - return `${randomInstance}/?q=${encodeURIComponent(url.searchParams.get("q"))}` - case "whoogle": - return `${randomInstance}/search?q=${encodeURIComponent(url.searchParams.get("q"))}` - case "librex": - return `${randomInstance}/search.php?q=${encodeURIComponent(url.searchParams.get("q"))}` - case "send": - return randomInstance - case "nitter": - let search = new URLSearchParams(url.search) - - search.delete("ref_src") - search.delete("ref_url") - - search = search.toString() - if (search !== "") search = `?${search}` - - if (url.host.split(".")[0] === "pbs" || url.host.split(".")[0] === "video") { - try { - const [, id, format, extra] = search.match(/(.*)\?format=(.*)&(.*)/) - const query = encodeURIComponent(`${id}.${format}?${extra}`) - return `${randomInstance}/pic${url.pathname}${query}` - } catch { - return `${randomInstance}/pic${url.pathname}${search}` - } - } - - if (url.pathname.split("/").includes("tweets")) return `${randomInstance}${url.pathname.replace("/tweets", "")}${search}` - if (url.host == "t.co") return `${randomInstance}/t.co${url.pathname}` - return `${randomInstance}${url.pathname}${search}` - case "yattee": - return url.href.replace(/^https?:\/{2}/, "yattee://") - case "freetube": - return `freetube://https://youtu.be${url.pathname}${url.search}`.replace(/watch\?v=/, "") - case "simplyTranslate": - return `${randomInstance}/${url.search}` - case "libreTranslate": - return `${randomInstance}/${url.search}` - .replace(/(?<=\/?)sl/, "source") - .replace(/(?<=&)tl/, "target") - .replace(/(?<=&)text/, "q") - case "osm": { - if (initiator && initiator.host === "earth.google.com") return - const travelModes = { - driving: "fossgis_osrm_car", - walking: "fossgis_osrm_foot", - bicycling: "fossgis_osrm_bike", - transit: "fossgis_osrm_car", // not implemented on OSM, default to car. - } - - function addressToLatLng(address) { - const xmlhttp = new XMLHttpRequest() - xmlhttp.open("GET", `https://nominatim.openstreetmap.org/search/${address}?format=json&limit=1`, false) - xmlhttp.send() - if (xmlhttp.status === 200) { - const json = JSON.parse(xmlhttp.responseText)[0] - if (json) { - console.log("json", json) - return [`${json.lat},${json.lon}`, `${json.boundingbox[2]},${json.boundingbox[1]},${json.boundingbox[3]},${json.boundingbox[0]}`] - } - } - console.info("Error: Status is " + xmlhttp.status) - } - - let mapCentre = "#" - let prefs = {} - - const mapCentreData = convertMapCentre() - if (mapCentreData[0] && mapCentreData[1] && mapCentreData[2]) mapCentre = `#map=${mapCentreData[0]}/${mapCentreData[1]}/${mapCentreData[2]}` - if (url.searchParams.get("layer")) prefs.layers = osmLayers[url.searchParams.get("layer")] - - if (url.pathname.includes("/embed")) { - // Handle Google Maps Embed API - // https://www.google.com/maps/embed/v1/place?key=AIzaSyD4iE2xVSpkLLOXoyqT-RuPwURN3ddScAI&q=Eiffel+Tower,Paris+France - //console.log("embed life") - - let query = "" - if (url.searchParams.has("q")) query = url.searchParams.get("q") - else if (url.searchParams.has("query")) query = url.searchParams.has("query") - else if (url.searchParams.has("pb")) - try { - query = url.searchParams.get("pb").split(/!2s(.*?)!/)[1] - } catch (error) { - console.error(error) - } // Unable to find map marker in URL. - - let [coords, boundingbox] = addressToLatLng(query) - prefs.bbox = boundingbox - prefs.marker = coords - prefs.layer = "mapnik" - let prefsEncoded = new URLSearchParams(prefs).toString() - return `${randomInstance}/export/embed.html?${prefsEncoded}` - } else if (url.pathname.includes("/dir")) { - // Handle Google Maps Directions - // https://www.google.com/maps/dir/?api=1&origin=Space+Needle+Seattle+WA&destination=Pike+Place+Market+Seattle+WA&travelmode=bicycling - - let travMod = url.searchParams.get("travelmode") - if (url.searchParams.has("travelmode")) prefs.engine = travelModes[travMod] - - let orgVal = url.searchParams.get("origin") - let destVal = url.searchParams.get("destination") - - let org = addressToLatLng(orgVal) - let dest = addressToLatLng(destVal) - prefs.route = `${org};${dest}` - - let prefsEncoded = new URLSearchParams(prefs).toString() - return `${randomInstance}/directions?${prefsEncoded}${mapCentre}` - } else if (url.pathname.includes("data=") && url.pathname.match(dataLatLngRegex)) { - // Get marker from data attribute - // https://www.google.com/maps/place/41%C2%B001'58.2%22N+40%C2%B029'18.2%22E/@41.032833,40.4862063,17z/data=!3m1!4b1!4m6!3m5!1s0x0:0xf64286eaf72fc49d!7e2!8m2!3d41.0328329!4d40.4883948 - //console.log("data life") - - let [, mlat, mlon] = url.pathname.match(dataLatLngRegex) - - return `${randomInstance}/search?query=${mlat}%2C${mlon}` - } else if (url.searchParams.has("ll")) { - // Get marker from ll param - // https://maps.google.com/?ll=38.882147,-76.99017 - //console.log("ll life") - - const [mlat, mlon] = url.searchParams.get("ll").split(",") - - return `${randomInstance}/search?query=${mlat}%2C${mlon}` - } else if (url.searchParams.has("viewpoint")) { - // Get marker from viewpoint param. - // https://www.google.com/maps/@?api=1&map_action=pano&viewpoint=48.857832,2.295226&heading=-45&pitch=38&fov=80 - //console.log("viewpoint life") - - const [mlat, mlon] = url.searchParams.get("viewpoint").split(",") - - return `${randomInstance}/search?query=${mlat}%2C${mlon}` - } else { - // Use query as search if present. - //console.log("normal life") - - let query - if (url.searchParams.has("q")) query = url.searchParams.get("q") - else if (url.searchParams.has("query")) query = url.searchParams.get("query") - else if (url.pathname.match(placeRegex)) query = url.pathname.match(placeRegex)[1] - - let prefsEncoded = new URLSearchParams(prefs).toString() - if (query) return `${randomInstance}/search?query="${query}${mapCentre}&${prefsEncoded}` - } - - let prefsEncoded = new URLSearchParams(prefs).toString() - console.log("mapCentre", mapCentre) - console.log("prefs", prefs) - console.log("prefsEncoded", prefsEncoded) - return `${randomInstance}/${mapCentre}&${prefsEncoded}` - } - case "facil": { - if (initiator && initiator.host === "earth.google.com") return - const travelModes = { - driving: "car", - walking: "pedestrian", - bicycling: "bicycle", - transit: "car", // not implemented on Facil, default to car. - } - const mapCentreData = convertMapCentre() - let mapCentre = "#" - if (mapCentreData[0] && mapCentreData[1] && mapCentreData[2]) mapCentre = `#${mapCentreData[0]}/${mapCentreData[1]}/${mapCentreData[2]}` - - if (url.pathname.includes("/embed")) { - // Handle Google Maps Embed API - // https://www.google.com/maps/embed/v1/place?key=AIzaSyD4iE2xVSpkLLOXoyqT-RuPwURN3ddScAI&q=Eiffel+Tower,Paris+France - //console.log("embed life") - - let query = "" - if (url.searchParams.has("q")) query = url.searchParams.get("q") - else if (url.searchParams.has("query")) query = url.searchParams.has("query") - else if (url.searchParams.has("pb")) - try { - query = url.searchParams.get("pb").split(/!2s(.*?)!/)[1] - } catch (error) { - console.error(error) - } // Unable to find map marker in URL. - - return `${randomInstance}/#q=${query}` - } else if (url.pathname.includes("/dir")) { - // Handle Google Maps Directions - // https://www.google.com/maps/dir/?api=1&origin=Space+Needle+Seattle+WA&destination=Pike+Place+Market+Seattle+WA&travelmode=bicycling - - let travMod = url.searchParams.get("travelmode") - - let orgVal = url.searchParams.get("origin") - let destVal = url.searchParams.get("destination") - - return `${randomInstance}/#q=${orgVal}%20to%20${destVal}%20by%20${travelModes[travMod]}` - } else if (url.pathname.includes("data=") && url.pathname.match(dataLatLngRegex)) { - // Get marker from data attribute - // https://www.google.com/maps/place/41%C2%B001'58.2%22N+40%C2%B029'18.2%22E/@41.032833,40.4862063,17z/data=!3m1!4b1!4m6!3m5!1s0x0:0xf64286eaf72fc49d!7e2!8m2!3d41.0328329!4d40.4883948 - //console.log("data life") - - let [, mlat, mlon] = url.pathname.match(dataLatLngRegex) - - return `${randomInstance}/#q=${mlat}%2C${mlon}` - } else if (url.searchParams.has("ll")) { - // Get marker from ll param - // https://maps.google.com/?ll=38.882147,-76.99017 - //console.log("ll life") - - const [mlat, mlon] = url.searchParams.get("ll").split(",") - - return `${randomInstance}/#q=${mlat}%2C${mlon}` - } else if (url.searchParams.has("viewpoint")) { - // Get marker from viewpoint param. - // https://www.google.com/maps/@?api=1&map_action=pano&viewpoint=48.857832,2.295226&heading=-45&pitch=38&fov=80 - //console.log("viewpoint life") - - const [mlat, mlon] = url.searchParams.get("viewpoint").split(",") - - return `${randomInstance}/#q=${mlat}%2C${mlon}` - } else { - // Use query as search if present. - //console.log("normal life") - - let query - if (url.searchParams.has("q")) query = url.searchParams.get("q") - else if (url.searchParams.has("query")) query = url.searchParams.get("query") - else if (url.pathname.match(placeRegex)) query = url.pathname.match(placeRegex)[1] - - if (query) return `${randomInstance}/${mapCentre}/Mpnk/${query}` - } - } - case "wikiless": - let GETArguments = [] - if (url.search.length > 0) { - let search = url.search.substring(1) //get rid of '?' - let argstrings = search.split("&") - for (let i = 0; i < argstrings.length; i++) { - let args = argstrings[i].split("=") - GETArguments.push([args[0], args[1]]) - } - } - - let link = `${randomInstance}${url.pathname}` - let urlSplit = url.host.split(".") - if (urlSplit[0] != "wikipedia" && urlSplit[0] != "www") { - if (urlSplit[0] == "m") GETArguments.push(["mobileaction", "toggle_view_mobile"]) - else GETArguments.push(["lang", urlSplit[0]]) - if (urlSplit[1] == "m") GETArguments.push(["mobileaction", "toggle_view_mobile"]) - // wikiless doesn't have mobile view support yet - } - for (let i = 0; i < GETArguments.length; i++) link += (i == 0 ? "?" : "&") + GETArguments[i][0] + "=" + GETArguments[i][1] - return link - - case "lingva": - let params_arr = url.search.split("&") - params_arr[0] = params_arr[0].substring(1) - let params = {} - for (let i = 0; i < params_arr.length; i++) { - let pair = params_arr[i].split("=") - params[pair[0]] = pair[1] - } - if (params.sl && params.tl && params.text) { - return `${randomInstance}/${params.sl}/${params.tl}/${params.text}` - } - return randomInstance - case "breezeWiki": - let wiki = url.hostname.match(/^[a-zA-Z0-9-]+(?=\.fandom\.com)/) - if (wiki == "www" || !wiki) wiki = "" - else wiki = "/" + wiki - if (url.href.search(/Special:Search\?query/) > -1) return `${randomInstance}${wiki}${url.pathname}${url.search}`.replace(/Special:Search\?query/, "search?q").replace(/\/wiki/, "") - else return `${randomInstance}${wiki}${url.pathname}${url.search}` - case "rimgo": - if (url.href.search(/^https?:\/{2}(?:[im]\.)?stack\./) > -1) return `${randomInstance}/stack${url.pathname}${url.search}` - else return `${randomInstance}${url.pathname}${url.search}` - case "libreddit": - const subdomain = url.hostname.match(/^(?:(?:external-)?preview|i)(?=\.redd\.it)/) - if (!subdomain) return `${randomInstance}${url.pathname}${url.search}` - switch (subdomain[0]) { - case "preview": - return `${randomInstance}/preview/pre${url.pathname}${url.search}` - case "external-preview": - return `${randomInstance}/preview/external-pre${url.pathname}${url.search}` - case "i": - return `${randomInstance}/img${url.pathname}` - } - case "teddit": - if (/^(?:(?:external-)?preview|i)\.redd\.it/.test(url.hostname)) { - if (url.search == "") return `${randomInstance}${url.pathname}?teddit_proxy=${url.hostname}` - else return `${randomInstance}${url.pathname}${url.search}&teddit_proxy=${url.hostname}` - } - return `${randomInstance}${url.pathname}${url.search}` - default: - return `${randomInstance}${url.pathname}${url.search}` - } -} - -function computeService(url, returnFrontend) { - return new Promise(resolve => { - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - const config = JSON.parse(configData) - browser.storage.local.get(["redirects", "options"], r => { - const redirects = r.redirects - const options = r.options - for (const service in config.services) { - if (regexArray(service, url, config)) { - resolve(service) - return - } else { - for (const frontend in config.services[service].frontends) { - if (all(service, frontend, options, config, redirects).includes(utils.protocolHost(url))) { - if (returnFrontend) resolve([service, frontend, utils.protocolHost(url)]) - else resolve(service) - return - } - } - } - } - resolve() - }) - }) - }) -} - -function switchInstance(url) { - return new Promise(async resolve => { - await init() - const protocolHost = utils.protocolHost(url) - for (const service in config.services) { - if (!all(service, null, options, config, redirects).includes(protocolHost)) continue - - let instancesList - if (Object.keys(config.services[service].frontends).length == 1) { - const frontend = Object.keys(config.services[service].frontends)[0] - instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] - if (instancesList.length === 0 && options.networkFallback) instancesList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] - } else { - const frontend = options[service].frontend - instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] - if (instancesList.length === 0 && options.networkFallback) instancesList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] - } - - let oldInstance - const i = instancesList.indexOf(protocolHost) - if (i > -1) { - oldInstance = instancesList[i] - instancesList.splice(i, 1) - } - if (instancesList.length === 0) { - resolve() - return - } - const randomInstance = utils.getRandomInstance(instancesList) - const oldUrl = `${oldInstance}${url.pathname}${url.search}` - // This is to make instance switching work when the instance depends on the pathname, eg https://darmarit.org/searx - // Doesn't work because of .includes array method, not a top priotiry atm - resolve(oldUrl.replace(oldInstance, randomInstance)) - return - } - resolve() - }) -} - -function reverse(url, urlString) { - return new Promise(async resolve => { - await init() - let protocolHost - if (!urlString) protocolHost = utils.protocolHost(url) - else protocolHost = url.match(/https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+/)[0] - for (const service in config.services) { - if (!all(service, null, options, config, redirects).includes(protocolHost)) continue - - switch (service) { - case "instagram": - case "youtube": - case "imdb": - case "imgur": - case "tiktok": - case "twitter": - case "reddit": - case "imdb": - case "reuters": - case "quora": - case "medium": - if (!urlString) resolve(config.services[service].url + url.pathname + url.search) - else resolve(url.replace(/https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+/, config.services[service].url)) - return - default: - resolve() - return - } - } - resolve() - }) -} - -function unifyPreferences(url, tabId) { - return new Promise(async resolve => { - await init() - const protocolHost = utils.protocolHost(url) - for (const service in config.services) { - for (const frontend in config.services[service].frontends) { - if (all(service, frontend, options, config, redirects).includes(protocolHost)) { - let instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] - if (options.networkFallback && options.network != "clearnet") instancesList.push(...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom) - - const frontendObject = config.services[service].frontends[frontend] - if ("cookies" in frontendObject.preferences) { - for (const cookie of frontendObject.preferences.cookies) { - await utils.copyCookie(url, instancesList, cookie) - } - } - if ("localstorage" in frontendObject.preferences) { - browser.storage.local.set({ tmp: [frontend, frontendObject.preferences.localstorage] }) - browser.tabs.executeScript(tabId, { - file: "/assets/javascripts/get-localstorage.js", - runAt: "document_start", - }) - for (const instance of instancesList) - browser.tabs.create({ url: instance }, tab => - browser.tabs.executeScript(tab.id, { - file: "/assets/javascripts/set-localstorage.js", - runAt: "document_start", - }) - ) - } - /* - if ("indexeddb" in frontendObject.preferences) { - } - if ("token" in frontendObject.preferences) { - } - */ - resolve(true) - return - } - } - } - }) -} - -function setRedirects(passedRedirects) { - return new Promise(resolve => { - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - browser.storage.local.get(/* [ */ "options" /* , "blacklists"] */, async r => { - let redirects = passedRedirects - let options = r.options - const config = JSON.parse(configData) - let targets = {} - for (const service in config.services) { - if (config.services[service].targets == "datajson") { - targets[service] = redirects[service] - delete redirects[service] - } - for (const frontend in config.services[service].frontends) { - if (config.services[service].frontends[frontend].instanceList) { - for (const network in config.networks) { - for (const instance of options[frontend][network].enabled) { - let i = redirects[frontend][network].indexOf(instance) - if (i < 0) options[frontend][network].enabled.splice(i, 1) - } - } - } - } - /* - for (const frontend in config.services[service].frontends) { - if (config.services[service].frontends[frontend].instanceList) { - for (const network in config.networks) { - options[frontend][network].enabled = redirects[frontend][network] - } - for (const blacklist in r.blacklists) { - for (const instance of blacklist) { - let i = options[frontend].clearnet.enabled.indexOf(instance) - if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) - } - } - } - } - */ - // The above will be implemented with https://github.com/libredirect/libredirect/issues/334 - } - for (const frontend in redirects) { - let exists = false - for (const service in config.services) if (config.services[service].frontends[frontend]) exists = true - if (!exists) delete redirects[frontend] - else for (const network in redirects[frontend]) if (!config.networks[network]) delete redirects[frontend][network] - } - browser.storage.local.set({ redirects, targets, options }, () => resolve()) - }) - }) - }) -} - -function initDefaults() { - return new Promise(resolve => { - fetch("/instances/data.json") - .then(response => response.text()) - .then(data => { - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - browser.storage.local.get(["options", "blacklists"], r => { - let redirects = JSON.parse(data) - let options = r.options - let targets = {} - let config = JSON.parse(configData) - const localstorage = {} - const latency = {} - for (const service in config.services) { - options[service] = {} - if (config.services[service].targets == "datajson") { - targets[service] = redirects[service] - delete redirects[service] - } - for (const defaultOption in config.services[service].options) options[service][defaultOption] = config.services[service].options[defaultOption] - for (const frontend in config.services[service].frontends) { - if (config.services[service].frontends[frontend].instanceList) { - options[frontend] = {} - for (const network in config.networks) { - options[frontend][network] = {} - options[frontend][network].enabled = JSON.parse(data)[frontend][network] - options[frontend][network].custom = [] - } - for (const blacklist in r.blacklists) { - for (const instance of r.blacklists[blacklist]) { - let i = options[frontend].clearnet.enabled.indexOf(instance) - if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) - } - } - } - } - } - browser.storage.local.set({ redirects, options, targets, latency, localstorage }) - resolve() - }) - }) - }) - }) -} - -function upgradeOptions() { - return new Promise(resolve => { - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - browser.storage.local.get(null, r => { - let options = r.options - let latency = {} - const config = JSON.parse(configData) - options.exceptions = r.exceptions - if (r.theme != "DEFAULT") options.theme = r.theme - options.popupServices = r.popupFrontends - let tmp = options.popupServices.indexOf("tikTok") - if (tmp > -1) { - options.popupServices.splice(tmp, 1) - options.popupServices.push("tiktok") - } - tmp = options.popupServices.indexOf("sendTarget") - if (tmp > -1) { - options.popupServices.splice(tmp, 1) - options.popupServices.push("sendFiles") - } - options.autoRedirect = r.autoRedirect - switch (r.onlyEmbeddedVideo) { - case "onlyNotEmbedded": - options.youtube.redirectType = "main_frame" - case "onlyEmbedded": - options.youtube.redirectType = "sub_frame" - case "both": - options.youtube.redirectType = "both" - } - for (const service in config.services) { - let oldService - switch (service) { - case "tiktok": - oldService = "tikTok" - break - case "sendFiles": - oldService = "sendTarget" - break - default: - oldService = service - } - options[service].enabled = !r["disable" + utils.camelCase(oldService)] - if (r[oldService + "Frontend"]) { - if (r[oldService + "Frontend"] == "yatte") options[service].frontend = "yattee" - else options[service].frontend = r[oldService + "Frontend"] - } - if (r[oldService + "RedirectType"]) options[service].redirectType = r[oldService + "RedirectType"] - if (r[oldService + "EmbedFrontend"] && (service != "youtube" || r[oldService + "EmbedFrontend"] == "invidious" || r[oldService + "EmbedFrontend"] == "piped")) - options[service].embedFrontend = r[oldService + "EmbedFrontend"] - for (const frontend in config.services[service].frontends) { - if (r[frontend + "Latency"]) latency[frontend] = r[frontend + "Latency"] - for (const network in config.networks) { - let protocol - if (network == "clearnet") protocol = "normal" - else protocol = network - if (r[frontend + utils.camelCase(protocol) + "RedirectsChecks"]) { - options[frontend][network].enabled = r[frontend + utils.camelCase(protocol) + "RedirectsChecks"] - options[frontend][network].custom = r[frontend + utils.camelCase(protocol) + "CustomRedirects"] - for (const instance of options[frontend][network].enabled) { - let i = r.redirects[frontend][network].indexOf(instance) - if (i < 0) options[frontend][network].enabled.splice(i, 1) - } - } - } - } - } - browser.storage.local.set({ options, latency }, () => resolve()) - }) - }) - }) -} - -function processUpdate() { - return new Promise(resolve => { - fetch("/instances/data.json") - .then(response => response.text()) - .then(data => { - fetch("/config/config.json") - .then(response => response.text()) - .then(configData => { - browser.storage.local.get(["options", "blacklists", "targets"], r => { - let redirects = JSON.parse(data) - let options = r.options - let targets = r.targets - let config = JSON.parse(configData) - for (const service in config.services) { - if (!options[service]) options[service] = {} - if (config.services[service].targets == "datajson") { - targets[service] = redirects[service] - delete redirects[service] - } - for (const defaultOption in config.services[service].options) { - if (options[service][defaultOption] === undefined) { - options[service][defaultOption] = config.services[service].options[defaultOption] - } - } - for (const frontend in config.services[service].frontends) { - if (config.services[service].frontends[frontend].instanceList) { - if (!options[frontend]) options[frontend] = {} - for (const network in config.networks) { - if (!options[frontend][network]) { - options[frontend][network] = {} - options[frontend][network].enabled = JSON.parse(data)[frontend][network] - options[frontend][network].custom = [] - if (network == "clearnet") { - for (const blacklist in r.blacklists) { - for (const instance of r.blacklists[blacklist]) { - let i = options[frontend].clearnet.enabled.indexOf(instance) - if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) - } - } - } - } else { - for (const instance of options[frontend][network].enabled) { - let i = redirects[frontend][network].indexOf(instance) - if (i < 0) options[frontend][network].enabled.splice(i, 1) - } - } - } - } - } - } - browser.storage.local.set({ redirects, options, targets }) - resolve() - }) - }) - }) - }) -} - -// For websites that have a strict policy that would not normally allow these frontends to be embedded within the website. -function modifyContentSecurityPolicy(details) { - let isChanged = false - if (details.type == "main_frame") { - for (const header in details.responseHeaders) { - if (details.responseHeaders[header].name == "content-security-policy") { - let instancesList = [] - for (const service in config.services) { - if (config.services[service].embeddable) { - for (const frontend in config.services[service].frontends) { - if (config.services[service].frontends[frontend].embeddable) { - for (const network in config.networks) { - instancesList.push(...options[frontend][network].enabled, ...options[frontend][network].custom) - } - } - } - } - } - let securityPolicyList = details.responseHeaders[header].value.split(";") - for (const i in securityPolicyList) securityPolicyList[i] = securityPolicyList[i].trim() - let newSecurity = "" - for (const item of securityPolicyList) { - if (item.trim() == "") continue - let regex = item.match(/([a-z-]{0,}) (.*)/) - if (regex == null) continue - let [, key, vals] = regex - if (key == "frame-src") vals = vals + " " + instancesList.join(" ") - newSecurity += key + " " + vals + "; " - } - - details.responseHeaders[header].value = newSecurity - isChanged = true - } - } - if (isChanged) return { responseHeaders: details.responseHeaders } - } -} - -export default { - redirect, - computeService, - switchInstance, - reverse, - unifyPreferences, - setRedirects, - initDefaults, - upgradeOptions, - processUpdate, - modifyContentSecurityPolicy, -} +window.browser = window.browser || window.chrome + +import utils from "./utils.js" + +let config, options, redirects, targets + +function init() { + return new Promise(async resolve => { + browser.storage.local.get(["options", "redirects", "targets"], r => { + options = r.options + redirects = r.redirects + targets = r.targets + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + config = JSON.parse(configData) + resolve() + }) + }) + }) +} + +init() +browser.storage.onChanged.addListener(init) + +function fetchFrontendInstanceList(service, frontend, redirects, options, config) { + let tmp = [] + if (config.services[service].frontends[frontend].instanceList) { + for (const network in config.networks) { + tmp.push(...redirects[network], ...options[frontend][network].custom) + } + } else if (config.services[service].frontends[frontend].singleInstance) tmp = config.services[service].frontends[frontend].singleInstance + return tmp +} + +function all(service, frontend, options, config, redirects) { + let instances = [] + if (!frontend) { + for (const frontend in config.services[service].frontends) { + instances.push(...fetchFrontendInstanceList(service, frontend, redirects[frontend], options, config)) + } + } else { + instances.push(...fetchFrontendInstanceList(service, frontend, redirects[frontend], options, config)) + } + return instances +} + +function regexArray(service, url, config) { + if (config.services[service].targets == "datajson") { + if (targets[service].includes(utils.protocolHost(url))) return true + } else { + const targetList = config.services[service].targets + for (const targetString in targetList) { + const target = new RegExp(targetList[targetString]) + if (target.test(url.href)) return true + } + } + return false +} + +function redirect(url, type, initiator, forceRedirection) { + if (type != "main_frame" && type != "sub_frame") return + let randomInstance + let frontend + for (const service in config.services) { + if (!forceRedirection && !options[service].enabled) continue + if (config.services[service].embeddable && type != options[service].redirectType && options[service].redirectType != "both") continue + if (!config.services[service].embeddable && type != "main_frame") continue + // let targets = new RegExp(config.services[service].targets.join("|"), "i") + + if (!regexArray(service, url, config)) continue + // if (initiator) { + // console.log(initiator.host) + // if (targets.test(initiator.host)) continue + // //if (all(service, null, options, config, redirects).includes(initiator.origin) && reverse(initiator) == url) return "BYPASSTAB" + // } + + if (Object.keys(config.services[service].frontends).length > 1) { + if (type == "sub_frame" && config.services[service].embeddable && !config.services[service].frontends[options[service].frontend].embeddable) frontend = options[service].embedFrontend + else frontend = options[service].frontend + } else frontend = Object.keys(config.services[service].frontends)[0] + + if (config.services[service].frontends[frontend].instanceList) { + let instanceList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] + if (instanceList.length === 0 && options.networkFallback) instanceList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] + if (instanceList.length === 0) return + randomInstance = utils.getRandomInstance(instanceList) + } else if (config.services[service].frontends[frontend].singleInstance) randomInstance = config.services[service].frontends[frontend].singleInstance + break + } + if (!frontend) return + + // Here is a (temperory) space for defining constants required in 2 or more switch cases. + // When possible, try have the two switch cases share all their code as done with searx and searxng. + // Do not do that when they do not share 100% of their code. + + const mapCentreRegex = /@(-?\d[0-9.]*),(-?\d[0-9.]*),(\d{1,2})[.z]/ + const dataLatLngRegex = /!3d(-?[0-9]{1,}.[0-9]{1,})!4d(-?[0-9]{1,}.[0-9]{1,})/ + const placeRegex = /\/place\/(.*)\// + function convertMapCentre() { + let [lat, lon, zoom] = [null, null, null] + if (url.pathname.match(mapCentreRegex)) { + // Set map centre if present + ;[lat, lon, zoom] = url.pathname.match(mapCentreRegex) + } else if (url.searchParams.has("center")) { + ;[lat, lon] = url.searchParams.get("center").split(",") + zoom = url.searchParams.get("zoom") ?? "17" + } + return [zoom, lon, lat] + } + + switch (frontend) { + // This is where all instance-specific code must be ran to convert the service url to one that can be understood by the frontend. + case "beatbump": + return `${randomInstance}${url.pathname}${url.search}` + .replace("/watch?v=", "/listen?id=") + .replace("/channel/", "/artist/") + .replace("/playlist?list=", "/playlist/VL") + .replace(/\/search\?q=.*/, searchQuery => searchQuery.replace("?q=", "/") + "?filter=all") + case "hyperpipe": + return `${randomInstance}${url.pathname}${url.search}`.replace(/\/search\?q=.*/, searchQuery => searchQuery.replace("?q=", "/")) + case "bibliogram": + const reservedPaths = ["u", "p", "privacy"] + if (url.pathname === "/" || reservedPaths.includes(url.pathname.split("/")[1])) return `${randomInstance}${url.pathname}${url.search}` + if (url.pathname.startsWith("/reel") || url.pathname.startsWith("/tv")) return `${randomInstance}/p${url.pathname.replace(/\/reel|\/tv/i, "")}${url.search}` + else return `${randomInstance}/u${url.pathname}${url.search}` // Likely a user profile, redirect to '/u/...' + case "lbryDesktop": + return url.href.replace(/^https?:\/{2}odysee\.com\//, "lbry://").replace(/:(?=[a-zA-Z0-9])/g, "#") + case "neuters": + if (url.pathname.startsWith("/article/") || url.pathname.startsWith("/pf/") || url.pathname.startsWith("/arc/") || url.pathname.startsWith("/resizer/")) return null + else if (url.pathname.endsWith("/")) return `${randomInstance}${url.pathname}` + else return `${randomInstance}${url.pathname}/` + case "searx": + case "searxng": + return `${randomInstance}/?q=${encodeURIComponent(url.searchParams.get("q"))}` + case "whoogle": + return `${randomInstance}/search?q=${encodeURIComponent(url.searchParams.get("q"))}` + case "librex": + return `${randomInstance}/search.php?q=${encodeURIComponent(url.searchParams.get("q"))}` + case "send": + return randomInstance + case "nitter": + let search = new URLSearchParams(url.search) + + search.delete("ref_src") + search.delete("ref_url") + + search = search.toString() + if (search !== "") search = `?${search}` + + if (url.host.split(".")[0] === "pbs" || url.host.split(".")[0] === "video") { + try { + const [, id, format, extra] = search.match(/(.*)\?format=(.*)&(.*)/) + const query = encodeURIComponent(`${id}.${format}?${extra}`) + return `${randomInstance}/pic${url.pathname}${query}` + } catch { + return `${randomInstance}/pic${url.pathname}${search}` + } + } + + if (url.pathname.split("/").includes("tweets")) return `${randomInstance}${url.pathname.replace("/tweets", "")}${search}` + if (url.host == "t.co") return `${randomInstance}/t.co${url.pathname}` + return `${randomInstance}${url.pathname}${search}` + case "yattee": + return url.href.replace(/^https?:\/{2}/, "yattee://") + case "freetube": + return `freetube://https://youtu.be${url.pathname}${url.search}`.replace(/watch\?v=/, "") + case "simplyTranslate": + return `${randomInstance}/${url.search}` + case "libreTranslate": + return `${randomInstance}/${url.search}` + .replace(/(?<=\/?)sl/, "source") + .replace(/(?<=&)tl/, "target") + .replace(/(?<=&)text/, "q") + case "osm": { + if (initiator && initiator.host === "earth.google.com") return + const travelModes = { + driving: "fossgis_osrm_car", + walking: "fossgis_osrm_foot", + bicycling: "fossgis_osrm_bike", + transit: "fossgis_osrm_car", // not implemented on OSM, default to car. + } + + function addressToLatLng(address) { + const xmlhttp = new XMLHttpRequest() + xmlhttp.open("GET", `https://nominatim.openstreetmap.org/search/${address}?format=json&limit=1`, false) + xmlhttp.send() + if (xmlhttp.status === 200) { + const json = JSON.parse(xmlhttp.responseText)[0] + if (json) { + console.log("json", json) + return [`${json.lat},${json.lon}`, `${json.boundingbox[2]},${json.boundingbox[1]},${json.boundingbox[3]},${json.boundingbox[0]}`] + } + } + console.info("Error: Status is " + xmlhttp.status) + } + + let mapCentre = "#" + let prefs = {} + + const mapCentreData = convertMapCentre() + if (mapCentreData[0] && mapCentreData[1] && mapCentreData[2]) mapCentre = `#map=${mapCentreData[0]}/${mapCentreData[1]}/${mapCentreData[2]}` + if (url.searchParams.get("layer")) prefs.layers = osmLayers[url.searchParams.get("layer")] + + if (url.pathname.includes("/embed")) { + // Handle Google Maps Embed API + // https://www.google.com/maps/embed/v1/place?key=AIzaSyD4iE2xVSpkLLOXoyqT-RuPwURN3ddScAI&q=Eiffel+Tower,Paris+France + //console.log("embed life") + + let query = "" + if (url.searchParams.has("q")) query = url.searchParams.get("q") + else if (url.searchParams.has("query")) query = url.searchParams.has("query") + else if (url.searchParams.has("pb")) + try { + query = url.searchParams.get("pb").split(/!2s(.*?)!/)[1] + } catch (error) { + console.error(error) + } // Unable to find map marker in URL. + + let [coords, boundingbox] = addressToLatLng(query) + prefs.bbox = boundingbox + prefs.marker = coords + prefs.layer = "mapnik" + let prefsEncoded = new URLSearchParams(prefs).toString() + return `${randomInstance}/export/embed.html?${prefsEncoded}` + } else if (url.pathname.includes("/dir")) { + // Handle Google Maps Directions + // https://www.google.com/maps/dir/?api=1&origin=Space+Needle+Seattle+WA&destination=Pike+Place+Market+Seattle+WA&travelmode=bicycling + + let travMod = url.searchParams.get("travelmode") + if (url.searchParams.has("travelmode")) prefs.engine = travelModes[travMod] + + let orgVal = url.searchParams.get("origin") + let destVal = url.searchParams.get("destination") + + let org = addressToLatLng(orgVal) + let dest = addressToLatLng(destVal) + prefs.route = `${org};${dest}` + + let prefsEncoded = new URLSearchParams(prefs).toString() + return `${randomInstance}/directions?${prefsEncoded}${mapCentre}` + } else if (url.pathname.includes("data=") && url.pathname.match(dataLatLngRegex)) { + // Get marker from data attribute + // https://www.google.com/maps/place/41%C2%B001'58.2%22N+40%C2%B029'18.2%22E/@41.032833,40.4862063,17z/data=!3m1!4b1!4m6!3m5!1s0x0:0xf64286eaf72fc49d!7e2!8m2!3d41.0328329!4d40.4883948 + //console.log("data life") + + let [, mlat, mlon] = url.pathname.match(dataLatLngRegex) + + return `${randomInstance}/search?query=${mlat}%2C${mlon}` + } else if (url.searchParams.has("ll")) { + // Get marker from ll param + // https://maps.google.com/?ll=38.882147,-76.99017 + //console.log("ll life") + + const [mlat, mlon] = url.searchParams.get("ll").split(",") + + return `${randomInstance}/search?query=${mlat}%2C${mlon}` + } else if (url.searchParams.has("viewpoint")) { + // Get marker from viewpoint param. + // https://www.google.com/maps/@?api=1&map_action=pano&viewpoint=48.857832,2.295226&heading=-45&pitch=38&fov=80 + //console.log("viewpoint life") + + const [mlat, mlon] = url.searchParams.get("viewpoint").split(",") + + return `${randomInstance}/search?query=${mlat}%2C${mlon}` + } else { + // Use query as search if present. + //console.log("normal life") + + let query + if (url.searchParams.has("q")) query = url.searchParams.get("q") + else if (url.searchParams.has("query")) query = url.searchParams.get("query") + else if (url.pathname.match(placeRegex)) query = url.pathname.match(placeRegex)[1] + + let prefsEncoded = new URLSearchParams(prefs).toString() + if (query) return `${randomInstance}/search?query="${query}${mapCentre}&${prefsEncoded}` + } + + let prefsEncoded = new URLSearchParams(prefs).toString() + console.log("mapCentre", mapCentre) + console.log("prefs", prefs) + console.log("prefsEncoded", prefsEncoded) + return `${randomInstance}/${mapCentre}&${prefsEncoded}` + } + case "facil": { + if (initiator && initiator.host === "earth.google.com") return + const travelModes = { + driving: "car", + walking: "pedestrian", + bicycling: "bicycle", + transit: "car", // not implemented on Facil, default to car. + } + const mapCentreData = convertMapCentre() + let mapCentre = "#" + if (mapCentreData[0] && mapCentreData[1] && mapCentreData[2]) mapCentre = `#${mapCentreData[0]}/${mapCentreData[1]}/${mapCentreData[2]}` + + if (url.pathname.includes("/embed")) { + // Handle Google Maps Embed API + // https://www.google.com/maps/embed/v1/place?key=AIzaSyD4iE2xVSpkLLOXoyqT-RuPwURN3ddScAI&q=Eiffel+Tower,Paris+France + //console.log("embed life") + + let query = "" + if (url.searchParams.has("q")) query = url.searchParams.get("q") + else if (url.searchParams.has("query")) query = url.searchParams.has("query") + else if (url.searchParams.has("pb")) + try { + query = url.searchParams.get("pb").split(/!2s(.*?)!/)[1] + } catch (error) { + console.error(error) + } // Unable to find map marker in URL. + + return `${randomInstance}/#q=${query}` + } else if (url.pathname.includes("/dir")) { + // Handle Google Maps Directions + // https://www.google.com/maps/dir/?api=1&origin=Space+Needle+Seattle+WA&destination=Pike+Place+Market+Seattle+WA&travelmode=bicycling + + let travMod = url.searchParams.get("travelmode") + + let orgVal = url.searchParams.get("origin") + let destVal = url.searchParams.get("destination") + + return `${randomInstance}/#q=${orgVal}%20to%20${destVal}%20by%20${travelModes[travMod]}` + } else if (url.pathname.includes("data=") && url.pathname.match(dataLatLngRegex)) { + // Get marker from data attribute + // https://www.google.com/maps/place/41%C2%B001'58.2%22N+40%C2%B029'18.2%22E/@41.032833,40.4862063,17z/data=!3m1!4b1!4m6!3m5!1s0x0:0xf64286eaf72fc49d!7e2!8m2!3d41.0328329!4d40.4883948 + //console.log("data life") + + let [, mlat, mlon] = url.pathname.match(dataLatLngRegex) + + return `${randomInstance}/#q=${mlat}%2C${mlon}` + } else if (url.searchParams.has("ll")) { + // Get marker from ll param + // https://maps.google.com/?ll=38.882147,-76.99017 + //console.log("ll life") + + const [mlat, mlon] = url.searchParams.get("ll").split(",") + + return `${randomInstance}/#q=${mlat}%2C${mlon}` + } else if (url.searchParams.has("viewpoint")) { + // Get marker from viewpoint param. + // https://www.google.com/maps/@?api=1&map_action=pano&viewpoint=48.857832,2.295226&heading=-45&pitch=38&fov=80 + //console.log("viewpoint life") + + const [mlat, mlon] = url.searchParams.get("viewpoint").split(",") + + return `${randomInstance}/#q=${mlat}%2C${mlon}` + } else { + // Use query as search if present. + //console.log("normal life") + + let query + if (url.searchParams.has("q")) query = url.searchParams.get("q") + else if (url.searchParams.has("query")) query = url.searchParams.get("query") + else if (url.pathname.match(placeRegex)) query = url.pathname.match(placeRegex)[1] + + if (query) return `${randomInstance}/${mapCentre}/Mpnk/${query}` + } + } + case "wikiless": + let GETArguments = [] + if (url.search.length > 0) { + let search = url.search.substring(1) //get rid of '?' + let argstrings = search.split("&") + for (let i = 0; i < argstrings.length; i++) { + let args = argstrings[i].split("=") + GETArguments.push([args[0], args[1]]) + } + } + + let link = `${randomInstance}${url.pathname}` + let urlSplit = url.host.split(".") + if (urlSplit[0] != "wikipedia" && urlSplit[0] != "www") { + if (urlSplit[0] == "m") GETArguments.push(["mobileaction", "toggle_view_mobile"]) + else GETArguments.push(["lang", urlSplit[0]]) + if (urlSplit[1] == "m") GETArguments.push(["mobileaction", "toggle_view_mobile"]) + // wikiless doesn't have mobile view support yet + } + for (let i = 0; i < GETArguments.length; i++) link += (i == 0 ? "?" : "&") + GETArguments[i][0] + "=" + GETArguments[i][1] + return link + + case "lingva": + let params_arr = url.search.split("&") + params_arr[0] = params_arr[0].substring(1) + let params = {} + for (let i = 0; i < params_arr.length; i++) { + let pair = params_arr[i].split("=") + params[pair[0]] = pair[1] + } + if (params.sl && params.tl && params.text) { + return `${randomInstance}/${params.sl}/${params.tl}/${params.text}` + } + return randomInstance + case "breezeWiki": + let wiki = url.hostname.match(/^[a-zA-Z0-9-]+(?=\.fandom\.com)/) + if (wiki == "www" || !wiki) wiki = "" + else wiki = "/" + wiki + if (url.href.search(/Special:Search\?query/) > -1) return `${randomInstance}${wiki}${url.pathname}${url.search}`.replace(/Special:Search\?query/, "search?q").replace(/\/wiki/, "") + else return `${randomInstance}${wiki}${url.pathname}${url.search}` + case "rimgo": + if (url.href.search(/^https?:\/{2}(?:[im]\.)?stack\./) > -1) return `${randomInstance}/stack${url.pathname}${url.search}` + else return `${randomInstance}${url.pathname}${url.search}` + case "libreddit": + const subdomain = url.hostname.match(/^(?:(?:external-)?preview|i)(?=\.redd\.it)/) + if (!subdomain) return `${randomInstance}${url.pathname}${url.search}` + switch (subdomain[0]) { + case "preview": + return `${randomInstance}/preview/pre${url.pathname}${url.search}` + case "external-preview": + return `${randomInstance}/preview/external-pre${url.pathname}${url.search}` + case "i": + return `${randomInstance}/img${url.pathname}` + } + case "teddit": + if (/^(?:(?:external-)?preview|i)\.redd\.it/.test(url.hostname)) { + if (url.search == "") return `${randomInstance}${url.pathname}?teddit_proxy=${url.hostname}` + else return `${randomInstance}${url.pathname}${url.search}&teddit_proxy=${url.hostname}` + } + return `${randomInstance}${url.pathname}${url.search}` + default: + return `${randomInstance}${url.pathname}${url.search}` + } +} + +function computeService(url, returnFrontend) { + return new Promise(resolve => { + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + const config = JSON.parse(configData) + browser.storage.local.get(["redirects", "options"], r => { + const redirects = r.redirects + const options = r.options + for (const service in config.services) { + if (regexArray(service, url, config)) { + resolve(service) + return + } else { + for (const frontend in config.services[service].frontends) { + if (all(service, frontend, options, config, redirects).includes(utils.protocolHost(url))) { + if (returnFrontend) resolve([service, frontend, utils.protocolHost(url)]) + else resolve(service) + return + } + } + } + } + resolve() + }) + }) + }) +} + +function switchInstance(url) { + return new Promise(async resolve => { + await init() + const protocolHost = utils.protocolHost(url) + for (const service in config.services) { + if (!all(service, null, options, config, redirects).includes(protocolHost)) continue + + let instancesList + if (Object.keys(config.services[service].frontends).length == 1) { + const frontend = Object.keys(config.services[service].frontends)[0] + instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] + if (instancesList.length === 0 && options.networkFallback) instancesList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] + } else { + const frontend = options[service].frontend + instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] + if (instancesList.length === 0 && options.networkFallback) instancesList = [...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom] + } + + let oldInstance + const i = instancesList.indexOf(protocolHost) + if (i > -1) { + oldInstance = instancesList[i] + instancesList.splice(i, 1) + } + if (instancesList.length === 0) { + resolve() + return + } + const randomInstance = utils.getRandomInstance(instancesList) + const oldUrl = `${oldInstance}${url.pathname}${url.search}` + // This is to make instance switching work when the instance depends on the pathname, eg https://darmarit.org/searx + // Doesn't work because of .includes array method, not a top priotiry atm + resolve(oldUrl.replace(oldInstance, randomInstance)) + return + } + resolve() + }) +} + +function reverse(url, urlString) { + return new Promise(async resolve => { + await init() + let protocolHost + if (!urlString) protocolHost = utils.protocolHost(url) + else protocolHost = url.match(/https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+/)[0] + for (const service in config.services) { + if (!all(service, null, options, config, redirects).includes(protocolHost)) continue + + switch (service) { + case "instagram": + case "youtube": + case "imdb": + case "imgur": + case "tiktok": + case "twitter": + case "reddit": + case "imdb": + case "reuters": + case "quora": + case "medium": + if (!urlString) resolve(config.services[service].url + url.pathname + url.search) + else resolve(url.replace(/https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+/, config.services[service].url)) + return + default: + resolve() + return + } + } + resolve() + }) +} + +function unifyPreferences(url, tabId) { + return new Promise(async resolve => { + await init() + const protocolHost = utils.protocolHost(url) + for (const service in config.services) { + for (const frontend in config.services[service].frontends) { + if (all(service, frontend, options, config, redirects).includes(protocolHost)) { + let instancesList = [...options[frontend][options.network].enabled, ...options[frontend][options.network].custom] + if (options.networkFallback && options.network != "clearnet") instancesList.push(...options[frontend].clearnet.enabled, ...options[frontend].clearnet.custom) + + const frontendObject = config.services[service].frontends[frontend] + if ("cookies" in frontendObject.preferences) { + for (const cookie of frontendObject.preferences.cookies) { + await utils.copyCookie(url, instancesList, cookie) + } + } + if ("localstorage" in frontendObject.preferences) { + browser.storage.local.set({ tmp: [frontend, frontendObject.preferences.localstorage] }) + browser.tabs.executeScript(tabId, { + file: "/assets/javascripts/get-localstorage.js", + runAt: "document_start", + }) + for (const instance of instancesList) + browser.tabs.create({ url: instance }, tab => + browser.tabs.executeScript(tab.id, { + file: "/assets/javascripts/set-localstorage.js", + runAt: "document_start", + }) + ) + } + /* + if ("indexeddb" in frontendObject.preferences) { + } + if ("token" in frontendObject.preferences) { + } + */ + resolve(true) + return + } + } + } + }) +} + +function setRedirects(passedRedirects) { + return new Promise(resolve => { + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + browser.storage.local.get(/* [ */ "options" /* , "blacklists"] */, async r => { + let redirects = passedRedirects + let options = r.options + const config = JSON.parse(configData) + let targets = {} + for (const service in config.services) { + if (config.services[service].targets == "datajson") { + targets[service] = redirects[service] + delete redirects[service] + } + for (const frontend in config.services[service].frontends) { + if (config.services[service].frontends[frontend].instanceList) { + for (const network in config.networks) { + for (const instance of options[frontend][network].enabled) { + let i = redirects[frontend][network].indexOf(instance) + if (i < 0) options[frontend][network].enabled.splice(i, 1) + } + } + } + } + /* + for (const frontend in config.services[service].frontends) { + if (config.services[service].frontends[frontend].instanceList) { + for (const network in config.networks) { + options[frontend][network].enabled = redirects[frontend][network] + } + for (const blacklist in r.blacklists) { + for (const instance of blacklist) { + let i = options[frontend].clearnet.enabled.indexOf(instance) + if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) + } + } + } + } + */ + // The above will be implemented with https://github.com/libredirect/libredirect/issues/334 + } + for (const frontend in redirects) { + let exists = false + for (const service in config.services) if (config.services[service].frontends[frontend]) exists = true + if (!exists) delete redirects[frontend] + else for (const network in redirects[frontend]) if (!config.networks[network]) delete redirects[frontend][network] + } + browser.storage.local.set({ redirects, targets, options }, () => resolve()) + }) + }) + }) +} + +function initDefaults() { + return new Promise(resolve => { + fetch("/instances/data.json") + .then(response => response.text()) + .then(data => { + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + browser.storage.local.get(["options", "blacklists"], r => { + let redirects = JSON.parse(data) + let options = r.options + let targets = {} + let config = JSON.parse(configData) + const localstorage = {} + const latency = {} + for (const service in config.services) { + options[service] = {} + if (config.services[service].targets == "datajson") { + targets[service] = redirects[service] + delete redirects[service] + } + for (const defaultOption in config.services[service].options) options[service][defaultOption] = config.services[service].options[defaultOption] + for (const frontend in config.services[service].frontends) { + if (config.services[service].frontends[frontend].instanceList) { + options[frontend] = {} + for (const network in config.networks) { + options[frontend][network] = {} + options[frontend][network].enabled = JSON.parse(data)[frontend][network] + options[frontend][network].custom = [] + } + for (const blacklist in r.blacklists) { + for (const instance of r.blacklists[blacklist]) { + let i = options[frontend].clearnet.enabled.indexOf(instance) + if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) + } + } + } + } + } + browser.storage.local.set({ redirects, options, targets, latency, localstorage }) + resolve() + }) + }) + }) + }) +} + +function upgradeOptions() { + return new Promise(resolve => { + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + browser.storage.local.get(null, r => { + let options = r.options + let latency = {} + const config = JSON.parse(configData) + options.exceptions = r.exceptions + if (r.theme != "DEFAULT") options.theme = r.theme + options.popupServices = r.popupFrontends + let tmp = options.popupServices.indexOf("tikTok") + if (tmp > -1) { + options.popupServices.splice(tmp, 1) + options.popupServices.push("tiktok") + } + tmp = options.popupServices.indexOf("sendTarget") + if (tmp > -1) { + options.popupServices.splice(tmp, 1) + options.popupServices.push("sendFiles") + } + options.autoRedirect = r.autoRedirect + switch (r.onlyEmbeddedVideo) { + case "onlyNotEmbedded": + options.youtube.redirectType = "main_frame" + case "onlyEmbedded": + options.youtube.redirectType = "sub_frame" + case "both": + options.youtube.redirectType = "both" + } + for (const service in config.services) { + let oldService + switch (service) { + case "tiktok": + oldService = "tikTok" + break + case "sendFiles": + oldService = "sendTarget" + break + default: + oldService = service + } + options[service].enabled = !r["disable" + utils.camelCase(oldService)] + if (r[oldService + "Frontend"]) { + if (r[oldService + "Frontend"] == "yatte") options[service].frontend = "yattee" + else options[service].frontend = r[oldService + "Frontend"] + } + if (r[oldService + "RedirectType"]) options[service].redirectType = r[oldService + "RedirectType"] + if (r[oldService + "EmbedFrontend"] && (service != "youtube" || r[oldService + "EmbedFrontend"] == "invidious" || r[oldService + "EmbedFrontend"] == "piped")) + options[service].embedFrontend = r[oldService + "EmbedFrontend"] + for (const frontend in config.services[service].frontends) { + if (r[frontend + "Latency"]) latency[frontend] = r[frontend + "Latency"] + for (const network in config.networks) { + let protocol + if (network == "clearnet") protocol = "normal" + else protocol = network + if (r[frontend + utils.camelCase(protocol) + "RedirectsChecks"]) { + options[frontend][network].enabled = r[frontend + utils.camelCase(protocol) + "RedirectsChecks"] + options[frontend][network].custom = r[frontend + utils.camelCase(protocol) + "CustomRedirects"] + for (const instance of options[frontend][network].enabled) { + let i = r.redirects[frontend][network].indexOf(instance) + if (i < 0) options[frontend][network].enabled.splice(i, 1) + } + } + } + } + } + browser.storage.local.set({ options, latency }, () => resolve()) + }) + }) + }) +} + +function processUpdate() { + return new Promise(resolve => { + fetch("/instances/data.json") + .then(response => response.text()) + .then(data => { + fetch("/config/config.json") + .then(response => response.text()) + .then(configData => { + browser.storage.local.get(["options", "blacklists", "targets"], r => { + let redirects = JSON.parse(data) + let options = r.options + let targets = r.targets + let config = JSON.parse(configData) + for (const service in config.services) { + if (!options[service]) options[service] = {} + if (config.services[service].targets == "datajson") { + targets[service] = redirects[service] + delete redirects[service] + } + for (const defaultOption in config.services[service].options) { + if (options[service][defaultOption] === undefined) { + options[service][defaultOption] = config.services[service].options[defaultOption] + } + } + for (const frontend in config.services[service].frontends) { + if (config.services[service].frontends[frontend].instanceList) { + if (!options[frontend]) options[frontend] = {} + for (const network in config.networks) { + if (!options[frontend][network]) { + options[frontend][network] = {} + options[frontend][network].enabled = JSON.parse(data)[frontend][network] + options[frontend][network].custom = [] + if (network == "clearnet") { + for (const blacklist in r.blacklists) { + for (const instance of r.blacklists[blacklist]) { + let i = options[frontend].clearnet.enabled.indexOf(instance) + if (i > -1) options[frontend].clearnet.enabled.splice(i, 1) + } + } + } + } else { + for (const instance of options[frontend][network].enabled) { + let i = redirects[frontend][network].indexOf(instance) + if (i < 0) options[frontend][network].enabled.splice(i, 1) + } + } + } + } + } + } + browser.storage.local.set({ redirects, options, targets }) + resolve() + }) + }) + }) + }) +} + +// For websites that have a strict policy that would not normally allow these frontends to be embedded within the website. +function modifyContentSecurityPolicy(details) { + let isChanged = false + if (details.type == "main_frame") { + for (const header in details.responseHeaders) { + if (details.responseHeaders[header].name == "content-security-policy") { + let instancesList = [] + for (const service in config.services) { + if (config.services[service].embeddable) { + for (const frontend in config.services[service].frontends) { + if (config.services[service].frontends[frontend].embeddable) { + for (const network in config.networks) { + instancesList.push(...options[frontend][network].enabled, ...options[frontend][network].custom) + } + } + } + } + } + let securityPolicyList = details.responseHeaders[header].value.split(";") + for (const i in securityPolicyList) securityPolicyList[i] = securityPolicyList[i].trim() + let newSecurity = "" + for (const item of securityPolicyList) { + if (item.trim() == "") continue + let regex = item.match(/([a-z-]{0,}) (.*)/) + if (regex == null) continue + let [, key, vals] = regex + if (key == "frame-src") vals = vals + " " + instancesList.join(" ") + newSecurity += key + " " + vals + "; " + } + + details.responseHeaders[header].value = newSecurity + isChanged = true + } + } + if (isChanged) return { responseHeaders: details.responseHeaders } + } +} + +export default { + redirect, + computeService, + switchInstance, + reverse, + unifyPreferences, + setRedirects, + initDefaults, + upgradeOptions, + processUpdate, + modifyContentSecurityPolicy, +} diff --git a/src/config/config.json b/src/config/config.json index ef598ea8..b1cbd036 100644 --- a/src/config/config.json +++ b/src/config/config.json @@ -1,610 +1,610 @@ -{ - "networks": { - "clearnet": { - "tld": "org", - "name": "Clearnet" - }, - "tor": { - "tld": "onion", - "name": "Tor" - }, - "i2p": { - "tld": "i2p", - "name": "I2P" - }, - "loki": { - "tld": "loki", - "name": "Lokinet" - } - }, - "services": { - "youtube": { - "frontends": { - "invidious": { - "preferences": { - "cookies": ["PREFS"], - "localstorage": ["dark_mode"] - }, - "name": "Invidious", - "embeddable": true, - "instanceList": true - }, - "piped": { - "preferences": { - "localstorage": [ - "bufferGoal", - "comments", - "disableLBRY", - "enabledCodecs", - "hl", - "homepage", - "instance", - "listen", - "minimizeDescription", - "playerAutoPlay", - "proxyLBRY", - "quality", - "region", - "selectedSkip", - "sponsorblock", - "theme", - "volume", - "watchHistory", - "localSubscriptions" - ] - }, - "name": "Piped", - "embeddable": true, - "instanceList": true - }, - "pipedMaterial": { - "preferences": { - "localstorage": ["PREFERENCES"] - }, - "name": "Piped-Material", - "embeddable": false, - "instanceList": true - }, - "cloudtube": { - "preferences": { - "token": "token", - "fetchEndpoint": "/api/settings", - "setEndpoint": "/settings" - }, - "name": "CloudTube", - "embeddable": false, - "instanceList": true - }, - "freetube": { - "name": "FreeTube", - "embeddable": false, - "instanceList": false - }, - "yattee": { - "name": "Yattee", - "embeddable": false, - "instanceList": false - } - }, - "targets": [ - "^https?:\\/{2}(?:www\\.|m\\.|)youtube.com(\\/|$)(?!iframe_api\\/|redirect\\/)", - "^https?:\\/{2}img\\.youtube.com\\/vi\\/.*\\/..*", - "^https?:\\/{2}(?:i|s)\\.ytimg.com\\/vi\\/.*\\/..*", - "^https?:\\/{2}(?:www\\.|)youtube.com\\/watch?v=..*", - "^https?:\\/{2}youtu\\.be\\/..*", - "^https?:\\/{2}(?:www\\.|)(youtube|youtube-nocookie)\\.com\\/embed\\/..*" - ], - "name": "Youtube", - "options": { - "enabled": true, - "redirectType": "both", - "frontend": "invidious", - "embedFrontend": "invidious" - }, - "imageType": "png", - "embeddable": true, - "url": "https://youtube.com" - }, - "youtubeMusic": { - "frontends": { - "beatbump": { - "preferences": { - "localstorage": ["settings"], - "indexeddb": "beatbump" - }, - "name": "Beatbump", - "instanceList": true - }, - "hyperpipe": { - "preferences": { - "localstorage": ["api", "authapi", "codec", "locale", "next", "pipedapi", "quality", "theme", "vol"], - "indexeddb": "hyperpipedb" - }, - "name": "Hyperpipe", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}music\\.youtube\\.com(\\/|$)"], - "name": "YT Music", - "options": { - "enabled": true, - "frontend": "beatbump" - }, - "imageType": "png", - "embeddable": false, - "url": "https://music.youtube.com" - }, - "twitter": { - "frontends": { - "nitter": { - "preferences": { - "cookies": [ - "autoplayGifs", - "bidiSupport", - "hideBanner", - "hidePins", - "hideReplies", - "hideTweetStats", - "hlsPlayback", - "infiniteScroll", - "mp4Playback", - "muteVideos", - "proxyVideos", - "replaceInstagram", - "replaceReddit", - "replaceTwitter", - "replaceYouTube", - "squareAvatars", - "theme" - ] - }, - "name": "Nitter", - "embeddable": true, - "instanceList": true - } - }, - "targets": [ - "^https?:\\/{2}(www\\.|mobile\\.|)twitter\\.com(\\/|$)", - "^https?:\\/{2}(pbs\\.|video\\.|)twimg\\.com(\\/|$)", - "^https?:\\/{2}platform\\.twitter\\.com/embed(\\/|$)", - "^https?:\\/{2}t\\.co(\\/|$)" - ], - "name": "Twitter", - "options": { - "enabled": true, - "redirectType": "both" - }, - "imageType": "png", - "embeddable": true, - "url": "https://twitter.com" - }, - "instagram": { - "frontends": { - "bibliogram": { - "preferences": { - "token": "token", - "fetchEndpoint": "/settings.json", - "setEndpoint": "/applysettings" - }, - "name": "Bibliogram", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(www\\.)?instagram\\.com\\/?(p\\/|$)"], - "name": "Instagram", - "options": { - "enabled": true - }, - "imageType": "png", - "embeddable": false, - "url": "https://instagram.com" - }, - "tiktok": { - "frontends": { - "proxiTok": { - "preferences": { - "cookies": ["api-test_endpoints", "theme"] - }, - "name": "ProxiTok", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(www\\.|)tiktok\\.com(\\/|$)"], - "name": "TikTok", - "options": { - "enabled": true - }, - "imageType": "png", - "embeddable": false, - "url": "https://tiktok.com" - }, - "reddit": { - "frontends": { - "libreddit": { - "preferences": { - "cookies": ["theme", "front_page", "layout", "wide", "post_sort", "comment_sort", "show_nsfw", "autoplay_videos", "use_hls", "hide_hls_notification", "subscriptions", "filters"] - }, - "name": "Libreddit", - "instanceList": true - }, - "teddit": { - "preferences": { - "cookies": [ - "collapse_child_comments", - "default_comment_sort", - "domain_instagram", - "domain_twitter", - "domain_youtube", - "flairs", - "highlight_controversial", - "nsfw_enabled", - "post_media_max_height", - "prefer_frontpage", - "show_large_gallery_images", - "show_upvoted_percentage", - "show_upvotes", - "subbed_subreddits", - "theme", - "videos_muted" - ] - }, - "name": "Teddit", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(www\\.|old\\.|np\\.|new\\.|amp\\.|)reddit\\.com(?=\\/u(ser)?\\/|\\/r\\/|\\/?$)", "^https?:\\/{2}(i|(external-)?preview)\\.redd\\.it"], - "name": "Reddit", - "options": { - "enabled": true, - "frontend": "libreddit" - }, - "imageType": "png", - "embeddable": false, - "url": "https://reddit.com" - }, - "imgur": { - "frontends": { - "rimgo": { - "name": "rimgo", - "embeddable": true, - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}([im]\\.)?(stack\\.)?imgur\\.(com|io)(\\/|$)"], - "name": "Imgur", - "options": { - "enabled": true, - "redirectType": "both" - }, - "imageType": "png", - "embeddable": true, - "url": "https://imgur.com" - }, - "wikipedia": { - "frontends": { - "wikiless": { - "preferences": { - "cookies": ["theme", "default_lang"] - }, - "name": "Wikiless", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(?:[a-z]+\\.)*wikipedia\\.org(\\/|$)"], - "name": "Wikipedia", - "options": { - "enabled": false - }, - "imageType": "svg", - "embeddable": false, - "url": "https://wikipedia.org" - }, - "medium": { - "frontends": { - "scribe": { - "name": "Scribe", - "instanceList": true - } - }, - "targets": [ - "(?:.*\\.)*(?<!(link\\.|cdn\\-images\\-\\d+\\.))medium\\.com(\\/.*)?$", - "^https?:\\/{2}towardsdatascience\\.com(\\/|$)", - "^https?:\\/{2}uxdesign\\.cc(\\/|$)", - "^https?:\\/{2}uxplanet\\.org(\\/|$)", - "^https?:\\/{2}betterprogramming\\.pub(\\/|$)", - "^https?:\\/{2}aninjusticemag\\.com(\\/|$)", - "^https?:\\/{2}betterhumans\\.pub(\\/|$)", - "^https?:\\/{2}psiloveyou\\.xyz(\\/|$)", - "^https?:\\/{2}entrepreneurshandbook\\.co(\\/|$)", - "^https?:\\/{2}blog\\.coinbase\\.com(\\/|$)", - "^https?:\\/{2}levelup\\.gitconnected\\.com(\\/|$)", - "^https?:\\/{2}javascript\\.plainenglish\\.io(\\/|$)", - "^https?:\\/{2}blog\\.bitsrc\\.io(\\/|$)", - "^https?:\\/{2}itnext\\.io(\\/|$)", - "^https?:\\/{2}codeburst\\.io(\\/|$)", - "^https?:\\/{2}infosecwriteups\\.com(\\/|$)", - "^https?:\\/{2}blog\\.devgenius\\.io(\\/|$)", - "^https?:\\/{2}writingcooperative\\.com(\\/|$)" - ], - "name": "Medium", - "options": { - "enabled": true - }, - "imageType": "svgMono", - "embeddable": false, - "url": "https://medium.com" - }, - "quora": { - "frontends": { - "quetre": { - "preferences": { - "localstorage": ["theme"] - }, - "name": "Quetre", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}([a-zA-Z0-9-]+\\.)*quora\\.com(\\/|$)"], - "name": "Quora", - "options": { - "enabled": true - }, - "imageType": "png", - "embeddable": false, - "url": "https://quora.com" - }, - "imdb": { - "frontends": { - "libremdb": { - "preferences": { - "localstorage": ["theme"] - }, - "name": "libremdb", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(?:www\\.|)imdb\\.com\\/title"], - "name": "IMDb", - "options": { - "enabled": true - }, - "imageType": "svg", - "embeddable": false, - "url": "https://imdb.com" - }, - "reuters": { - "frontends": { - "neuters": { - "name": "Neuters", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(www\\.|)reuters\\.com(\\/|$)"], - "name": "Reuters", - "options": { - "enabled": false - }, - "imageType": "svg", - "embeddable": false, - "url": "https://reuters.com" - }, - "fandom": { - "frontends": { - "breezeWiki": { - "name": "BreezeWiki", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}(?:[a-zA-Z0-9-]+\\.)?fandom\\.com(?=\\/wiki|\\/?$)"], - "name": "Fandom", - "options": { - "enabled": true - }, - "imageType": "svg", - "embeddable": false, - "url": "https://fandom.com" - }, - "peertube": { - "frontends": { - "simpleertube": { - "name": "SimpleerTube", - "instanceList": true - } - }, - "targets": "datajson", - "name": "PeerTube", - "options": { - "enabled": false - }, - "imageType": "svg", - "embeddable": false, - "url": "https://joinpeertube.org" - }, - "lbry": { - "frontends": { - "librarian": { - "preferences": { - "cookies": ["nsfw", "theme"], - "localstorage": ["autoplay", "autoplayNextVid", "collapseComments", "plyr", "sb_categories", "showRelated"] - }, - "name": "Librarian", - "embeddable": true, - "instanceList": true - }, - "lbryDesktop": { - "name": "LBRY Desktop", - "embeddable": false, - "instanceList": false - } - }, - "targets": ["^https?:\\/{2}odysee\\.com(\\/|$)", "^https?:\\/{2}lbry\\.tv(\\/|$)"], - "name": "LBRY", - "options": { - "enabled": true, - "frontend": "librarian", - "redirectType": "both", - "embedFrontend": "librarian" - }, - "imageType": "png", - "embeddable": true, - "url": "https://odysee.com" - }, - "search": { - "frontends": { - "searx": { - "preferences": { - "cookies": [ - "advanced_search", - "autocomplete", - "categories", - "disabled_engines", - "disabled_plugins", - "doi_resolver", - "enabled_engines", - "enabled_plugins", - "image_proxy", - "language", - "locale", - "method", - "oscar-style", - "results_on_new_tab", - "safesearch", - "theme", - "tokens" - ] - }, - "name": "SearX", - "instanceList": true - }, - "searxng": { - "preferences": { - "cookies": [ - "autocomplete", - "categories", - "center_alignment", - "disabled_engines", - "disabled_plugins", - "doi_resolver", - "enabled_plugins", - "enabled_engines", - "image_proxy", - "infinite_scroll", - "language", - "locale", - "maintab", - "method", - "query_in_title", - "results_on_new_tab", - "safesearch", - "simple_style", - "theme", - "tokens" - ] - }, - "name": "SearXNG", - "instanceList": true - }, - "whoogle": { - "name": "Whoogle", - "instanceList": true - }, - "librex": { - "preferences": { - "cookies": ["bibliogram", "disable_frontends", " disable_special", "invidious", "libreddit", "nitter", "proxitok", "save", "theme", "wikiless"] - }, - "name": "LibreX", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}search\\.libredirect\\.invalid"], - "name": "Search", - "options": { - "enabled": true, - "frontend": "searxng" - }, - "imageType": "svgMono", - "embeddable": false, - "url": "https://search.libredirect.invalid" - }, - "translate": { - "frontends": { - "simplyTranslate": { - "preferences": { - "cookies": ["from_lang", "to_lang", "tts_enabled", "use_text_fields"] - }, - "name": "SimplyTranslate", - "instanceList": true - }, - "lingva": { - "preferences": { - "localstorage": ["isauto", "source", "target", "chakra-ui-color-mode"] - }, - "name": "Lingva Translate", - "instanceList": true - }, - "libreTranslate": { - "name": "LibreTranslate", - "instanceList": true - } - }, - "targets": ["^https?:\\/{2}translate\\.google(\\.[a-z]{2,3}){1,2}\\/", "^https?:\\/{2}translate\\.libredirect\\.invalid"], - "name": "Translate", - "options": { - "enabled": true, - "frontend": "simplyTranslate" - }, - "imageType": "svgMono", - "embeddable": false, - "url": "https://translate.libredirect.invalid" - }, - "maps": { - "frontends": { - "facil": { - "name": "FacilMap", - "instanceList": true - }, - "osm": { - "name": "OpenStreetMap", - "instanceList": false, - "singleInstance": "https://www.openstreetmap.org" - } - }, - "targets": ["^https?:\\/{2}maps\\.libredirect\\.invalid", "^https?:\\/{2}(((www|maps)\\.)?(google\\.).*(\\/maps)|maps\\.(google\\.).*)"], - "name": "Maps", - "options": { - "enabled": true, - "frontend": "osm" - }, - "imageType": "svgMono", - "embeddable": false, - "url": "https://maps.libredirect.invalid" - }, - "sendFiles": { - "frontends": { - "send": { - "name": "Send", - "instanceList": "true" - } - }, - "targets": ["^https?:\\/{2}send\\.libredirect\\.invalid", "^https?:\\/{2}send\\.firefox\\.com\\/?$", "^https?:\\/{2}sendfiles\\.online\\/?$"], - "name": "Send Files", - "options": { - "enabled": true - }, - "imageType": "svgMono", - "embeddable": false, - "url": "https://send.libredirect.invalid" - } - }, - "blacklist": { - "cloudflare": { - "color": "red" - }, - "authenticate": { - "color": "orange" - }, - "offline": { - "color": "grey" - } - } -} +{ + "networks": { + "clearnet": { + "tld": "org", + "name": "Clearnet" + }, + "tor": { + "tld": "onion", + "name": "Tor" + }, + "i2p": { + "tld": "i2p", + "name": "I2P" + }, + "loki": { + "tld": "loki", + "name": "Lokinet" + } + }, + "services": { + "youtube": { + "frontends": { + "invidious": { + "preferences": { + "cookies": ["PREFS"], + "localstorage": ["dark_mode"] + }, + "name": "Invidious", + "embeddable": true, + "instanceList": true + }, + "piped": { + "preferences": { + "localstorage": [ + "bufferGoal", + "comments", + "disableLBRY", + "enabledCodecs", + "hl", + "homepage", + "instance", + "listen", + "minimizeDescription", + "playerAutoPlay", + "proxyLBRY", + "quality", + "region", + "selectedSkip", + "sponsorblock", + "theme", + "volume", + "watchHistory", + "localSubscriptions" + ] + }, + "name": "Piped", + "embeddable": true, + "instanceList": true + }, + "pipedMaterial": { + "preferences": { + "localstorage": ["PREFERENCES"] + }, + "name": "Piped-Material", + "embeddable": false, + "instanceList": true + }, + "cloudtube": { + "preferences": { + "token": "token", + "fetchEndpoint": "/api/settings", + "setEndpoint": "/settings" + }, + "name": "CloudTube", + "embeddable": false, + "instanceList": true + }, + "freetube": { + "name": "FreeTube", + "embeddable": false, + "instanceList": false + }, + "yattee": { + "name": "Yattee", + "embeddable": false, + "instanceList": false + } + }, + "targets": [ + "^https?:\\/{2}(?:www\\.|m\\.|)youtube.com(\\/|$)(?!iframe_api\\/|redirect\\/)", + "^https?:\\/{2}img\\.youtube.com\\/vi\\/.*\\/..*", + "^https?:\\/{2}(?:i|s)\\.ytimg.com\\/vi\\/.*\\/..*", + "^https?:\\/{2}(?:www\\.|)youtube.com\\/watch?v=..*", + "^https?:\\/{2}youtu\\.be\\/..*", + "^https?:\\/{2}(?:www\\.|)(youtube|youtube-nocookie)\\.com\\/embed\\/..*" + ], + "name": "Youtube", + "options": { + "enabled": true, + "redirectType": "both", + "frontend": "invidious", + "embedFrontend": "invidious" + }, + "imageType": "png", + "embeddable": true, + "url": "https://youtube.com" + }, + "youtubeMusic": { + "frontends": { + "beatbump": { + "preferences": { + "localstorage": ["settings"], + "indexeddb": "beatbump" + }, + "name": "Beatbump", + "instanceList": true + }, + "hyperpipe": { + "preferences": { + "localstorage": ["api", "authapi", "codec", "locale", "next", "pipedapi", "quality", "theme", "vol"], + "indexeddb": "hyperpipedb" + }, + "name": "Hyperpipe", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}music\\.youtube\\.com(\\/|$)"], + "name": "YT Music", + "options": { + "enabled": true, + "frontend": "beatbump" + }, + "imageType": "png", + "embeddable": false, + "url": "https://music.youtube.com" + }, + "twitter": { + "frontends": { + "nitter": { + "preferences": { + "cookies": [ + "autoplayGifs", + "bidiSupport", + "hideBanner", + "hidePins", + "hideReplies", + "hideTweetStats", + "hlsPlayback", + "infiniteScroll", + "mp4Playback", + "muteVideos", + "proxyVideos", + "replaceInstagram", + "replaceReddit", + "replaceTwitter", + "replaceYouTube", + "squareAvatars", + "theme" + ] + }, + "name": "Nitter", + "embeddable": true, + "instanceList": true + } + }, + "targets": [ + "^https?:\\/{2}(www\\.|mobile\\.|)twitter\\.com(\\/|$)", + "^https?:\\/{2}(pbs\\.|video\\.|)twimg\\.com(\\/|$)", + "^https?:\\/{2}platform\\.twitter\\.com/embed(\\/|$)", + "^https?:\\/{2}t\\.co(\\/|$)" + ], + "name": "Twitter", + "options": { + "enabled": true, + "redirectType": "both" + }, + "imageType": "png", + "embeddable": true, + "url": "https://twitter.com" + }, + "instagram": { + "frontends": { + "bibliogram": { + "preferences": { + "token": "token", + "fetchEndpoint": "/settings.json", + "setEndpoint": "/applysettings" + }, + "name": "Bibliogram", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(www\\.)?instagram\\.com\\/?(p\\/|$)"], + "name": "Instagram", + "options": { + "enabled": true + }, + "imageType": "png", + "embeddable": false, + "url": "https://instagram.com" + }, + "tiktok": { + "frontends": { + "proxiTok": { + "preferences": { + "cookies": ["api-test_endpoints", "theme"] + }, + "name": "ProxiTok", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(www\\.|)tiktok\\.com(\\/|$)"], + "name": "TikTok", + "options": { + "enabled": true + }, + "imageType": "png", + "embeddable": false, + "url": "https://tiktok.com" + }, + "reddit": { + "frontends": { + "libreddit": { + "preferences": { + "cookies": ["theme", "front_page", "layout", "wide", "post_sort", "comment_sort", "show_nsfw", "autoplay_videos", "use_hls", "hide_hls_notification", "subscriptions", "filters"] + }, + "name": "Libreddit", + "instanceList": true + }, + "teddit": { + "preferences": { + "cookies": [ + "collapse_child_comments", + "default_comment_sort", + "domain_instagram", + "domain_twitter", + "domain_youtube", + "flairs", + "highlight_controversial", + "nsfw_enabled", + "post_media_max_height", + "prefer_frontpage", + "show_large_gallery_images", + "show_upvoted_percentage", + "show_upvotes", + "subbed_subreddits", + "theme", + "videos_muted" + ] + }, + "name": "Teddit", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(www\\.|old\\.|np\\.|new\\.|amp\\.|)reddit\\.com(?=\\/u(ser)?\\/|\\/r\\/|\\/?$)", "^https?:\\/{2}(i|(external-)?preview)\\.redd\\.it"], + "name": "Reddit", + "options": { + "enabled": true, + "frontend": "libreddit" + }, + "imageType": "png", + "embeddable": false, + "url": "https://reddit.com" + }, + "imgur": { + "frontends": { + "rimgo": { + "name": "rimgo", + "embeddable": true, + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}([im]\\.)?(stack\\.)?imgur\\.(com|io)(\\/|$)"], + "name": "Imgur", + "options": { + "enabled": true, + "redirectType": "both" + }, + "imageType": "png", + "embeddable": true, + "url": "https://imgur.com" + }, + "wikipedia": { + "frontends": { + "wikiless": { + "preferences": { + "cookies": ["theme", "default_lang"] + }, + "name": "Wikiless", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(?:[a-z]+\\.)*wikipedia\\.org(\\/|$)"], + "name": "Wikipedia", + "options": { + "enabled": false + }, + "imageType": "svg", + "embeddable": false, + "url": "https://wikipedia.org" + }, + "medium": { + "frontends": { + "scribe": { + "name": "Scribe", + "instanceList": true + } + }, + "targets": [ + "(?:.*\\.)*(?<!(link\\.|cdn\\-images\\-\\d+\\.))medium\\.com(\\/.*)?$", + "^https?:\\/{2}towardsdatascience\\.com(\\/|$)", + "^https?:\\/{2}uxdesign\\.cc(\\/|$)", + "^https?:\\/{2}uxplanet\\.org(\\/|$)", + "^https?:\\/{2}betterprogramming\\.pub(\\/|$)", + "^https?:\\/{2}aninjusticemag\\.com(\\/|$)", + "^https?:\\/{2}betterhumans\\.pub(\\/|$)", + "^https?:\\/{2}psiloveyou\\.xyz(\\/|$)", + "^https?:\\/{2}entrepreneurshandbook\\.co(\\/|$)", + "^https?:\\/{2}blog\\.coinbase\\.com(\\/|$)", + "^https?:\\/{2}levelup\\.gitconnected\\.com(\\/|$)", + "^https?:\\/{2}javascript\\.plainenglish\\.io(\\/|$)", + "^https?:\\/{2}blog\\.bitsrc\\.io(\\/|$)", + "^https?:\\/{2}itnext\\.io(\\/|$)", + "^https?:\\/{2}codeburst\\.io(\\/|$)", + "^https?:\\/{2}infosecwriteups\\.com(\\/|$)", + "^https?:\\/{2}blog\\.devgenius\\.io(\\/|$)", + "^https?:\\/{2}writingcooperative\\.com(\\/|$)" + ], + "name": "Medium", + "options": { + "enabled": true + }, + "imageType": "svgMono", + "embeddable": false, + "url": "https://medium.com" + }, + "quora": { + "frontends": { + "quetre": { + "preferences": { + "localstorage": ["theme"] + }, + "name": "Quetre", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}([a-zA-Z0-9-]+\\.)*quora\\.com(\\/|$)"], + "name": "Quora", + "options": { + "enabled": true + }, + "imageType": "png", + "embeddable": false, + "url": "https://quora.com" + }, + "imdb": { + "frontends": { + "libremdb": { + "preferences": { + "localstorage": ["theme"] + }, + "name": "libremdb", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(?:www\\.|)imdb\\.com\\/title"], + "name": "IMDb", + "options": { + "enabled": true + }, + "imageType": "svg", + "embeddable": false, + "url": "https://imdb.com" + }, + "reuters": { + "frontends": { + "neuters": { + "name": "Neuters", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(www\\.|)reuters\\.com(\\/|$)"], + "name": "Reuters", + "options": { + "enabled": false + }, + "imageType": "svg", + "embeddable": false, + "url": "https://reuters.com" + }, + "fandom": { + "frontends": { + "breezeWiki": { + "name": "BreezeWiki", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}(?:[a-zA-Z0-9-]+\\.)?fandom\\.com(?=\\/wiki|\\/?$)"], + "name": "Fandom", + "options": { + "enabled": true + }, + "imageType": "svg", + "embeddable": false, + "url": "https://fandom.com" + }, + "peertube": { + "frontends": { + "simpleertube": { + "name": "SimpleerTube", + "instanceList": true + } + }, + "targets": "datajson", + "name": "PeerTube", + "options": { + "enabled": false + }, + "imageType": "svg", + "embeddable": false, + "url": "https://joinpeertube.org" + }, + "lbry": { + "frontends": { + "librarian": { + "preferences": { + "cookies": ["nsfw", "theme"], + "localstorage": ["autoplay", "autoplayNextVid", "collapseComments", "plyr", "sb_categories", "showRelated"] + }, + "name": "Librarian", + "embeddable": true, + "instanceList": true + }, + "lbryDesktop": { + "name": "LBRY Desktop", + "embeddable": false, + "instanceList": false + } + }, + "targets": ["^https?:\\/{2}odysee\\.com(\\/|$)", "^https?:\\/{2}lbry\\.tv(\\/|$)"], + "name": "LBRY", + "options": { + "enabled": true, + "frontend": "librarian", + "redirectType": "both", + "embedFrontend": "librarian" + }, + "imageType": "png", + "embeddable": true, + "url": "https://odysee.com" + }, + "search": { + "frontends": { + "searx": { + "preferences": { + "cookies": [ + "advanced_search", + "autocomplete", + "categories", + "disabled_engines", + "disabled_plugins", + "doi_resolver", + "enabled_engines", + "enabled_plugins", + "image_proxy", + "language", + "locale", + "method", + "oscar-style", + "results_on_new_tab", + "safesearch", + "theme", + "tokens" + ] + }, + "name": "SearX", + "instanceList": true + }, + "searxng": { + "preferences": { + "cookies": [ + "autocomplete", + "categories", + "center_alignment", + "disabled_engines", + "disabled_plugins", + "doi_resolver", + "enabled_plugins", + "enabled_engines", + "image_proxy", + "infinite_scroll", + "language", + "locale", + "maintab", + "method", + "query_in_title", + "results_on_new_tab", + "safesearch", + "simple_style", + "theme", + "tokens" + ] + }, + "name": "SearXNG", + "instanceList": true + }, + "whoogle": { + "name": "Whoogle", + "instanceList": true + }, + "librex": { + "preferences": { + "cookies": ["bibliogram", "disable_frontends", " disable_special", "invidious", "libreddit", "nitter", "proxitok", "save", "theme", "wikiless"] + }, + "name": "LibreX", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}search\\.libredirect\\.invalid"], + "name": "Search", + "options": { + "enabled": true, + "frontend": "searxng" + }, + "imageType": "svgMono", + "embeddable": false, + "url": "https://search.libredirect.invalid" + }, + "translate": { + "frontends": { + "simplyTranslate": { + "preferences": { + "cookies": ["from_lang", "to_lang", "tts_enabled", "use_text_fields"] + }, + "name": "SimplyTranslate", + "instanceList": true + }, + "lingva": { + "preferences": { + "localstorage": ["isauto", "source", "target", "chakra-ui-color-mode"] + }, + "name": "Lingva Translate", + "instanceList": true + }, + "libreTranslate": { + "name": "LibreTranslate", + "instanceList": true + } + }, + "targets": ["^https?:\\/{2}translate\\.google(\\.[a-z]{2,3}){1,2}\\/", "^https?:\\/{2}translate\\.libredirect\\.invalid"], + "name": "Translate", + "options": { + "enabled": true, + "frontend": "simplyTranslate" + }, + "imageType": "svgMono", + "embeddable": false, + "url": "https://translate.libredirect.invalid" + }, + "maps": { + "frontends": { + "facil": { + "name": "FacilMap", + "instanceList": true + }, + "osm": { + "name": "OpenStreetMap", + "instanceList": false, + "singleInstance": "https://www.openstreetmap.org" + } + }, + "targets": ["^https?:\\/{2}maps\\.libredirect\\.invalid", "^https?:\\/{2}(((www|maps)\\.)?(google\\.).*(\\/maps)|maps\\.(google\\.).*)"], + "name": "Maps", + "options": { + "enabled": true, + "frontend": "osm" + }, + "imageType": "svgMono", + "embeddable": false, + "url": "https://maps.libredirect.invalid" + }, + "sendFiles": { + "frontends": { + "send": { + "name": "Send", + "instanceList": "true" + } + }, + "targets": ["^https?:\\/{2}send\\.libredirect\\.invalid", "^https?:\\/{2}send\\.firefox\\.com\\/?$", "^https?:\\/{2}sendfiles\\.online\\/?$"], + "name": "Send Files", + "options": { + "enabled": true + }, + "imageType": "svgMono", + "embeddable": false, + "url": "https://send.libredirect.invalid" + } + }, + "blacklist": { + "cloudflare": { + "color": "red" + }, + "authenticate": { + "color": "orange" + }, + "offline": { + "color": "grey" + } + } +} diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py index 9c0543ca..8a0258c0 100644 --- a/src/instances/get_instances.py +++ b/src/instances/get_instances.py @@ -12,12 +12,15 @@ import socket mightyList = {} config = {} -startRegex = r"https?:\/{2}(?:[^\s\/]+\.)+" +startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*" endRegex = "(?:\/[^\s\/]+)*\/?" torRegex = startRegex + "onion" + endRegex i2pRegex = startRegex + "i2p" + endRegex lokiRegex = startRegex + "loki" + endRegex -authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)+[a-zA-Z0-9]+" + endRegex +authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex + +# 2.0 because Libredirect is currently on version 2.x.x +headers = {'User-Agent': 'Libredirect-instance-fetcher/2.0'} with open('./src/config/config.json', 'rt') as tmp: config['networks'] = json.load(tmp)['networks'] @@ -92,7 +95,8 @@ def is_cloudflare(url): instance_bin_masked = instance_bin[:mask] if cloudflare_bin_masked == instance_bin_masked: - print(url + ' is behind ' + Fore.RED + 'cloudflare' + Style.RESET_ALL) + print(url + ' is behind ' + Fore.RED + + 'cloudflare' + Style.RESET_ALL) return True return False @@ -100,11 +104,13 @@ def is_cloudflare(url): def is_authenticate(url): try: if re.match(authRegex, url): - print(url + ' requires ' + Fore.RED + 'authentication' + Style.RESET_ALL) + print(url + ' requires ' + Fore.RED + + 'authentication' + Style.RESET_ALL) return True - r = requests.get(url, timeout=5) + r = requests.get(url, timeout=5, headers=headers) if 'www-authenticate' in r.headers: - print(url + ' requires ' + Fore.RED + 'authentication' + Style.RESET_ALL) + print(url + ' requires ' + Fore.RED + + 'authentication' + Style.RESET_ALL) return True except Exception: return False @@ -113,7 +119,7 @@ def is_authenticate(url): def is_offline(url): try: - r = requests.get(url, timeout=5) + r = requests.get(url, timeout=5, headers=headers) if r.status_code >= 400: print(url + ' is ' + Fore.RED + 'offline' + Style.RESET_ALL) print("Status code") @@ -126,9 +132,12 @@ def is_offline(url): def fetchCache(frontend, name): - with open('./src/instances/data.json') as file: - mightyList[frontend] = json.load(file)[frontend] - print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + name) + try: + with open('./src/instances/data.json') as file: + mightyList[frontend] = json.load(file)[frontend] + print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + name) + except Exception: + print(Fore.RED + 'Failed' + Style.RESET_ALL + ' to get cached ' + name) def fetchFromFile(frontend, name): @@ -139,7 +148,7 @@ def fetchFromFile(frontend, name): def fetchJsonList(frontend, name, url, urlItem, jsonObject): try: - r = requests.get(url) + r = requests.get(url, headers=headers) rJson = json.loads(r.text) if jsonObject: rJson = rJson['instances'] @@ -178,7 +187,7 @@ def fetchJsonList(frontend, name, url, urlItem, jsonObject): def fetchRegexList(frontend, name, url, regex): try: - r = requests.get(url) + r = requests.get(url, headers=headers) _list = {} for network in config['networks']: _list[network] = [] @@ -205,23 +214,32 @@ def fetchRegexList(frontend, name, url, regex): def fetchTextList(frontend, name, url, prepend): try: - r = requests.get(url) - tmp = r.text.strip().split('\n') - _list = {} for network in config['networks']: _list[network] = [] - for item in tmp: - item = prepend + item - if re.search(torRegex, item): - _list['tor'].append(item) - elif re.search(i2pRegex, item): - _list['i2p'].append(item) - elif re.search(lokiRegex, item): - _list['loki'].append(item) - else: - _list['clearnet'].append(item) + if type(url) == dict: + for network in config['networks']: + if url[network] is not None: + r = requests.get(url[network], headers=headers) + tmp = r.text.strip().split('\n') + for item in tmp: + item = prepend[network] + item + _list[network].append(item) + else: + r = requests.get(url, headers=headers) + tmp = r.text.strip().split('\n') + + for item in tmp: + item = prepend + item + if re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) mightyList[frontend] = _list print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) except Exception: @@ -239,7 +257,7 @@ def invidious(): _list['tor'] = [] _list['i2p'] = [] _list['loki'] = [] - r = requests.get(url) + r = requests.get(url, headers=headers) rJson = json.loads(r.text) for instance in rJson: if instance[1]['type'] == 'https': @@ -265,13 +283,13 @@ def piped(): _list['i2p'] = [] _list['loki'] = [] r = requests.get( - 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') + 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md', headers=headers) tmp = re.findall( r'(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) for item in tmp: try: - url = requests.get(item, timeout=5).url + url = requests.get(item, timeout=5, headers=headers).url if url.strip("/") == item: continue else: @@ -287,7 +305,8 @@ def piped(): def pipedMaterial(): - fetchRegexList('pipedMaterial', 'Piped-Material', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production") + fetchRegexList('pipedMaterial', 'Piped-Material', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', + r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production") def cloudtube(): @@ -295,15 +314,18 @@ def cloudtube(): def proxitok(): - fetchRegexList('proxiTok', 'ProxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))") + fetchRegexList('proxiTok', 'ProxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))") def send(): - fetchRegexList('send', 'Send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', r"- ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}") + fetchRegexList('send', 'Send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', + r"- ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}") def nitter(): - fetchRegexList('nitter', 'Nitter', 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md', r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))") + fetchRegexList('nitter', 'Nitter', 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md', + r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))") def bibliogram(): @@ -311,65 +333,53 @@ def bibliogram(): def libreddit(): - fetchJsonList('libreddit', 'Libreddit', 'https://github.com/ferritreader/libreddit-instances/raw/master/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) + fetchJsonList('libreddit', 'Libreddit', 'https://github.com/ferritreader/libreddit-instances/raw/master/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) def teddit(): - fetchJsonList('teddit', 'Teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) + fetchJsonList('teddit', 'Teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def wikiless(): - fetchJsonList('wikiless', 'Wikiless', 'https://wikiless.org/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) + fetchJsonList('wikiless', 'Wikiless', 'https://wikiless.org/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def scribe(): - fetchJsonList('scribe', 'Scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False) + fetchJsonList('scribe', 'Scribe', + 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False) def quetre(): - fetchRegexList('quetre', 'Quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/README.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") + fetchRegexList('quetre', 'Quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/README.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") def libremdb(): - fetchRegexList('libremdb', 'libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/README.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") + fetchRegexList('libremdb', 'libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/README.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") def simpleertube(): - fetchTextList('simpleertube', 'SimpleerTube', 'https://simple-web.org/instances/simpleertube', 'https://') + fetchTextList('simpleertube', 'SimpleerTube', {'clearnet': 'https://simple-web.org/instances/simpleertube', 'tor': 'https://simple-web.org/instances/simpleertube_onion', + 'i2p': 'https://simple-web.org/instances/simpleertube_i2p', 'loki': None}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) def simplytranslate(): - r = requests.get('https://simple-web.org/instances/simplytranslate') - simplyTranslateList = {} - simplyTranslateList['clearnet'] = [] - for item in r.text.strip().split('\n'): - simplyTranslateList['clearnet'].append('https://' + item) - - r = requests.get('https://simple-web.org/instances/simplytranslate_onion') - simplyTranslateList['tor'] = [] - for item in r.text.strip().split('\n'): - simplyTranslateList['tor'].append('http://' + item) - - r = requests.get('https://simple-web.org/instances/simplytranslate_i2p') - simplyTranslateList['i2p'] = [] - for item in r.text.strip().split('\n'): - simplyTranslateList['i2p'].append('http://' + item) - - r = requests.get('https://simple-web.org/instances/simplytranslate_loki') - simplyTranslateList['loki'] = [] - for item in r.text.strip().split('\n'): - simplyTranslateList['loki'].append('http://' + item) - - mightyList['simplyTranslate'] = simplyTranslateList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SimplyTranslate') + fetchTextList('simplyTranslate', 'SimplyTranslate', {'clearnet': 'https://simple-web.org/instances/simplytranslate', 'tor': 'https://simple-web.org/instances/simplytranslate_onion', + 'i2p': 'https://simple-web.org/instances/simplytranslate_i2p', 'loki': 'https://simple-web.org/instances/simplytranslate_loki'}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) def linvgatranslate(): - fetchJsonList('lingva', 'LingvaTranslate', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False) + fetchJsonList('lingva', 'LingvaTranslate', + 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False) def searx_searxng(): - r = requests.get('https://searx.space/data/instances.json') + r = requests.get( + 'https://searx.space/data/instances.json', headers=headers) rJson = json.loads(r.text) searxList = {} searxList['clearnet'] = [] @@ -404,19 +414,23 @@ def searx_searxng(): def whoogle(): - fetchTextList('whoogle', 'Whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/misc/instances.txt', '') + fetchRegexList('whoogle', 'Whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md', + r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ") def librex(): - fetchJsonList('librex', 'LibreX', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True) + fetchJsonList('librex', 'LibreX', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', + {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True) def rimgo(): - fetchJsonList('rimgo', 'rimgo', 'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) + fetchJsonList('rimgo', 'rimgo', 'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def librarian(): - fetchJsonList('librarian', 'Librarian', 'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json', 'url', True) + fetchJsonList('librarian', 'Librarian', + 'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json', 'url', True) def neuters(): @@ -428,7 +442,8 @@ def beatbump(): def hyperpipe(): - fetchJsonList('hyperpipe', 'Hyperpipe', 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False) + fetchJsonList('hyperpipe', 'Hyperpipe', + 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False) def facil(): @@ -436,17 +451,19 @@ def facil(): def libreTranslate(): - fetchRegexList('libreTranslate', 'LibreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|") + fetchRegexList('libreTranslate', 'LibreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', + r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|") def breezeWiki(): - fetchRegexList('breezeWiki', 'BreezeWiki', 'https://gitdab.com/cadence/breezewiki-docs/raw/branch/main/docs.scrbl', r"\(\"[^\n\s\r\t\f\v\"]+\" \"https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*\" \"(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*)\"\)") + fetchRegexList('breezeWiki', 'BreezeWiki', 'https://gitdab.com/cadence/breezewiki-docs/raw/branch/main/docs.scrbl', + r"\(\"[^\n\s\r\t\f\v\"]+\" \"https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*\" \"(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*)\"\)") def peertube(): try: r = requests.get( - 'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt') + 'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt', headers=headers) rJson = json.loads(r.text) myList = ['https://search.joinpeertube.org'] |