Added cloudflare detection #99

4 files changed, 106 insertions, 12 deletions

diff --git a/src/instances/cloudflare.json b/src/instances/cloudflare.json
new file mode 100644
index 00000000..9918a9e2
--- /dev/null
+++ b/src/instances/cloudflare.json
@@ -0,0 +1,53 @@
+[
+  "https://invidious.kavin.rocks",
+  "https://invidious-us.kavin.rocks",
+  "https://invidious.lunar.icu",
+  "https://send.silkky.cloud",
+  "https://nhanh.cloud",
+  "https://nitter.domain.glass",
+  "https://nitter.actionsack.com",
+  "https://birdsite.xanny.family",
+  "https://nitter.moomoo.me",
+  "https://nittereu.moomoo.me",
+  "https://nitter.alefvanoon.xyz",
+  "https://notabird.site",
+  "https://nitter.silkky.cloud",
+  "https://fuckthesacklers.network",
+  "https://nitter.govt.land",
+  "https://nitter.winscloud.net",
+  "https://twtr.bch.bar",
+  "https://nitter.lunar.icu",
+  "https://bib.actionsack.com",
+  "https://biblio.alefvanoon.xyz",
+  "https://libreddit.silkky.cloud",
+  "https://libreddit.domain.glass",
+  "https://libreddit.jamiethalacker.dev",
+  "https://r.nf",
+  "https://libreddit.alefvanoon.xyz",
+  "https://libreddit.flux.industries",
+  "https://libreddit.hu",
+  "https://lr.stilic.ml",
+  "https://reddi.tk",
+  "https://r.walkx.org",
+  "https://libreddit.yonalee.eu",
+  "https://libreddit.winscloud.net",
+  "https://libreddit.lunar.icu",
+  "https://teddit.domain.glass",
+  "https://teddit.alefvanoon.xyz",
+  "https://wikiless.alefvanoon.xyz",
+  "https://wiki.604kph.xyz",
+  "https://wikiless.lunar.icu",
+  "https://st.alefvanoon.xyz",
+  "https://translate.syncpundit.com",
+  "https://searx.josie.lol",
+  "https://searx.kujonello.cf",
+  "https://searx.org",
+  "https://searx.roughs.ru",
+  "https://searx.run",
+  "https://searx.tk",
+  "https://s.alefvanoon.xyz",
+  "https://search.albony.xyz",
+  "https://whoogle.lunar.icu",
+  "https://rimgo.lunar.icu",
+  "https://i.actionsack.com"
+]
\ No newline at end of file
diff --git a/src/instances/data.json b/src/instances/data.json
index d1212eaf..3ee84a5d 100644
--- a/src/instances/data.json
+++ b/src/instances/data.json
@@ -341,7 +341,6 @@
       "https://search.st8.at",
       "https://search.stinpriza.org",
       "https://search.trom.tf",
-      "https://search.vojkovic.xyz",
       "https://search.zdechov.net",
       "https://searx.bissisoft.com",
       "https://searx.divided-by-zero.eu",
@@ -405,6 +404,7 @@
     "i2p": [],
     "normal": [
       "https://darmarit.org/searx",
+      "https://etsi.me",
       "https://northboot.xyz",
       "https://paulgo.io",
       "https://s.zhaocloud.net",
@@ -412,12 +412,14 @@
       "https://search.mdosch.de",
       "https://search.neet.works",
       "https://search.ononoki.org",
+      "https://search.vojkovic.xyz",
       "https://search.zzls.xyz",
       "https://searx.be",
       "https://searx.ebnar.xyz",
       "https://searx.esmailelbob.xyz",
       "https://searx.fmac.xyz",
       "https://searx.gnous.eu",
+      "https://searx.jaska.cc",
       "https://searx.mha.fi",
       "https://searx.namejeff.xyz",
       "https://searx.prvcy.eu",
@@ -468,6 +470,7 @@
     ]
   },
   "peertube": [
+    "https://videos.rabbit-company.com",
     "https://video.paradigmthreat.net",
     "https://video.lincolncyber.com",
     "https://vid.twhtv.club",
@@ -925,7 +928,6 @@
     "https://tube.toontoet.nl",
     "https://video.gyt.is",
     "https://peertube.0x5e.eu",
-    "https://turkum.me",
     "https://peertube.jensdiemer.de",
     "https://tube.futuretic.fr",
     "https://libra.syntazia.org",
@@ -1067,7 +1069,6 @@
     "https://videos.pzelawski.xyz",
     "https://peertube.zoz-serv.org",
     "https://videos.stadtfabrikanten.org",
-    "https://archive.vidicon.org",
     "https://peertube.gargantia.fr",
     "https://tube.melonbread.xyz",
     "https://tube.grap.coop",
@@ -1404,7 +1405,6 @@
     "https://peertube.uno",
     "https://tube.ksl-bmx.de",
     "https://tube.tchncs.de",
-    "https://peertube.kosebamse.com",
     "https://yunopeertube.myddns.me",
     "https://peertube.anon-kenkai.com",
     "https://tube.maiti.info",
@@ -1448,7 +1448,6 @@
     "https://peertube.we-keys.fr",
     "https://artitube.artifaille.fr",
     "https://peertube.fr",
-    "https://peertube.nayya.org",
     "https://peertube.amicale.net",
     "https://aperi.tube",
     "https://tube.ac-lyon.fr",
diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py
index 8ecec164..6194c6bb 100644
--- a/src/instances/get_instances.py
+++ b/src/instances/get_instances.py
@@ -7,6 +7,8 @@ from bs4 import BeautifulSoup
 import re
 from colorama import Fore, Back, Style
 from urllib.parse import urlparse
+import socket
+import subprocess
 
 mightyList = {}
 
@@ -22,6 +24,39 @@ def filterLastSlash(urlList):
     return tmp
 
 
+def init_cloudflare():
+    r = requests.get('https://www.cloudflare.com/ips-v4')
+    myList = []
+    for i in r.text.split('\n'):
+        out = subprocess.run(
+            ["sh", "./src/instances/get_possible_ips.sh", i],
+            capture_output=True,
+            text=True
+        )
+        myList += out.stdout.splitlines()
+    print(Fore.GREEN + 'Fetched ' +
+          Fore.RED + 'Cloudflare IPs' +
+          Style.RESET_ALL)
+    return myList
+
+
+cloudflare_ips = init_cloudflare()
+
+
+def is_cloudflare(url):
+    href = urlparse(url)
+    ip = ''
+    try:
+        ip = socket.gethostbyname(href.hostname)
+    except:
+        return False
+
+    if ip in cloudflare_ips:
+        return True
+    else:
+        return False
+
+
 # Invidious
 r = requests.get('https://api.invidious.io/instances.json')
 rJson = json.loads(r.text)
@@ -281,11 +316,12 @@ print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Peertube')
 def isValid(url):  # This code is contributed by avanitrachhadiya2155
     try:
         result = urlparse(url)
-        return all([result.scheme, result.netloc, result.path])
+        return all([result.scheme, result.netloc])
     except:
         return False
 
 
+cloudflareMightyList = []
 for k1, v1 in mightyList.items():
     if type(mightyList[k1]) is dict:
         for k2, v2 in mightyList[k1].items():
@@ -293,17 +329,22 @@ for k1, v1 in mightyList.items():
                 if (not isValid(instance)):
                     mightyList[k1][k2].remove(instance)
                     print("removed " + instance)
+                else:
+                    if (is_cloudflare(instance)):
+                        cloudflareMightyList.append(instance)
+                        
 
-    elif type(mightyList[k1]) is list:
-        for instance in mightyList[k1]:
-            if (not isValid(instance)):
-                mightyList[k1].remove(instance)
-                print("removed " + instance)
 
 # Writing to file
 json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
 with open('./src/instances/data.json', 'w') as outfile:
     outfile.write(json_object)
+print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
+
+json_object = json.dumps(cloudflareMightyList, ensure_ascii=False, indent=2)
+with open('./src/instances/cloudflare.json', 'w') as outfile:
+    outfile.write(json_object)
+print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/cloudflare.json')
+
 # print(json_object)
 
-print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
diff --git a/src/instances/get_possible_ips.sh b/src/instances/get_possible_ips.sh
new file mode 100644
index 00000000..81347c5b
--- /dev/null
+++ b/src/instances/get_possible_ips.sh
@@ -0,0 +1 @@
+nmap -sL -n $1 | awk '/Nmap scan report/{print $NF}'
\ No newline at end of file