Clarifying authenticate instances #311

3 files changed, 70 insertions, 20 deletions

diff --git a/src/instances/blacklist.json b/src/instances/blacklist.json
new file mode 100644
index 00000000..d02bbec6
--- /dev/null
+++ b/src/instances/blacklist.json
@@ -0,0 +1,52 @@
+{
+  "cloudflare": [
+    "https://invidious.kavin.rocks",
+    "https://invidious.lunar.icu",
+    "https://piped.kavin.rocks",
+    "https://piped.tokhmi.xyz",
+    "https://piped.moomoo.me",
+    "https://piped.syncpundit.com",
+    "https://piped.mha.fi",
+    "https://piped.shimul.me",
+    "https://nitter.domain.glass",
+    "https://birdsite.xanny.family",
+    "https://nitter.moomoo.me",
+    "https://notabird.site",
+    "https://fuckthesacklers.network",
+    "https://nitter.winscloud.net",
+    "https://twtr.bch.bar",
+    "https://nitter.lunar.icu",
+    "https://twitter.dr460nf1r3.org",
+    "https://bib.actionsack.com",
+    "https://libreddit.domain.glass",
+    "https://r.nf",
+    "https://libreddit.flux.industries",
+    "https://lr.oversold.host",
+    "https://libreddit.hu",
+    "https://lr.stilic.ml",
+    "https://reddi.tk",
+    "https://r.walkx.org",
+    "https://libreddit.yonalee.eu",
+    "https://libreddit.winscloud.net",
+    "https://libreddit.lunar.icu",
+    "https://r.ahwx.org",
+    "https://reddit.dr460nf1r3.org",
+    "https://teddit.domain.glass",
+    "https://wiki.604kph.xyz",
+    "https://wikiless.lunar.icu",
+    "https://lingva.lunar.icu",
+    "https://searx.josie.lol",
+    "https://searx.org",
+    "https://searx.run",
+    "https://searx.albony.xyz",
+    "https://search.albony.xyz",
+    "https://search.garudalinux.org",
+    "https://search.dr460nf1r3.org",
+    "https://whoogle.lunar.icu",
+    "https://rimgo.lunar.icu",
+    "https://i.actionsack.com"
+  ],
+  "authenticate": [
+    "https://nitter.nixnet.services"
+  ]
+}
\ No newline at end of file
diff --git a/src/instances/data.json b/src/instances/data.json
index b521b48a..19f7d217 100644
--- a/src/instances/data.json
+++ b/src/instances/data.json
@@ -329,7 +329,8 @@
       "https://tl.vern.cc",
       "https://translate.slipfox.xyz",
       "https://st.privacydev.net",
-      "https://translate.beparanoid.de"
+      "https://translate.beparanoid.de",
+      "https://translate.priv.pw"
     ],
     "tor": [
       "http://fyng2tsmzmvxmojzbbwmfnsn2lrcyftf4cw6rk5j2v2huliazud3fjid.onion",
@@ -367,7 +368,6 @@
     "normal": [
       "https://anon.sx",
       "https://dynabyte.ca",
-      "https://engo.mint.lgbt",
       "https://jsearch.pw",
       "https://nibblehole.com",
       "https://search.antonkling.se",
@@ -383,14 +383,12 @@
       "https://searx.dresden.network",
       "https://searx.fossencdi.org",
       "https://searx.gnu.style",
-      "https://searx.jaska.cc",
       "https://searx.josie.lol",
       "https://searx.kujonello.cf",
       "https://searx.mastodontech.de",
       "https://searx.mxchange.org",
       "https://searx.nakhan.net",
       "https://searx.netzspielplatz.de",
-      "https://searx.ninja",
       "https://searx.nixnet.services",
       "https://searx.openhoofd.nl",
       "https://searx.org",
@@ -404,7 +402,6 @@
       "https://searx.sp-codes.de",
       "https://searx.stuehieyr.com",
       "https://searx.theanonymouse.xyz",
-      "https://searx.tk",
       "https://searx.tux.land",
       "https://searx.tuxcloud.net",
       "https://searx.tyil.nl",
@@ -419,7 +416,6 @@
       "https://sx.catgirl.cloud",
       "https://timdor.noip.me/searx",
       "https://www.gruble.de",
-      "https://yuno.zecircle.xyz/yunohost/sso/?r=aHR0cHM6Ly9zZWFyeC56ZWNpcmNsZS54eXov",
       "https://searx.roflcopter.fr"
     ]
   },
@@ -435,6 +431,7 @@
     ],
     "i2p": [],
     "normal": [
+      "https://asowneryt.cloudns.nz",
       "https://darmarit.org/searx",
       "https://etsi.me",
       "https://northboot.xyz",
@@ -442,6 +439,7 @@
       "https://procurx.pt",
       "https://s.zhaocloud.net",
       "https://search.bus-hit.me",
+      "https://search.chemicals-in-the-water.eu",
       "https://search.mdosch.de",
       "https://search.neet.works",
       "https://search.ononoki.org",
@@ -485,7 +483,8 @@
       "https://www.whooglesearch.ml",
       "https://whoogle.dcs0.hu",
       "https://whoogle.esmailelbob.xyz",
-      "https://whoogle.lunar.icu"
+      "https://whoogle.lunar.icu",
+      "https://whoogle.privacydev.net"
     ],
     "tor": [],
     "i2p": []
@@ -589,14 +588,12 @@
     "https://video.linc.systems",
     "https://video.sftblw.moe",
     "https://peertube.dsmouse.net",
-    "https://tube.govital.net",
     "https://ptube.horsentiers.fr",
     "https://media.reseaumilitant.org",
     "https://video.cnt.social",
     "https://peertube.okko.io",
     "https://video.mentality.rip",
     "https://tube.03281.fspfc.org",
-    "https://videos.casually.cat",
     "https://tube.linc.systems",
     "https://artefac-paris.tv",
     "https://peertube.ketchup.noho.st",
@@ -967,10 +964,8 @@
     "https://peertube.myrasp.eu",
     "https://watch.softinio.com",
     "https://peertube.plataformess.org",
-    "https://tv1.gomntu.space",
     "https://tube.arthack.nz",
     "https://tv.atmx.ca",
-    "https://tube.superseriousbusiness.org",
     "https://kino.kompot.si",
     "https://tube.kockatoo.org",
     "https://peertube.cabaal.net",
@@ -1095,7 +1090,6 @@
     "https://videos.thisishowidontdisappear.com",
     "https://video.islameye.com",
     "https://tube.kotur.org",
-    "https://v.szy.io",
     "https://peertube.euskarabildua.eus",
     "https://video.veloma.org",
     "https://vidcommons.org",
@@ -1259,7 +1253,6 @@
     "https://tube.rita.moe",
     "https://tuktube.com",
     "https://v.basspistol.org",
-    "https://mojotube.net",
     "https://mytube.kn-cloud.de",
     "https://tube.nuagelibre.fr",
     "https://video.nogafam.es",
diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py
index b2937d7e..1758ed01 100644
--- a/src/instances/get_instances.py
+++ b/src/instances/get_instances.py
@@ -57,7 +57,6 @@ def is_cloudflare(url):
         cloudflare_bin = ip2bin(cloudflare_ip)
 
         mask = int(cloudflare_ip_mask.split('/')[1])
-
         cloudflare_bin_masked = cloudflare_bin[:mask]
         instance_bin_masked = instance_bin[:mask]
 
@@ -71,6 +70,7 @@ def is_authenticate(url):
     try:
         r = requests.get(url, timeout=5)
         if 'www-authenticate' in r.headers:
+            print(url + ' is ' + Fore.RED + 'authenticate' + Style.RESET_ALL)
             return True
     except:
         return False
@@ -376,7 +376,8 @@ searx_searxng()
 whoogle()
 rimgo()
 
-blocklist = []
+cloudflare = []
+authenticate = []
 for k1, v1 in mightyList.items():
     if type(mightyList[k1]) is dict:
         for k2, v2 in mightyList[k1].items():
@@ -386,12 +387,16 @@ for k1, v1 in mightyList.items():
                     print("removed " + instance)
                 else:
                     if not instance.endswith('.onion') and not instance.endswith('.i2p') and is_cloudflare(instance):
-                        blocklist.append(instance)
+                        cloudflare.append(instance)
                     if not instance.endswith('.onion') and not instance.endswith('.i2p') and is_authenticate(instance):
-                        blocklist.append(instance)
+                        authenticate.append(instance)
 
 peertube()
 
+blacklist = {
+    'cloudflare': cloudflare,
+    'authenticate': authenticate
+}
 
 # Writing to file
 json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
@@ -399,9 +404,9 @@ with open('./src/instances/data.json', 'w') as outfile:
     outfile.write(json_object)
 print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json')
 
-json_object = json.dumps(blocklist, ensure_ascii=False, indent=2)
-with open('./src/instances/blocklist.json', 'w') as outfile:
+json_object = json.dumps(blacklist, ensure_ascii=False, indent=2)
+with open('./src/instances/blacklist.json', 'w') as outfile:
     outfile.write(json_object)
-print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blocklist')
+print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json')
 
 # print(json_object)