Merge pull request 'Squashed a few bugs' (#10) from BobIsMyManager/libredirect:master into master

Reviewed-on: https://codeberg.org/LibRedirect/libredirect/pulls/10

6 files changed, 158 insertions, 56 deletions

diff --git a/src/instances/beatbump.json b/src/instances/beatbump.json
new file mode 100644
index 00000000..4c7f65a6
--- /dev/null
+++ b/src/instances/beatbump.json
@@ -0,0 +1,6 @@
+{
+	"normal": ["https://beatbump.ml"],
+	"tor": [],
+	"i2p": [],
+	"loki": []
+}
diff --git a/src/instances/cloudtube.json b/src/instances/cloudtube.json
new file mode 100644
index 00000000..402adc39
--- /dev/null
+++ b/src/instances/cloudtube.json
@@ -0,0 +1,6 @@
+{
+	"normal": ["https://tube.cadence.moe", "https://yt.beparanoid.de"],
+	"tor": ["http://yt.prnoid54e44a4bduq5due64jkk7wcnkxcp5kv3juncm7veptjcqudgyd.onion"],
+	"i2p": [],
+	"loki": []
+}
diff --git a/src/instances/facil.json b/src/instances/facil.json
new file mode 100644
index 00000000..798a9cfb
--- /dev/null
+++ b/src/instances/facil.json
@@ -0,0 +1,6 @@
+{
+	"normal": ["https://facilmap.org"],
+	"tor": [],
+	"i2p": [],
+	"loki": []
+}
diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py
index b8970ee8..5edbfb68 100644
--- a/src/instances/get_instances.py
+++ b/src/instances/get_instances.py
@@ -12,6 +12,12 @@ import subprocess
 
 mightyList = {}
 
+startRegex = "https?:\/{2}(?:[^\s\/]+\.)+"
+endRegex = "(?:\/[^\s\/]+)*\/?"
+torRegex = startRegex + "onion" + endRegex
+i2pRegex = startRegex + "i2p" + endRegex
+lokiRegex = startRegex + "loki" + endRegex
+authRegex = "https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)+[a-zA-Z0-9]+" + endRegex
 
 def filterLastSlash(urlList):
     tmp = {}
@@ -65,16 +71,19 @@ def is_cloudflare(url):
         instance_bin_masked = instance_bin[:mask]
 
         if cloudflare_bin_masked == instance_bin_masked:
-            print(url + ' is ' + Fore.RED + 'cloudflare' + Style.RESET_ALL)
+            print(url + ' is behind ' + Fore.RED + 'cloudflare' + Style.RESET_ALL)
             return True
     return False
 
 
 def is_authenticate(url):
     try:
+        if re.match(authRegex, url):
+            print(url + ' requires ' + Fore.RED + 'authentication' + Style.RESET_ALL)
+            return True
         r = requests.get(url, timeout=5)
         if 'www-authenticate' in r.headers:
-            print(url + ' is ' + Fore.RED + 'authenticate' + Style.RESET_ALL)
+            print(url + ' requires ' + Fore.RED + 'authentication' + Style.RESET_ALL)
             return True
     except:
         return False
@@ -99,9 +108,26 @@ def invidious():
 
 
 def piped():
-    json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
-    with open('./src/instances/piped.json') as file:
-        mightyList['piped'] = json.load(file)
+    r = requests.get(
+        'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md')
+
+    tmp = re.findall(
+        '(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text)
+    _list = {}
+    _list['normal'] = []
+    _list['tor'] = []
+    _list['i2p'] = []
+    _list['loki'] = []
+    for item in tmp:
+        try:
+            url = requests.get(item, timeout=5).url
+            if url.strip("/") == item:
+                continue
+            else:
+                _list['normal'].append(url)
+        except:
+            continue
+    mightyList['piped'] = _list
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Piped')
 
 
@@ -110,7 +136,7 @@ def pipedMaterial():
         'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md')
 
     tmp = re.findall(
-            r"\| (https?:\/{2}(?:.+\.)+[a-zA-Z0-9]*) +\|", r.text)
+            r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\|", r.text)
     pipedMaterialList = {}
     pipedMaterialList['normal'] = []
     pipedMaterialList['tor'] = []
@@ -122,6 +148,13 @@ def pipedMaterial():
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'pipedMaterial')
 
 
+def cloudtube():
+    json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
+    with open('./src/instances/cloudtube.json') as file:
+        mightyList['cloudtube'] = json.load(file)
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'CloudTube')
+
+
 def proxitok():
     r = requests.get(
         'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md')
@@ -156,30 +189,29 @@ def send():
 
 
 def nitter():
-    r = requests.get('https://github.com/zedeus/nitter/wiki/Instances')
-    soup = BeautifulSoup(r.text, 'html.parser')
-    markdownBody = soup.find(class_='markdown-body')
-    tables = markdownBody.find_all('table')
-    tables.pop(3)
-    tables.pop(3)
+    r = requests.get('https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md')
+    tmp = re.findall(
+        r"(?:(?:\| \[(?:\S+\.)+[a-zA-Z]+\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z]+)\/?\) (?:\((?:\S+ ?\S*)\) )? *\| (?:✅|🇩🇪) +\|(?:(?:\n)|(?: (?:❌)|(?: ✅)|(?: ❓)|(?: \[))))|(?:-   \[(?:\S+\.)+(?:(?:i2p)|(?:loki))\]\((https?:\/{2}(?:\S+\.)(?:(?:i2p)|(?:loki)))\/?\)))", r.text)
+
     nitterList = {}
     nitterList['normal'] = []
     nitterList['tor'] = []
     nitterList['i2p'] = []
     nitterList['loki'] = []
-    for table in tables:
-        tbody = table.find('tbody')
-        trs = tbody.find_all('tr')
-        for tr in trs:
-            td = tr.find('td')
-            a = td.find('a')
-            url = a.contents[0]
-            if url.endswith('.onion'):
-                url = 'http://' + url
-                nitterList['tor'].append(url)
+    for item in tmp:
+        for i in item:
+            if i == '':
+                continue
             else:
-                url = 'https://' + url
-                nitterList['normal'].append(url)
+                item = i
+        if re.search(torRegex, item):
+            nitterList['tor'].append(item)
+        elif re.search(i2pRegex, item):
+            nitterList['i2p'].append(item)
+        elif re.search(lokiRegex, item):
+            nitterList['loki'].append(item)
+        else:
+            nitterList['normal'].append(item)
     mightyList['nitter'] = nitterList
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Nitter')
 
@@ -211,7 +243,7 @@ def libreddit():
         r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text)
 
     for item in tmp:
-        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+        if re.search(torRegex, item):
             libredditList['tor'].append(item)
         else:
             libredditList['normal'].append(item)
@@ -292,7 +324,7 @@ def quetre():
 
 
     for item in tmp:
-        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+        if re.search(torRegex, item):
             _list['tor'].append(item)
         else:
             _list['normal'].append(item)
@@ -316,7 +348,7 @@ def libremdb():
     for item in tmp:
         if item.strip() == "":
             continue
-        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+        if re.search(torRegex, item):
             _list['tor'].append(item)
         else:
             _list['normal'].append(item)
@@ -324,6 +356,19 @@ def libremdb():
     mightyList['libremdb'] = _list
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Libremdb')
 
+def simpleertube():
+    r = requests.get('https://simple-web.org/instances/simpleertube')
+    _list = {}
+    _list['normal'] = []
+    _list['tor'] = []
+    _list['i2p'] = []
+    _list['loki'] = []
+    for item in r.text.strip().split('\n'):
+        _list['normal'].append('https://' + item)
+
+    mightyList['simpleertube'] = _list
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SimpleerTube')
+
 
 def simplytranslate():
     r = requests.get('https://simple-web.org/instances/simplytranslate')
@@ -381,12 +426,12 @@ def searx_searxng():
     searxngList['i2p'] = []
     searxngList['loki'] = []
     for item in rJson['instances']:
-        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item[:-1]):
+        if re.search(torRegex, item[:-1]):
             if (rJson['instances'][item].get('generator') == 'searxng'):
                 searxngList['tor'].append(item[:-1])
             else:
                 searxList['tor'].append(item[:-1])
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item[:-1]):
+        elif re.search(torRegex, item[:-1]):
             if (rJson['instances'][item].get('generator') == 'searxng'):
                 searxngList['i2p'].append(item[:-1])
             else:
@@ -412,9 +457,9 @@ def whoogle():
     whoogleList['i2p'] = []
     whoogleList['loki'] = []
     for item in tmpList:
-        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+        if re.search(torRegex, item):
             whoogleList['tor'].append(item)
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item):
+        elif re.search(torRegex, item):
             whoogleList['i2p'].append(item)
         else:
             whoogleList['normal'].append(item)
@@ -437,9 +482,9 @@ def librex():
     for item in tmp:
         if item.strip() == "":
             continue
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+        elif re.search(torRegex, item):
             _list['tor'].append(item)
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item):
+        elif re.search(i2pRegex, item):
             _list['i2p'].append(item)
         else:
             _list['normal'].append(item)
@@ -481,16 +526,64 @@ def librarian():
         url = item['url']
         if url.strip() == "":
             continue
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", url):
+        elif re.search(torRegex, url):
             librarianList['tor'].append(url)
-        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", url):
+        elif re.search(i2pRegex, url):
             librarianList['i2p'].append(url)
+        elif re.search(lokiRegex, url):
+            librarianList['loki'].append(url)
         else:
             librarianList['normal'].append(url)
     mightyList['librarian'] = librarianList
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Librarian')
 
 
+def neuters():
+    json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
+    with open('./src/instances/neuters.json') as file:
+        mightyList['neuters'] = json.load(file)
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Neuters')
+
+
+def beatbump():
+    json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
+    with open('./src/instances/beatbump.json') as file:
+        mightyList['beatbump'] = json.load(file)
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Beatbump')
+
+
+def hyperpipe():
+    r = requests.get(
+        'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json')
+    rJson = json.loads(r.text)
+    hyperpipeList = {}
+    hyperpipeList['normal'] = []
+    hyperpipeList['tor'] = []
+    hyperpipeList['i2p'] = []
+    hyperpipeList['loki'] = []
+    for item in rJson:
+        url = item['url']
+        if url.strip() == "":
+            continue
+        elif re.search(torRegex, url):
+            hyperpipeList['tor'].append(url)
+        elif re.search(i2pRegex, url):
+            hyperpipeList['i2p'].append(url)
+        elif re.search(lokiRegex, url):
+            hyperpipeList['loki'].append(url)
+        else:
+            hyperpipeList['normal'].append(url)
+    mightyList['hyperpipe'] = hyperpipeList
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Hyperpipe')
+
+
+def facil():
+    json_object = json.dumps(mightyList, ensure_ascii=False, indent=2)
+    with open('./src/instances/facil.json') as file:
+        mightyList['facil'] = json.load(file)
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'FacilMap')
+
+
 def peertube():
     r = requests.get(
         'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt')
@@ -515,6 +608,7 @@ def isValid(url):  # This code is contributed by avanitrachhadiya2155
 invidious()
 piped()
 pipedMaterial()
+cloudtube()
 proxitok()
 send()
 nitter()
@@ -532,6 +626,11 @@ whoogle()
 librex()
 rimgo()
 librarian()
+neuters()
+beatbump()
+hyperpipe()
+facil()
+simpleertube()
 mightyList = filterLastSlash(mightyList)
 
 cloudflare = []
diff --git a/src/instances/neuters.json b/src/instances/neuters.json
new file mode 100644
index 00000000..9e224a52
--- /dev/null
+++ b/src/instances/neuters.json
@@ -0,0 +1,6 @@
+{
+	"normal": ["https://neuters.de"],
+	"tor": [],
+	"i2p": [],
+	"loki": []
+}
diff --git a/src/instances/piped.json b/src/instances/piped.json
deleted file mode 100644
index 2f742ff9..00000000
--- a/src/instances/piped.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "normal": [
-    "https://piped.kavin.rocks",
-    "https://piped.tokhmi.xyz",
-    "https://piped.moomoo.me",
-    "https://il.ax",
-    "https://piped.syncpundit.com",
-    "https://piped.mha.fi",
-    "https://piped.shimul.me",
-    "https://pipedus.palash.dev/",
-    "https://yt.jae.fi",
-    "https://piped.mint.lgbt",
-    "https://piped.privacy.com.de",
-    "https://piped.esmailelbob.xyz"
-  ],
-  "tor": [
-    "http://piped2bbch4xslbl2ckr6k62q56kon56ffowxaqzy42ai22a4sash3ad.onion"
-  ],
-  "i2p": [],
-  "loki": []
-}
\ No newline at end of file