Merge branch 'BobIsMyManager-librex'

1 files changed, 112 insertions, 14 deletions

diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py
index 2227a39b..b8970ee8 100644
--- a/src/instances/get_instances.py
+++ b/src/instances/get_instances.py
@@ -87,6 +87,8 @@ def invidious():
     invidiousList = {}
     invidiousList['normal'] = []
     invidiousList['tor'] = []
+    invidiousList['i2p'] = []
+    invidiousList['loki'] = []
     for instance in rJson:
         if instance[1]['type'] == 'https':
             invidiousList['normal'].append(instance[1]['uri'])
@@ -103,6 +105,23 @@ def piped():
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Piped')
 
 
+def pipedMaterial():
+    r = requests.get(
+        'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md')
+
+    tmp = re.findall(
+            r"\| (https?:\/{2}(?:.+\.)+[a-zA-Z0-9]*) +\|", r.text)
+    pipedMaterialList = {}
+    pipedMaterialList['normal'] = []
+    pipedMaterialList['tor'] = []
+    pipedMaterialList['i2p'] = []
+    pipedMaterialList['loki'] = []
+    for item in tmp:
+        pipedMaterialList['normal'].append(item)
+    mightyList['pipedMaterial'] = pipedMaterialList
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'pipedMaterial')
+
+
 def proxitok():
     r = requests.get(
         'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md')
@@ -112,6 +131,8 @@ def proxitok():
     proxiTokList = {}
     proxiTokList['normal'] = []
     proxiTokList['tor'] = []
+    proxiTokList['i2p'] = []
+    proxiTokList['loki'] = []
     for item in tmp:
         proxiTokList['normal'].append(re.sub(r'/$', '', item))
     mightyList['proxiTok'] = proxiTokList
@@ -126,6 +147,8 @@ def send():
     sendList = {}
     sendList['normal'] = []
     sendList['tor'] = []
+    sendList['i2p'] = []
+    sendList['loki'] = []
     for item in tmp:
         sendList['normal'].append(item)
     mightyList['send'] = sendList
@@ -142,6 +165,8 @@ def nitter():
     nitterList = {}
     nitterList['normal'] = []
     nitterList['tor'] = []
+    nitterList['i2p'] = []
+    nitterList['loki'] = []
     for table in tables:
         tbody = table.find('tbody')
         trs = tbody.find_all('tr')
@@ -160,11 +185,13 @@ def nitter():
 
 
 def bibliogram():
-    r = requests.get('https://bibliogram.pussthecat.org/api/instances')
+    r = requests.get('https://bibliogram.art/api/instances')
     rJson = json.loads(r.text)
     bibliogramList = {}
     bibliogramList['normal'] = []
     bibliogramList['tor'] = []
+    bibliogramList['i2p'] = []
+    bibliogramList['loki'] = []
     for item in rJson['data']:
         bibliogramList['normal'].append(item['address'])
     mightyList['bibliogram'] = bibliogramList
@@ -177,13 +204,14 @@ def libreddit():
     libredditList = {}
     libredditList['normal'] = []
     libredditList['tor'] = []
+    libredditList['i2p'] = []
+    libredditList['loki'] = []
 
     tmp = re.findall(
         r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text)
 
-
     for item in tmp:
-        if item.endswith('.onion'):
+        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
             libredditList['tor'].append(item)
         else:
             libredditList['normal'].append(item)
@@ -198,6 +226,8 @@ def teddit():
     tedditList = {}
     tedditList['normal'] = []
     tedditList['tor'] = []
+    tedditList['i2p'] = []
+    tedditList['loki'] = []
     for item in rJson:
         url = item['url']
         if url != '':
@@ -218,13 +248,17 @@ def wikiless():
     wikilessList['normal'] = []
     wikilessList['tor'] = []
     wikilessList['i2p'] = []
+    wikilessList['loki'] = []
     for item in rJson:
         if 'url' in item:
-            wikilessList['normal'].append(item['url'])
+            if item['url'].strip() != "":
+                wikilessList['normal'].append(item['url'])
         if 'onion' in item:
-            wikilessList['tor'].append(item['onion'])
+            if item['onion'].strip() != "":
+                wikilessList['tor'].append(item['onion'])
         if 'i2p' in item:
-            wikilessList['i2p'].append(item['i2p'])
+            if item['i2p'].strip() != "":
+                wikilessList['i2p'].append(item['i2p'])
     mightyList['wikiless'] = wikilessList
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Wikiless')
 
@@ -236,6 +270,8 @@ def scribe():
     scribeList = {}
     scribeList['normal'] = []
     scribeList['tor'] = []
+    scribeList['i2p'] = []
+    scribeList['loki'] = []
     for item in rJson:
         scribeList['normal'].append(item)
     mightyList['scribe'] = scribeList
@@ -248,13 +284,15 @@ def quetre():
     _list = {}
     _list['normal'] = []
     _list['tor'] = []
+    _list['i2p'] = []
+    _list['loki'] = []
 
     tmp = re.findall(
         r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text)
 
 
     for item in tmp:
-        if item.endswith('.onion'):
+        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
             _list['tor'].append(item)
         else:
             _list['normal'].append(item)
@@ -268,6 +306,8 @@ def libremdb():
     _list = {}
     _list['normal'] = []
     _list['tor'] = []
+    _list['i2p'] = []
+    _list['loki'] = []
 
     tmp = re.findall(
         r"\| ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)*\|*[A-Z]{0,}.*\|.*\|", r.text)
@@ -276,7 +316,7 @@ def libremdb():
     for item in tmp:
         if item.strip() == "":
             continue
-        if item.endswith('.onion'):
+        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
             _list['tor'].append(item)
         else:
             _list['normal'].append(item)
@@ -318,6 +358,8 @@ def linvgatranslate():
     lingvaList = {}
     lingvaList['normal'] = []
     lingvaList['tor'] = []
+    lingvaList['i2p'] = []
+    lingvaList['loki'] = []
     for item in rJson:
         lingvaList['normal'].append(item)
 
@@ -329,20 +371,22 @@ def searx_searxng():
     r = requests.get('https://searx.space/data/instances.json')
     rJson = json.loads(r.text)
     searxList = {}
+    searxList['normal'] = []
     searxList['tor'] = []
     searxList['i2p'] = []
-    searxList['normal'] = []
+    searxList['loki'] = []
     searxngList = {}
+    searxngList['normal'] = []
     searxngList['tor'] = []
     searxngList['i2p'] = []
-    searxngList['normal'] = []
+    searxngList['loki'] = []
     for item in rJson['instances']:
-        if item[:-1].endswith('.onion'):
+        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item[:-1]):
             if (rJson['instances'][item].get('generator') == 'searxng'):
                 searxngList['tor'].append(item[:-1])
             else:
                 searxList['tor'].append(item[:-1])
-        elif item[:-1].endswith('.i2p'):
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item[:-1]):
             if (rJson['instances'][item].get('generator') == 'searxng'):
                 searxngList['i2p'].append(item[:-1])
             else:
@@ -366,10 +410,11 @@ def whoogle():
     whoogleList['normal'] = []
     whoogleList['tor'] = []
     whoogleList['i2p'] = []
+    whoogleList['loki'] = []
     for item in tmpList:
-        if item.endswith('.onion'):
+        if re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
             whoogleList['tor'].append(item)
-        elif item.endswith('.i2p'):
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item):
             whoogleList['i2p'].append(item)
         else:
             whoogleList['normal'].append(item)
@@ -377,6 +422,31 @@ def whoogle():
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Whoogle')
 
 
+def librex():
+    r = requests.get(
+        'https://raw.githubusercontent.com/hnhx/librex/main/README.md')
+    _list = {}
+    _list['normal'] = []
+    _list['tor'] = []
+    _list['i2p'] = []
+    _list['loki'] = []
+
+    tmp = re.findall(
+            r"\| {1,2}\[(?:(?:[a-zA-Z0-9]+\.)+[a-zA-Z]{2,}|✅)\]\((https?:\/{2}(?:[a-zA-Z0-9]+\.)+[a-zA-Z0-9]{2,})", r.text)
+
+    for item in tmp:
+        if item.strip() == "":
+            continue
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", item):
+            _list['tor'].append(item)
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", item):
+            _list['i2p'].append(item)
+        else:
+            _list['normal'].append(item)
+    mightyList['librex'] = _list
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Librex')
+
+
 def rimgo():
     r = requests.get(
         'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json')
@@ -385,6 +455,7 @@ def rimgo():
     rimgoList['normal'] = []
     rimgoList['tor'] = []
     rimgoList['i2p'] = []
+    rimgoList['loki'] = []
     for item in rJson:
         if 'url' in item:
             rimgoList['normal'].append(item['url'])
@@ -396,6 +467,30 @@ def rimgo():
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Rimgo')
 
 
+def librarian():
+    r = requests.get(
+        'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json')
+    rJson = json.loads(r.text)
+    librarianList = {}
+    librarianList['normal'] = []
+    librarianList['tor'] = []
+    librarianList['i2p'] = []
+    librarianList['loki'] = []
+    instances = rJson['instances']
+    for item in instances:
+        url = item['url']
+        if url.strip() == "":
+            continue
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+onion(?:\/[a-zA-Z0-9]+)*\/?", url):
+            librarianList['tor'].append(url)
+        elif re.search("https?:\/{2}(?:[a-zA-Z0-9]+\.)+i2p(?:\/[a-zA-Z0-9]+)*\/?", url):
+            librarianList['i2p'].append(url)
+        else:
+            librarianList['normal'].append(url)
+    mightyList['librarian'] = librarianList
+    print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Librarian')
+
+
 def peertube():
     r = requests.get(
         'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt')
@@ -419,6 +514,7 @@ def isValid(url):  # This code is contributed by avanitrachhadiya2155
 
 invidious()
 piped()
+pipedMaterial()
 proxitok()
 send()
 nitter()
@@ -433,7 +529,9 @@ simplytranslate()
 linvgatranslate()
 searx_searxng()
 whoogle()
+librex()
 rimgo()
+librarian()
 mightyList = filterLastSlash(mightyList)
 
 cloudflare = []