diff options
Diffstat (limited to 'src/instances')
| -rw-r--r-- | src/instances/beatbump.json | 4 | ||||
| -rw-r--r-- | src/instances/bibliogram.json | 2 | ||||
| -rw-r--r-- | src/instances/blacklist.json | 29 | ||||
| -rw-r--r-- | src/instances/cloudtube.json | 2 | ||||
| -rw-r--r-- | src/instances/data.json | 291 | ||||
| -rw-r--r-- | src/instances/facil.json | 2 | ||||
| -rw-r--r-- | src/instances/get_instances.py | 640 | ||||
| -rw-r--r-- | src/instances/neuters.json | 2 |
8 files changed, 409 insertions, 563 deletions
diff --git a/src/instances/beatbump.json b/src/instances/beatbump.json index e09273dd..c5c78123 100644 --- a/src/instances/beatbump.json +++ b/src/instances/beatbump.json @@ -1,6 +1,6 @@ { - "normal": ["https://beatbump.ml", "https://beatbump.esmailelbob.xyz"], - "tor": ["http://beatbump.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion"], + "clearnet": ["https://beatbump.ml", "https://beatbump.esmailelbob.xyz"], + "tor": ["http://beatbump.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion"], "i2p": [], "loki": [] } diff --git a/src/instances/bibliogram.json b/src/instances/bibliogram.json index d72f2129..b08aa7a0 100644 --- a/src/instances/bibliogram.json +++ b/src/instances/bibliogram.json @@ -1,5 +1,5 @@ { - "normal": ["https://bibliogram.1d4.us", "https://bibliogram.froth.zone", "https://ig.tokhmi.xyz", "https://ig.beparanoid.de", "https://bibliogram.priv.pw"], + "clearnet": ["https://bibliogram.1d4.us", "https://bibliogram.froth.zone", "https://ig.tokhmi.xyz", "https://ig.beparanoid.de", "https://bibliogram.priv.pw"], "tor": [], "i2p": [], "loki": [] diff --git a/src/instances/blacklist.json b/src/instances/blacklist.json index bedaa748..8bf43ebf 100644 --- a/src/instances/blacklist.json +++ b/src/instances/blacklist.json @@ -7,11 +7,9 @@ "https://piped.moomoo.me", "https://piped.mha.fi", "https://de-piped.shimul.me", - "https://pipedus.palash.dev", "https://watch.whatever.social", "https://piped.garudalinux.org", "https://y.rivo.lol", - "https://cringe.whatever.social", "https://nitter.domain.glass", "https://birdsite.xanny.family", "https://notabird.site", @@ -32,15 +30,15 @@ "https://libreddit.domain.glass", "https://r.nf", "https://libreddit.hu", - "https://lr.stilic.ml", "https://reddi.tk", "https://r.walkx.org", - "https://libreddit.yonalee.eu", - "https://libreddit.winscloud.net", "https://r.ahwx.org", "https://reddit.dr460nf1r3.org", "https://libreddit.encrypted-data.xyz", "https://libreddit.eu.org", + "https://libreddit.cachyos.org", + "https://futureddit.gq", + "https://libreddit.freedit.eu", "https://teddit.domain.glass", "https://teddit.httpjames.space", "https://teddit.encrypted-data.xyz", @@ -48,11 +46,13 @@ "https://tedd.it", "https://wiki.privacytools.io", "https://teddit.rawbit.ninja", + "https://wiki.604kph.xyz", "https://wiki.privacytools.io", "https://wikiless.rawbit.ninja", "https://scribe.rawbit.ninja", "https://lingva.garudalinux.org", "https://translate.dr460nf1r3.org", + "https://libretranslate.com", "https://searx.run", "https://searx.org", "https://search.garudalinux.org", @@ -68,31 +68,32 @@ ], "offline": [ "https://invidious.rhyshl.live", - "https://piped.esmailelbob.xyz", "https://proxitok.odyssey346.dev", "https://nhanh.cloud", "https://ntr.odyssey346.dev", - "https://libredd.it", - "https://libreddit.notyourcomputer.net", + "https://futureddit.gq", "https://teddit.ggc-project.de", - "https://teddit.nautolan.racing", - "https://quetreus.herokuapp.com", + "https://teddit.froth.zone", + "https://teddit.tokhmi.xyz", + "https://wiki.privacytools.io", "https://quetre.odyssey346.dev", - "https://translate.northboot.xyz", - "https://translate.priv.pw", "https://st.odyssey346.dev", + "https://translate.projectsegfau.lt", "https://jsearch.pw", "https://searx.gnu.style", "https://searx.semipvt.com", "https://etsi.me", + "https://search.chemicals-in-the-water.eu", "https://search.zzls.xyz", "https://searx.ericaftereric.top", "https://searx.fmac.xyz", "https://www.webrats.xyz", + "https://gowogle.voring.me", "https://whoogle.esmailelbob.xyz", + "https://search.wef.lol", "https://i.bcow.xyz", - "https://i.actionsack.com", "https://lbry.bcow.xyz", - "https://beatbump.ml" + "https://beatbump.ml", + "https://bw.odyssey346.dev" ] } \ No newline at end of file diff --git a/src/instances/cloudtube.json b/src/instances/cloudtube.json index 2c55b083..8742d09c 100644 --- a/src/instances/cloudtube.json +++ b/src/instances/cloudtube.json @@ -1,5 +1,5 @@ { - "normal": ["https://tube.cadence.moe"], + "clearnet": ["https://tube.cadence.moe"], "tor": [], "i2p": [], "loki": [] diff --git a/src/instances/data.json b/src/instances/data.json index a5e6cb1f..a5a7177e 100644 --- a/src/instances/data.json +++ b/src/instances/data.json @@ -1,6 +1,6 @@ { "invidious": { - "normal": [ + "clearnet": [ "https://yewtu.be", "https://vid.puffyan.us", "https://invidious.snopyta.org", @@ -14,34 +14,35 @@ "https://invidious.flokinet.to", "https://inv.bp.projectsegfau.lt", "https://inv.vern.cc", - "https://invidious.projectsegfau.lt", "https://invidious.nerdvpn.de", + "https://invidious.projectsegfau.lt", "https://invidious.rhyshl.live", "https://inv.privacy.com.de", "https://invidious.slipfox.xyz", "https://invidious.esmailelbob.xyz", "https://youtube.076.ne.jp", - "https://invidious.weblibre.org", "https://invidious.namazso.eu" ], "tor": [ - "http://am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd.onion", - "http://inv.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", - "http://invidious.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion", - "http://euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd.onion", - "http://u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad.onion", - "http://osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd.onion", - "http://grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion", - "http://kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion", "http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion", "http://c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion", - "http://ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid.onion" + "http://grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion", + "http://osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd.onion", + "http://u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad.onion", + "http://euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd.onion", + "http://invidious.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://inv.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd.onion", + "http://ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid.onion", + "http://kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion" + ], + "i2p": [ + "http://verni6dr4qxjgjumnvesxerh5rvhv6oy5ddeibaqy5d7tgbiiyfa.b32.i2p" ], - "i2p": [], "loki": [] }, "piped": { - "normal": [ + "clearnet": [ "https://piped.kavin.rocks", "https://piped.tokhmi.xyz", "https://piped.moomoo.me", @@ -57,29 +58,26 @@ "https://piped.esmailelbob.xyz", "https://piped.projectsegfau.lt", "https://piped.privacydev.net", - "https://piped.palveluntarjoaja.eu", "https://piped.smnz.de", "https://piped.adminforge.de", - "https://watch.whatevertinfoil.de" + "https://watch.whatevertinfoil.de", + "https://piped.qdi.fi" ], "tor": [], "i2p": [], "loki": [] }, "pipedMaterial": { - "normal": [ - "https://piped-material.১.net", - "https://piped-material.ftp.sh", - "https://piped-staging.ftp.sh", - "https://ui.piped.১.net", - "https://ng.piped.১.net" + "clearnet": [ + "https://piped-material.xn--17b.net", + "https://piped-material.ftp.sh" ], "tor": [], "i2p": [], "loki": [] }, "cloudtube": { - "normal": [ + "clearnet": [ "https://tube.cadence.moe" ], "tor": [], @@ -87,12 +85,11 @@ "loki": [] }, "proxiTok": { - "normal": [ + "clearnet": [ "https://proxitok.pabloferreiro.es", "https://proxitok.pussthecat.org", "https://tok.habedieeh.re", "https://proxitok.esmailelbob.xyz", - "https://cringe.whatever.social", "https://proxitok.dhusch.de", "https://proxitok.privacydev.net", "https://proxitok.odyssey346.dev", @@ -100,12 +97,14 @@ "https://tok.adminforge.de", "https://proxitok.manasiwibi.com" ], - "tor": [], + "tor": [ + "http://vywqfflneajejuhg7o5iklqvzemu2fcdrb3gtkvnyqsho6qin5svdsad.onion" + ], "i2p": [], "loki": [] }, "send": { - "normal": [ + "clearnet": [ "https://send.vis.ee", "https://send.zcyph.cc", "https://send.turingpoint.de", @@ -129,9 +128,9 @@ "loki": [] }, "nitter": { - "normal": [ + "clearnet": [ "https://nitter.net", - "https://nitter.42l.fr", + "https://nitter.lacontrevoie.fr", "https://nitter.pussthecat.org", "https://nitter:nitter@nitter.nixnet.services", "https://nitter.fdn.fr", @@ -207,7 +206,12 @@ "https://nitter.smnz.de", "https://nitter.twei.space", "https://nitter.inpt.fr", - "https://nitter.d420.de" + "https://nitter.d420.de", + "https://nitter.caioalonso.com", + "https://nitter.at", + "https://nitter.drivet.xyz", + "https://nitter.pw", + "https://nitter.nicfab.eu" ], "tor": [ "http://nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion", @@ -233,7 +237,11 @@ "http://wiio4sgs4247brk7hj6qck2jxnvldwfdbguigc5ivpxrsegnliyfvuqd.onion", "http://qwikxx2erhx6qrymued6ox2qkf2yeogjwypqvzoif4fqkljixasr6oid.onion", "http://4g47cxugkohbweao2x66nnxxfoe3k7gdfzxej537nhdbwr522sbjxeqd.onion", - "http://tw.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion" + "http://nt.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://tw.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion", + "http://r2eqimhkvxboaltbdsectoo3hkf476pyemsdykclexzajbfx5v6ojlyd.onion", + "http://li7snkj6oituazbkr5clmilccwwumhd2dntbhttxomy4dfakeeoar4qd.onion", + "http://a5xyidyppowvblric6k6nixgf2eqwnb7zzeaarj7slqbv7tb6ip5t3ad.onion" ], "i2p": [ "http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p", @@ -247,7 +255,7 @@ ] }, "bibliogram": { - "normal": [ + "clearnet": [ "https://bibliogram.1d4.us", "https://bibliogram.froth.zone", "https://ig.tokhmi.xyz", @@ -259,97 +267,65 @@ "loki": [] }, "libreddit": { - "normal": [ - "https://libredd.it", + "clearnet": [ "https://libreddit.spike.codes", - "https://libreddit.dothq.co", "https://libreddit.kavin.rocks", "https://reddit.invak.id", - "https://reddit.phii.me", "https://lr.riverside.rocks", "https://libreddit.strongthany.cc", - "https://libreddit.database.red", "https://libreddit.privacy.com.de", "https://libreddit.domain.glass", - "https://libreddit.sugoma.tk", - "https://libreddit.jamiethalacker.dev", - "https://reddit.artemislena.eu", "https://r.nf", - "https://libreddit.some-things.org", "https://reddit.stuehieyr.com", "https://lr.mint.lgbt", - "https://libreddit.igna.rocks", - "https://libreddit.autarkic.org", - "https://libreddit.flux.industries", "https://libreddit.drivet.xyz", - "https://lr.oversold.host", "https://libreddit.de", "https://libreddit.pussthecat.org", - "https://libreddit.mutahar.rocks", "https://libreddit.northboot.xyz", "https://leddit.xyz", - "https://de.leddit.xyz", - "https://lr.cowfee.moe", "https://libreddit.hu", "https://libreddit.totaldarkness.net", "https://libreddit.esmailelbob.xyz", "https://lr.vern.cc", "https://libreddit.nl", - "https://lr.stilic.ml", "https://reddi.tk", - "https://libreddit.bus-hit.me", - "https://libreddit.datatunnel.xyz", - "https://libreddit.crewz.me", "https://r.walkx.org", "https://libreddit.kylrth.com", - "https://libreddit.yonalee.eu", - "https://libreddit.winscloud.net", "https://libreddit.tiekoetter.com", "https://reddit.rtrace.io", - "https://libreddit.lunar.icu", "https://libreddit.privacydev.net", - "https://libreddit.notyourcomputer.net", "https://r.ahwx.org", "https://bob.fr.to", - "https://reddit.beparanoid.de", "https://libreddit.dcs0.hu", "https://reddit.dr460nf1r3.org", "https://rd.jae.su", - "https://libreddit.mha.fi", "https://libreddit.foss.wtf", "https://libreddit.encrypted-data.xyz", "https://libreddit.eu.org", - "https://l.opnxng.com" + "https://l.opnxng.com", + "https://libreddit.cachyos.org", + "https://rd.funami.tech", + "https://libreddit.projectsegfau.lt", + "https://futureddit.gq", + "https://lr.slipfox.xyz", + "https://libreddit.oxymagnesium.com", + "https://reddit.utsav2.dev", + "https://libreddit.freedit.eu" ], "tor": [ - "http://spjmllawtheisznfs7uryhxumin26ssv2draj7oope3ok3wuhy43eoyd.onion", - "http://fwhhsbrbltmrct5hshrnqlqygqvcgmnek3cnka55zj4y7nuus5muwyyd.onion", - "http://kphht2jcflojtqte4b4kyx7p2ahagv4debjj32nre67dxz7y57seqwyd.onion", - "http://inytumdgnri7xsqtvpntjevaelxtgbjqkuqhtf6txxhwbll2fwqtakqd.onion", - "http://liredejj74h5xjqr2dylnl5howb2bpikfowqoveub55ru27x43357iid.onion", - "http://kzhfp3nvb4qp575vy23ccbrgfocezjtl5dx66uthgrhu7nscu6rcwjyd.onion", "http://ecue64ybzvn6vjzl37kcsnwt4ycmbsyf74nbttyg7rkc3t3qwnj7mcyd.onion", - "http://ledditqo2mxfvlgobxnlhrkq4dh34jss6evfkdkb2thlvy6dn4f4gpyd.onion", - "http://libredoxhxwnmsb6dvzzd35hmgzmawsq5i764es7witwhddvpc2razid.onion", - "http://libreddit.2syis2nnyytz6jnusnjurva4swlaizlnleiks5mjp46phuwjbdjqwgqd.onion", - "http://ol5begilptoou34emq2sshf3may3hlblvipdjtybbovpb7c7zodxmtqd.onion", - "http://lbrdtjaj7567ptdd4rv74lv27qhxfkraabnyphgcvptl64ijx2tijwid.onion", - "http://libreddit.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion", - "http://reddit.prnoid54e44a4bduq5due64jkk7wcnkxcp5kv3juncm7veptjcqudgyd.onion", - "http://inz6tbezfwzexva6dize4cqraj2tjdhygxabmcgysccesvw2pybzhbyd.onion", - "http://libreddit.micohauwkjbyw5meacrb4ipicwvwg4xtzl7y7viv53kig2mdcsvwkyyd.onion", - "http://lr.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + "http://libreddit.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://ojwp2gtj7dq7scd7gnbac6wp53tklgsicteabrnx2pr7zai64wriiaad.onion" ], "i2p": [], "loki": [] }, "teddit": { - "normal": [ + "clearnet": [ "https://teddit.net", "https://teddit.ggc-project.de", "https://teddit.zaggy.nl", "https://teddit.namazso.eu", - "https://teddit.nautolan.racing", "https://teddit.tinfoil-hat.net", "https://teddit.domain.glass", "https://snoo.ioens.is", @@ -386,11 +362,16 @@ "http://teddit.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion", "http://24fympskbrdgbf4afuvhqwwl2tv3y2vwxg5t2ktozd4j5b3fob5ntzyd.onion" ], - "i2p": [], + "i2p": [ + "http://xugoqcf2pftm76vbznx4xuhrzyb5b6zwpizpnw2hysexjdn5l2tq.b32.i2p", + "http://teddit.i2p", + "http://vzeiwzi7ogwl3ijrfek4fbtwhvamxcpyqoc3s4vcgnhlp54s5clq.b32.i2p", + "http://verncco2oaxjikammz4pi7umzp673cme6zuemx7yeeewspwrw3va.b32.i2p" + ], "loki": [] }, "wikiless": { - "normal": [ + "clearnet": [ "https://wikiless.org", "https://wikiless.sethforprivacy.com", "https://wiki.604kph.xyz", @@ -423,7 +404,7 @@ "loki": [] }, "scribe": { - "normal": [ + "clearnet": [ "https://scribe.rip", "https://scribe.nixnet.services", "https://scribe.citizen4.eu", @@ -439,7 +420,7 @@ "loki": [] }, "quetre": { - "normal": [ + "clearnet": [ "https://quetre.iket.me", "https://qr.vern.cc", "https://quetre.pussthecat.org", @@ -454,17 +435,31 @@ "http://quetre.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", "http://qr.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" ], - "i2p": [], + "i2p": [ + "http://vernnflenvsqccuanaun7yydnmturi4jkyxlyzhn6ultpje66c3q.b32.i2p" + ], "loki": [] }, "libremdb": { - "normal": [], - "tor": [], - "i2p": [], + "clearnet": [ + "https://libremdb.iket.me", + "https://libremdb.pussthecat.org", + "https://libremdbeu.herokuapp.com", + "https://lmdb.tokhmi.xyz", + "https://libremdb.esmailelbob.xyz", + "https://ld.vern.cc" + ], + "tor": [ + "http://libremdb.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://ld.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [ + "http://vernz3ubrntql4wrgyrssd6u3qzi36zrhz2agbo6vibzbs5olk2q.b32.i2p" + ], "loki": [] }, "simplyTranslate": { - "normal": [ + "clearnet": [ "https://simplytranslate.org", "https://st.tokhmi.xyz", "https://translate.josias.dev", @@ -499,7 +494,7 @@ ] }, "lingva": { - "normal": [ + "clearnet": [ "https://lingva.ml", "https://translate.igna.ooo", "https://lingva.pussthecat.org", @@ -514,8 +509,26 @@ "i2p": [], "loki": [] }, + "libreTranslate": { + "clearnet": [ + "https://libretranslate.com", + "https://libretranslate.de", + "https://translate.argosopentech.com", + "https://translate.api.skitzen.com", + "https://translate.fortytwo-it.com", + "https://translate.terraprint.co", + "https://lt.vern.cc" + ], + "tor": [ + "http://lt.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [ + "http://vernf45n7mxwqnp5riaax7p67pwcl7wcefdcnqqvim7ckdx4264a.b32.i2p" + ], + "loki": [] + }, "searx": { - "normal": [ + "clearnet": [ "https://dynabyte.ca", "https://jsearch.pw", "https://search.ethibox.fr", @@ -532,6 +545,7 @@ "https://searx.nakhan.net", "https://searx.netzspielplatz.de", "https://searx.nixnet.services", + "https://searx.rimkus.it", "https://searx.ru", "https://searx.run", "https://searx.semipvt.com", @@ -554,13 +568,12 @@ ], "tor": [ "http://3afisqjw2rxm6z7mmstyt5rx75qfqrgxnkzftknbp2vhipr2nrmrjdyd.onion", - "http://w5rl6wsd7mzj4bdkbuqvzidet5osdsm5jhg2f7nvfidakfq5exda5wid.onion", + "http://privateoz3u5utrimal2edr56j3r5caakektxxgixigdkycuxigvquid.onion", "http://yra4tke2pwcnatxjkufpw6kvebu3h3ti2jca2lcdpgx3mpwol326lzid.onion", "http://z5vawdol25vrmorm4yydmohsd4u6rdoj2sylvoi3e3nqvxkvpqul7bqd.onion", - "http://searxdr3pqz4nydgnqocsia2xbywptxbkympa2emn7zlgggrir4bkfad.onion", "http://zbuc3bbzbfdqqo2x46repx2ddajbha6fpsjeeptjhhhhzji3zopxdqyd.onion", "http://f4qfqajszpx5b7itzxt6mb7kj4ktpgbdq7lq6xaiqyqx6a7de3epptad.onion", - "http://searxfilowxokbogygrigir4wqxfxqzuxofxgdon7dg6rsii4yxzytyd.onion" + "http://searx.micohauwkjbyw5meacrb4ipicwvwg4xtzl7y7viv53kig2mdcsvwkyyd.onion" ], "i2p": [ "http://ransack.i2p", @@ -569,11 +582,10 @@ "loki": [] }, "searxng": { - "normal": [ + "clearnet": [ "https://darmarit.org/searx", "https://de.xcxc.ml", "https://etsi.me", - "https://icanfindit.online", "https://jackgoss.xyz", "https://metasearch.nl", "https://northboot.xyz", @@ -583,6 +595,7 @@ "https://priv.au", "https://privatus.live", "https://s.frlt.one", + "https://s.trung.fun", "https://s.zhaocloud.net", "https://saber.tk", "https://search.0relay.com", @@ -628,13 +641,13 @@ "https://searx.orion-hub.fr", "https://searx.priv.pw", "https://searx.prvcy.eu", - "https://searx.rimkus.it", "https://searx.sethforprivacy.com", "https://searx.sev.monster", "https://searx.slipfox.xyz/searx", "https://searx.tiekoetter.com", "https://searx.zcyph.cc", - "https://searxng.nicfab.it", + "https://searxng.dupa.edu.pl", + "https://searxng.nicfab.eu", "https://searxng.zackptg5.com", "https://serx.ml", "https://sh0.it", @@ -648,14 +661,15 @@ "https://xo.wtf" ], "tor": [ + "http://w5rl6wsd7mzj4bdkbuqvzidet5osdsm5jhg2f7nvfidakfq5exda5wid.onion", "http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion", - "http://privateoz3u5utrimal2edr56j3r5caakektxxgixigdkycuxigvquid.onion", "http://b6sxmon57qza6dt36li7huabie5ntrvjr4q5rc2vvbn4hqvzd4phrvyd.onion", "http://searxngg6zleq6ceboe5ltkyo4hyrb3aaycrgzmrljv3jjlb5vcytead.onion", "http://gbat2pbpg7ys3fi3pbp64667tt5x66mg45xok35bxdw7v55brm7a27yd.onion", "http://b2c7fvbhnfvwnl6oh2tydhzfx4i37kmmrycq42heqbbe7wovq3uzenid.onion", + "http://searxdr3pqz4nydgnqocsia2xbywptxbkympa2emn7zlgggrir4bkfad.onion", "http://searx.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", - "http://searx.micohauwkjbyw5meacrb4ipicwvwg4xtzl7y7viv53kig2mdcsvwkyyd.onion", + "http://searxfilowxokbogygrigir4wqxfxqzuxofxgdon7dg6rsii4yxzytyd.onion", "http://searx.privpw3tndpkw6pnp3g727zfgfdzbu3k6a7chv226s3xymv2p4eiuqyd.onion", "http://rq2w52kyrif3xpfihkgjnhqm3a5aqhoikpv72z3drpjglfzc2wr5z4yd.onion", "http://fub6vgedgeadlu3ctskrpkcqjruh76tckwtj5swfhyblgml2tzgzckqd.onion/searx", @@ -667,7 +681,7 @@ "loki": [] }, "whoogle": { - "normal": [ + "clearnet": [ "https://gowogle.voring.me", "https://s.tokhmi.xyz", "https://search.albony.xyz", @@ -687,10 +701,11 @@ "loki": [] }, "librex": { - "normal": [ + "clearnet": [ "https://librex.beparanoid.de", "https://librex.extravi.dev", "https://search.davidovski.xyz", + "https://librex.kitscomputer.tk", "https://search.funami.tech", "https://librex.catalyst.sx", "https://search.madreyk.xyz", @@ -698,8 +713,7 @@ ], "tor": [ "http://librex.2356uhnbpv5nk3bni5bv6jg2cd6lgj664kwx3lhyelstpttpyv4kk2qd.onion", - "http://ncblhz7q4sfbf755bdbhebfzxcpypz7ewafgi4agatecojz7pln4i3id.onion", - "http://librex.so2mpiyfo4cje7bof5v52y3cvjyo2haxpqfvut4sr6gj2ul4mddx2jid.onion" + "http://ncblhz7q4sfbf755bdbhebfzxcpypz7ewafgi4agatecojz7pln4i3id.onion" ], "i2p": [ "http://rra33hiaf6nmby7jfpqe2gqmng3jnzkvbu2n7jgce7vbhoyuhzya.b32.i2p" @@ -707,7 +721,7 @@ "loki": [] }, "rimgo": { - "normal": [ + "clearnet": [ "https://i.bcow.xyz", "https://rimgo.pussthecat.org", "https://rimgo.totaldarkness.net", @@ -729,7 +743,7 @@ "loki": [] }, "librarian": { - "normal": [ + "clearnet": [ "https://lbry.bcow.xyz", "https://odysee.076.ne.jp", "https://librarian.pussthecat.org", @@ -752,7 +766,7 @@ "loki": [] }, "neuters": { - "normal": [ + "clearnet": [ "https://neuters.de" ], "tor": [], @@ -760,18 +774,18 @@ "loki": [] }, "beatbump": { - "normal": [ + "clearnet": [ "https://beatbump.ml", "https://beatbump.esmailelbob.xyz" ], "tor": [ - "http://beatbump.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion" + "http://beatbump.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion" ], "i2p": [], "loki": [] }, "hyperpipe": { - "normal": [ + "clearnet": [ "https://hyperpipe.surge.sh", "https://hyperpipe.esmailelbob.xyz", "https://listen.whatever.social", @@ -784,7 +798,7 @@ "loki": [] }, "facil": { - "normal": [ + "clearnet": [ "https://facilmap.org" ], "tor": [], @@ -792,7 +806,7 @@ "loki": [] }, "simpleertube": { - "normal": [ + "clearnet": [ "https://tube.simple-web.org", "https://simpleertube.esmailelbob.xyz", "https://stube.tokhmi.xyz", @@ -802,11 +816,30 @@ "i2p": [], "loki": [] }, + "breezeWiki": { + "clearnet": [ + "https://breezewiki.com", + "https://breezewiki.pussthecat.org", + "https://bw.odyssey346.dev", + "https://bw.vern.cc", + "https://breezewiki.esmailelbob.xyz" + ], + "tor": [], + "i2p": [], + "loki": [] + }, "peertube": [ + "https://search.joinpeertube.org", + "https://video.bmu.cloud", + "https://freedomadultgames.video", + "https://tbh.co-shaoghal.net", + "https://tube.kansanvalta.org", + "https://ytube.retronerd.at", + "https://tube.media-techport.de", + "https://notobono.de", "https://video.vegafjord.me", "https://peertube-us.howlround.com", "https://peertube-eu.howlround.com", - "https://guntube.net", "https://pt.freedomwolf.cc", "https://vr360tube.online", "https://bodavr.com", @@ -818,7 +851,6 @@ "https://videos.im.allmendenetz.de", "https://tube.cyberia.club", "https://casstream.nohost.me", - "https://peertube.as62430.net", "https://peertube.askan.info", "https://apertatube.net", "https://beetoons.tv", @@ -834,7 +866,6 @@ "https://tube-test.apps.education.fr", "https://tube-sciences-technologies.apps.education.fr", "https://tube-institutionnel.apps.education.fr", - "https://trutube.online", "https://tube-cycle-3.apps.education.fr", "https://video.manicphase.me", "https://tubulus.openlatin.org", @@ -855,7 +886,6 @@ "https://tube.reseau-canope.fr", "https://frentube.myyouniverse.observer", "https://tube-maternelle.apps.education.fr", - "https://peertube.onionstorm.net", "https://dev.dollarvigilante.tv", "https://video.trankil.info", "https://video.uriopss-pdl.fr", @@ -863,13 +893,11 @@ "https://video.3cmr.fr", "https://peertube.miguelcr.me", "https://video.thinkof.name", - "https://video.comun.al", "https://video.occm.cc", "https://tube-action-educative.apps.education.fr", "https://jahve.pl", "https://videos.yesil.club", "https://peertube.adjutor.xyz", - "https://peertube.terranout.mine.nu", "https://tube.hunterjozwiak.com", "https://tube-numerique-educatif.apps.education.fr", "https://tube.itsg.host", @@ -904,7 +932,6 @@ "https://seka.pona.la", "https://syop.tv", "https://watch.thelema.social", - "https://tube.miegl.cz", "https://mov.clov.fr", "https://video.vaku.org.ua", "https://videos.trom.lt", @@ -938,13 +965,10 @@ "https://videos.tormentasolar.win", "https://tube.nestor.coop", "https://live.oldskool.fi", - "https://dytube.com", - "https://tube.thierrytalbert.fr", "https://peertube.informaction.info", "https://tube.ac-amiens.fr", "https://tube.alado.space", "https://tube.network.europa.eu", - "https://vnchich.com", "https://pt.maciej.website", "https://peertube.frontmediatique.fr", "https://peertube.bgzashtita.es", @@ -970,16 +994,13 @@ "https://ocfedtest.hosted.spacebear.ee", "https://video.lono.space", "https://mirrored.rocks", - "https://videopen.net", "https://demo.lioncast.org", "https://peertube.get-racing.de", "https://pierre.tube", "https://mirametube.fr", "https://vidz.dou.bet", - "https://videos.redeyes.site", "https://video.hostpath.de", "https://free-streams.com", - "https://tube.pawelko.net", "https://video.livecchi.cloud", "https://peertube.gsugambit.com", "https://pt.nospy.net", @@ -989,19 +1010,15 @@ "https://videos-passages.huma-num.fr", "https://video.linc.systems", "https://video.sftblw.moe", - "https://peertube.dsmouse.net", "https://ptube.horsentiers.fr", "https://video.cnt.social", "https://tube.03281.fspfc.org", - "https://tube.linc.systems", "https://peertube.ketchup.noho.st", "https://yt.x1337x.fr", "https://videos.rabbit-company.com", "https://video.paradigmthreat.net", - "https://video.lincolncyber.com", "https://vid.twhtv.club", "https://video.retroedge.tech", - "https://truvitv.com", "https://pt.ilyamikcoder.com", "https://peertube.sensin.eu", "https://video.sadmin.io", @@ -1023,8 +1040,6 @@ "https://video.snug.moe", "https://video.avant-le-strike.buzz", "https://videos.ritimo.org", - "https://videos.ephphatha.church", - "https://video.cerclearistote.com", "https://video.greenmycity.eu", "https://przej.me", "https://pt.mezzo.moe", @@ -1036,7 +1051,6 @@ "https://bee-tube.fr", "https://vid.prometheus.systems", "https://tube.nx12.net", - "https://redundant2.peertube.support", "https://peertube.ignifi.me", "https://tv.based.quest", "https://peertube.beardedtek.com", @@ -1055,7 +1069,6 @@ "https://peertube.chatinbit.com", "https://peertube.ffs2play.fr", "https://peertube.swarm.solvingmaz.es", - "https://film.node9.org", "https://peertube.mi-site.net", "https://raptube.antipub.org", "https://video.cm-en-transition.fr", @@ -1071,7 +1084,6 @@ "https://peertube.revelin.fr", "https://peertube.ti-fr.com", "https://video.turbo.chat", - "https://peertube.am-networks.fr", "https://video.chbmeyer.de", "https://video.rs-einrich.de", "https://p2ptv.ru", @@ -1089,7 +1101,6 @@ "https://polskijutub.mkljczk.pl", "https://peertube.noussommes.org", "https://exode.me", - "https://megatube.lilomoino.fr", "https://video.anartist.org", "https://peertube.home.x0r.fr", "https://peertube.marud.fr", @@ -1131,7 +1142,6 @@ "https://tube.childrenshealthdefense.eu", "https://stream.litera.tools", "https://peertube.kriom.net", - "https://peertube.grosist.fr", "https://peertube.gemlog.ca", "https://nettube.uc-netcorsoft.de", "https://live.solari.com", @@ -1142,7 +1152,6 @@ "https://www.orion-hub.fr", "https://tv.orion-serv.fr", "https://video.interru.io", - "https://tube.cnr.it", "https://peertube.dtmf.ca", "https://tube.ponsonaille.fr", "https://tube.int5.net", @@ -1160,12 +1169,9 @@ "https://video.antopie.org", "https://vtr.chikichiki.tube", "https://fedimovie.com", - "https://videos.thinkerview.com", "https://tube.doctors4covidethics.org", - "https://yoba.tv", "https://tube.mediainformationcenter.de", "https://peertube.communecter.org", - "https://peertube.librosphere.fr", "https://queermotion.org", "https://video.audiovisuel-participatif.org", "https://peertube.vip", @@ -1188,7 +1194,6 @@ "https://video.windfluechter.org", "https://pocketnetpeertube10.nohost.me", "https://tube.io18.top", - "https://peertube.remerge.net", "https://tube.geekyboo.net", "https://notretube.asselma.eu", "https://canal.facil.services", @@ -1296,7 +1301,6 @@ "https://peertube.alpharius.io", "https://ptb.lunarviews.net", "https://ovaltube.codinglab.ch", - "https://video.wilkie.how", "https://videos.ahp-numerique.fr", "https://auf1.eu", "https://tube.toontoet.nl", @@ -1389,7 +1393,6 @@ "https://video.lespoesiesdheloise.fr", "https://peertube.luga.at", "https://peertube.roflcopter.fr", - "https://ptube.rousset.nom.fr", "https://peertube.swrs.net", "https://tube.shanti.cafe", "https://videos.cloudron.io", @@ -1426,7 +1429,6 @@ "https://videos.john-livingston.fr", "https://melsungen.peertube-host.de", "https://evangelisch.video", - "https://tube.anufrij.de", "https://videos.mastodont.cat", "https://media.undeadnetwork.de", "https://tube.dragonpsi.xyz", @@ -1477,8 +1479,6 @@ "https://video.mycrowd.ca", "https://kodcast.com", "https://video.altertek.org", - "https://ruraletv.ovh", - "https://videos.weblib.re", "https://tube.oisux.org", "https://peertube.louisematic.site", "https://clap.nerv-project.eu", @@ -1496,7 +1496,6 @@ "https://fotogramas.politicaconciencia.org", "https://peertube.pl", "https://peertube.manalejandro.com", - "https://www4.mir.inter21.net", "https://video.csc49.fr", "https://tube.wolfe.casa", "https://video.dresden.network", @@ -1520,14 +1519,12 @@ "https://tube.nox-rhea.org", "https://peertube.securitymadein.lu", "https://tube.rita.moe", - "https://tuktube.com", "https://mytube.kn-cloud.de", "https://tube.nuagelibre.fr", "https://video.nogafam.es", "https://peertube.stream", "https://videos.leslionsfloorball.fr", "https://player.ojamajo.moe", - "https://ftsi.ru", "https://video.cigliola.com", "https://xxx.noho.st", "https://peertube.stefofficiel.me", @@ -1621,7 +1618,6 @@ "https://peertube.debian.social", "https://tube.piweb.be", "https://peertube.su", - "https://video.hackers.town", "https://tube.fdn.fr", "https://peertube.demonix.fr", "https://videos.hauspie.fr", @@ -1667,7 +1663,6 @@ "https://video.vny.fr", "https://peervideo.club", "https://tube.taker.fr", - "https://peertube.co.uk", "https://video.fitchfamily.org", "https://video.fdlibre.eu", "https://peer.philoxweb.be", diff --git a/src/instances/facil.json b/src/instances/facil.json index 798a9cfb..6c89eb17 100644 --- a/src/instances/facil.json +++ b/src/instances/facil.json @@ -1,5 +1,5 @@ { - "normal": ["https://facilmap.org"], + "clearnet": ["https://facilmap.org"], "tor": [], "i2p": [], "loki": [] diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py index 695569ce..b36cf7ab 100644 --- a/src/instances/get_instances.py +++ b/src/instances/get_instances.py @@ -1,36 +1,57 @@ # Note: Run this script from the root of the repo +import traceback +import logging import requests import json from urllib.parse import urlparse -from bs4 import BeautifulSoup import re -from colorama import Fore, Back, Style -from urllib.parse import urlparse +from colorama import Fore, Style import socket -import subprocess mightyList = {} +config = {} -startRegex = "https?:\/{2}(?:[^\s\/]+\.)+" +startRegex = r"https?:\/{2}(?:[^\s\/]+\.)+" endRegex = "(?:\/[^\s\/]+)*\/?" torRegex = startRegex + "onion" + endRegex i2pRegex = startRegex + "i2p" + endRegex lokiRegex = startRegex + "loki" + endRegex -authRegex = "https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)+[a-zA-Z0-9]+" + endRegex +authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)+[a-zA-Z0-9]+" + endRegex + +with open('./src/config/config.json', 'rt') as tmp: + config['networks'] = json.load(tmp)['networks'] + def filterLastSlash(urlList): tmp = {} - for x in urlList: - tmp[x] = {} - for y in urlList[x]: - tmp[x][y] = [] - for z in urlList[x][y]: - if z.endswith('/'): - tmp[x][y].append(z[:-1]) - print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + z) + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + if url.endswith('/'): + tmp[frontend][network].append(url[:-1]) + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) else: - tmp[x][y].append(z) + tmp[frontend][network].append(url) + return tmp + + +def idnaEncode(urlList): + tmp = {} + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + try: + encodedUrl = url.encode("idna").decode("utf8") + tmp[frontend][network].append(encodedUrl) + if (encodedUrl != url): + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) + except Exception: + tmp[frontend][network].append(url) return tmp @@ -58,7 +79,7 @@ def is_cloudflare(url): instance_ip = socket.gethostbyname(urlparse(url).hostname) if instance_ip is None: return False - except: + except Exception: return False instance_bin = ip2bin(instance_ip) @@ -85,10 +106,11 @@ def is_authenticate(url): if 'www-authenticate' in r.headers: print(url + ' requires ' + Fore.RED + 'authentication' + Style.RESET_ALL) return True - except: + except Exception: return False return False + def is_offline(url): try: r = requests.get(url, timeout=5) @@ -99,298 +121,229 @@ def is_offline(url): return True else: return False - except: + except Exception: return False +def fetchCache(frontend, name): + with open('./src/instances/data.json') as file: + mightyList[frontend] = json.load(file)[frontend] + print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + name) + + +def fetchFromFile(frontend, name): + with open('./src/instances/' + frontend + '.json') as file: + mightyList[frontend] = json.load(file) + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + + +def fetchJsonList(frontend, name, url, urlItem, jsonObject): + try: + r = requests.get(url) + rJson = json.loads(r.text) + if jsonObject: + rJson = rJson['instances'] + _list = {} + for network in config['networks']: + _list[network] = [] + if type(urlItem) == dict: + for item in rJson: + for network in config['networks']: + if urlItem[network] is not None: + if urlItem[network] in item and item[urlItem[network]] is not None: + if item[urlItem[network]].strip() != '': + _list[network].append(item[urlItem[network]]) + else: + for item in rJson: + tmpItem = item + if urlItem is not None: + tmpItem = item[urlItem] + if tmpItem.strip() == '': + continue + elif re.search(torRegex, tmpItem): + _list['tor'].append(tmpItem) + elif re.search(i2pRegex, tmpItem): + _list['i2p'].append(tmpItem) + elif re.search(lokiRegex, tmpItem): + _list['loki'].append(tmpItem) + else: + _list['clearnet'].append(tmpItem) + + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def fetchRegexList(frontend, name, url, regex): + try: + r = requests.get(url) + _list = {} + for network in config['networks']: + _list[network] = [] + + tmp = re.findall(regex, r.text) + + for item in tmp: + if item.strip() == "": + continue + elif re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def fetchTextList(frontend, name, url, prepend): + try: + r = requests.get(url) + tmp = r.text.strip().split('\n') + + _list = {} + for network in config['networks']: + _list[network] = [] + + for item in tmp: + item = prepend + item + if re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + def invidious(): - r = requests.get('https://api.invidious.io/instances.json') - rJson = json.loads(r.text) - invidiousList = {} - invidiousList['normal'] = [] - invidiousList['tor'] = [] - invidiousList['i2p'] = [] - invidiousList['loki'] = [] - for instance in rJson: - if instance[1]['type'] == 'https': - invidiousList['normal'].append(instance[1]['uri']) - elif instance[1]['type'] == 'onion': - invidiousList['tor'].append(instance[1]['uri']) - mightyList['invidious'] = invidiousList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Invidious') + name = 'Invidious' + frontend = 'invidious' + url = 'https://api.invidious.io/instances.json' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get(url) + rJson = json.loads(r.text) + for instance in rJson: + if instance[1]['type'] == 'https': + _list['clearnet'].append(instance[1]['uri']) + elif instance[1]['type'] == 'onion': + _list['tor'].append(instance[1]['uri']) + elif instance[1]['type'] == 'i2p': + _list['i2p'].append(instance[1]['uri']) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) def piped(): - r = requests.get( - 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') - - tmp = re.findall( - '(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) - _list = {} - _list['normal'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - for item in tmp: - try: - url = requests.get(item, timeout=5).url - if url.strip("/") == item: + frontend = 'piped' + name = 'Piped' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get( + 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md') + + tmp = re.findall( + r'(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) + for item in tmp: + try: + url = requests.get(item, timeout=5).url + if url.strip("/") == item: + continue + else: + _list['clearnet'].append(url) + except Exception: + logging.error(traceback.format_exc()) continue - else: - _list['normal'].append(url) - except: - continue - mightyList['piped'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Piped') + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) def pipedMaterial(): - r = requests.get( - 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md') - - tmp = re.findall( - r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\|", r.text) - pipedMaterialList = {} - pipedMaterialList['normal'] = [] - pipedMaterialList['tor'] = [] - pipedMaterialList['i2p'] = [] - pipedMaterialList['loki'] = [] - for item in tmp: - pipedMaterialList['normal'].append(item) - mightyList['pipedMaterial'] = pipedMaterialList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'pipedMaterial') + fetchRegexList('pipedMaterial', 'Piped-Material', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production") def cloudtube(): - json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) - with open('./src/instances/cloudtube.json') as file: - mightyList['cloudtube'] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'CloudTube') + fetchFromFile('cloudtube', 'Cloudtube') def proxitok(): - r = requests.get( - 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md') - - tmp = re.findall( - r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text) - proxiTokList = {} - proxiTokList['normal'] = [] - proxiTokList['tor'] = [] - proxiTokList['i2p'] = [] - proxiTokList['loki'] = [] - for item in tmp: - proxiTokList['normal'].append(re.sub(r'/$', '', item)) - mightyList['proxiTok'] = proxiTokList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'ProxiTok') + fetchRegexList('proxiTok', 'ProxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))") def send(): - r = requests.get( - 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md') - tmp = re.findall( - r"- ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}", r.text) - sendList = {} - sendList['normal'] = [] - sendList['tor'] = [] - sendList['i2p'] = [] - sendList['loki'] = [] - for item in tmp: - sendList['normal'].append(item) - mightyList['send'] = sendList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Send') + fetchRegexList('send', 'Send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', r"- ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}") def nitter(): - r = requests.get('https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md') - tmp = re.findall( - r"(?:(?:\| \[(?:\S+\.)+[a-zA-Z]+\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z]+)\/?\) (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,3} +\|(?:(?:\n)|(?: (?:❌)|(?: ✅)|(?: ❓)|(?: \[))))|(?:- \[(?:\S+\.)+(?:(?:i2p)|(?:loki))\]\((https?:\/{2}(?:\S+\.)(?:(?:i2p)|(?:loki)))\/?\)))", r.text) - - nitterList = {} - nitterList['normal'] = [] - nitterList['tor'] = [] - nitterList['i2p'] = [] - nitterList['loki'] = [] - for item in tmp: - for i in item: - if i == '': - continue - else: - item = i - if re.search(torRegex, item): - nitterList['tor'].append(item) - elif re.search(i2pRegex, item): - nitterList['i2p'].append(item) - elif re.search(lokiRegex, item): - nitterList['loki'].append(item) - else: - nitterList['normal'].append(item) - mightyList['nitter'] = nitterList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Nitter') + fetchRegexList('nitter', 'Nitter', 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md', r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))") def bibliogram(): - json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) - with open('./src/instances/bibliogram.json') as file: - mightyList['bibliogram'] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Bibliogram') + fetchFromFile('bibliogram', 'Bibliogram') def libreddit(): - r = requests.get( - 'https://github.com/libbacon/libreddit-instances/raw/master/instances.json') - rJson = json.loads(r.text) - libredditList = {} - libredditList['normal'] = [] - libredditList['tor'] = [] - libredditList['i2p'] = [] - libredditList['loki'] = [] - - if 'instances' not in rJson: - mightyList['libreddit'] = libredditList - print(Fore.RED + 'Failed to fetch ' + Style.RESET_ALL + 'LibReddit') - return - - for item in rJson['instances']: - if 'url' in item: - url = item['url'] - libredditList['normal'].append(url) - elif 'onion' in item: - onion = item['onion'] - libredditList['tor'].append(onion) - elif 'i2p' in item: - i2p = item['i2p'] - libredditList['i2p'].append(i2p) - - mightyList['libreddit'] = libredditList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'LibReddit') + fetchJsonList('libreddit', 'Libreddit', 'https://github.com/libbacon/libreddit-instances/raw/master/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) def teddit(): - r = requests.get( - 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json') - rJson = json.loads(r.text) - tedditList = {} - tedditList['normal'] = [] - tedditList['tor'] = [] - tedditList['i2p'] = [] - tedditList['loki'] = [] - for item in rJson: - url = item['url'] - if url != '': - tedditList['normal'].append(url) - if 'onion' in item: - onion = item['onion'] - if onion != '': - tedditList['tor'].append(onion) - - mightyList['teddit'] = tedditList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Teddit') + fetchJsonList('teddit', 'Teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def wikiless(): - r = requests.get('https://wikiless.org/instances.json') - rJson = json.loads(r.text) - wikilessList = {} - wikilessList['normal'] = [] - wikilessList['tor'] = [] - wikilessList['i2p'] = [] - wikilessList['loki'] = [] - for item in rJson: - if 'url' in item: - if item['url'].strip() != "": - wikilessList['normal'].append(item['url']) - if 'onion' in item: - if item['onion'].strip() != "": - wikilessList['tor'].append(item['onion']) - if 'i2p' in item: - if item['i2p'].strip() != "": - wikilessList['i2p'].append(item['i2p']) - mightyList['wikiless'] = wikilessList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Wikiless') + fetchJsonList('wikiless', 'Wikiless', 'https://wikiless.org/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def scribe(): - r = requests.get( - 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json') - rJson = json.loads(r.text) - scribeList = {} - scribeList['normal'] = [] - scribeList['tor'] = [] - scribeList['i2p'] = [] - scribeList['loki'] = [] - for item in rJson: - scribeList['normal'].append(item) - mightyList['scribe'] = scribeList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Scribe') + fetchJsonList('scribe', 'Scribe', 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False) def quetre(): - r = requests.get( - 'https://raw.githubusercontent.com/zyachel/quetre/main/README.md') - _list = {} - _list['normal'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - - tmp = re.findall( - r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text) - - - for item in tmp: - if re.search(torRegex, item): - _list['tor'].append(item) - else: - _list['normal'].append(item) - mightyList['quetre'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Quetre') + fetchRegexList('quetre', 'Quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/README.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") def libremdb(): - r = requests.get( - 'https://raw.githubusercontent.com/zyachel/libremdb/main/README.md') - _list = {} - _list['normal'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - - tmp = re.findall( - r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|", r.text) - - - for item in tmp: - if item.strip() == "": - continue - if re.search(torRegex, item): - _list['tor'].append(item) - else: - _list['normal'].append(item) + fetchRegexList('libremdb', 'libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/README.md', r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") - mightyList['libremdb'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Libremdb') def simpleertube(): - r = requests.get('https://simple-web.org/instances/simpleertube') - _list = {} - _list['normal'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - for item in r.text.strip().split('\n'): - _list['normal'].append('https://' + item) - - mightyList['simpleertube'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SimpleerTube') + fetchTextList('simpleertube', 'SimpleerTube', 'https://simple-web.org/instances/simpleertube', 'https://') def simplytranslate(): r = requests.get('https://simple-web.org/instances/simplytranslate') simplyTranslateList = {} - simplyTranslateList['normal'] = [] + simplyTranslateList['clearnet'] = [] for item in r.text.strip().split('\n'): - simplyTranslateList['normal'].append('https://' + item) + simplyTranslateList['clearnet'].append('https://' + item) r = requests.get('https://simple-web.org/instances/simplytranslate_onion') simplyTranslateList['tor'] = [] @@ -412,31 +365,19 @@ def simplytranslate(): def linvgatranslate(): - r = requests.get( - 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json') - rJson = json.loads(r.text) - lingvaList = {} - lingvaList['normal'] = [] - lingvaList['tor'] = [] - lingvaList['i2p'] = [] - lingvaList['loki'] = [] - for item in rJson: - lingvaList['normal'].append(item) - - mightyList['lingva'] = lingvaList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'LinvgaTranslate') + fetchJsonList('lingva', 'LingvaTranslate', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False) def searx_searxng(): r = requests.get('https://searx.space/data/instances.json') rJson = json.loads(r.text) searxList = {} - searxList['normal'] = [] + searxList['clearnet'] = [] searxList['tor'] = [] searxList['i2p'] = [] searxList['loki'] = [] searxngList = {} - searxngList['normal'] = [] + searxngList['clearnet'] = [] searxngList['tor'] = [] searxngList['i2p'] = [] searxngList['loki'] = [] @@ -453,9 +394,9 @@ def searx_searxng(): searxList['i2p'].append(item[:-1]) else: if (rJson['instances'][item].get('generator') == 'searxng'): - searxngList['normal'].append(item[:-1]) + searxngList['clearnet'].append(item[:-1]) else: - searxList['normal'].append(item[:-1]) + searxList['clearnet'].append(item[:-1]) mightyList['searx'] = searxList mightyList['searxng'] = searxngList @@ -463,160 +404,66 @@ def searx_searxng(): def whoogle(): - r = requests.get( - 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/misc/instances.txt') - tmpList = r.text.strip().split('\n') - whoogleList = {} - whoogleList['normal'] = [] - whoogleList['tor'] = [] - whoogleList['i2p'] = [] - whoogleList['loki'] = [] - for item in tmpList: - if re.search(torRegex, item): - whoogleList['tor'].append(item) - elif re.search(torRegex, item): - whoogleList['i2p'].append(item) - else: - whoogleList['normal'].append(item) - mightyList['whoogle'] = whoogleList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Whoogle') + fetchTextList('whoogle', 'Whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/misc/instances.txt', '') def librex(): - r = requests.get( - 'https://raw.githubusercontent.com/hnhx/librex/main/README.md') - _list = {} - _list['normal'] = [] - _list['tor'] = [] - _list['i2p'] = [] - _list['loki'] = [] - - tmp = re.findall( - r"\| {1,2}\[(?:(?:[a-zA-Z0-9]+\.)+[a-zA-Z]{2,}|✅)\]\((https?:\/{2}(?:[a-zA-Z0-9]+\.)+[a-zA-Z0-9]{2,})", r.text) - - for item in tmp: - if item.strip() == "": - continue - elif re.search(torRegex, item): - _list['tor'].append(item) - elif re.search(i2pRegex, item): - _list['i2p'].append(item) - else: - _list['normal'].append(item) - mightyList['librex'] = _list - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Librex') + fetchJsonList('librex', 'LibreX', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True) def rimgo(): - r = requests.get( - 'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json') - rJson = json.loads(r.text) - rimgoList = {} - rimgoList['normal'] = [] - rimgoList['tor'] = [] - rimgoList['i2p'] = [] - rimgoList['loki'] = [] - for item in rJson: - if 'url' in item: - rimgoList['normal'].append(item['url']) - if 'onion' in item: - rimgoList['tor'].append(item['onion']) - if 'i2p' in item: - rimgoList['i2p'].append(item['i2p']) - mightyList['rimgo'] = rimgoList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Rimgo') + fetchJsonList('rimgo', 'rimgo', 'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json', {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) def librarian(): - r = requests.get( - 'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json') - rJson = json.loads(r.text) - librarianList = {} - librarianList['normal'] = [] - librarianList['tor'] = [] - librarianList['i2p'] = [] - librarianList['loki'] = [] - instances = rJson['instances'] - for item in instances: - url = item['url'] - if url.strip() == "": - continue - elif re.search(torRegex, url): - librarianList['tor'].append(url) - elif re.search(i2pRegex, url): - librarianList['i2p'].append(url) - elif re.search(lokiRegex, url): - librarianList['loki'].append(url) - else: - librarianList['normal'].append(url) - mightyList['librarian'] = librarianList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Librarian') + fetchJsonList('librarian', 'Librarian', 'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json', 'url', True) def neuters(): - json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) - with open('./src/instances/neuters.json') as file: - mightyList['neuters'] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Neuters') + fetchFromFile('neuters', 'Neuters') def beatbump(): - json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) - with open('./src/instances/beatbump.json') as file: - mightyList['beatbump'] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Beatbump') + fetchFromFile('beatbump', 'Beatbump') def hyperpipe(): - r = requests.get( - 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json') - rJson = json.loads(r.text) - hyperpipeList = {} - hyperpipeList['normal'] = [] - hyperpipeList['tor'] = [] - hyperpipeList['i2p'] = [] - hyperpipeList['loki'] = [] - for item in rJson: - url = item['url'] - if url.strip() == "": - continue - elif re.search(torRegex, url): - hyperpipeList['tor'].append(url) - elif re.search(i2pRegex, url): - hyperpipeList['i2p'].append(url) - elif re.search(lokiRegex, url): - hyperpipeList['loki'].append(url) - else: - hyperpipeList['normal'].append(url) - mightyList['hyperpipe'] = hyperpipeList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Hyperpipe') + fetchJsonList('hyperpipe', 'Hyperpipe', 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False) def facil(): - json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) - with open('./src/instances/facil.json') as file: - mightyList['facil'] = json.load(file) - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'FacilMap') + fetchFromFile('facil', 'FacilMap') + + +def libreTranslate(): + fetchRegexList('libreTranslate', 'LibreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|") + + +def breezeWiki(): + fetchRegexList('breezeWiki', 'BreezeWiki', 'https://gitdab.com/cadence/breezewiki-docs/raw/branch/main/docs.scrbl', r"\(\"[^\n\s\r\t\f\v\"]+\" \"https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*\" \"(https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+(?:\/[^\s\/]+)*)\"\)") def peertube(): - r = requests.get( - 'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt') - rJson = json.loads(r.text) + try: + r = requests.get( + 'https://instances.joinpeertube.org/api/v1/instances?start=0&count=1045&sort=-createdAt') + rJson = json.loads(r.text) - myList = [] - for k in rJson['data']: - myList.append('https://'+k['host']) + myList = ['https://search.joinpeertube.org'] + for k in rJson['data']: + myList.append('https://'+k['host']) - mightyList['peertube'] = myList - print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'PeerTube') + mightyList['peertube'] = myList + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'PeerTube') + except Exception: + fetchCache('peertube', 'PeerTube') def isValid(url): # This code is contributed by avanitrachhadiya2155 try: result = urlparse(url) return all([result.scheme, result.netloc]) - except: + except Exception: return False @@ -636,6 +483,7 @@ quetre() libremdb() simplytranslate() linvgatranslate() +libreTranslate() searx_searxng() whoogle() librex() @@ -646,7 +494,9 @@ beatbump() hyperpipe() facil() simpleertube() +breezeWiki() mightyList = filterLastSlash(mightyList) +mightyList = idnaEncode(mightyList) cloudflare = [] authenticate = [] diff --git a/src/instances/neuters.json b/src/instances/neuters.json index 9e224a52..8023079b 100644 --- a/src/instances/neuters.json +++ b/src/instances/neuters.json @@ -1,5 +1,5 @@ { - "normal": ["https://neuters.de"], + "clearnet": ["https://neuters.de"], "tor": [], "i2p": [], "loki": [] |