From 1d36d40335e4a3235b1148dc7e74d88402d0a2a7 Mon Sep 17 00:00:00 2001 From: GhostSec Date: Wed, 6 Aug 2025 07:51:57 -0400 Subject: [PATCH] `Enhanced User-Agent Handling and Tor Configuration Updates` --- .DS_Store | Bin 0 -> 6148 bytes CHANGELOG.md | 35 ++ README.md | 35 ++ banner/banner.py | 2 +- darkdump.py | 84 +++- headers/__pycache__/agents.cpython-313.pyc | Bin 0 -> 31569 bytes headers/agents.py | 480 ++++++++++++++------- requirements.txt | 1 + test_headers.py | 147 +++++++ 9 files changed, 613 insertions(+), 171 deletions(-) create mode 100644 .DS_Store create mode 100644 CHANGELOG.md create mode 100644 headers/__pycache__/agents.cpython-313.pyc create mode 100644 test_headers.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..58f143779294f3230838ff2d37e78a1229226a03 GIT binary patch literal 6148 zcmeHKPfrs;6n|5yZ9!y#RuQAw*oz4yLIVb445id4F$O{j76t2WcW5_kXPVt@D-e>N z_2dWe1NaS0yl~TlUcDLbUi9kCH#1xLQ}sxQ*_X`x-n=*O&CYMP^Ev=Pat&(|Km&j{ zHi6Mz?EWCaFItyG0!s{$$Q%@0@XVH1H=VB8p&?`-WZ<+hAbz_sr~(B8EbM>(ez}RB zw`Se3J%3{2Gl}#>&zy~+^u;y`Ro*K2hTrgVhWkW}RycRej&7K%Hf@!(Pua@Di0{yRyqoJe9<*$fwS(zG_;quXv)j?_B@+0c~(-crZDb zN~JDb9Jw@ld9^;=MHysPJ=$FxVBZO8)chSiaver7wjEc(uR&rU8dI4 z1JlZO9p@wu3^Lt4mr^G4EpBdl)Gr>=*Gk%eS9ZDMEOF2FZO+!#J!E^qr?myP$#KOd zGQC(AE^n2(9%5Haam{dDgH>p5d&4nI>MXJx_kCU~+GWnvlMl$WI9zh6NSU7fp}0(E zA9R%KO3@s!u#jus$E?>lX&uQ1MNBAs1Qo&;8qF_?pgunJqS1FzsUyn%P{0lvU@ z_(@_!BiG0nnId<{EV)k>$YW9_RbmrYC5Uuh^%XVl^u$B$*sE?9T;_0AL63-vLK8o1 z22H0jSt#>bUBpJN&deP4)p$OyJ`Yjr+8_6m@8^#Ct$+IK)q~vdO{{SGC7?9kclsYS ze8P}{kbx6sK%5UcHi6jy+Y05;ft|bpAV$$F1$A+k5HWgy*#O%L(St%%D544_!4-q3 zaJ1{eyljANg(@6~Tp9BOE0f@cLgeab*X48|S%qOAG7vIwlz|>K)I|Ni`}h0*(V%b? zG7vKGUok+U`9eO2TMBAx*XBg6)v-OpCPd6@E0iJF1?5;Oq7;u{lY%~7D2Ukr+X~Tw Q68;g8G=yEqz=<;O3#EYPW&i*H literal 0 HcmV?d00001 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f9fd04f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +## [3.1.0] – 2025-07-17 + +### Added +* **User-Agent API demo** – `test_headers.py` showcasing every helper on the new `Headers` class. +* **Browser selection CLI switch** – `-b / --browser` option in `darkdump.py` to pick a UA family (Chrome, Firefox, IE / Edge, Opera, Safari, Mobile). + +### Changed +#### `headers/agents.py` +* Refactored into a fully-featured **`Headers`** manager: + * Categorised >200 UA strings by browser family. + * Fixed one malformed UA and removed duplicates. + * New helpers: + * `get_random_agent()` – any UA. + * `get_random_by_browser(browser_type)` – browser-specific UA. + * `get_random_by_os(os_type)` – OS-specific UA. + * `get_modern_agent()` – modern (2022+) UA. + * Added doc-strings, type hints and runtime validation. + * Maintains legacy `user_agents` list for backward compatibility. + +#### `darkdump.py` +* Integrates the new `Headers` API: + * Picks UA through the helper methods. + * Accepts `-b / --browser` argument. + * Debug mode now prints the selected UA. +* Minor docs & argument-parsing improvements. + +### Documentation +* **README.md** updated: + * Describes the new UA system and CLI switch. + * Adds example commands and usage notes. + +### Notes +* Version bumped to **3.1.0** to reflect backward-compatible feature upgrade. diff --git a/README.md b/README.md index 4c09927..1173760 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Darkdump is a OSINT interface for carrying out deep web investgations written in 1) ``git clone https://github.com/josh0xA/darkdump``
2) ``cd darkdump``
3) ``python3 -m pip install -r requirements.txt``
+ • Make sure `PySocks` is installed (it is now listed in `requirements.txt`). 4) ``python3 darkdump.py --help``
### Tor Configuration @@ -32,10 +33,18 @@ Replace `[YourHashedPasswordHere]` with a hashed password which can be generated Linux: `sudo systemctl start tor.service`
MacOS: `brew services start tor`
+> **Port note:** +> • **Tor Browser** exposes a SOCKS5 proxy on **`localhost:9150`** (this is the default that Darkdump now uses). +> • The system Tor daemon typically listens on **`localhost:9050`**. +> • If you are running the daemon on 9050 instead of Tor Browser, either edit `Configuration.__socks5init__` in `darkdump.py` or start Tor Browser so the 9150 proxy is available. + +Darkdump will test the connection automatically and print an error if the proxy cannot be reached. + ### Example Queries: `python3 darkdump.py -q "hacking" -a 10 --scrape --proxy` - search for 10 links and scrape each site
`python3 darkdump.py -q "free movies" -a 25` - don't scrape, just return 25 links for that query (does not require tor)
`python3 darkdump.py -q "marketplaces" -a 15 --scrape --proxy -i` - search for 10 links and scrape each site as well as find and store images. +`python3 darkdump.py -q "cryptography" -a 5 --scrape --proxy -b firefox` – scrape 5 results while forcing a Firefox-specific User-Agent. ## Menu ``` @@ -61,8 +70,34 @@ options: -p, --proxy use tor proxy for scraping -i, --images scrape images and visual content from the site -s, --scrape scrape the actual site for content and look for keywords + -b {chrome,firefox,ie,edge,opera,safari,mobile}, --browser {chrome,firefox,ie,edge,opera,safari,mobile} + specify the browser family to use when randomly + choosing a User-Agent header (default is a completely + random choice) + +``` + +## Enhanced User-Agent Handling 🚀 +Darkdump ships with an overhauled **`headers/agents.py`** module that now: +* Organises more than 200 User-Agent strings by browser family (Chrome, Firefox, IE / Edge, Opera, Safari, Mobile). +* Provides convenience helpers: + * `Headers.get_random_agent()` – any UA + * `Headers.get_random_by_browser('chrome')` – browser-specific UA + * `Headers.get_random_by_os('windows')` – OS-specific UA + * `Headers.get_modern_agent()` – modern (2022+) UA +* Fixes malformed strings and removes duplicates. + +Use the new **`-b / --browser`** CLI switch (see Menu above) to restrict the UA that Darkdump advertises during requests. + +For a hands-on tour of the new API, run the helper script: + +```bash +python test_headers.py ``` + +It prints examples for every helper method and shows category counts. + ## Visual

diff --git a/banner/banner.py b/banner/banner.py index f7e3294..e201219 100644 --- a/banner/banner.py +++ b/banner/banner.py @@ -28,7 +28,7 @@ class Banner(object): def LoadDarkdumpBanner(self): try: from termcolor import cprint, colored - banner = ''' + banner = r''' _ _ _ __ __| | __ _ _ __| | ____| |_ _ _ __ ___ _ __ / / / _` |/ _` | '__| |/ / _` | | | | '_ ` _ \| '_ \ / / diff --git a/darkdump.py b/darkdump.py index fc42f84..c649712 100644 --- a/darkdump.py +++ b/darkdump.py @@ -89,7 +89,8 @@ class Configuration: descriptions = [] urls = [] - __socks5init__ = "socks5h://localhost:9050" + # Default Tor Browser SOCKS port is 9150 (9050 is used by the system daemon) + __socks5init__ = "socks5h://localhost:9150" __darkdump_api__ = "https://ahmia.fi/search/?q=" class Platform(object): @@ -120,15 +121,18 @@ def clean_screen(self): else: os.system('cls') else: pass - def check_tor_connection(self, proxy_config): + def check_tor_connection(self, proxy_config, *, debug: bool = False): test_url = 'http://api.ipify.org' try: response = requests.get(test_url, proxies=proxy_config, timeout=10) print(f"{Colors.BOLD + Colors.G}Tor service is active. {Colors.END}") print(f"{Colors.BOLD + Colors.P}Current IP Address via Tor: {Colors.END}{response.text}") return True # Connection was successful - except: - print(f"{Colors.BOLD + Colors.R} Tor is inactive or not configured properly. Cannot scrape. {Colors.END}") + except Exception as exc: + print(f"{Colors.BOLD + Colors.R}Tor is inactive or not configured properly. Cannot scrape.{Colors.END}") + if debug: + # Provide the underlying reason when debug flag is set + print(f"{Colors.BOLD + Colors.R}[DEBUG] Tor connectivity error: {exc}{Colors.END}") return False class Darkdump(object): @@ -220,9 +224,50 @@ def extract_document_links(self, soup): return links - def crawl(self, query, amount, use_proxy=False, scrape_sites=False, scrape_images=False, debug_mode=False): - headers = {'User-Agent': random.choice(Headers.user_agents)} - proxy_config = {'http': 'socks5h://localhost:9050', 'https': 'socks5h://localhost:9050'} if use_proxy else {} + def crawl( + self, + query, + amount, + use_proxy: bool = False, + scrape_sites: bool = False, + scrape_images: bool = False, + debug_mode: bool = False, + browser_type: str | None = None, + ): + """ + Crawl Ahmia results and optionally scrape target onion sites. + + Parameters + ---------- + browser_type : str | None + If provided, restrict the randomly-selected User-Agent header to the + specified browser family (chrome, firefox, ie, edge, opera, safari, + mobile). Falls back to a completely random User-Agent when omitted. + """ + # Determine an appropriate user-agent string + if browser_type: + try: + user_agent = Headers.get_random_by_browser(browser_type) + except ValueError: + # Fallback to a fully random UA if an invalid browser_type slips + # through (should be prevented by argparse choices). + user_agent = random.choice(Headers.user_agents) + else: + user_agent = random.choice(Headers.user_agents) + + # ------------------------------------------------------------------ # + # Debug helper – show chosen User-Agent when debug mode is enabled + # ------------------------------------------------------------------ # + if debug_mode: + browser_lbl = browser_type if browser_type else "random" + print(f"{Colors.BOLD}{Colors.C}[DEBUG] Using User-Agent ({browser_lbl}): " + f"{Colors.END}{user_agent}") + + headers = {"User-Agent": user_agent} + proxy_config = ( + {'http': Configuration.__socks5init__, 'https': Configuration.__socks5init__} + if use_proxy else {} + ) # Fetching the initial search page try: @@ -236,10 +281,12 @@ def crawl(self, query, amount, use_proxy=False, scrape_sites=False, scrape_image seen_urls = set() # This set will store URLs to avoid duplicates - if scrape_sites: - if Platform(True).check_tor_connection(proxy_config) == False: return + if scrape_sites: + # Forward the debug flag so we reveal connection errors when requested + if not Platform(True).check_tor_connection(proxy_config, debug=debug_mode): + return - for idx, result in enumerate(second_results[:min(amount + 1, len(second_results))], start=1): + for idx, result in enumerate(second_results[:min(amount, len(second_results))]): site_url = result.find('cite').text if "http://" not in site_url and "https://" not in site_url: site_url = "http://" + site_url @@ -317,6 +364,13 @@ def darkdump_main(): parser.add_argument("-i", "--images", help="scrape images and visual content from the site", action="store_true") parser.add_argument("-s", "--scrape", help="scrape the actual site for content and look for keywords", action="store_true") parser.add_argument("-d", "--debug", help="enable debug output", action="store_true") + parser.add_argument( + "-b", "--browser", + help=("specify browser type for the User-Agent header " + "(chrome, firefox, ie, edge, opera, safari, mobile)"), + choices=['chrome', 'firefox', 'ie', 'edge', 'opera', 'safari', 'mobile'], + type=str + ) args = parser.parse_args() @@ -338,7 +392,15 @@ def darkdump_main(): if args.query: print(f"Searching For: {args.query} and showing {args.amount} results...\nIndexing is viable, skipping dead onions.\n") - Darkdump().crawl(args.query, args.amount, use_proxy=args.proxy, scrape_sites=args.scrape, scrape_images=args.images, debug_mode=args.debug) + Darkdump().crawl( + args.query, + args.amount, + use_proxy=args.proxy, + scrape_sites=args.scrape, + scrape_images=args.images, + debug_mode=args.debug, + browser_type=args.browser + ) else: print("[~] Note: No query arguments were passed. Please supply a query to search.") diff --git a/headers/__pycache__/agents.cpython-313.pyc b/headers/__pycache__/agents.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4df524d6c3049952c260afbf84ec6300778192cb GIT binary patch literal 31569 zcmc&-TW}lKd0v1dNP?nn)Wvpd=}2*6p&~&nu7v0~rYJ~)&6@#IvSY;v2rNlhAiyj@ zNyId%(yNn9+Fa_`X*;!FI@9E(p6N^{nRcd=@N?o+@2 z>;-2Rz!HE(7ZP_D*ggO4{O9}sbN1|>2cx5237>!V?`-_z7bWQz^u_q}>jw`~ICxD; zORAK1rQOfDR9Cw1hO6i{F66)VseRb@KiB{4fI9H3NA)~As180mqz*CZ5$UXy9(YVj zdm?%nb$H$F8_Z6Eg?B!=%;pL?rBqS8v*oQFwXnWXW#@7}77PT#?AdZN^ z*tzP4!p^4k5@&s!NnXhoy@e8}Da<&DBEjuB_0H+>Ya<$7ES$ z6{T48fQe0R!s^O@7lIn0!tVLSLDku8Rz(llB?1$ zB>7soSS;Tr)pF%hzCij`E_l7mIF(&3-%_|PTJ@I7Rfx*X%`1Qlq1CE>s-s|l7~yu5sUCY4~xG+Rn7UQ1p|Tw-Tu z()fOMoLx^YUtL^TX1F3Xv#|UkTfEF>7G7k}B^NG@v&0KasYE)>7E|8j{L)-9f!~q~ zvvVt#k_%VZMU-1ugbOF($RN7B$cUj%Dw#mJ%WOW8n!O6jGZ&L{$>kTvy_b{A3q**IE+z2e#RMdqxj2{5SV5`TxtZkrIJ-16KXZkoOR+^z zNqOlqO*(e{YJz?uelz$#yPRBHAT?$e7nW1_Iu0#U%VyE*$#i0z&7_iPGRWoB;{3Rm z%!v}94k9SGkkF`*skwp78o?!GuoWn6$iglqX68T()w94=G%ogge@URTu;0F|uPeI2?AT4?HHl z(jVb8Q1%FZAH8vypX*N_w46JNbBD}xhb`yEOnOHwzaL8fo zz$>dlZ0`)q6#;{2(JI@_ma^*{Gl7j;YWY@yqZhzQW!ZIrBO?eZt!r0bU0z-S;C*fz zm{+03NARi}DnQSwNJaxMP5e!S z4hkO-Ih(tQXpsk=ZEk_$YN1%D?r=P3${l#627!46sF>yS0OtTfk}tzeR$+keD$pA& ztHm9^*Y|QQKVPmDipA_?#2;Yit{0%^?FvInkNSf?HnX)=RIV$l&lReZkx{x&@souE~HAzbrd3kz>##9u4`U(FPMLRJ++;li)PFzIZ(vwky;-v(H(W zy&%ieY*vL|6rxj67K+P3zZ{VLP1^10mZY5=j)b6)pdS@;vSO1z^sZP}e%Q5sA@2WC zG`mov_PBO!YY#5lruOzSCO{zI4+g?apU&XgS~(v;he!uGmf~gn+ipoB<^mA`1p%vR{rhOc*0XxozR*=qB+@$PO-h zx+lUIFd1+ewlId8ISXT&34?=FLaCqdNs7;80p{^O%WjGcf^gLnTW(K2o-K0W?(71@9m|E(U6+I1r5Z!-85=s@6@82-J7vqQ$Ey(6-^+DC30e?V2uXAm+p?$ZINuTr6s7Nph?= zOzKTBhh9t1?Fm)YupWnohw9-Jn1Z1|+#iYsZG5^&52spnQ)EXYYBd&&qk)Nhx9>&t5Ut>cj&7(yZGbKlazXijyRc=fIlVxVhhA{|9DlU4VV0VMq@4ZR@LL40CQ~jodnDsjuF8>u5kdqK88OYf79t3}Ck0Gvt(e zw(FWuKdpl%+@f>m;D5b(yOFCk=Nk^p9Q@A_6CW!6c35NR=QUiNQjWHs-UBi@B%f9` zLXV%uidtl8JsPfy&gxWiG}$YYchGMi8dpDL9h|KGAq2>1_Z=>W zYT73@f)RPR?>bQnI{UlUUkF>^7Pu+fhA#xO_0W1ll<|f_NSWk77tyyJuJ6^DLVWxp z5(hhbw&Z}d?YYU0QN&29EhdD!hI9TL!Yoza@5x4H2bkd_l z&&aA>%*m9!2IP>I>{1RTni9FLwT~i1l-6U&d%9y|L?dMSV>l5GhA!9cSh9D=A<#%> zszo&GaLhrD2%RIIhqi_M7VW#&48Uqg0Yd=2ZHFcub5On4g`t0;(kR*JZWy2=_s3YP zz=Q%&H`NvWP}OC-`xdt)Q+xVkV=%ep?b*g?vT&P}2Ccg>BOJv!w$OlK%Ug+UxwP=T z-nn#x{dz30mvogNB~2)LZ$J?aM0G7S5~)6)$K_cJ%WHX zccZYqIVqcC)p9s!OzE)xXeJ*-x@H|R7ZjtLp&#aE4emmr#w>>9wY7f?N#c)|A>|>> z8g?;HMvi1+_S9gnq@F=j_Rf7| zNZIm7%8)vzA1OncEcTHyq^Q@#(08=ysStYhgqqSJvqU3v_)42RmFcGE3I1P=#_INLQvG-L;gINLQvz`p}Uob8$- zpG9Usr5%ZA`fk@00pJcY(pkR^(~OznMIiWmI+Zgv(uH~&In84rP^ z809jNKr9%e$=_?`rEGNrGtIkK1f&H;rmc!lb{&mlY$N8YJ!2>WeOo9(N;hj_kfvI# z{ed$DyD4}$F$kD8fZl*Q}PF!W_+@3~$q&Q|Fm-=BaswQ{>5DbfsZaZi0VtrrENB3_pQ$ zI<}Qg>#IfGhz+fUV`)FZjCe+l{B7yo1~m!k4D%iHLsQLNyL~41K&CRo^Y4+*u zlET7%piRx$WX)pOefi4rMNBgl>I}T>J}vuW(RcuStv>y_!5fv?0_p5H#i!L6{%7Kp zh)1T?1-EI1Q377Hu$!;;rXg5s2u3KgxDl@hi(r>8rstQhUlhb3(L&*=sNJDLu`uoo z1Z5b<+GIjeg4)eW30;CJo_(z;`D9{hID@zy=8UnYTul7x6q3vz0*`-Zp(8kKFo9uwD4e(SV z1ib!~vru+R;`V~YlD^tcx+BoIqi3O`LH)xA5))?u7pD2_P~6X zg8nIgBp8ZA94jcBEvt1CwL1zosl_XscsF@x$d4!Y;*nSr?|*Qv&Su{BU}S>|ZVQKm z-io;GG0eP<^7K$FfIhtNlacSH+ekmI^=)JskI;+Ht3rLla6F>WO%)oL@(GXD)PBDW zQKt@*%?i9N>sD|EI=gQgX+rHA&QjPbrnyI9x<5I?L&>XbbAOufb8nfNs!*s^6F__D z=?RUIo8Ho86?=LwI$PBaf0m}!OqrlHHWj2kny>b!&ZOBpBX2dPaf1GsKQa}a zLXV`44{=ertAr>KY~*}4$bY^3a9S2D{fXvk59rysKo?h0rXacX(0UGS|Y!WL7*^Z zu_2DtzPMi!w7Cn(z#cJi}AIfNeI0IN)V( z?0m?{yz>Qs!DG#hkzzTBiD~>c+GN~61+cLf$DOI##zghDk1f$VoywCGLh$s=HM~6q zZ=Ioe9Ju-RDUpP58-k|i@L{-Dvd#o98lKJ(uPGGR5s!mA(QMoS4Ah<$g%I>G?y%)5 z&N6&jEH5ab&B5eq7?W2}6|mK2_LhQ|T;251-8*O676Y}AO(_ZGaqBZWC0aNl*JecF z1Omo=(+(r@_*6-~fazRCZuIO(9revsL!3VHzZ@Y)zr zkZ7>>yhz9i@cmIgMwvh}uLv}oJttUeXJ5lMcF&mtLfLBNUCbwQU_291kSSei3> zj3(0J{%B}QKmwq5cyFvNh+fuY$UyBWjW^wV72%%zVPzUXWUvp?91IbNcvG7dB^R_e z?PqVV&FagwuHhb8EBOcKtykn;aKD!s+Lh4t!szFC|RD&(< z8V%DD69_G)1hhb11g*Yo^&&2UMTM4l(*ZbCL|<+NzI%lxfWwuG&bH{q9|-sw{?LO@ zp_fc1ZqR5S2aH%m?;A6kM{^z&s6A(-G3ejgaw%{_R}0l9reD@+wkV?u!yEcub_HeHXm(cJ^-qqX0*)Sl43 zsk_8)TJ_isZ`TQ4TuM#G;kxv1AGwqG^v~A+<=lb3ocr%uba%>xG7q!sYq5ZJYJgCc zy*-un|84cw9bB$4MuO;{Z_w7UEG%X7yy46)t5zY z72jNk5*SMibo0*0X7`H*h-#b1Dh>>>cOkPPn8%Nx-wsy3bmNFQwP8~MQ`i9vfA(W|or30~`Ht(Qn zVlC}pt{}`|2$KvyG-1BfnJ_oY6F1dqHg{!WZY2~51olIeFLxjc?#-;J6KfC!)x|C; z6+EkD%JM8`>^-D9vfi?6>6oT#YIUT^0=J2XZu6mPs`m8`EMhc}eUrlKyw)jSZ4J|K zUbY|3Tl?A`mZ2yM%jA`b9Nu_LzLLhW1KP{|wN{jzpNoZ`v-=BP@P{6ZbNFs*)KqBC zR;^1UG$c71nA3gNF3e5-?&scx>7*$E`8cv_=DAkj3oYye!hCy=sshH-EmvKZ$OrADd`G;y!R%f<8*Q-0T5qo6#PkMeL9S_Lp##^C(&&T3ul!^W| z`3jY^rFU)lPCabC)@Ke|@Lek0!K=ZWUgq;5)$NdN_m5UOTltSIj;G~Ta(EooiXOZz z-HK!a&gmA0CeX`uga+#08Hv3dp9wWyKN$_6c&&{*+MS{%dBP7_oSyo?|T){j2E0b6SfbM;*V||kH?d%TLFPOZf zFC9~v#CjvoV)cvkh|hnKC9l$Z*Oi*ZfNT+OlD~ng^GV`r+#={sb)0p+Cl;K zNA$%R{Pr`Sf&`V5km9cNtHaXa6MypgFMs}x^qZM)XLb)w+#8yBK!&Uz(AQST(Ao+) z(e|_|2CtzZ{Q8=-;!272M}_e*R-Y)jR$OTpty__0lJt^Er&X=-+VoIq>!(qhpylp@ z){vDBeY4Q&o5n9$){$I%wIf_Ed@Qv3yO8UEKtw;Z&>HB17FGhWe=M{-UC=sgrzG0z zC9FlFebR#~u7g<9B-CecQe3{_Y#DA~G?|u7jn)LovI^X(uvW-f7Vkh~n^D)X<`1x`v{bR9H;-c+_w zfplDE$h)m1RJ9D4Nv>gmBP?44>7j!~Z!C(UFLILidkJXh*{}Mbu9VMRcNdg;KdDq7 z;0o0}nmTolrcCWbvIX4>tTyY{ZAL8~V5uq>^e3{gI+kWLEJB6lqKNo7)DP;k%Zdk}&cYD1KMEe@c4fiJzYIy>rsH zd-D9N2YxT}^_1)1Ygp=|+XXM?n;JpJuD*3|h=*NYVR-SE7{^!x(ymvbC zcJb|(?@Q7p*A+K*r>?qz@ncCk9!W7azttVoe_W!#Y=S-L>z^^{n!Dnj zL=9naBe+P(op!Id35NNyMk6TaUUJVL(iSu-^_AR<{{+zQ;~>@z)&4!G{w1htFpHI! zI*=7^1z9~-ssKXwG(3gN4tV6UBeIT2OyaWN1ec+%Whbgm7I_9#`BG5X^Bu_!$soH| zr;4DM9uYX8k#M=8p(*#FmyD&ar1Sv}_M;-$uk@YM>obow2kQ;$Uxl5K06ZbRWQO5X ziy(@iZ-$X4;1>*kCHGp#N4|sExh?|_2N+>FCrj|rn}Ha2Fn-DxnCdp<+}38c^Ei^t z(XL1iE!!$P_85aY{Yx~tG)sAi2SgSzDPLH%x7y>By{t9i8B6+n1Wqdvep zeDy)SgIgcem8$narMgG!Al3V69JqGGh&eh#gqv`SaHI1@G|A@Z{byjqR~sZ-A@-Cj zK95Rkji~?#Nd+EA>QUO5!N%9GMFH~9x_diYKZK?|Y;JV_z%mj+}mXeDZG=zq`153Y)i9zqj+_ zi{Jm9H`VVfzPfnt6gKaU9(!Y8cl2@dvITzGogW9k{{`c+1?{rI-O&a$g1z8BxT_jqadX!+iO@_!vV_3qe-pN>8G&e)T?W4?O_eE)gqIPXH}-G>j}KP{bJ za((b0rxx(1;BOIsFIS#{;s45eDcJwF9&CPa*n26~|ASK=+D>~fJ=Ooi;eOg4@zB=q zO}P4h_>>3RA3b_F;pzX;ga=zoPmw^U4}TBp*u3mQPeh{iM(rPj9M{V(Gw7U=q)pdL zRE~dETw(N2u)f_;8X?I5EhDcrVGf~e{y*9w4dBkptAL~-AZ_VX%Eqn)( z*WLaA@=kPHkr9SLqe2uJyQ{Un;iu4lRqkx5PuI_j>6j*+)?)on70bD7vGTOvy!drA z!pd)e(Et3B8l&VHJM_NfnLPN`%))LVb_+}DCXJ9P9{RGaTnZ9br*UrV0D zhko5BjUM{;>HB@s@QM2pF8=Qm(&5M3pwMS_51qd^bpC_)AIF6iN;tlLYUUh32OISQ zzApGq)Q2*e65f`P$<)1>3z=5yQGqJHL3CWD?zrHUw|l*%|gn8{#O-%LgY z5J)Pc%^BL9r42wqQlFp=g{JzjQo{1TSTs_py9)IIK2B2~;mbj4D^u3{(BL$Se}YOy zX){Abk7qJin-+`QX0la`4bcj{6)+{yd8W>rj!FQ+Hxv#lCt%k8X3S*pEPjzy<^hxQ*Ye-w`0HcrJ+|{uSVV-IehQf z%uhyU?z^Q^*W9l@_lD~$3-1j`$IpX==g58c;DGXs=&Hhxec6SDgWY%6$pDzpgx>oqh?9?q7ECOO6kW6A$e8xd(0>X(l6U`Ml~C z>|pqvN_{L}E>$zP)k0OttnO5m9IfL?Sw!7~R1d3gssvPOKWzxpRf2BK=+=s;4?U&% z&(rGjxCHeeRR}Ea_qklI_uR)_{SW+-YvkR>PTlSMSLx(0q|twu{JWC>*Zm2PYy5qQ Iwp_;l18yU&L;wH) literal 0 HcmV?d00001 diff --git a/headers/agents.py b/headers/agents.py index 9a7482b..d6f9cd5 100644 --- a/headers/agents.py +++ b/headers/agents.py @@ -24,37 +24,39 @@ import sys sys.dont_write_bytecode = True +import random +from typing import List, Dict, Optional, Union + class Headers: - user_agents = [ + """ + A class for managing and providing user agent strings for HTTP requests. + + This class organizes user agents by browser type and provides methods to: + - Get random user agents + - Filter user agents by browser type or operating system + - Access modern/recent user agents + + The class maintains backward compatibility with existing code that uses + the user_agents list directly. + """ + + # Chrome user agents + chrome_agents = [ 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36', - 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)', - 'Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36', - 'Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01', 'Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', - 'Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5', 'Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/536.5 (KHTML like Gecko) Chrome/19.0.1084.56 Safari/1EA69', @@ -64,194 +66,354 @@ class Headers: 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10', - 'Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36', 'Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5', + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22', + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19', + # Modern Chrome agents (2022+) + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', + ] + + # Firefox user agents + firefox_agents = [ 'Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01', + 'Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01', + 'Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0', + 'Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00', + 'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00', + # Modern Firefox agents (2022+) + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:111.0) Gecko/20100101 Firefox/111.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/109.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:110.0) Gecko/20100101 Firefox/110.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/109.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:110.0) Gecko/20100101 Firefox/110.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:113.0) Gecko/20100101 Firefox/113.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:114.0) Gecko/20100101 Firefox/114.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:116.0) Gecko/20100101 Firefox/116.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:117.0) Gecko/20100101 Firefox/117.0', + ] + + # IE/Edge user agents + ie_edge_agents = [ + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205', + 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)', + 'Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)', 'Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)', - 'Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)', 'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)', + 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', + 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)', + 'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)', + 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)', + 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)', + 'Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)', + 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393', + # Modern Edge agents (2022+) + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.78', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.57', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.78', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.57', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.44', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.48', + ] + + # Opera user agents + opera_agents = [ + 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01', + 'Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01', + 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; ja) Opera 11.00', + 'Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01', + 'Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01', 'Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51', 'Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01', 'Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', 'Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00', 'Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.7.39 Version/11.00', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)', 'Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', - 'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11', 'Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)', - 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11', 'Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10', 'Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)', - 'Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01', 'Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; pl) Opera 11.00', 'Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50', 'Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14', 'Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7', - 'AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19', 'Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; fr) Opera 11.00', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Opera/9.80 (Windows NT 5.1; U; it) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', 'Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02', - 'Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1;', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; pl) Opera 11.00', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', - 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36', 'Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1', 'Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00', 'Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00', - 'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3', 'Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7', 'Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36', 'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00', - 'Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Mozilla/5.0 (Windows NT 6.0; U; ja; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)', 'Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36', - 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36', - 'Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36', - 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11', - 'Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872', - 'Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)', - 'Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30', - 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221', - 'Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)', - 'Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01', + 'Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1', + 'Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50', 'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16', + # Modern Opera agents (2022+) + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 OPR/95.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 OPR/96.0.0.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 OPR/95.0.0.0', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 OPR/95.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 OPR/97.0.0.0', + ] + + # Safari user agents + safari_agents = [ + 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10', + # Modern Safari agents (2022+) + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 Safari/604.1', + 'Mozilla/5.0 (iPad; CPU OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Mobile/15E148 Safari/604.1', + ] + + # Mobile user agents + mobile_agents = [ + 'Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9', + 'Mozilla/5.0 (Linux; U; Android 2.3.5; zh-cn; HTC_IncredibleS_S710e Build/GRJ90) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', + 'Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', + 'Mozilla/5.0 (Linux; U; Android 2.3.3; ko-kr; LG-LU3000 Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', + 'Mozilla/5.0 (Linux; U; Android 2.3.4; fr-fr; HTC Desire Build/GRJ22) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', + 'Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile', + 'Mozilla/5.0 (Linux; U; Android 2.3.3; en-us; HTC_DesireS_S510e Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', + 'Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari', + 'Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30', 'Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21', 'Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F', - 'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19', - 'Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5', - 'Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)', - 'Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)', - 'Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00', - 'Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36', - 'Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36', - 'Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10', - 'Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00', - 'Mozilla/5.0 (Linux; U; Android 2.3.3; zh-tw; HTC_Pyramid Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari', - 'Mozilla/5.0 (Windows NT 5.1; U; pl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00', - 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11', - 'Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00', - 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', - 'Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)', - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36', - 'Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00', 'Mozilla/5.0 (Linux; U; Android 2.3.4; en-us; T-Mobile myTouch 3G Slide Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19', - 'Mozilla/5.0 (Windows NT 5.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00' - ] \ No newline at end of file + # Modern Mobile agents (2022+) + 'Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36', + 'Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19A346 Safari/602.1', + 'Mozilla/5.0 (iPhone14,6; U; CPU iPhone OS 15_4 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19E241 Safari/602.1', + ] + + # Combine all user agents into a single list for backward compatibility + user_agents = ( + chrome_agents + + firefox_agents + + ie_edge_agents + + opera_agents + + safari_agents + + mobile_agents + ) + + # Fix the incomplete user agent string + for i, agent in enumerate(user_agents): + if agent == 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7': + user_agents[i] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20' + + @classmethod + def get_random_agent(cls) -> str: + """ + Returns a random user agent string from the full list. + + Returns: + str: A random user agent string. + """ + return random.choice(cls.user_agents) + + @classmethod + def get_random_by_browser(cls, browser_type: str) -> str: + """ + Returns a random user agent for a specific browser type. + + Args: + browser_type (str): The browser type to get a user agent for. + Valid values: 'chrome', 'firefox', 'ie', 'edge', 'opera', 'safari', 'mobile'. + + Returns: + str: A random user agent string for the specified browser. + + Raises: + ValueError: If an invalid browser type is provided. + """ + browser_type = browser_type.lower() + + if browser_type == 'chrome': + return random.choice(cls.chrome_agents) + elif browser_type == 'firefox': + return random.choice(cls.firefox_agents) + elif browser_type in ['ie', 'edge']: + return random.choice(cls.ie_edge_agents) + elif browser_type == 'opera': + return random.choice(cls.opera_agents) + elif browser_type == 'safari': + return random.choice(cls.safari_agents) + elif browser_type == 'mobile': + return random.choice(cls.mobile_agents) + else: + raise ValueError(f"Invalid browser type: {browser_type}. Valid types are: chrome, firefox, ie, edge, opera, safari, mobile") + + @classmethod + def get_random_by_os(cls, os_type: str) -> str: + """ + Returns a random user agent for a specific operating system. + + Args: + os_type (str): The OS type to get a user agent for. + Valid values: 'windows', 'mac', 'linux', 'android', 'ios'. + + Returns: + str: A random user agent string for the specified OS. + + Raises: + ValueError: If an invalid OS type is provided. + """ + os_type = os_type.lower() + + # Filter agents by OS + if os_type == 'windows': + filtered_agents = [ua for ua in cls.user_agents if 'Windows' in ua] + elif os_type == 'mac': + filtered_agents = [ua for ua in cls.user_agents if 'Mac OS' in ua] + elif os_type == 'linux': + filtered_agents = [ua for ua in cls.user_agents if 'Linux' in ua and 'Android' not in ua] + elif os_type == 'android': + filtered_agents = [ua for ua in cls.user_agents if 'Android' in ua] + elif os_type == 'ios': + filtered_agents = [ua for ua in cls.user_agents if 'iPhone' in ua or 'iPad' in ua] + else: + raise ValueError(f"Invalid OS type: {os_type}. Valid types are: windows, mac, linux, android, ios") + + if not filtered_agents: + return cls.get_random_agent() + + return random.choice(filtered_agents) + + @classmethod + def get_modern_agent(cls) -> str: + """ + Returns a random modern user agent (2022 or newer). + + Returns: + str: A random modern user agent string. + """ + modern_agents = [ + ua for ua in cls.user_agents + if any(marker in ua for marker in [ + 'Chrome/10', 'Chrome/11', 'Chrome/12', 'Chrome/13', 'Chrome/14', 'Chrome/15', 'Chrome/16', 'Chrome/17', + 'Firefox/10', 'Firefox/11', 'Firefox/12', 'Firefox/13', 'Firefox/14', 'Firefox/15', 'Firefox/16', 'Firefox/17', + 'Version/16', 'Android 13', 'iPhone OS 15', 'iPhone OS 16', + 'OPR/95', 'OPR/96', 'OPR/97', + 'Edg/10', 'Edg/11', 'Edg/12' + ]) + ] + + if not modern_agents: + return cls.get_random_agent() + + return random.choice(modern_agents) diff --git a/requirements.txt b/requirements.txt index 97ef6cb..ce87305 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ argparse>=1.4.0 beautifulsoup4>=4.11.1 nltk>=3.5 textblob>=0.15.3 +PySocks>=1.7.1 diff --git a/test_headers.py b/test_headers.py new file mode 100644 index 0000000..524cd4d --- /dev/null +++ b/test_headers.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +''' +test_headers.py - Test script for the Headers class in darkdump + +This script demonstrates all the functionality of the enhanced Headers class, +including getting random user agents, filtering by browser type or OS, and +accessing modern user agents. + +Usage: + python test_headers.py + +Author: Josh Schiavone (Enhanced by Factory AI) +License: MIT +''' + +import sys +import random +from headers.agents import Headers + +def print_header(title): + """Print a formatted section header.""" + print("\n" + "=" * 80) + print(f" {title}") + print("=" * 80) + +def print_example(description, value): + """Print an example with description and value.""" + print(f"\n{description}:") + print(f" {value}") + +def test_random_agents(): + """Demonstrate getting random user agents.""" + print_header("RANDOM USER AGENTS") + + # Get a single random user agent + print_example("Random user agent", Headers.get_random_agent()) + + # Get multiple random user agents + print("\nMultiple random user agents:") + for i in range(3): + print(f" {i+1}. {Headers.get_random_agent()}") + +def test_browser_specific_agents(): + """Demonstrate getting browser-specific user agents.""" + print_header("BROWSER-SPECIFIC USER AGENTS") + + browsers = ['chrome', 'firefox', 'ie', 'edge', 'opera', 'safari', 'mobile'] + + for browser in browsers: + print_example(f"Random {browser.upper()} user agent", Headers.get_random_by_browser(browser)) + + # Demonstrate error handling with an invalid browser type + print("\nError handling with invalid browser type:") + try: + Headers.get_random_by_browser('invalid_browser') + except ValueError as e: + print(f" Error: {e}") + +def test_os_specific_agents(): + """Demonstrate getting OS-specific user agents.""" + print_header("OS-SPECIFIC USER AGENTS") + + os_types = ['windows', 'mac', 'linux', 'android', 'ios'] + + for os_type in os_types: + print_example(f"Random {os_type.upper()} user agent", Headers.get_random_by_os(os_type)) + + # Demonstrate error handling with an invalid OS type + print("\nError handling with invalid OS type:") + try: + Headers.get_random_by_os('invalid_os') + except ValueError as e: + print(f" Error: {e}") + +def test_modern_agents(): + """Demonstrate getting modern user agents.""" + print_header("MODERN USER AGENTS") + + print_example("Random modern user agent", Headers.get_modern_agent()) + + print("\nMultiple modern user agents:") + for i in range(3): + print(f" {i+1}. {Headers.get_modern_agent()}") + +def test_direct_access(): + """Demonstrate direct access to user agent lists.""" + print_header("DIRECT ACCESS TO USER AGENT LISTS") + + # Show total counts of user agents by category + print("\nTotal user agents by category:") + print(f" All user agents: {len(Headers.user_agents)}") + print(f" Chrome agents: {len(Headers.chrome_agents)}") + print(f" Firefox agents: {len(Headers.firefox_agents)}") + print(f" IE/Edge agents: {len(Headers.ie_edge_agents)}") + print(f" Opera agents: {len(Headers.opera_agents)}") + print(f" Safari agents: {len(Headers.safari_agents)}") + print(f" Mobile agents: {len(Headers.mobile_agents)}") + + # Show a random example from each category + print("\nRandom examples from each category:") + print(f" Chrome: {random.choice(Headers.chrome_agents)}") + print(f" Firefox: {random.choice(Headers.firefox_agents)}") + print(f" IE/Edge: {random.choice(Headers.ie_edge_agents)}") + print(f" Opera: {random.choice(Headers.opera_agents)}") + print(f" Safari: {random.choice(Headers.safari_agents)}") + print(f" Mobile: {random.choice(Headers.mobile_agents)}") + +def test_practical_usage(): + """Demonstrate practical usage in HTTP requests.""" + print_header("PRACTICAL USAGE IN HTTP REQUESTS") + + print(""" +# Example of using Headers with the requests library: + +import requests +from headers.agents import Headers + +# Get a random Chrome user agent +headers = {'User-Agent': Headers.get_random_by_browser('chrome')} + +# Make a request with the Chrome user agent +response = requests.get('https://example.com', headers=headers) + +# For scraping that requires modern browsers +headers = {'User-Agent': Headers.get_modern_agent()} +response = requests.get('https://example.com', headers=headers) + +# For mobile-specific content +headers = {'User-Agent': Headers.get_random_by_browser('mobile')} +response = requests.get('https://example.com', headers=headers) +""") + +def main(): + """Run all test functions.""" + print("\nTesting the Headers class functionality from darkdump\n") + + test_random_agents() + test_browser_specific_agents() + test_os_specific_agents() + test_modern_agents() + test_direct_access() + test_practical_usage() + + print("\nAll tests completed successfully!") + +if __name__ == "__main__": + main()