From 80c495416ead8b82024b3093d017f8d7f148e438 Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 08:50:38 +0530 Subject: [PATCH 1/8] Add official provider logo crawler --- .../logo-crawler/collect_official_logos.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .github/logo-crawler/collect_official_logos.py diff --git a/.github/logo-crawler/collect_official_logos.py b/.github/logo-crawler/collect_official_logos.py new file mode 100644 index 0000000..18c6409 --- /dev/null +++ b/.github/logo-crawler/collect_official_logos.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +import argparse,csv,hashlib,json,re,zipfile +from concurrent.futures import ThreadPoolExecutor,as_completed +from pathlib import Path +from urllib.parse import urljoin,urlparse +import requests +from bs4 import BeautifulSoup +try: + from ddgs import DDGS +except Exception: + DDGS=None + +UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126 Safari/537.36' +BAD={'wikipedia.org','facebook.com','instagram.com','linkedin.com','youtube.com','trustpilot.com','investopedia.com','tradingview.com','tradezella.com','tradersync.com','tradesviz.com','brokerchooser.com','forexbrokers.com'} +HEAD={'User-Agent':UA,'Accept':'text/html,application/xhtml+xml,image/avif,image/webp,image/png,image/svg+xml,*/*;q=.8'} + +def slug(s): + s=s.lower();s=re.sub(r'[^a-z0-9]+','-',s).strip('-');return s or 'provider' +def bad(u): + h=urlparse(u).netloc.lower().removeprefix('www.');return any(h==x or h.endswith('.'+x) for x in BAD) +def official(name,provided=''): + if provided and not bad(provided): return provided,'provided' + if DDGS: + for q in (f'{name} official website',f'{name} broker official',f'{name} prop firm official'): + try: + for x in DDGS().text(q,max_results=6): + u=x.get('href') or x.get('url') or '' + if u.startswith('http') and not bad(u): + h=urlparse(u).netloc.lower(); n=re.sub(r'[^a-z0-9]','',name.lower()) + if any(t in re.sub(r'[^a-z0-9]','',h) for t in re.findall(r'[a-z0-9]{4,}',n)) or re.sub(r'[^a-z0-9]','',h.split('.')[0]) in n: + return f'{urlparse(u).scheme}://{urlparse(u).netloc}/','search' + except Exception: pass + return '','unresolved' +def candidates(site): + r=requests.get(site,headers=HEAD,timeout=18,allow_redirects=True) + r.raise_for_status(); soup=BeautifulSoup(r.text,'lxml'); out=[] + for tag in soup.find_all(['link','meta','img']): + u='' + if tag.name=='link' and any(x in ' '.join(tag.get('rel',[])).lower() for x in ('icon','logo','apple-touch')): u=tag.get('href','') + elif tag.name=='meta' and tag.get('property','').lower() in ('og:image','twitter:image'): u=tag.get('content','') + elif tag.name=='img': + txt=' '.join([tag.get('alt',''),tag.get('class','') if isinstance(tag.get('class',''),str) else ' '.join(tag.get('class',[])),tag.get('id','')]).lower() + if 'logo' in txt or 'brand' in txt: u=tag.get('src') or tag.get('data-src') or '' + if u: out.append(urljoin(r.url,u)) + for p in ('/logo.svg','/assets/logo.svg','/images/logo.svg','/static/logo.svg','/favicon.svg','/favicon.png','/favicon.ico'): + out.append(urljoin(r.url,p)) + seen=[] + for u in out: + if u.startswith('http') and u not in seen and not bad(u): seen.append(u) + return seen[:40],r.url +def score(u,ctype,size): + s=0; x=u.lower() + if 'logo' in x:s+=8 + if x.endswith('.svg') or 'svg' in ctype:s+=6 + if any(k in x for k in ('header','brand','primary')):s+=3 + if any(k in x for k in ('white','mono','footer','small','icon','favicon')):s-=3 + if size>1500:s+=2 + return s +def one(row,out): + name=row['Provider Name']; typ=row.get('Provider Type',''); site,src=official(name,row.get('Official URL','')) + rec={'rank':row.get('Rank',''),'provider':name,'slug':slug(name),'type':typ,'official_url':site,'status':'unresolved','source':'','local_file':'','notes':src} + if not site:return rec + try: urls,site2=candidates(site); rec['official_url']=site2 + except Exception as e: rec['notes']+=';site:'+type(e).__name__; return rec + best=None + for u in urls: + try: + rr=requests.get(u,headers=HEAD,timeout=15,allow_redirects=True) + if rr.status_code!=200 or len(rr.content)<150: continue + ct=rr.headers.get('content-type','').split(';')[0].lower() + if not (ct.startswith('image/') or u.lower().endswith(('.svg','.png','.jpg','.jpeg','.webp','.ico'))): continue + sc=score(rr.url,ct,len(rr.content)) + if best is None or sc>best[0]: best=(sc,rr.url,ct,rr.content) + except Exception: pass + if not best:return rec + sc,u,ct,data=best + ext={ 'image/svg+xml':'.svg','image/png':'.png','image/jpeg':'.jpg','image/webp':'.webp','image/x-icon':'.ico','image/vnd.microsoft.icon':'.ico'}.get(ct) + if not ext: + ext=Path(urlparse(u).path).suffix.lower(); ext=ext if ext in ('.svg','.png','.jpg','.jpeg','.webp','.ico') else '.bin' + folder=out/'logos';folder.mkdir(parents=True,exist_ok=True); fn=f"{slug(name)}{ext}";(folder/fn).write_bytes(data) + rec.update(status='downloaded',source=u,local_file='logos/'+fn,sha256=hashlib.sha256(data).hexdigest(),bytes=len(data),score=sc) + return rec +def write(out,res): + cols=['rank','provider','slug','type','official_url','status','source','local_file','bytes','sha256','score','notes'] + with open(out/'manifest.csv','w',newline='',encoding='utf-8') as f: + w=csv.DictWriter(f,fieldnames=cols,extrasaction='ignore');w.writeheader();w.writerows(res) + (out/'manifest.json').write_text(json.dumps(res,indent=2,ensure_ascii=False),encoding='utf-8') + with zipfile.ZipFile(str(out)+'.zip','w',zipfile.ZIP_DEFLATED) as z: + for p in out.rglob('*'): + if p.is_file():z.write(p,p.relative_to(out)) +def main(): + ap=argparse.ArgumentParser();ap.add_argument('--input',required=True);ap.add_argument('--output',required=True);ap.add_argument('--start',type=int,default=1);ap.add_argument('--end',type=int,default=9999);ap.add_argument('--workers',type=int,default=8);a=ap.parse_args() + out=Path(a.output);out.mkdir(parents=True,exist_ok=True) + rows=list(csv.DictReader(open(a.input,encoding='utf-8-sig')));rows=[r for r in rows if a.start<=int(r.get('Rank') or 0)<=a.end] + res=[] + with ThreadPoolExecutor(max_workers=a.workers) as ex: + fm={ex.submit(one,r,out):r for r in rows} + for i,f in enumerate(as_completed(fm),1): + try:x=f.result() + except Exception as e: + r=fm[f];x={'rank':r.get('Rank',''),'provider':r.get('Provider Name',''),'slug':slug(r.get('Provider Name','')),'type':r.get('Provider Type',''),'status':'unresolved','notes':'worker:'+type(e).__name__} + print(f"[{i}/{len(rows)}] {x['provider']} -> {x['status']}");res.append(x) + res.sort(key=lambda x:int(x.get('rank') or 0));write(out,res) +if __name__=='__main__':main() From 9ed2d6b518cada71dd17f5b3a4635accb82e19fb Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 08:50:50 +0530 Subject: [PATCH 2/8] Add logo crawler dependencies --- .github/logo-crawler/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/logo-crawler/requirements.txt diff --git a/.github/logo-crawler/requirements.txt b/.github/logo-crawler/requirements.txt new file mode 100644 index 0000000..80d6194 --- /dev/null +++ b/.github/logo-crawler/requirements.txt @@ -0,0 +1,4 @@ +requests==2.32.4 +beautifulsoup4==4.13.4 +lxml==6.0.0 +ddgs==9.5.5 From e2d01cde3e8834bc7978b2b29794f50bbaa11843 Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 08:50:59 +0530 Subject: [PATCH 3/8] Add official logo collection workflow --- .../collect-official-provider-logos.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/collect-official-provider-logos.yml diff --git a/.github/workflows/collect-official-provider-logos.yml b/.github/workflows/collect-official-provider-logos.yml new file mode 100644 index 0000000..a093231 --- /dev/null +++ b/.github/workflows/collect-official-provider-logos.yml @@ -0,0 +1,66 @@ +name: Collect official provider logos + +on: + pull_request: + branches: [master] + paths: + - '.github/logo-crawler/**' + - '.github/workflows/collect-official-provider-logos.yml' + +permissions: + contents: read + +jobs: + collect: + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - part: 1 + start: 1 + end: 55 + - part: 2 + start: 56 + end: 110 + - part: 3 + start: 111 + end: 165 + - part: 4 + start: 166 + end: 220 + - part: 5 + start: 221 + end: 275 + - part: 6 + start: 276 + end: 330 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: .github/logo-crawler/requirements.txt + - name: Install crawler dependencies + run: | + python -m pip install --upgrade pip + pip install -r .github/logo-crawler/requirements.txt + - name: Collect official website logos + run: | + python .github/logo-crawler/collect_official_logos.py \ + --input .github/logo-crawler/providers_330.csv \ + --output official_logo_output \ + --start ${{ matrix.start }} \ + --end ${{ matrix.end }} \ + --workers 8 + - name: Upload logo package shard + uses: actions/upload-artifact@v4 + with: + name: official-provider-logos-part-${{ matrix.part }} + path: | + official_logo_output/** + official_logo_output.zip + if-no-files-found: error + retention-days: 7 From 4ec647a1348cf7133c960b32f8fff7aff1552708 Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 08:52:04 +0530 Subject: [PATCH 4/8] Add compressed provider dataset --- .github/logo-crawler/providers_330.csv.gz.b64 | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/logo-crawler/providers_330.csv.gz.b64 diff --git a/.github/logo-crawler/providers_330.csv.gz.b64 b/.github/logo-crawler/providers_330.csv.gz.b64 new file mode 100644 index 0000000..51471a4 --- /dev/null +++ b/.github/logo-crawler/providers_330.csv.gz.b64 @@ -0,0 +1 @@ +H4sICHBYVGoAA3Byb3ZpZGVyc18zMzAuY3N2AM1cy3LjuJLdV0T9AzcTN2Iiu0qiRD2mV3q6VGVZvpaqyn13EAVZaPHVfNjWjVnMR8zfzOL+y/zA/MJkAiRFSSBdC1tydHSUiXMSmQRBIB+g/u9//nXHvC3chv6jWPHQuGEu318tdgGHniNYxCPjs3HjxzyC2XotbMEc4/vd9ccPdejZsXhchGyF2NgP+TMyB+Oh0Q/9LQ8VbCjc+E+jyP74wYTeyhUhdjZl4ZbH+h6OKB8/NKBXxc+YpG4v1ITeI5OatUKPTBlJQj96+78zmY8fLOg7CV/yMNxVmXtCwl7ytr05LejvliKGQbgLYt8YPdsb5j1w1Uoiuz6iHz+0YcACETPnk+27On0pbAwQ/vihAwMR74yJt+LPWnaOog66UMyPH7owmA6q7qsAk+h0kHE3cRxE//H589PT0yfbtV3VTOZ+xvlRg4EvvCVOoZNbzQGk1eUVf9YbnULIM8GWzyME+kd4D8atw+K1H7ooMvfX8RMLec5BgQYMWfhE8rqecwyZTRj25sYv9L6n4UAULvC9UQ3qGi+pWwuGyZZFth/stBbkIHJbMLyP5XSr1H+fT8+MjrJtGImHTWyzQKcmw1BG/omzhoQ6MHID/wkJ08SJxW8sinicymAHmfachSJdGD17PNLOkRTBF7sG4/5cR6FmxOswxiXGwQn4guachkImjGd3o/uyN0E2EYg3uS78vRfCPhowXkxntMYFxliQArpEoAnjJBK+V/UOHDFQyoLx/WCq5WI7GfJsS70tbLgVD3qmgQgZep//O7kioTaMn2kalQjJ+YZk9e+zmnBmB676o5SmvYsijPwuXPHQ3ojtQKsoBXF9QSV7Jq7CNbhy/CUuPbehcLWr6gGOEnW4mlWN75VfWGIKVBQ14QvzYm5XiR8xUKoBk8pFLUeJ3IQJyoeMtiiej1D19NRIoOmF1v1ANyxkP7IofrFLSUKBFkxC3xvfay1HBJ89KVMc5Lfh22zR1NIVgJwOXNP7HwWcr14wpEBEwS5ex3wsPObZ2odNsJHiaFaRjbtvDa6n+juR7ciow3S/deiI2dRQb3WRjNImwrG9+YXlW/J+yxfwohj204Cp77u+/8LYKBKKu4qNkk1spCf+kz1Wr98FGopZMF00q/k8Zqm5TTI4v5RXKIy9tLAX61d7sQ56kVcojL204YY9MN3YUzvybnpXPSJ24EZ4f6Yd4LIYJyG6dxm3AGXOgUdtcq8KU8+g2YUbP1zp5wQhananHHS/ajDr3Qx7OvqMeSsyTxGQW4eZHWtvhNpVfybc8iDAdzP2Pe2EVvCcYOy7SEbpBm0hf3I7vq8c9ZyEIk34e4IrlNxIq2T2LBSy8JJHapevnpJ7Hoq14M5fCm/j+y+95XseirXp0pdjoRuRPYjcDszZs2/0KYCoVpHzcBjpbymCPXRhLtArZoMNC+PKIVE8QxKpk4IYutI1mD+JKPorwQDlJVP2RBSsw2Jg1mq1St0pBekmLDbC2/ph9CTcahniGX5ooD5arApy+yvCqNsGLIS9dYXj6MY8x5CJ68Rk5pZvZ4gWNtACF2VxdfCDKOZB9YTNSCTvB3PJR+mWFOLXvr19YfYWeSjYVg0YWcboOmmNLuCklS5TNsp3FB5GP5nWf1bo3yID8Uw6PLp6YuQ+trqKPI9V59UT5YCKYZiaI/wfPHxpa9jzUKyuxupFBztjoYgpL/xHhnP0aGXdA8hrwE++THBiVPecklCgiX87cVkEnGPItOBe69NiK6ItoLtbbdgLmjMWirSh9yy0cfezwOfEEERWB4NnZm/7aG91hI0ko8iSIfaxJHbYhf5QvvHaboaGxEg4ZWH8jIHLbWUIcFtQ0KnDl3GlRzousk1At17ny5Gz32nA/aKvHXVsRrwJt04SWbgU6faqFEKeBT9w/2AP2qecQchrwY9Fle0FFNlttaZV8A9wlOjA7ffy0CDHkNmFr0kUV3RNcLbvdPH+J/2ZcRX6iTbULaDIxkDzfhZw7cqDDgZBMoiSHOSbFLhtfKH3cVFCgVIm5aEURpX3w6FeAtuR0UT/eMnDWL+x5hgyLZhvcb3H9VXHzDFkYnxwM57czPQBgrcWni97bKPL6aG/V/W0jxgo1cFwngIqfaBPCCUT7ns3ixHRu/muYNbNspU6hdOlGa+IS1miGvCFr490FUCkOgx5KB61aRQJEAkDxT/0EblsJwpGhePqsLAAk0BTej4VAgUY742u3IK4BbdJGDg8u2nt+3DIILEWzP9KcEtdqTBKMK1/kHKMnET+0bEcdYdr8GJQlRcowiTQgd7qkeK3VXV69ohDol2IMYbdrbWTffSM0bGHhrLVSshd/3//678Nz48NZnjMxa5C/8kQnhFvuGEnYci9mIK+gMciRp/qyQ+3S9/fyrwhThwW7SrsGyFsZItHva42ZP4o+FPVXeUkKWTC6N8Xd73hCDAawL3nM64bZHi+MacwvRC/Zfnier0B5KI6uH/P7c0TW5ZJK/R3YzE0ei5OZLkPG8y2/cTDCeWKhxD3/JUR+1lH1HtTjbHatEt6XhDjJ46XfCyUixxdTe5mZfQhvxIqickfhHrp6hhZorONi44xWj2UKipyZHQpL6UEddKmjecBJ+WqrIMcJ3oHcP1+eirjKpCIXeh5D9wxZl6pZZIwk0Fb3azBcMO80tsnLItY0XPxMFSlHLZZh+/oFeNiWiKYokQ1wQqYiFgZ1TIkjGOU8kiI0lT4nzHnON3xleBRmfgxj/I/1FSQpA6b8GWIa9jL/UlaVOyOWo56s+CbH7PtL3R3wiPxlsx4OPjCz9AVd0qnkCIpDsm1Yd7/lTFB1pHGDvmVkTEUIcbepROjQCEhjEbxbeXbigmyJ1BloQY3ySNzSx91ihIVvZA/RnfzMqYEyaXYpWtvw6S6G7oZVDkqvQFiSAJJ4ByajK9/ZQod0UgYfUu2i12DvIBdmaCkKAbJWDCtZ/m+0mdaL2QEp/V9PrDeaMEdf/Sd5KV0aMYiGZwRNnPYEjfKtOxVvqQqXkYj6TSUNO54kCxx7EoX81fcnxpdGIecVy7SewJVoWrwhYUPIWeP+DpeMy9a+U+lE/I1TW1SDc51cWaUactgIuMEnd4aaXx6HKcilCHF6iBzg7VqzqqDzQaQDjeJyro6ginJp1oyPvXSlOnwJWfusbhMk/cRSBPlkkMilnKQqRxaovkYl3l91VTU3QLa5QYOZyeJ0Fd7OtkgctRkk6Z8/PCNoHjOsPa+5RvZQNo6MBSR9ErOoq4LtxuKdoLsLT5SlqLZWkBJ2kM+lV9rMOBUjrn1BaoqrHtHnZWwqAt6MZYhK7vnA5Bq43RdcOQtE/rpwndS7d8viFYDvoVsi1HoMSltJk4TZt/uTwjURqiF/cUPXHOoQDUTBzfj0f3ghEGN0m97lq+31YYrdDk/Cf+ESe1CVbY78C2h2vypxYlNzcTppqCs05ycANhDVP2uwRV38QU71amaiVOnm1nLmr3mLtdZMb9FYx5HMS45Op4CiNegZ/BwOqaqlRhNEsDx0fWDzZQ+ErGbqrVkJKo72JG1E6sFX3YBDx3xVyJWJ8wiRuw2rP5YnaqXjYR35AGJ0emAqHMTkkOvEnc1g5Y20wEC3HkWp2qojVB1QmM4OGW85jvfNvHt8ujNuRVBVCzTq1aDmslTKpBIrCEJfHXDn+MjqbSRWE1YbLhFPtaekraQQ67+wQbD+rfUEWtb6GYFG5avMSqxtBd/1ZtvQS/ASDQt06X3+FbK2tDr9fTlNwUQSXrSfqhNvSuEDkndT2Z3f0h+V2VnnzjGgqr6X5rBLXLoKEotP8pUfnrpgEBCOCunFVmVwbS4Lk8Li3IHY/qr6cF+oMkdHDJIrIEPwXaSiAr+VVmHExIJNxW1ufMTpKcIXZUeZyHw9wMm9WMpcLARfF0qakiY3pU9l4RbCv7ha4c4x4japsvSikGOEbVDbmy0KS0dSlSZQibtuSTcpaTraK4/MKQQOnlUw4sxxgV6mkSIRilfmlclGVwFEZFyvbd3k+n3EsUyQ51E6pCN4pFYA64Wg3TmGqUnCk84JNqkMypbv1Roj1IkThf5CHct+MJDX//CEqLq5YqT/SGHl/LE+CLkTpTuGMweJoG2OteSlfd1yeU9TAId+JqD+gx+ASaBLmBoQyV+bTY8hegcGQbZaNzEw2gIN1lP++yPGCRWh7lwA4frFeQYUU1YDBstS5sFlACRGrDgDi99bgTG6WPbE0kQtxzBw7reDgmp55axSEaWakskJEAkqsiKqpw0wcVqcIFN8lSYpfVpnzee8/BR2PqVsJRLXXXgB8MQvfQMcAElehcbQoGRbUmFSmFo8o/e3aR3M6fyglmvIZI4cWXF6oBAQnX4KbxVhFtfxdw8ppCgCbMoCPlO/xRyjKi4I7CHspkmIfWEMxbJNGHok0tbfdzwhEOidF6ofMMrbHFmvQVXg0lV0aEIkwB6/d/09yHbidKhM3MVXRZQouN77nsPGz/UFuByjA531mAqVizS65eQGseMRTJ1uHX48650i5Joflp2z80u8veUzpb6Nq71VaXfIkHuq3hdeDSUxr25mi2q5trEe/Dj4kwzm3CTuEt092ZentWqsqKCTd0VDoHhliDXaU8eYtCv/OVk6iw99YG7X2mNZPx8K6lt8pOZXZoMTVGiok/p4Etu/0Ku8oRI3qZsO3DLTBPDTFmMXVQl2hQlmwBFAToYXJNxfWmSDjF65t8X3yUbnQz0N3hpSSBFiWrK42weL81EZzCRG3DXH6Rp6XRTo3epRLLAFRmXCpD9gWrdd0B90wEQ5sSbSG5/ZX0qjtoisa8DEerFgv50lprmh9eivPRCPJHyHOEp8RbMbT8WzBCVFbWclD8fmmHOLlVcPmH2FBJCt1SEUXX6NSeQAAbKMjkskxJliXDJUB9cmM0a9L4afQxmSm36KlHi1ildty01RYFENA9OFmeDXZrS13APjyHnPVDn6MBy/09ROogpStQmuJ8kq4zrKpS4Fsx7c1yYnIopgYyUQBLp+jJ36SBfWQlTOg6uOupnNtvwlaEX4kq44s342pvc9OgAvqJL2Q7MN77v7Zix3JEv88giUbpgZfjvRioku8jWmZfUv2Kkblo1HLbpoLRq+6q66pRukIpKF+VX1WfCnDtrtcicRWED5v5YvLCMvKrGJvRsdHLOM56WOra/o3LEOfThKzz85e3qVTW3YcidmB1nk4sZ1yIsTzVgQw5THx2Qh9Du+4vTxGgGEK8LP2cz4zT1ia3SEW3VKBsc0DF0XZpYAcSjkwS67PBrjozKfW/QT3xbNQ3oM3sb0GHM0mfwqgqbMgU9fxKxvXk7TVmxjUoYkdSljoGYLQv+wZe3bPf2uv/JlwHbpVU+s9WCn+yfItTMP9VMHIzdxuN96PHGlciH9XqpVOVGdmDAPA+7fvvSoNnqYnAcuuwsyto1kB+B3L+pkjo63qtlQklSfT36VbWZMN0ZqjqS69vXGaY7Be3r3Ga7oTwxsd4ViHHWRIwmjDp5LPoGRYu8Ct45/MbYzIs0p3fyqoPWguvEFqt8zr3hPTqkKFZ68ttsZ19nFDTf8CcjwKvf1niVq/9kTNzAD1EHC9hS0Ae0hoiMFUczXfS+V+QBky0Rd3Dnxuv0tOHfIvwrTvBOgvQAzid56x3oh+RXqHlxfv1duPJZnM3YNPN8bis6Nfl1v3FFBxUF885vQB0KI6C+wEmi85tBH1s+ck/zIpzJgAaFZzEui+L8upvqRxSyL42wdx6KxD2/IRbVrvczQcTnNyEN4GXN/Nb3nfNb0Ibx/WT8x3kVZ6v0+hn3vmx57nRgeNMzLrFC5meWPbaW6nOburBgW07p3bWIL7VudmswYqFnqlze2bWnR/6HbHd+3SbMPP49uMjIF49eovuaBAffX5vdBnxX8+IyZiWBVJ6b04R+4nD6ZOki5iyV8twcS355K/+/4LOLUb/8//DRtWDk0M8tlJ4UOqOJ9PMsqqgSrpUdn1gQSDPbcM254/uXcRSK89+Rdhw51N0O3PEld6KLjh+ZF0ozsuHLzOvCF99h6XeSZ166GrVa4aesLuP2N2r0M0Ge/LmMy8xxssGE3l8Ju9QINKCXrJhN/WQndM5uQ1N923sx/Vb2yz6X8V8atVaWeKNZcDEr5Gm8LO44v/oOTLwoxgdxuVexSwkQefThQs+gXoNrFm4vNgL1evah9oXWIzoYFVJ+61L6s4Pe0v85v3p0UkfTyz1+C3qx60cXG/0W/JxfSnc7S4gfP/o5Dxh9IG2kH5eRMQ8hcz8vQzpIyr043P1ubDkPjJWg+qQdG0tusyTihoijXI1BbKGMDv0EO7SZhyLrNbrY69B3pblreRBKdq3s6sBhHt0YM1c4u/djYDf7CR3jmiWevTn7o6PPrtnuUhuXWS98BqIpEVz26Zhm/gXKuzOtkW+37860ZuqUvzOzrMMCwfsbt1YhpHx/1rWPM9rvz8TOQSHq3ZnXpRrse7OqcZhKeFfG/T93Ai9qA10AAA== \ No newline at end of file From baa152bf73f0cfcaea82b649c24b34469028bdab Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 08:52:19 +0530 Subject: [PATCH 5/8] Decode provider dataset before logo collection --- .github/workflows/collect-official-provider-logos.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/collect-official-provider-logos.yml b/.github/workflows/collect-official-provider-logos.yml index a093231..6876ce3 100644 --- a/.github/workflows/collect-official-provider-logos.yml +++ b/.github/workflows/collect-official-provider-logos.yml @@ -47,6 +47,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r .github/logo-crawler/requirements.txt + - name: Restore provider CSV + run: | + base64 -d .github/logo-crawler/providers_330.csv.gz.b64 | gzip -d > .github/logo-crawler/providers_330.csv - name: Collect official website logos run: | python .github/logo-crawler/collect_official_logos.py \ From 9289ecdd22bc4b3f0264c6d8cd822452b876a24e Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 10:02:38 +0530 Subject: [PATCH 6/8] Add DOM-based official SVG logo extractor --- .github/logo-crawler/collect_dom_svg_logos.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/logo-crawler/collect_dom_svg_logos.py diff --git a/.github/logo-crawler/collect_dom_svg_logos.py b/.github/logo-crawler/collect_dom_svg_logos.py new file mode 100644 index 0000000..f357baf --- /dev/null +++ b/.github/logo-crawler/collect_dom_svg_logos.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +import argparse, asyncio, csv, hashlib, json, re, zipfile +from pathlib import Path +from urllib.parse import urljoin, urlparse +from playwright.async_api import async_playwright + +BAD_WORDS = ('favicon','icon','apple-touch','sprite','loader','flag','social','footer') + +def slug(s): + return re.sub(r'[^a-z0-9]+','-',s.lower()).strip('-') or 'provider' + +def valid_svg(data: bytes) -> bool: + head=data[:5000].lstrip().lower() + return b'=100: s+=3 + if meta.get('height',0)>=24: s+=2 + if any(w in text for w in BAD_WORDS): s-=12 + if 'white' in text or 'mono' in text: s-=2 + return s + +async def fetch_svg(request, url): + try: + r=await request.get(url, timeout=20000) + if not r.ok: return None + data=await r.body() + if valid_svg(data): return data + except Exception: + return None + return None + +async def process(browser, row, outdir, sem): + async with sem: + name=row.get('Provider Name','').strip() + site=row.get('Official URL','').strip() + rec={'rank':row.get('Rank',''),'provider':name,'slug':slug(name),'official_url':site,'status':'unresolved','source':'','local_file':'','sha256':'','notes':''} + if not site: + rec['notes']='missing_official_url'; return rec + ctx=await browser.new_context(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126 Safari/537.36', viewport={'width':1440,'height':1000}, ignore_https_errors=True) + page=await ctx.new_page() + try: + await page.goto(site, wait_until='domcontentloaded', timeout=45000) + await page.wait_for_timeout(2500) + final=page.url + rec['official_url']=final + items=await page.evaluate(""" + () => { + const out=[]; + const vis=e=>{const r=e.getBoundingClientRect(),s=getComputedStyle(e);return r.width>8&&r.height>8&&s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0'}; + document.querySelectorAll('svg').forEach(e=>{const p=e.closest('a,div,header,nav,span');const t=[e.getAttribute('aria-label')||'',e.id||'',e.className?.baseVal||'',p?.id||'',p?.className||'',p?.getAttribute?.('aria-label')||''].join(' ');if(/logo|brand/i.test(t))out.push({kind:'inline',html:e.outerHTML,alt:e.getAttribute('aria-label')||'',id:e.id||'',cls:(e.className&&e.className.baseVal)||'',visible:vis(e),width:e.getBoundingClientRect().width,height:e.getBoundingClientRect().height,src:location.href});}); + document.querySelectorAll('img,source').forEach(e=>{const src=e.currentSrc||e.src||e.getAttribute('srcset')?.split(',').pop()?.trim().split(' ')[0]||'';const t=[e.alt||'',e.id||'',e.className||'',src,e.closest('a,div,header,nav')?.className||''].join(' ');if(src&&(/logo|brand/i.test(t)||/\.svg(?:\?|$)/i.test(src)))out.push({kind:'img',src:new URL(src,location.href).href,alt:e.alt||'',id:e.id||'',cls:e.className||'',visible:vis(e),width:e.getBoundingClientRect().width,height:e.getBoundingClientRect().height});}); + document.querySelectorAll('*').forEach(e=>{const bg=getComputedStyle(e).backgroundImage;const m=bg&&bg.match(/url\(["']?(.*?)["']?\)/);const t=[e.id||'',e.className||''].join(' ');if(m&&/logo|brand/i.test(t+ ' '+m[1]))out.push({kind:'bg',src:new URL(m[1],location.href).href,alt:'',id:e.id||'',cls:e.className||'',visible:vis(e),width:e.getBoundingClientRect().width,height:e.getBoundingClientRect().height});}); + return out; + } + """) + candidates=[] + for m in items: + data=None + if m.get('kind')=='inline': + data=m.get('html','').encode('utf-8') + if not valid_svg(data): data=None + elif str(m.get('src','')).startswith('data:image/svg+xml'): + import urllib.parse, base64 + src=m['src'] + try: + payload=src.split(',',1)[1] + data=base64.b64decode(payload) if ';base64' in src[:80] else urllib.parse.unquote_to_bytes(payload) + except Exception: data=None + else: + data=await fetch_svg(ctx.request, m.get('src','')) + if data: candidates.append((score(m),m,data)) + if not candidates: + rec['notes']='no_valid_svg_found'; return rec + candidates.sort(key=lambda x:x[0], reverse=True) + sc,m,data=candidates[0] + folder=outdir/'logos'; folder.mkdir(parents=True,exist_ok=True) + fn=f"{slug(name)}.svg"; (folder/fn).write_bytes(data) + rec.update(status='downloaded',source=m.get('src',final),local_file='logos/'+fn,sha256=hashlib.sha256(data).hexdigest(),score=sc,notes=m.get('kind','')) + return rec + except Exception as e: + rec['notes']=type(e).__name__; return rec + finally: + await ctx.close() + +async def main_async(a): + rows=list(csv.DictReader(open(a.input,encoding='utf-8-sig'))) + rows=[r for r in rows if a.start<=int(r.get('Rank') or 0)<=a.end] + out=Path(a.output); out.mkdir(parents=True,exist_ok=True) + async with async_playwright() as p: + browser=await p.chromium.launch(headless=True,args=['--no-sandbox']) + sem=asyncio.Semaphore(a.workers) + res=await asyncio.gather(*(process(browser,r,out,sem) for r in rows)) + await browser.close() + res.sort(key=lambda x:int(x.get('rank') or 0)) + cols=['rank','provider','slug','official_url','status','source','local_file','sha256','score','notes'] + with open(out/'manifest.csv','w',newline='',encoding='utf-8') as f: + w=csv.DictWriter(f,fieldnames=cols,extrasaction='ignore');w.writeheader();w.writerows(res) + (out/'manifest.json').write_text(json.dumps(res,indent=2,ensure_ascii=False),encoding='utf-8') + with zipfile.ZipFile(str(out)+'.zip','w',zipfile.ZIP_DEFLATED) as z: + for pth in out.rglob('*'): + if pth.is_file(): z.write(pth,pth.relative_to(out)) + +if __name__=='__main__': + ap=argparse.ArgumentParser();ap.add_argument('--input',required=True);ap.add_argument('--output',required=True);ap.add_argument('--start',type=int,default=1);ap.add_argument('--end',type=int,default=9999);ap.add_argument('--workers',type=int,default=4) + asyncio.run(main_async(ap.parse_args())) From 5811a74f5d36dade8b29d5f3c2055e856dc7a450 Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 10:02:52 +0530 Subject: [PATCH 7/8] Use Playwright for DOM SVG extraction --- .github/logo-crawler/requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/logo-crawler/requirements.txt b/.github/logo-crawler/requirements.txt index 80d6194..2e874d6 100644 --- a/.github/logo-crawler/requirements.txt +++ b/.github/logo-crawler/requirements.txt @@ -1,4 +1 @@ -requests==2.32.4 -beautifulsoup4==4.13.4 -lxml==6.0.0 -ddgs==9.5.5 +playwright==1.52.0 From 98f2dc3704e1657d1614023e7e1a9c60b7f1317a Mon Sep 17 00:00:00 2001 From: Raj Meena Date: Mon, 13 Jul 2026 10:03:09 +0530 Subject: [PATCH 8/8] Run DOM-based official SVG logo extraction --- .../collect-official-provider-logos.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/collect-official-provider-logos.yml b/.github/workflows/collect-official-provider-logos.yml index 6876ce3..5e48c45 100644 --- a/.github/workflows/collect-official-provider-logos.yml +++ b/.github/workflows/collect-official-provider-logos.yml @@ -1,4 +1,4 @@ -name: Collect official provider logos +name: Collect official provider SVG logos on: pull_request: @@ -43,27 +43,28 @@ jobs: python-version: '3.12' cache: pip cache-dependency-path: .github/logo-crawler/requirements.txt - - name: Install crawler dependencies + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r .github/logo-crawler/requirements.txt + playwright install --with-deps chromium - name: Restore provider CSV run: | base64 -d .github/logo-crawler/providers_330.csv.gz.b64 | gzip -d > .github/logo-crawler/providers_330.csv - - name: Collect official website logos + - name: Extract official DOM SVG logos run: | - python .github/logo-crawler/collect_official_logos.py \ + python .github/logo-crawler/collect_dom_svg_logos.py \ --input .github/logo-crawler/providers_330.csv \ - --output official_logo_output \ + --output official_svg_output \ --start ${{ matrix.start }} \ --end ${{ matrix.end }} \ - --workers 8 - - name: Upload logo package shard + --workers 4 + - name: Upload SVG package shard uses: actions/upload-artifact@v4 with: - name: official-provider-logos-part-${{ matrix.part }} + name: official-provider-svg-part-${{ matrix.part }} path: | - official_logo_output/** - official_logo_output.zip + official_svg_output/** + official_svg_output.zip if-no-files-found: error retention-days: 7