Support for list like top 250

[webysther]

Looks like the list also have the json inside the page: https://www.imdb.com/pt/chart/top/

This snippet return all top 250:

I build in this way to be easy to load and check in one step in python, this also allow get the movie details easily.

import requests, json, jmespath
from lxml import html

url = f"https://www.imdb.com/pt/chart/top/"

resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
if resp.status_code != 200:
    logger.error("Error fetching %s: %s", url, resp.status_code)
    raise Exception(f"Error fetching {url}")

tree = html.fromstring(resp.content)
script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')

if not script:
    logger.error("No script found with id '__NEXT_DATA__'")
    raise Exception("No script found with id '__NEXT_DATA__'")

raw_json = json.loads(script[0])

processed_json = jmespath.search("props.pageProps.pageData.chartTitles.edges", raw_json)

movies = {}

for item in processed_json:
    movies.update({
        item['node']['id']: {
            'rank': item['currentRank'],
            'title': item['node'].get('titleText', {}).get('text'),
            'year': item['node'].get('releaseYear', {}).get('year'),
            'rating': item['node'].get('ratingsSummary', {}).get('aggregateRating'),
            'ratingCount': item['node'].get('ratingsSummary', {}).get('voteCount')
        }
    })

json_dumps = json.dumps(movies, indent=2, ensure_ascii=False)

print(json_dumps)

The data returned:

{
  "tt0111161": {
    "rank": 1,
    "title": "Um Sonho de Liberdade",
    "year": 1994,
    "rating": 9.3,
    "ratingCount": 3088087
  },
  "tt0068646": {
    "rank": 2,
    "title": "O Poderoso Chefão",
    "year": 1972,
    "rating": 9.2,
    "ratingCount": 2153202
  }
  ...
}

[tveronesi]

Hi @webysther,

Absolutely! There are a lot of other aggregate information types on IMDb that could be interesting to grab and handle, like showtimes, top this week lists, most popular celebrities, top picks, suggestions, and much more. That’s definitely something worth exploring and could add a lot of value.

For now, I’m focusing mainly on the parents guides for titles, but I’ll keep this idea on my list for a future milestone. Thanks for bringing it up!

[webysther]

For who find this issue, this script get a user list with pagination (+250 items):

import requests, json, jmespath, os
from lxml import html

url = "https://www.imdb.com/pt/list/ls024863935/?sort=release_date%2Cdesc"
xpath = "props.pageProps.mainColumnData.list.titleListItemSearch."
headers={"User-Agent": "Mozilla/5.0"}

def get_html(url):
    resp = requests.get(
        url, 
        headers={"User-Agent": "Mozilla/5.0"}
    )
    if resp.status_code != 200:
        print("Error fetching: %s" % (resp.status_code))
        exit(1)
    tree = html.fromstring(resp.content)
    script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')
    return json.loads(script[0])


def process_json(raw_json):
    next_page = jmespath.search(f"{xpath}pageInfo.hasNextPage", raw_json)
    processed_json = jmespath.search(xpath+"edges[].listItem.{id: id, year:releaseYear.year}", raw_json)
    return next_page, processed_json

raw_json = get_html(url)
total = jmespath.search(f"{xpath}total", raw_json)

next_page, processed_json = process_json(raw_json)

movies = {}
while next_page:
    print(len(movies), "of", total)
    for item in processed_json:
        movies.update({
            item['id']: item['year']
        })
        
    raw_json = get_html(url+f"&page={len(movies)//250+1}")
    next_page, processed_json = process_json(raw_json)

for item in processed_json:
    movies.update({
        item['id']: item['year']
    })

print(total)

movies_sorted = dict(
    sorted(movies.items(), key=lambda item: int(item[1]), reverse=True)
)

with open('./data/list_1001_movies_before_die.json', 'w', encoding='utf-8') as file:
    json.dump(movies_sorted, file, indent=2)