stats/scraper.py at master · frosty-dev/stats · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
from ss13stats.db import Server, GlobalStat, ServerStat, db_ext

from bs4 import BeautifulSoup

from datetime import datetime

import re

import requests

BYOND_GAME_URL = "http://www.byond.com/games/Exadv1/SpaceStation13"

# Sorry for whoever has to deal with this regex in the future
PLAYER_COUNT_REGEX = re.compile(r" ([0-9]{1,3}|No) players?(?:\.|[\s]*\[See list\])")

document = requests.get(BYOND_GAME_URL).text

soup = BeautifulSoup(document, 'html.parser')

now = datetime.utcnow().replace(second=0, microsecond=0) # Get current time truncated to the minute

fan_count = int(soup.find(id="hub_overview").tr.td.div.div.span.a.text.split(" ")[0])

live_games = soup.find_all("div", class_="live_game_entry")

total_player_count = 0

tracked_world_ids = []

for server in Server.query.all():
	server.player_count = 0 # reset all servers to zero so if they're not on the list they don't stay at whatever they were at
	server.online = 0 # assume all previously tracked servers not in the hub list are offline

for game in live_games:
	server_url = game.div.div.span.nobr.text

	server_id = int(server_url.split(".")[-1])

	if server_id in tracked_world_ids: # to ignore stupid byond hub ghost server duplicates
		continue

	tracked_world_ids.append(server_id)

	player_count_parse = PLAYER_COUNT_REGEX.search(game.div.div.text).groups(0)[0]

	if game.div.div.find("b"):
		title = game.div.div.find("b").text
	else:
		title = server_url

	player_count = int(player_count_parse) if player_count_parse != "No" else 0

	server = Server.query.get(server_id)

	if not server:
		server = Server(
			id = server_id,
			first_seen = now,
			last_seen = now,
			title = title,
			player_count = player_count
		)

		db_ext.session.add(server)

	server.last_seen = now
	server.online = 1
	server.title = title
	server.player_count = player_count

	server_stat = ServerStat(
		timestamp = now,
		server_id = server_id,
		player_count = player_count
	)

	db_ext.session.add(server_stat)

	total_player_count += player_count

db_ext.session.add(GlobalStat(
	timestamp = now,
	type = "SERVER_COUNT",
	value = len(live_games)
))

db_ext.session.add(GlobalStat(
	timestamp = now,
	type = "PLAYER_COUNT",
	value = total_player_count
))

db_ext.session.add(GlobalStat(
	timestamp = now,
	type = "FAN_COUNT",
	value = fan_count
))

db_ext.session.commit()