You're welcome! It was a fun hyperfixation project. I ended up making the script so easy to use I decided to just scrape every other endpoint too, so if anyone wants it, here's a full dump of every endpoint, it's only like 4x bigger:
https://mega.nz/file/YF4F3bCS#pkS8Ki9QuucMGJF65YwGUE-NQZ78QEWs73fmF71qa18
And if anyone wants to do their own scraping to get more up to date data later, just pip install:
python-dotenv==1.2.2
Requests==2.34.2
tqdm==4.67.1
Put API keys in .env or export env vars:
CLIENT_ID=<client_id>
# Provide to fetch new token
CLIENT_SECRET=<client_secret>
# Optional, provide to reuse existing access token, secret will not be used
ACCESS_TOKEN=<access_token>
And just run python dump.py games or any other endpoint in the api docs like release_dates etc. It outputs the json and a simple log to an output folder wherever you ran it. No error handling or checkpointing so if it fails partway through you don't get anything, but I didn't have a single error the whole time.
usage: dump.py [-h] api_route
IGDB Dump Script
positional arguments:
api_route The API route to scrape, eg: games or game_time_to_beats
options:
-h, --help show this help message and exit
dump.py:
import argparse
import json
import logging
import os
import pathlib
import time
from dotenv import dotenv_values
import requests
from tqdm import tqdm
API_PAGE_SIZE = 500
OUT_DIR = "output"
config = {
**dotenv_values(".env"),
**os.environ,
}
# Set up flags / args
parser = argparse.ArgumentParser(
prog="dump.py", description="IGDB Dump Script"
)
parser.add_argument(
"api_route",
help="The API route to scrape, eg: games or game_time_to_beats",
)
args = parser.parse_args()
# Create out dir
pathlib.Path.mkdir(OUT_DIR, parents=False, exist_ok=True)
# Set up logging to the route's file
tqdmHandler = logging.StreamHandler(tqdm)
tqdmHandler.terminator = ""
logging.basicConfig(
level=logging.INFO,
format="%(message)s",
handlers=[
logging.FileHandler(f"{OUT_DIR}/{args.api_route}.log"),
tqdmHandler
],
)
# Check for existing json to prevent overwriting existing dumps
outFile = f"{OUT_DIR}/{args.api_route}.json"
if pathlib.Path(outFile).exists():
print(f"Existing json found {outFile}, please move or remove it before proceeding")
exit(1)
if config['CLIENT_ID'] and config['ACCESS_TOKEN']:
logging.info("Using CLIENT_ID and existing ACCESS_TOKEN")
elif config['CLIENT_ID'] and config['CLIENT_SECRET'] and not config['ACCESS_TOKEN']:
logging.info("Fetching new access token...")
response = requests.post(
url="https://id.twitch.tv/oauth2/token",
params={
"client_id": config['CLIENT_ID'],
"client_secret": config['CLIENT_SECRET'],
"grant_type": "client_credentials"
},
timeout=30
)
config['ACCESS_TOKEN'] = response.json()['access_token']
else:
logging.info("Missing CLIENT_ID and CLIENT_SECRET or ACCESS_TOKEN")
exit(1)
# Re-check access token in case fetch failed
if config['CLIENT_ID'] and config['ACCESS_TOKEN']:
items = []
offset = 0
logging.info(f"Fetching batches of {API_PAGE_SIZE} on endpoint {args.api_route}")
with tqdm() as pbar:
while True:
response = requests.post(
url=f"https://api.igdb.com/v4/%7Bargs.api_route%7D",
headers={
"Client-ID": config['CLIENT_ID'],
"Authorization": f"Bearer {config['ACCESS_TOKEN']}"
},
data=f"fields *; limit {API_PAGE_SIZE}; offset {offset};",
timeout=30
)
newItems = response.json()
fetchCount = len(newItems)
pbar.update(fetchCount)
if fetchCount != API_PAGE_SIZE:
logging.info(f"WARN: Requested {API_PAGE_SIZE}, got {fetchCount}")
offset += API_PAGE_SIZE
items.extend(newItems)
if fetchCount < API_PAGE_SIZE:
logging.info("Received partial page, ending")
break
time.sleep(1)
logging.info(f"Total fetched: {len(items)}")
with open(outFile, "w", encoding="utf-8") as file:
logging.info("Writing to json...")
json.dump(items, file, ensure_ascii=False, indent=2)
# Print some stats
logging.info(f"\nChecking json output: {args.api_route}.json")
entries = []
with open(outFile, "r", encoding="utf-8") as file:
entries = json.load(file)
logging.info(f"{len(entries)} entries in json")
entryDict = {}
for entry in entries:
entryDict.update({entry['id']: entry})
logging.info(f"{len(entryDict)} unique IDs in json")
else:
logging.error("Client ID or Access Token not available")
exit(1)