Classes/Bases/Skills/Uniques to DB
This commit is contained in:
mikx
221
importer.py
Normal file
221
importer.py
Normal file
@@ -0,0 +1,221 @@
|
||||
import json
|
||||
import requests
|
||||
|
||||
def download_and_parse_items(url):
|
||||
"""
|
||||
Downloads item bases JSON from the given URL and parses it
|
||||
into a flat structure suitable for MySQL insertion.
|
||||
"""
|
||||
print(f"Downloading base items JSON from: {url}")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
print("Download completed. Parsing JSON structure...")
|
||||
raw_data = response.json()
|
||||
parsed_items = []
|
||||
|
||||
for item_id, item_val in raw_data.items():
|
||||
# Extrapolate requirements
|
||||
reqs = item_val.get("requirements") or {}
|
||||
req_level = reqs.get("level")
|
||||
req_strength = reqs.get("strength")
|
||||
req_dexterity = reqs.get("dexterity")
|
||||
req_intelligence = reqs.get("intelligence")
|
||||
|
||||
# Extrapolate visual identity
|
||||
vis = item_val.get("visual_identity") or {}
|
||||
vis_id = vis.get("id")
|
||||
vis_dds = vis.get("dds_file")
|
||||
|
||||
# Serialize nested lists/dicts to JSON strings
|
||||
properties = json.dumps(item_val.get("properties")) if item_val.get("properties") is not None else None
|
||||
tags = json.dumps(item_val.get("tags")) if item_val.get("tags") is not None else None
|
||||
implicits = json.dumps(item_val.get("implicits")) if item_val.get("implicits") is not None else None
|
||||
skills_granted = json.dumps(item_val.get("skills_granted")) if item_val.get("skills_granted") is not None else None
|
||||
|
||||
# Prepare tuple matching schema order:
|
||||
# id, name, item_class, inherits_from, domain, drop_level,
|
||||
# inventory_width, inventory_height, release_state,
|
||||
# req_level, req_strength, req_dexterity, req_intelligence,
|
||||
# visual_identity_id, visual_identity_dds,
|
||||
# properties, tags, implicits, skills_granted
|
||||
parsed_item = (
|
||||
item_id,
|
||||
item_val.get("name", "Unknown Name"),
|
||||
item_val.get("item_class"),
|
||||
item_val.get("inherits_from"),
|
||||
item_val.get("domain", "undefined"),
|
||||
item_val.get("drop_level"),
|
||||
item_val.get("inventory_width"),
|
||||
item_val.get("inventory_height"),
|
||||
item_val.get("release_state"),
|
||||
req_level,
|
||||
req_strength,
|
||||
req_dexterity,
|
||||
req_intelligence,
|
||||
vis_id,
|
||||
vis_dds,
|
||||
properties,
|
||||
tags,
|
||||
implicits,
|
||||
skills_granted
|
||||
)
|
||||
|
||||
parsed_items.append(parsed_item)
|
||||
|
||||
print(f"Successfully parsed {len(parsed_items)} items from JSON.")
|
||||
return parsed_items
|
||||
|
||||
def download_and_parse_skills(url):
|
||||
"""
|
||||
Downloads skill_gems.json from the given URL and parses it
|
||||
into a flat structure suitable for MySQL insertion.
|
||||
"""
|
||||
print(f"Downloading skill gems JSON from: {url}")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
print("Download completed. Parsing JSON structure...")
|
||||
raw_data = response.json()
|
||||
parsed_skills = []
|
||||
|
||||
for skill_id, skill_val in raw_data.items():
|
||||
base_item = skill_val.get("base_item") or {}
|
||||
display_name = base_item.get("display_name", "Unknown Name")
|
||||
release_state = base_item.get("release_state")
|
||||
|
||||
req_weights = skill_val.get("requirement_weights") or {}
|
||||
req_dex = req_weights.get("dexterity", 0)
|
||||
req_int = req_weights.get("intelligence", 0)
|
||||
req_str = req_weights.get("strength", 0)
|
||||
|
||||
# List structures to be JSON serialized
|
||||
crafting_types = json.dumps(skill_val.get("crafting_types")) if skill_val.get("crafting_types") is not None else None
|
||||
grants_skills = json.dumps(skill_val.get("grants_skills")) if skill_val.get("grants_skills") is not None else None
|
||||
recommended_supports = json.dumps(skill_val.get("recommended_supports")) if skill_val.get("recommended_supports") is not None else None
|
||||
tags = json.dumps(skill_val.get("tags")) if skill_val.get("tags") is not None else None
|
||||
|
||||
# Prepare tuple matching schema order:
|
||||
# id, display_name, release_state, color, crafting_level, gem_type,
|
||||
# icon_dds_file, skill_name, support_name, support_text,
|
||||
# tutorial_video, ui_image,
|
||||
# req_weight_dexterity, req_weight_intelligence, req_weight_strength,
|
||||
# crafting_types, grants_skills, recommended_supports, tags
|
||||
parsed_skill = (
|
||||
skill_id,
|
||||
display_name,
|
||||
release_state,
|
||||
skill_val.get("color"),
|
||||
skill_val.get("crafting_level"),
|
||||
skill_val.get("gem_type"),
|
||||
skill_val.get("icon_dds_file"),
|
||||
skill_val.get("skill_name"),
|
||||
skill_val.get("support_name"),
|
||||
skill_val.get("support_text"),
|
||||
skill_val.get("tutorial_video"),
|
||||
skill_val.get("ui_image"),
|
||||
req_dex,
|
||||
req_int,
|
||||
req_str,
|
||||
crafting_types,
|
||||
grants_skills,
|
||||
recommended_supports,
|
||||
tags
|
||||
)
|
||||
|
||||
parsed_skills.append(parsed_skill)
|
||||
|
||||
print(f"Successfully parsed {len(parsed_skills)} skill gems from JSON.")
|
||||
return parsed_skills
|
||||
|
||||
def download_and_parse_item_classes(url):
|
||||
"""
|
||||
Downloads item_classes.json from the given URL and parses it
|
||||
into a flat structure suitable for MySQL insertion.
|
||||
"""
|
||||
print(f"Downloading item classes JSON from: {url}")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
print("Download completed. Parsing JSON structure...")
|
||||
raw_data = response.json()
|
||||
parsed_classes = []
|
||||
|
||||
for class_id, class_val in raw_data.items():
|
||||
influence_tags = json.dumps(class_val.get("influence_tags")) if class_val.get("influence_tags") is not None else None
|
||||
|
||||
# Prepare tuple matching schema order:
|
||||
# id, category, category_id, name, influence_tags
|
||||
parsed_class = (
|
||||
class_id,
|
||||
class_val.get("category"),
|
||||
class_val.get("category_id"),
|
||||
class_val.get("name", "Unknown Name"),
|
||||
influence_tags
|
||||
)
|
||||
|
||||
parsed_classes.append(parsed_class)
|
||||
|
||||
print(f"Successfully parsed {len(parsed_classes)} item classes from JSON.")
|
||||
return parsed_classes
|
||||
|
||||
def download_and_parse_uniques(url):
|
||||
"""
|
||||
Downloads uniques.json from the given URL and parses it
|
||||
into a flat structure suitable for MySQL insertion.
|
||||
"""
|
||||
print(f"Downloading uniques JSON from: {url}")
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
print("Download completed. Parsing JSON structure...")
|
||||
raw_data = response.json()
|
||||
parsed_uniques = []
|
||||
|
||||
for item_id, item_val in raw_data.items():
|
||||
# Extrapolate visual identity
|
||||
vis = item_val.get("visual_identity") or {}
|
||||
vis_id = vis.get("id")
|
||||
vis_dds = vis.get("dds_file")
|
||||
|
||||
# Prepare tuple matching schema order:
|
||||
# id, name, unique_id, item_class,
|
||||
# inventory_width, inventory_height, is_alternate_art,
|
||||
# visual_identity_id, visual_identity_dds,
|
||||
# renamed_version, base_version
|
||||
parsed_unique = (
|
||||
item_id,
|
||||
item_val.get("name", "Unknown Name"),
|
||||
item_val.get("id"),
|
||||
item_val.get("item_class"),
|
||||
item_val.get("inventory_width"),
|
||||
item_val.get("inventory_height"),
|
||||
item_val.get("is_alternate_art"),
|
||||
vis_id,
|
||||
vis_dds,
|
||||
item_val.get("renamed_version"),
|
||||
item_val.get("base_version")
|
||||
)
|
||||
|
||||
parsed_uniques.append(parsed_unique)
|
||||
|
||||
print(f"Successfully parsed {len(parsed_uniques)} uniques from JSON.")
|
||||
return parsed_uniques
|
||||
|
||||
|
||||
Reference in New Issue
Block a user