349 lines
12 KiB
Python
349 lines
12 KiB
Python
import os
|
|
import json
|
|
import mysql.connector
|
|
from mysql.connector import errorcode
|
|
|
|
def get_db_connection(host, port, user, password, db_name=None):
|
|
"""Establishes connection to MySQL. Optionally connects to a specific database."""
|
|
config = {
|
|
'host': host,
|
|
'port': port,
|
|
'user': user,
|
|
'password': password,
|
|
}
|
|
if db_name:
|
|
config['database'] = db_name
|
|
return mysql.connector.connect(**config)
|
|
|
|
def init_db(host, port, user, password, db_name):
|
|
"""Initializes the database and the item_bases table."""
|
|
try:
|
|
# Check if database exists by attempting connection
|
|
conn = get_db_connection(host, port, user, password, db_name)
|
|
conn.close()
|
|
except mysql.connector.Error as err:
|
|
if err.errno == errorcode.ER_BAD_DB_ERROR:
|
|
# Database doesn't exist, connect without database name to create it
|
|
print(f"Database '{db_name}' not found. Creating database...")
|
|
conn = get_db_connection(host, port, user, password)
|
|
cursor = conn.cursor()
|
|
cursor.execute(f"CREATE DATABASE `{db_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
|
|
cursor.close()
|
|
conn.close()
|
|
else:
|
|
raise err
|
|
|
|
# Connect to the target database to verify / create table
|
|
conn = get_db_connection(host, port, user, password, db_name)
|
|
cursor = conn.cursor()
|
|
|
|
# Create main item_bases table with JSON support
|
|
table_query = """
|
|
CREATE TABLE IF NOT EXISTS item_bases (
|
|
id VARCHAR(255) PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL,
|
|
item_class VARCHAR(100),
|
|
inherits_from VARCHAR(255),
|
|
domain VARCHAR(50),
|
|
drop_level INT,
|
|
inventory_width INT,
|
|
inventory_height INT,
|
|
release_state VARCHAR(50),
|
|
req_level INT,
|
|
req_strength INT,
|
|
req_dexterity INT,
|
|
req_intelligence INT,
|
|
visual_identity_id VARCHAR(255),
|
|
visual_identity_dds VARCHAR(255),
|
|
properties JSON,
|
|
tags JSON,
|
|
implicits JSON,
|
|
skills_granted JSON,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
|
"""
|
|
cursor.execute(table_query)
|
|
|
|
# Create skill_gems table with JSON support
|
|
skill_table_query = """
|
|
CREATE TABLE IF NOT EXISTS skill_gems (
|
|
id VARCHAR(255) PRIMARY KEY,
|
|
display_name VARCHAR(255),
|
|
release_state VARCHAR(50),
|
|
color VARCHAR(50),
|
|
crafting_level INT,
|
|
gem_type VARCHAR(50),
|
|
icon_dds_file VARCHAR(255),
|
|
skill_name VARCHAR(255),
|
|
support_name VARCHAR(255),
|
|
support_text TEXT,
|
|
tutorial_video VARCHAR(255),
|
|
ui_image VARCHAR(255),
|
|
req_weight_dexterity INT,
|
|
req_weight_intelligence INT,
|
|
req_weight_strength INT,
|
|
crafting_types JSON,
|
|
grants_skills JSON,
|
|
recommended_supports JSON,
|
|
tags JSON,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
|
"""
|
|
cursor.execute(skill_table_query)
|
|
|
|
# Create item_classes table with JSON support
|
|
classes_table_query = """
|
|
CREATE TABLE IF NOT EXISTS item_classes (
|
|
id VARCHAR(255) PRIMARY KEY,
|
|
category VARCHAR(100),
|
|
category_id VARCHAR(100),
|
|
name VARCHAR(255) NOT NULL,
|
|
influence_tags JSON,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
|
"""
|
|
cursor.execute(classes_table_query)
|
|
|
|
# Create item_uniques table
|
|
uniques_table_query = """
|
|
CREATE TABLE IF NOT EXISTS item_uniques (
|
|
id VARCHAR(255) PRIMARY KEY,
|
|
name VARCHAR(255) NOT NULL,
|
|
unique_id VARCHAR(255),
|
|
item_class VARCHAR(100),
|
|
inventory_width INT,
|
|
inventory_height INT,
|
|
is_alternate_art BOOLEAN,
|
|
visual_identity_id VARCHAR(255),
|
|
visual_identity_dds VARCHAR(255),
|
|
renamed_version VARCHAR(255),
|
|
base_version VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
|
"""
|
|
cursor.execute(uniques_table_query)
|
|
|
|
conn.commit()
|
|
cursor.close()
|
|
print("Database tables 'item_bases', 'skill_gems', 'item_classes', and 'item_uniques' verified/created successfully.")
|
|
return conn
|
|
|
|
def bulk_upsert_items(conn, items_data):
|
|
"""Upserts list of items into database using batch processing."""
|
|
if not items_data:
|
|
return 0
|
|
|
|
query = """
|
|
INSERT INTO item_bases (
|
|
id, name, item_class, inherits_from, domain, drop_level,
|
|
inventory_width, inventory_height, release_state,
|
|
req_level, req_strength, req_dexterity, req_intelligence,
|
|
visual_identity_id, visual_identity_dds,
|
|
properties, tags, implicits, skills_granted
|
|
) VALUES (
|
|
%s, %s, %s, %s, %s, %s,
|
|
%s, %s, %s,
|
|
%s, %s, %s, %s,
|
|
%s, %s,
|
|
%s, %s, %s, %s
|
|
) ON DUPLICATE KEY UPDATE
|
|
name = VALUES(name),
|
|
item_class = VALUES(item_class),
|
|
inherits_from = VALUES(inherits_from),
|
|
domain = VALUES(domain),
|
|
drop_level = VALUES(drop_level),
|
|
inventory_width = VALUES(inventory_width),
|
|
inventory_height = VALUES(inventory_height),
|
|
release_state = VALUES(release_state),
|
|
req_level = VALUES(req_level),
|
|
req_strength = VALUES(req_strength),
|
|
req_dexterity = VALUES(req_dexterity),
|
|
req_intelligence = VALUES(req_intelligence),
|
|
visual_identity_id = VALUES(visual_identity_id),
|
|
visual_identity_dds = VALUES(visual_identity_dds),
|
|
properties = VALUES(properties),
|
|
tags = VALUES(tags),
|
|
implicits = VALUES(implicits),
|
|
skills_granted = VALUES(skills_granted)
|
|
"""
|
|
|
|
cursor = conn.cursor()
|
|
batch_size = 500
|
|
total_upserted = len(items_data)
|
|
|
|
print(f"Upserting {total_upserted} items in batches of {batch_size}...")
|
|
for i in range(0, total_upserted, batch_size):
|
|
chunk = items_data[i:i + batch_size]
|
|
cursor.executemany(query, chunk)
|
|
conn.commit()
|
|
print(f"Upserted {min(i + batch_size, total_upserted)} / {total_upserted} items...")
|
|
|
|
cursor.close()
|
|
return total_upserted
|
|
|
|
def clear_items_table(conn):
|
|
"""Truncates the item_bases table to empty all existing items."""
|
|
cursor = conn.cursor()
|
|
cursor.execute("TRUNCATE TABLE item_bases")
|
|
conn.commit()
|
|
cursor.close()
|
|
print("Table 'item_bases' cleared successfully.")
|
|
|
|
def clear_skills_table(conn):
|
|
"""Truncates the skill_gems table to empty all existing skill gems."""
|
|
cursor = conn.cursor()
|
|
cursor.execute("TRUNCATE TABLE skill_gems")
|
|
conn.commit()
|
|
cursor.close()
|
|
print("Table 'skill_gems' cleared successfully.")
|
|
|
|
def bulk_upsert_skills(conn, skills_data):
|
|
"""Upserts list of skill gems into database using batch processing."""
|
|
if not skills_data:
|
|
return 0
|
|
|
|
query = """
|
|
INSERT INTO skill_gems (
|
|
id, display_name, release_state, color, crafting_level, gem_type,
|
|
icon_dds_file, skill_name, support_name, support_text,
|
|
tutorial_video, ui_image,
|
|
req_weight_dexterity, req_weight_intelligence, req_weight_strength,
|
|
crafting_types, grants_skills, recommended_supports, tags
|
|
) VALUES (
|
|
%s, %s, %s, %s, %s, %s,
|
|
%s, %s, %s, %s,
|
|
%s, %s,
|
|
%s, %s, %s,
|
|
%s, %s, %s, %s
|
|
) ON DUPLICATE KEY UPDATE
|
|
display_name = VALUES(display_name),
|
|
release_state = VALUES(release_state),
|
|
color = VALUES(color),
|
|
crafting_level = VALUES(crafting_level),
|
|
gem_type = VALUES(gem_type),
|
|
icon_dds_file = VALUES(icon_dds_file),
|
|
skill_name = VALUES(skill_name),
|
|
support_name = VALUES(support_name),
|
|
support_text = VALUES(support_text),
|
|
tutorial_video = VALUES(tutorial_video),
|
|
ui_image = VALUES(ui_image),
|
|
req_weight_dexterity = VALUES(req_weight_dexterity),
|
|
req_weight_intelligence = VALUES(req_weight_intelligence),
|
|
req_weight_strength = VALUES(req_weight_strength),
|
|
crafting_types = VALUES(crafting_types),
|
|
grants_skills = VALUES(grants_skills),
|
|
recommended_supports = VALUES(recommended_supports),
|
|
tags = VALUES(tags)
|
|
"""
|
|
|
|
cursor = conn.cursor()
|
|
batch_size = 500
|
|
total_upserted = len(skills_data)
|
|
|
|
print(f"Upserting {total_upserted} skill gems in batches of {batch_size}...")
|
|
for i in range(0, total_upserted, batch_size):
|
|
chunk = skills_data[i:i + batch_size]
|
|
cursor.executemany(query, chunk)
|
|
conn.commit()
|
|
print(f"Upserted {min(i + batch_size, total_upserted)} / {total_upserted} skill gems...")
|
|
|
|
cursor.close()
|
|
return total_upserted
|
|
|
|
def clear_item_classes_table(conn):
|
|
"""Truncates the item_classes table to empty all existing item classes."""
|
|
cursor = conn.cursor()
|
|
cursor.execute("TRUNCATE TABLE item_classes")
|
|
conn.commit()
|
|
cursor.close()
|
|
print("Table 'item_classes' cleared successfully.")
|
|
|
|
def bulk_upsert_item_classes(conn, classes_data):
|
|
"""Upserts list of item classes into database using batch processing."""
|
|
if not classes_data:
|
|
return 0
|
|
|
|
query = """
|
|
INSERT INTO item_classes (
|
|
id, category, category_id, name, influence_tags
|
|
) VALUES (
|
|
%s, %s, %s, %s, %s
|
|
) ON DUPLICATE KEY UPDATE
|
|
category = VALUES(category),
|
|
category_id = VALUES(category_id),
|
|
name = VALUES(name),
|
|
influence_tags = VALUES(influence_tags)
|
|
"""
|
|
|
|
cursor = conn.cursor()
|
|
batch_size = 500
|
|
total_upserted = len(classes_data)
|
|
|
|
print(f"Upserting {total_upserted} item classes in batches of {batch_size}...")
|
|
for i in range(0, total_upserted, batch_size):
|
|
chunk = classes_data[i:i + batch_size]
|
|
cursor.executemany(query, chunk)
|
|
conn.commit()
|
|
print(f"Upserted {min(i + batch_size, total_upserted)} / {total_upserted} item classes...")
|
|
|
|
cursor.close()
|
|
return total_upserted
|
|
|
|
def clear_uniques_table(conn):
|
|
"""Truncates the item_uniques table to empty all existing uniques."""
|
|
cursor = conn.cursor()
|
|
cursor.execute("TRUNCATE TABLE item_uniques")
|
|
conn.commit()
|
|
cursor.close()
|
|
print("Table 'item_uniques' cleared successfully.")
|
|
|
|
def bulk_upsert_uniques(conn, uniques_data):
|
|
"""Upserts list of unique items into database using batch processing."""
|
|
if not uniques_data:
|
|
return 0
|
|
|
|
query = """
|
|
INSERT INTO item_uniques (
|
|
id, name, unique_id, item_class,
|
|
inventory_width, inventory_height, is_alternate_art,
|
|
visual_identity_id, visual_identity_dds,
|
|
renamed_version, base_version
|
|
) VALUES (
|
|
%s, %s, %s, %s,
|
|
%s, %s, %s,
|
|
%s, %s,
|
|
%s, %s
|
|
) ON DUPLICATE KEY UPDATE
|
|
name = VALUES(name),
|
|
unique_id = VALUES(unique_id),
|
|
item_class = VALUES(item_class),
|
|
inventory_width = VALUES(inventory_width),
|
|
inventory_height = VALUES(inventory_height),
|
|
is_alternate_art = VALUES(is_alternate_art),
|
|
visual_identity_id = VALUES(visual_identity_id),
|
|
visual_identity_dds = VALUES(visual_identity_dds),
|
|
renamed_version = VALUES(renamed_version),
|
|
base_version = VALUES(base_version)
|
|
"""
|
|
|
|
cursor = conn.cursor()
|
|
batch_size = 500
|
|
total_upserted = len(uniques_data)
|
|
|
|
print(f"Upserting {total_upserted} uniques in batches of {batch_size}...")
|
|
for i in range(0, total_upserted, batch_size):
|
|
chunk = uniques_data[i:i + batch_size]
|
|
cursor.executemany(query, chunk)
|
|
conn.commit()
|
|
print(f"Upserted {min(i + batch_size, total_upserted)} / {total_upserted} uniques...")
|
|
|
|
cursor.close()
|
|
return total_upserted
|
|
|
|
|
|
|