175 lines
6.4 KiB
Python
175 lines
6.4 KiB
Python
import os
|
|
import json
|
|
import re
|
|
|
|
# Paths configuration
|
|
VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database"
|
|
VANILLA_TEXT_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\resources\text_en"
|
|
|
|
# Target scan directory for Phase 1 ingestion
|
|
VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records")
|
|
|
|
JSON_OUTPUT_DIR = r".\vanilla_json_mirror"
|
|
TEXT_OUTPUT_DIR = os.path.join(JSON_OUTPUT_DIR, "resources", "text_en")
|
|
|
|
def clean_dbr_value(val):
|
|
"""
|
|
Cleans trailing commas and whitespace.
|
|
Returns None if the value is a standard engine zero-default or empty field,
|
|
otherwise returns the cleaned string value.
|
|
"""
|
|
val = val.rstrip(",").strip()
|
|
|
|
# Filter out empty fields or zero defaults (0, 0.0, 0.000000)
|
|
if val == "" or val == "0" or re.match(r"^0\.0+$", val):
|
|
return None
|
|
return val
|
|
|
|
def parse_text_file(file_path):
|
|
"""Parses a single .txt file with key=value format into a dictionary."""
|
|
file_data = {}
|
|
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
# Skip empty lines and comments (lines that don't contain =)
|
|
if not line or "=" not in line:
|
|
continue
|
|
|
|
# Split at the first = to isolate the key and value
|
|
key, value = line.split("=", 1)
|
|
key = key.strip()
|
|
value = value.strip()
|
|
|
|
# Only store non-empty values
|
|
if key and value:
|
|
file_data[key] = value
|
|
|
|
return file_data
|
|
|
|
def ingest_text_files():
|
|
"""Ingests all .txt files from Grim Dawn resources/text_en and converts to JSON."""
|
|
if not os.path.exists(VANILLA_TEXT_DIR):
|
|
print(f"Warning: Text resources directory not found at: {VANILLA_TEXT_DIR}")
|
|
return 0
|
|
|
|
print(f"\nStarting text file ingestion from: {VANILLA_TEXT_DIR}")
|
|
|
|
# Create output directory if it doesn't exist
|
|
os.makedirs(TEXT_OUTPUT_DIR, exist_ok=True)
|
|
|
|
processed_files = 0
|
|
|
|
# Get all .txt files in the text_en directory
|
|
for filename in os.listdir(VANILLA_TEXT_DIR):
|
|
if not filename.lower().endswith('.txt'):
|
|
continue
|
|
|
|
file_path = os.path.join(VANILLA_TEXT_DIR, filename)
|
|
|
|
# Parse the text file
|
|
text_data = parse_text_file(file_path)
|
|
|
|
# Create output JSON filename (replace .txt with .json)
|
|
json_filename = os.path.splitext(filename)[0] + ".json"
|
|
json_output_path = os.path.join(TEXT_OUTPUT_DIR, json_filename)
|
|
|
|
# Write to JSON
|
|
with open(json_output_path, "w", encoding="utf-8") as json_file:
|
|
json.dump(text_data, json_file, indent=2)
|
|
|
|
processed_files += 1
|
|
print(f" Converted: {filename} -> {json_filename}")
|
|
|
|
return processed_files
|
|
|
|
def parse_dbr_file(file_path):
|
|
"""Parses a single .dbr file into a clean, sparse key-value dictionary."""
|
|
file_data = {}
|
|
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or "," not in line:
|
|
continue
|
|
|
|
# Split exactly at the first comma to isolate the key
|
|
key, rest = line.split(",", 1)
|
|
key = key.strip()
|
|
|
|
cleaned_val = clean_dbr_value(rest)
|
|
if cleaned_val is not None:
|
|
file_data[key] = cleaned_val
|
|
|
|
return file_data
|
|
|
|
def main():
|
|
if not os.path.exists(VANILLA_RECORDS_DIR):
|
|
print(f"Error: Vanilla records directory not found at: {VANILLA_RECORDS_DIR}")
|
|
return
|
|
|
|
print(f"Starting database ingestion from: {VANILLA_RECORDS_DIR}")
|
|
print("Grouping files by directory level...")
|
|
|
|
processed_directories = 0
|
|
total_files_mapped = 0
|
|
|
|
# Walk the directory tree
|
|
for root, dirs, files in os.walk(VANILLA_RECORDS_DIR):
|
|
# Filter for only .dbr files in the current folder
|
|
dbr_files = [f for f in files if f.lower().endswith('.dbr')]
|
|
|
|
if not dbr_files:
|
|
continue
|
|
|
|
# This will hold the map of filename -> sparse content dictionary for this specific folder
|
|
directory_map = {}
|
|
|
|
for filename in dbr_files:
|
|
full_path = os.path.join(root, filename)
|
|
sparse_content = parse_dbr_file(full_path)
|
|
|
|
# Keep track of the file even if it's completely empty after cleaning,
|
|
# so the compiler knows the file exists in vanilla.
|
|
|
|
# Determine the relative path back to the base database directory
|
|
rel_path = os.path.relpath(root, VANILLA_DB_DIR)
|
|
|
|
# Reconstruct the standard internal game engine path format
|
|
# e.g., "records/skills/playerclass01/willtolive1.dbr"
|
|
normalized_game_key = os.path.join(rel_path, filename).replace(os.sep, "/")
|
|
|
|
# Save using the full normalized path as the dictionary key
|
|
directory_map[normalized_game_key] = sparse_content
|
|
|
|
total_files_mapped += 1
|
|
|
|
# Determine the relative path to recreate the mirror structure
|
|
rel_path = os.path.relpath(root, VANILLA_DB_DIR)
|
|
|
|
# Define our output directory mirror path
|
|
target_output_dir = os.path.join(JSON_OUTPUT_DIR, rel_path)
|
|
os.makedirs(target_output_dir, exist_ok=True)
|
|
|
|
# Use the name of the parent folder as the JSON filename
|
|
# e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json
|
|
folder_name = os.path.basename(root) if rel_path != "." else "root"
|
|
json_output_path = os.path.join(target_output_dir, f"records.json")
|
|
|
|
# Save out the consolidated directory map
|
|
with open(json_output_path, "w", encoding="utf-8") as json_file:
|
|
json.dump(directory_map, json_file, indent=2)
|
|
|
|
processed_directories += 1
|
|
|
|
print(f"\nPhase 1 Complete (Records).")
|
|
print(f"Processed {processed_directories} directories.")
|
|
print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.")
|
|
|
|
# Phase 2: Ingest text files
|
|
text_files_processed = ingest_text_files()
|
|
print(f"\nPhase 2 Complete (Text Resources).")
|
|
print(f"Converted {text_files_processed} text files to JSON.")
|
|
|
|
print(f"\nSuccess! Full Ingestion Complete.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|