From b964e4d5785e692f3eefc79830628958a721cf68 Mon Sep 17 00:00:00 2001 From: deceivedhornet Date: Thu, 11 Jun 2026 01:45:22 -0400 Subject: [PATCH 01/10] Create ingest_database.py --- ingest_database.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 ingest_database.py diff --git a/ingest_database.py b/ingest_database.py new file mode 100644 index 0000000..715336a --- /dev/null +++ b/ingest_database.py @@ -0,0 +1,95 @@ +import os +import json +import re + +# Paths configuration +VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database\records" +JSON_OUTPUT_DIR = r".\vanilla_json_mirror" + +def clean_dbr_value(val): + """ + Cleans trailing commas and whitespace. + Returns None if the value is a standard engine zero-default or empty field, + otherwise returns the cleaned string value. + """ + val = val.rstrip(",").strip() + + # Filter out empty fields or zero defaults (0, 0.0, 0.000000) + if val == "" or val == "0" or re.match(r"^0\.0+$", val): + return None + return val + +def parse_dbr_file(file_path): + """Parses a single .dbr file into a clean, sparse key-value dictionary.""" + file_data = {} + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + if not line or "," not in line: + continue + + # Split exactly at the first comma to isolate the key + key, rest = line.split(",", 1) + key = key.strip() + + cleaned_val = clean_dbr_value(rest) + if cleaned_val is not None: + file_data[key] = cleaned_val + + return file_data + +def main(): + if not os.path.exists(VANILLA_DB_DIR): + print(f"Error: Vanilla database directory not found at: {VANILLA_DB_DIR}") + return + + print(f"Starting database ingestion from: {VANILLA_DB_DIR}") + print("Grouping files by directory level...") + + processed_directories = 0 + total_files_mapped = 0 + + # Walk the directory tree + for root, dirs, files in os.walk(VANILLA_DB_DIR): + # Filter for only .dbr files in the current folder + dbr_files = [f for f in files if f.lower().endswith('.dbr')] + + if not dbr_files: + continue + + # This will hold the map of filename -> sparse content dictionary for this specific folder + directory_map = {} + + for filename in dbr_files: + full_path = os.path.join(root, filename) + sparse_content = parse_dbr_file(full_path) + + # Keep track of the file even if it's completely empty after cleaning, + # so the compiler knows the file exists in vanilla. + directory_map[filename] = sparse_content + total_files_mapped += 1 + + # Determine the relative path to recreate the mirror structure + rel_path = os.path.relpath(root, VANILLA_DB_DIR) + + # Define our output directory mirror path + target_output_dir = os.path.join(JSON_OUTPUT_DIR, rel_path) + os.makedirs(target_output_dir, exist_ok=True) + + # Use the name of the parent folder as the JSON filename + # e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json + folder_name = os.path.basename(root) if rel_path != "." else "root" + json_output_path = os.path.join(target_output_dir, f"{folder_name}.json") + + # Save out the consolidated directory map + with open(json_output_path, "w", encoding="utf-8") as json_file: + json.dump(directory_map, json_file, indent=2) + + processed_directories += 1 + + print(f"\nSuccess! Phase 1 Ingestion Complete.") + print(f"Processed {processed_directories} directories.") + print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.") + +if __name__ == "__main__": + main() From 25dca33e84c3cdcb796a8b520df7b6c26ad1c96b Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 01:46:19 -0400 Subject: [PATCH 02/10] ignore generate dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..935c5ba --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vanilla_json_mirror/ From fb54809a870f1bb68f15bb9cb0ee99f381165fce Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 01:52:47 -0400 Subject: [PATCH 03/10] Call all jsons "records,json" --- ingest_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest_database.py b/ingest_database.py index 715336a..e4e88da 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -79,7 +79,7 @@ def main(): # Use the name of the parent folder as the JSON filename # e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json folder_name = os.path.basename(root) if rel_path != "." else "root" - json_output_path = os.path.join(target_output_dir, f"{folder_name}.json") + json_output_path = os.path.join(target_output_dir, f"records.json") # Save out the consolidated directory map with open(json_output_path, "w", encoding="utf-8") as json_file: From 804e1b990c272deac3461b15bd483be567ac4896 Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 02:06:56 -0400 Subject: [PATCH 04/10] Include whole file path for searchability of references to .mbr --- ingest_database.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ingest_database.py b/ingest_database.py index e4e88da..ee31a26 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -3,7 +3,11 @@ import json import re # Paths configuration -VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database\records" +VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database" + +# Target scan directory for Phase 1 ingestion +VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records") + JSON_OUTPUT_DIR = r".\vanilla_json_mirror" def clean_dbr_value(val): @@ -39,18 +43,18 @@ def parse_dbr_file(file_path): return file_data def main(): - if not os.path.exists(VANILLA_DB_DIR): - print(f"Error: Vanilla database directory not found at: {VANILLA_DB_DIR}") + if not os.path.exists(VANILLA_RECORDS_DIR): + print(f"Error: Vanilla records directory not found at: {VANILLA_RECORDS_DIR}") return - print(f"Starting database ingestion from: {VANILLA_DB_DIR}") + print(f"Starting database ingestion from: {VANILLA_RECORDS_DIR}") print("Grouping files by directory level...") processed_directories = 0 total_files_mapped = 0 # Walk the directory tree - for root, dirs, files in os.walk(VANILLA_DB_DIR): + for root, dirs, files in os.walk(VANILLA_RECORDS_DIR): # Filter for only .dbr files in the current folder dbr_files = [f for f in files if f.lower().endswith('.dbr')] @@ -66,7 +70,17 @@ def main(): # Keep track of the file even if it's completely empty after cleaning, # so the compiler knows the file exists in vanilla. - directory_map[filename] = sparse_content + + # Determine the relative path back to the base database directory + rel_path = os.path.relpath(root, VANILLA_DB_DIR) + + # Reconstruct the standard internal game engine path format + # e.g., "records/skills/playerclass01/willtolive1.dbr" + normalized_game_key = os.path.join(rel_path, filename).replace(os.sep, "/") + + # Save using the full normalized path as the dictionary key + directory_map[normalized_game_key] = sparse_content + total_files_mapped += 1 # Determine the relative path to recreate the mirror structure From ec8c37f00d759686c0f88d97517841868aa494d3 Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 03:05:47 -0400 Subject: [PATCH 05/10] Also ingest resources/text_en into .jsons --- ingest_database.py | 68 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/ingest_database.py b/ingest_database.py index ee31a26..271efd6 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -4,11 +4,13 @@ import re # Paths configuration VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database" +VANILLA_TEXT_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\resources\text_en" # Target scan directory for Phase 1 ingestion VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records") JSON_OUTPUT_DIR = r".\vanilla_json_mirror" +TEXT_OUTPUT_DIR = os.path.join(JSON_OUTPUT_DIR, "resources", "text_en") def clean_dbr_value(val): """ @@ -23,6 +25,63 @@ def clean_dbr_value(val): return None return val +def parse_text_file(file_path): + """Parses a single .txt file with key=value format into a dictionary.""" + file_data = {} + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + # Skip empty lines and comments (lines that don't contain =) + if not line or "=" not in line: + continue + + # Split at the first = to isolate the key and value + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + + # Only store non-empty values + if key and value: + file_data[key] = value + + return file_data + +def ingest_text_files(): + """Ingests all .txt files from Grim Dawn resources/text_en and converts to JSON.""" + if not os.path.exists(VANILLA_TEXT_DIR): + print(f"Warning: Text resources directory not found at: {VANILLA_TEXT_DIR}") + return 0 + + print(f"\nStarting text file ingestion from: {VANILLA_TEXT_DIR}") + + # Create output directory if it doesn't exist + os.makedirs(TEXT_OUTPUT_DIR, exist_ok=True) + + processed_files = 0 + + # Get all .txt files in the text_en directory + for filename in os.listdir(VANILLA_TEXT_DIR): + if not filename.lower().endswith('.txt'): + continue + + file_path = os.path.join(VANILLA_TEXT_DIR, filename) + + # Parse the text file + text_data = parse_text_file(file_path) + + # Create output JSON filename (replace .txt with .json) + json_filename = os.path.splitext(filename)[0] + ".json" + json_output_path = os.path.join(TEXT_OUTPUT_DIR, json_filename) + + # Write to JSON + with open(json_output_path, "w", encoding="utf-8") as json_file: + json.dump(text_data, json_file, indent=2) + + processed_files += 1 + print(f" Converted: {filename} -> {json_filename}") + + return processed_files + def parse_dbr_file(file_path): """Parses a single .dbr file into a clean, sparse key-value dictionary.""" file_data = {} @@ -101,9 +160,16 @@ def main(): processed_directories += 1 - print(f"\nSuccess! Phase 1 Ingestion Complete.") + print(f"\nPhase 1 Complete (Records).") print(f"Processed {processed_directories} directories.") print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.") + + # Phase 2: Ingest text files + text_files_processed = ingest_text_files() + print(f"\nPhase 2 Complete (Text Resources).") + print(f"Converted {text_files_processed} text files to JSON.") + + print(f"\nSuccess! Full Ingestion Complete.") if __name__ == "__main__": main() From 1c8f41f4c99c03ea186d54d064eaa219b1196da0 Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 01:45:22 -0400 Subject: [PATCH 06/10] Create ingest_database.py --- ingest_database.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 ingest_database.py diff --git a/ingest_database.py b/ingest_database.py new file mode 100644 index 0000000..715336a --- /dev/null +++ b/ingest_database.py @@ -0,0 +1,95 @@ +import os +import json +import re + +# Paths configuration +VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database\records" +JSON_OUTPUT_DIR = r".\vanilla_json_mirror" + +def clean_dbr_value(val): + """ + Cleans trailing commas and whitespace. + Returns None if the value is a standard engine zero-default or empty field, + otherwise returns the cleaned string value. + """ + val = val.rstrip(",").strip() + + # Filter out empty fields or zero defaults (0, 0.0, 0.000000) + if val == "" or val == "0" or re.match(r"^0\.0+$", val): + return None + return val + +def parse_dbr_file(file_path): + """Parses a single .dbr file into a clean, sparse key-value dictionary.""" + file_data = {} + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + if not line or "," not in line: + continue + + # Split exactly at the first comma to isolate the key + key, rest = line.split(",", 1) + key = key.strip() + + cleaned_val = clean_dbr_value(rest) + if cleaned_val is not None: + file_data[key] = cleaned_val + + return file_data + +def main(): + if not os.path.exists(VANILLA_DB_DIR): + print(f"Error: Vanilla database directory not found at: {VANILLA_DB_DIR}") + return + + print(f"Starting database ingestion from: {VANILLA_DB_DIR}") + print("Grouping files by directory level...") + + processed_directories = 0 + total_files_mapped = 0 + + # Walk the directory tree + for root, dirs, files in os.walk(VANILLA_DB_DIR): + # Filter for only .dbr files in the current folder + dbr_files = [f for f in files if f.lower().endswith('.dbr')] + + if not dbr_files: + continue + + # This will hold the map of filename -> sparse content dictionary for this specific folder + directory_map = {} + + for filename in dbr_files: + full_path = os.path.join(root, filename) + sparse_content = parse_dbr_file(full_path) + + # Keep track of the file even if it's completely empty after cleaning, + # so the compiler knows the file exists in vanilla. + directory_map[filename] = sparse_content + total_files_mapped += 1 + + # Determine the relative path to recreate the mirror structure + rel_path = os.path.relpath(root, VANILLA_DB_DIR) + + # Define our output directory mirror path + target_output_dir = os.path.join(JSON_OUTPUT_DIR, rel_path) + os.makedirs(target_output_dir, exist_ok=True) + + # Use the name of the parent folder as the JSON filename + # e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json + folder_name = os.path.basename(root) if rel_path != "." else "root" + json_output_path = os.path.join(target_output_dir, f"{folder_name}.json") + + # Save out the consolidated directory map + with open(json_output_path, "w", encoding="utf-8") as json_file: + json.dump(directory_map, json_file, indent=2) + + processed_directories += 1 + + print(f"\nSuccess! Phase 1 Ingestion Complete.") + print(f"Processed {processed_directories} directories.") + print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.") + +if __name__ == "__main__": + main() From a8d882f30f8de98cb20fdb3acb17f8b532088e07 Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 01:46:19 -0400 Subject: [PATCH 07/10] ignore generate dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..935c5ba --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vanilla_json_mirror/ From 248998dbfe9e22ba68ef44ad7529a1c3a7b3eb4e Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 01:52:47 -0400 Subject: [PATCH 08/10] Call all jsons "records,json" --- ingest_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest_database.py b/ingest_database.py index 715336a..e4e88da 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -79,7 +79,7 @@ def main(): # Use the name of the parent folder as the JSON filename # e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json folder_name = os.path.basename(root) if rel_path != "." else "root" - json_output_path = os.path.join(target_output_dir, f"{folder_name}.json") + json_output_path = os.path.join(target_output_dir, f"records.json") # Save out the consolidated directory map with open(json_output_path, "w", encoding="utf-8") as json_file: From eeeb56f1e5f1f152b5a4b813eb2dc3752de9b3ed Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 02:06:56 -0400 Subject: [PATCH 09/10] Include whole file path for searchability of references to .mbr --- ingest_database.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ingest_database.py b/ingest_database.py index e4e88da..ee31a26 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -3,7 +3,11 @@ import json import re # Paths configuration -VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database\records" +VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database" + +# Target scan directory for Phase 1 ingestion +VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records") + JSON_OUTPUT_DIR = r".\vanilla_json_mirror" def clean_dbr_value(val): @@ -39,18 +43,18 @@ def parse_dbr_file(file_path): return file_data def main(): - if not os.path.exists(VANILLA_DB_DIR): - print(f"Error: Vanilla database directory not found at: {VANILLA_DB_DIR}") + if not os.path.exists(VANILLA_RECORDS_DIR): + print(f"Error: Vanilla records directory not found at: {VANILLA_RECORDS_DIR}") return - print(f"Starting database ingestion from: {VANILLA_DB_DIR}") + print(f"Starting database ingestion from: {VANILLA_RECORDS_DIR}") print("Grouping files by directory level...") processed_directories = 0 total_files_mapped = 0 # Walk the directory tree - for root, dirs, files in os.walk(VANILLA_DB_DIR): + for root, dirs, files in os.walk(VANILLA_RECORDS_DIR): # Filter for only .dbr files in the current folder dbr_files = [f for f in files if f.lower().endswith('.dbr')] @@ -66,7 +70,17 @@ def main(): # Keep track of the file even if it's completely empty after cleaning, # so the compiler knows the file exists in vanilla. - directory_map[filename] = sparse_content + + # Determine the relative path back to the base database directory + rel_path = os.path.relpath(root, VANILLA_DB_DIR) + + # Reconstruct the standard internal game engine path format + # e.g., "records/skills/playerclass01/willtolive1.dbr" + normalized_game_key = os.path.join(rel_path, filename).replace(os.sep, "/") + + # Save using the full normalized path as the dictionary key + directory_map[normalized_game_key] = sparse_content + total_files_mapped += 1 # Determine the relative path to recreate the mirror structure From bf23ed23bde490eadfd1cc63eaeb321d9b308b2e Mon Sep 17 00:00:00 2001 From: Owl Flora Date: Thu, 11 Jun 2026 03:05:47 -0400 Subject: [PATCH 10/10] Also ingest resources/text_en into .jsons --- ingest_database.py | 68 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/ingest_database.py b/ingest_database.py index ee31a26..271efd6 100644 --- a/ingest_database.py +++ b/ingest_database.py @@ -4,11 +4,13 @@ import re # Paths configuration VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database" +VANILLA_TEXT_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\resources\text_en" # Target scan directory for Phase 1 ingestion VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records") JSON_OUTPUT_DIR = r".\vanilla_json_mirror" +TEXT_OUTPUT_DIR = os.path.join(JSON_OUTPUT_DIR, "resources", "text_en") def clean_dbr_value(val): """ @@ -23,6 +25,63 @@ def clean_dbr_value(val): return None return val +def parse_text_file(file_path): + """Parses a single .txt file with key=value format into a dictionary.""" + file_data = {} + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + # Skip empty lines and comments (lines that don't contain =) + if not line or "=" not in line: + continue + + # Split at the first = to isolate the key and value + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + + # Only store non-empty values + if key and value: + file_data[key] = value + + return file_data + +def ingest_text_files(): + """Ingests all .txt files from Grim Dawn resources/text_en and converts to JSON.""" + if not os.path.exists(VANILLA_TEXT_DIR): + print(f"Warning: Text resources directory not found at: {VANILLA_TEXT_DIR}") + return 0 + + print(f"\nStarting text file ingestion from: {VANILLA_TEXT_DIR}") + + # Create output directory if it doesn't exist + os.makedirs(TEXT_OUTPUT_DIR, exist_ok=True) + + processed_files = 0 + + # Get all .txt files in the text_en directory + for filename in os.listdir(VANILLA_TEXT_DIR): + if not filename.lower().endswith('.txt'): + continue + + file_path = os.path.join(VANILLA_TEXT_DIR, filename) + + # Parse the text file + text_data = parse_text_file(file_path) + + # Create output JSON filename (replace .txt with .json) + json_filename = os.path.splitext(filename)[0] + ".json" + json_output_path = os.path.join(TEXT_OUTPUT_DIR, json_filename) + + # Write to JSON + with open(json_output_path, "w", encoding="utf-8") as json_file: + json.dump(text_data, json_file, indent=2) + + processed_files += 1 + print(f" Converted: {filename} -> {json_filename}") + + return processed_files + def parse_dbr_file(file_path): """Parses a single .dbr file into a clean, sparse key-value dictionary.""" file_data = {} @@ -101,9 +160,16 @@ def main(): processed_directories += 1 - print(f"\nSuccess! Phase 1 Ingestion Complete.") + print(f"\nPhase 1 Complete (Records).") print(f"Processed {processed_directories} directories.") print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.") + + # Phase 2: Ingest text files + text_files_processed = ingest_text_files() + print(f"\nPhase 2 Complete (Text Resources).") + print(f"Converted {text_files_processed} text files to JSON.") + + print(f"\nSuccess! Full Ingestion Complete.") if __name__ == "__main__": main()