Also ingest resources/text_en into .jsons

.gitignore

@@ -0,0 +1 @@
+vanilla_json_mirror/

ingest_database.py

@@ -3,8 +3,14 @@ import json
 import re
 
 # Paths configuration
-VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database\records"
+VANILLA_DB_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\database"
+VANILLA_TEXT_DIR = r"D:\SteamLibrary\steamapps\common\Grim Dawn\resources\text_en"
+
+# Target scan directory for Phase 1 ingestion
+VANILLA_RECORDS_DIR = os.path.join(VANILLA_DB_DIR, "records")
+
 JSON_OUTPUT_DIR = r".\vanilla_json_mirror"
+TEXT_OUTPUT_DIR = os.path.join(JSON_OUTPUT_DIR, "resources", "text_en")
 
 def clean_dbr_value(val):
     """
@@ -19,6 +25,63 @@ def clean_dbr_value(val):
         return None
     return val
 
+def parse_text_file(file_path):
+    """Parses a single .txt file with key=value format into a dictionary."""
+    file_data = {}
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            line = line.strip()
+            # Skip empty lines and comments (lines that don't contain =)
+            if not line or "=" not in line:
+                continue
+            
+            # Split at the first = to isolate the key and value
+            key, value = line.split("=", 1)
+            key = key.strip()
+            value = value.strip()
+            
+            # Only store non-empty values
+            if key and value:
+                file_data[key] = value
+                
+    return file_data
+
+def ingest_text_files():
+    """Ingests all .txt files from Grim Dawn resources/text_en and converts to JSON."""
+    if not os.path.exists(VANILLA_TEXT_DIR):
+        print(f"Warning: Text resources directory not found at: {VANILLA_TEXT_DIR}")
+        return 0
+    
+    print(f"\nStarting text file ingestion from: {VANILLA_TEXT_DIR}")
+    
+    # Create output directory if it doesn't exist
+    os.makedirs(TEXT_OUTPUT_DIR, exist_ok=True)
+    
+    processed_files = 0
+    
+    # Get all .txt files in the text_en directory
+    for filename in os.listdir(VANILLA_TEXT_DIR):
+        if not filename.lower().endswith('.txt'):
+            continue
+        
+        file_path = os.path.join(VANILLA_TEXT_DIR, filename)
+        
+        # Parse the text file
+        text_data = parse_text_file(file_path)
+        
+        # Create output JSON filename (replace .txt with .json)
+        json_filename = os.path.splitext(filename)[0] + ".json"
+        json_output_path = os.path.join(TEXT_OUTPUT_DIR, json_filename)
+        
+        # Write to JSON
+        with open(json_output_path, "w", encoding="utf-8") as json_file:
+            json.dump(text_data, json_file, indent=2)
+        
+        processed_files += 1
+        print(f"  Converted: {filename} -> {json_filename}")
+    
+    return processed_files
+
 def parse_dbr_file(file_path):
     """Parses a single .dbr file into a clean, sparse key-value dictionary."""
     file_data = {}
@@ -39,18 +102,18 @@ def parse_dbr_file(file_path):
     return file_data
 
 def main():
-    if not os.path.exists(VANILLA_DB_DIR):
-        print(f"Error: Vanilla database directory not found at: {VANILLA_DB_DIR}")
+    if not os.path.exists(VANILLA_RECORDS_DIR):
+        print(f"Error: Vanilla records directory not found at: {VANILLA_RECORDS_DIR}")
         return
 
-    print(f"Starting database ingestion from: {VANILLA_DB_DIR}")
+    print(f"Starting database ingestion from: {VANILLA_RECORDS_DIR}")
     print("Grouping files by directory level...")
     
     processed_directories = 0
     total_files_mapped = 0
 
     # Walk the directory tree
-    for root, dirs, files in os.walk(VANILLA_DB_DIR):
+    for root, dirs, files in os.walk(VANILLA_RECORDS_DIR):
         # Filter for only .dbr files in the current folder
         dbr_files = [f for f in files if f.lower().endswith('.dbr')]
         
@@ -66,7 +129,17 @@ def main():
             
             # Keep track of the file even if it's completely empty after cleaning,
             # so the compiler knows the file exists in vanilla.
-            directory_map[filename] = sparse_content
+
+            # Determine the relative path back to the base database directory
+            rel_path = os.path.relpath(root, VANILLA_DB_DIR)
+
+            # Reconstruct the standard internal game engine path format
+            # e.g., "records/skills/playerclass01/willtolive1.dbr"
+            normalized_game_key = os.path.join(rel_path, filename).replace(os.sep, "/")
+
+            # Save using the full normalized path as the dictionary key
+            directory_map[normalized_game_key] = sparse_content
+
             total_files_mapped += 1
 
         # Determine the relative path to recreate the mirror structure
@@ -79,7 +152,7 @@ def main():
         # Use the name of the parent folder as the JSON filename
         # e.g., .../items/gearfeet/ becomes .../items/gearfeet/gearfeet.json
         folder_name = os.path.basename(root) if rel_path != "." else "root"
-        json_output_path = os.path.join(target_output_dir, f"{folder_name}.json")
+        json_output_path = os.path.join(target_output_dir, f"records.json")
         
         # Save out the consolidated directory map
         with open(json_output_path, "w", encoding="utf-8") as json_file:
@@ -87,9 +160,16 @@ def main():
             
         processed_directories += 1
 
-    print(f"\nSuccess! Phase 1 Ingestion Complete.")
+    print(f"\nPhase 1 Complete (Records).")
     print(f"Processed {processed_directories} directories.")
     print(f"Mapped a total of {total_files_mapped} files into sparse JSON endpoints.")
+    
+    # Phase 2: Ingest text files
+    text_files_processed = ingest_text_files()
+    print(f"\nPhase 2 Complete (Text Resources).")
+    print(f"Converted {text_files_processed} text files to JSON.")
+    
+    print(f"\nSuccess! Full Ingestion Complete.")
 
 if __name__ == "__main__":
     main()