diff --git a/simulation/regional/RegionalChunkedFileStore.gd b/simulation/regional/RegionalChunkedFileStore.gd new file mode 100644 index 0000000..75a6412 --- /dev/null +++ b/simulation/regional/RegionalChunkedFileStore.gd @@ -0,0 +1,954 @@ +class_name RegionalChunkedFileStore +extends RefCounted + +const SCHEMA_VERSION := 1 +const FORMAT_ID := "regional_chunked_file_store" +const GENERATIONS_DIRECTORY := "generations" +const MANIFEST_FILE_NAME := "manifest.bin" +const CURRENT_POINTER_FILE_NAME := "current.bin" +const PREVIOUS_POINTER_FILE_NAME := "current.previous.bin" +const TEMP_POINTER_FILE_NAME := "current.next.bin" +const GENERATION_PREFIX := "generation_" +const GENERATION_DIGITS := 20 +const MAX_BASE_PATH_LENGTH := 2048 +const MAX_CHUNK_ID_LENGTH := 256 +const MAX_CHUNKS := 100_000 +const MAX_GENERATIONS := 4096 +const MAX_MANIFEST_BYTES := 8 * 1024 * 1024 +const MAX_CHUNK_BYTES := 32 * 1024 * 1024 +const MAX_TOTAL_GENERATION_BYTES := 64 * 1024 * 1024 +const MAX_POINTER_BYTES := 4096 + +var _base_directory: String +var _absolute_base_directory := "" +var _last_error := "" +var _last_telemetry: Dictionary = {} + + +func _init(base_directory: String) -> void: + _base_directory = base_directory + _absolute_base_directory = ProjectSettings.globalize_path(base_directory).simplify_path() + _begin_telemetry("idle") + + +func save(archive: RegionalChunkedPersistence) -> bool: + _last_error = "" + _begin_telemetry("save") + if not _base_directory_is_safe(): + return _fail("The caller-supplied regional save directory is unsafe") + if archive == null: + return _fail("Regional chunk archive is missing") + var validation_started := Time.get_ticks_usec() + var archive_bundle := archive.to_dictionary() + var revalidated := RegionalChunkedPersistence.from_dictionary(archive_bundle) + _add_validation_time(validation_started) + if revalidated == null or revalidated.checksum() != archive.checksum(): + return _fail("Regional chunk archive failed pre-write validation") + var chunk_ids := archive.get_chunk_ids() + if chunk_ids.is_empty() or chunk_ids.size() > MAX_CHUNKS: + return _fail("Regional chunk count exceeds the filesystem contract") + if not _ensure_directories(): + return false + var generation_ids := _list_generation_ids() + if generation_ids.size() >= MAX_GENERATIONS: + return _fail("Regional generation limit reached; explicit maintenance is required") + var generation_id := _next_generation_id(generation_ids) + if generation_id.is_empty(): + return _fail("Could not allocate a bounded regional generation ID") + var final_generation_path := _generation_path(generation_id) + var temporary_generation_path := _allocate_temporary_generation_path(generation_id) + if temporary_generation_path.is_empty(): + return _fail("Could not allocate a temporary regional generation directory") + var directory_error := DirAccess.make_dir_recursive_absolute(temporary_generation_path) + if directory_error != OK and directory_error != ERR_ALREADY_EXISTS: + return _fail("Could not create the temporary regional generation directory") + + var serialized_chunks: Array[PackedByteArray] = [] + var chunk_files: Array[Dictionary] = [] + var generation_bytes := 0 + var archive_manifest := archive.get_manifest() + var archive_descriptors: Array = archive_manifest["chunks"] + for index in range(chunk_ids.size()): + var chunk_id := chunk_ids[index] + if chunk_id.length() > MAX_CHUNK_ID_LENGTH: + return _fail("Regional chunk ID exceeds the bounded file-store contract") + var chunk := archive.get_chunk(StringName(chunk_id)) + if chunk.is_empty() or not _is_primitive_tree(chunk): + return _fail("Regional chunk payload is not primitive-only") + var serialized := var_to_bytes(chunk) + if serialized.is_empty() or serialized.size() > MAX_CHUNK_BYTES: + return _fail("Regional chunk exceeds the per-file size limit") + generation_bytes += serialized.size() + if generation_bytes > MAX_TOTAL_GENERATION_BYTES: + return _fail("Regional generation exceeds the total size limit") + serialized_chunks.append(serialized) + var archive_descriptor: Dictionary = archive_descriptors[index] + ( + chunk_files + . append( + { + "chunk_id": chunk_id, + "file_name": _chunk_file_name(index), + "serialized_bytes": serialized.size(), + "file_checksum": _bytes_checksum(serialized), + "canonical_checksum": String(archive_descriptor["checksum"]), + } + ) + ) + _last_telemetry["serialized_bytes"] += serialized.size() + + var generation_manifest := { + "schema_version": SCHEMA_VERSION, + "format_id": FORMAT_ID, + "generation_id": generation_id, + "archive_checksum": archive.checksum(), + "envelope_checksum": String(archive_manifest["envelope_checksum"]), + "archive_manifest": archive_manifest, + "chunk_files": chunk_files, + } + if not _is_primitive_tree(generation_manifest): + return _fail("Regional generation manifest is not primitive-only") + var manifest_bytes := var_to_bytes(generation_manifest) + if manifest_bytes.is_empty() or manifest_bytes.size() > MAX_MANIFEST_BYTES: + return _fail("Regional generation manifest exceeds the size limit") + generation_bytes += manifest_bytes.size() + if generation_bytes > MAX_TOTAL_GENERATION_BYTES: + return _fail("Regional generation exceeds the total size limit") + _last_telemetry["serialized_bytes"] += manifest_bytes.size() + + for index in range(serialized_chunks.size()): + var chunk_path := temporary_generation_path.path_join(_chunk_file_name(index)) + if not _write_bytes(chunk_path, serialized_chunks[index]): + return _fail("Could not write regional chunk file %d" % index) + var manifest_path := temporary_generation_path.path_join(MANIFEST_FILE_NAME) + if not _write_bytes(manifest_path, manifest_bytes): + return _fail("Could not write the regional generation manifest") + var readback := _read_generation_at_path(temporary_generation_path, generation_id, "") + if readback == null or readback.checksum() != archive.checksum(): + return _fail("Temporary regional generation failed exact readback validation") + if DirAccess.rename_absolute(temporary_generation_path, final_generation_path) != OK: + return _fail("Could not atomically install the regional generation") + var pointer := { + "schema_version": SCHEMA_VERSION, + "format_id": FORMAT_ID, + "generation_id": generation_id, + "manifest_checksum": _bytes_checksum(manifest_bytes), + "archive_checksum": archive.checksum(), + } + if not _install_current_pointer(pointer): + return false + _last_telemetry["generation_id"] = generation_id + _last_telemetry["succeeded"] = true + return true + + +func load_archive() -> RegionalChunkedPersistence: + _last_error = "" + _begin_telemetry("load_archive") + if not _base_directory_is_safe(): + _fail("The caller-supplied regional save directory is unsafe") + return null + var pointer := _read_current_pointer() + var pointer_generation_id := String(pointer.get("generation_id", "")) + var pointer_manifest_checksum := String(pointer.get("manifest_checksum", "")) + var pointer_archive_checksum := String(pointer.get("archive_checksum", "")) + var generation_ids := _list_generation_ids() + generation_ids.reverse() + for generation_id: String in generation_ids: + var expected_manifest_checksum := ( + pointer_manifest_checksum if generation_id == pointer_generation_id else "" + ) + var archive := _read_generation(generation_id, expected_manifest_checksum) + if archive == null: + continue + _last_telemetry["generation_id"] = generation_id + _last_telemetry["recovered_generation"] = ( + generation_id != pointer_generation_id + or pointer_archive_checksum != archive.checksum() + or _last_telemetry["pointer_mismatch"] + ) + _last_telemetry["succeeded"] = true + return archive + _fail("No valid regional generation could be recovered") + return null + + +func load_active_location_metadata(location_id: StringName) -> Dictionary: + _last_error = "" + _begin_telemetry("load_active_location_metadata") + _last_telemetry["partial_load"] = true + if not _base_directory_is_safe() or location_id.is_empty(): + _fail("Regional active-location request is invalid") + return {} + var pointer := _read_current_pointer() + var pointer_generation_id := String(pointer.get("generation_id", "")) + var pointer_manifest_checksum := String(pointer.get("manifest_checksum", "")) + var pointer_archive_checksum := String(pointer.get("archive_checksum", "")) + var generation_ids := _list_generation_ids() + generation_ids.reverse() + for generation_id: String in generation_ids: + var expected_manifest_checksum := ( + pointer_manifest_checksum if generation_id == pointer_generation_id else "" + ) + var partial := _read_active_location_at_generation( + generation_id, location_id, expected_manifest_checksum + ) + if bool(partial.get("location_absent", false)): + _fail("The latest recoverable regional generation does not contain the location") + return {} + if partial.is_empty(): + continue + _last_telemetry["generation_id"] = generation_id + _last_telemetry["recovered_generation"] = ( + generation_id != pointer_generation_id + or pointer_archive_checksum != String(partial["archive_checksum"]) + or _last_telemetry["pointer_mismatch"] + ) + _last_telemetry["succeeded"] = true + return partial + _fail("No regional generation contains valid metadata for the active location") + return {} + + +func prune_old_generations(keep_valid_generations: int = 2) -> int: + _last_error = "" + _begin_telemetry("prune_old_generations") + if not _base_directory_is_safe(): + _fail("The caller-supplied regional save directory is unsafe") + return 0 + if keep_valid_generations < 2: + _fail("Conservative pruning must keep at least two valid generations") + return 0 + var pointer := _read_current_pointer() + var pointer_generation_id := String(pointer.get("generation_id", "")) + var valid_generations: Array[Dictionary] = [] + var generation_ids := _list_generation_ids() + generation_ids.reverse() + for generation_id: String in generation_ids: + var archive := _read_generation(generation_id, "") + if archive != null: + ( + valid_generations + . append( + { + "generation_id": generation_id, + "archive": archive, + } + ) + ) + var protected_ids: Dictionary = {} + for index in range(mini(keep_valid_generations, valid_generations.size())): + protected_ids[String(valid_generations[index]["generation_id"])] = true + if not pointer_generation_id.is_empty(): + protected_ids[pointer_generation_id] = true + var pruned := 0 + for candidate: Dictionary in valid_generations: + var generation_id := String(candidate["generation_id"]) + if protected_ids.has(generation_id): + continue + var archive: RegionalChunkedPersistence = candidate["archive"] + if not _remove_valid_generation(generation_id, archive): + _fail("Conservative regional generation pruning stopped on an unexpected layout") + _last_telemetry["generations_pruned"] = pruned + return pruned + pruned += 1 + _last_telemetry["generations_pruned"] = pruned + _last_telemetry["succeeded"] = true + return pruned + + +func get_base_directory() -> String: + return _base_directory + + +func get_generations_directory() -> String: + return _base_directory.path_join(GENERATIONS_DIRECTORY) + + +func get_current_pointer_path() -> String: + return _base_directory.path_join(CURRENT_POINTER_FILE_NAME) + + +func get_generation_ids() -> Array[String]: + return _list_generation_ids() if _base_directory_is_safe() else [] + + +func get_generation_directory(generation_id: String) -> String: + if not _is_generation_id(generation_id): + return "" + return get_generations_directory().path_join(generation_id) + + +func get_last_error() -> String: + return _last_error + + +func get_last_telemetry() -> Dictionary: + return _last_telemetry.duplicate(true) + + +func _read_generation( + generation_id: String, expected_manifest_checksum: String +) -> RegionalChunkedPersistence: + if not _is_generation_id(generation_id): + return null + return _read_generation_at_path( + _generation_path(generation_id), generation_id, expected_manifest_checksum + ) + + +func _read_generation_at_path( + generation_path: String, generation_id: String, expected_manifest_checksum: String +) -> RegionalChunkedPersistence: + _last_telemetry["candidates_validated"] += 1 + var manifest_bytes := _read_bytes( + generation_path.path_join(MANIFEST_FILE_NAME), MAX_MANIFEST_BYTES + ) + if manifest_bytes.is_empty(): + return null + if ( + not expected_manifest_checksum.is_empty() + and _bytes_checksum(manifest_bytes) != expected_manifest_checksum + ): + _last_telemetry["pointer_mismatch"] = true + var validation_started := Time.get_ticks_usec() + var decoded_manifest: Variant = bytes_to_var(manifest_bytes) + if ( + not decoded_manifest is Dictionary + or not _is_primitive_tree(decoded_manifest) + or not _generation_manifest_is_valid(decoded_manifest, generation_id) + ): + _add_validation_time(validation_started) + return null + var generation_manifest: Dictionary = decoded_manifest + _add_validation_time(validation_started) + var chunks: Array[Dictionary] = [] + var generation_bytes := manifest_bytes.size() + for raw_entry: Variant in generation_manifest["chunk_files"]: + var entry: Dictionary = raw_entry + var chunk := _read_chunk_file(generation_path, entry) + if chunk.is_empty(): + return null + generation_bytes += int(entry["serialized_bytes"]) + if generation_bytes > MAX_TOTAL_GENERATION_BYTES: + return null + chunks.append(chunk) + var archive_manifest: Dictionary = generation_manifest["archive_manifest"] + var bundle := { + "schema_version": RegionalChunkedPersistence.SCHEMA_VERSION, + "manifest": archive_manifest.duplicate(true), + "chunks": chunks, + } + validation_started = Time.get_ticks_usec() + var archive := RegionalChunkedPersistence.from_dictionary(bundle) + _add_validation_time(validation_started) + if ( + archive == null + or archive.checksum() != String(generation_manifest["archive_checksum"]) + or ( + String(archive.get_manifest()["envelope_checksum"]) + != String(generation_manifest["envelope_checksum"]) + ) + ): + return null + return archive + + +func _read_active_location_at_generation( + generation_id: StringName, location_id: StringName, expected_manifest_checksum: String +) -> Dictionary: + if not _is_generation_id(String(generation_id)): + return {} + _last_telemetry["candidates_validated"] += 1 + var generation_path := _generation_path(String(generation_id)) + var manifest_bytes := _read_bytes( + generation_path.path_join(MANIFEST_FILE_NAME), MAX_MANIFEST_BYTES + ) + if manifest_bytes.is_empty(): + return {} + if ( + not expected_manifest_checksum.is_empty() + and _bytes_checksum(manifest_bytes) != expected_manifest_checksum + ): + _last_telemetry["pointer_mismatch"] = true + var validation_started := Time.get_ticks_usec() + var decoded_manifest: Variant = bytes_to_var(manifest_bytes) + if ( + not decoded_manifest is Dictionary + or not _is_primitive_tree(decoded_manifest) + or not _generation_manifest_is_valid(decoded_manifest, String(generation_id)) + ): + _add_validation_time(validation_started) + return {} + var generation_manifest: Dictionary = decoded_manifest + var archive_manifest: Dictionary = generation_manifest["archive_manifest"] + var archive_descriptors: Array = archive_manifest["chunks"] + var chunk_entries: Array = generation_manifest["chunk_files"] + var location_chunk_id := "location:%s" % location_id + var global_entry: Dictionary = {} + var location_entry: Dictionary = {} + for index in range(archive_descriptors.size()): + var descriptor: Dictionary = archive_descriptors[index] + if String(descriptor["chunk_id"]) == RegionalChunkedPersistence.GLOBAL_CHUNK_ID: + global_entry = chunk_entries[index] + elif String(descriptor["chunk_id"]) == location_chunk_id: + location_entry = chunk_entries[index] + _add_validation_time(validation_started) + if global_entry.is_empty(): + return {} + var global_chunk := _read_chunk_file(generation_path, global_entry) + if ( + global_chunk.is_empty() + or not _chunk_header_is_valid(global_chunk) + or String(global_chunk.get("chunk_type", "")) != RegionalChunkedPersistence.CHUNK_GLOBAL + or not global_chunk.get("payload") is Dictionary + ): + return {} + var global_payload: Dictionary = global_chunk["payload"] + if not global_payload.get("location_ids") is Array: + return {} + if String(location_id) not in global_payload["location_ids"]: + return {"location_absent": true} + if location_entry.is_empty(): + return {} + var location_chunk := _read_chunk_file(generation_path, location_entry) + if ( + location_chunk.is_empty() + or not _chunk_header_is_valid(location_chunk) + or String(location_chunk.get("chunk_type", "")) != RegionalChunkedPersistence.CHUNK_LOCATION + or String(location_chunk.get("scope_id", "")) != String(location_id) + or not location_chunk.get("payload") is Dictionary + ): + return {} + var location_payload: Dictionary = location_chunk["payload"] + if ( + String(global_payload.get("world_id", "")) != String(archive_manifest["world_id"]) + or String(location_payload.get("world_id", "")) != String(archive_manifest["world_id"]) + or String(location_payload.get("location_id", "")) != String(location_id) + ): + return {} + var loaded_chunk_ids: Array[String] = [ + RegionalChunkedPersistence.GLOBAL_CHUNK_ID, location_chunk_id + ] + var unloaded_chunk_ids: Array[String] = [] + var mobile_group_refs: Array[Dictionary] = [] + for raw_descriptor: Variant in archive_descriptors: + var descriptor: Dictionary = raw_descriptor + var chunk_id := String(descriptor["chunk_id"]) + if chunk_id not in loaded_chunk_ids: + unloaded_chunk_ids.append(chunk_id) + if ( + String(descriptor["chunk_type"]) == RegionalChunkedPersistence.CHUNK_MOBILE_GROUP + and String(descriptor["location_id"]) == String(location_id) + ): + ( + mobile_group_refs + . append( + { + "group_id": String(descriptor["scope_id"]), + "chunk_id": chunk_id, + } + ) + ) + return { + "schema_version": RegionalChunkedPersistence.SCHEMA_VERSION, + "world_id": String(archive_manifest["world_id"]), + "active_location_id": String(location_id), + "loaded_chunk_ids": loaded_chunk_ids, + "unloaded_chunk_ids": unloaded_chunk_ids, + "global_index": global_payload.duplicate(true), + "active_location": location_payload.duplicate(true), + "mobile_group_refs": mobile_group_refs, + "envelope_checksum": String(archive_manifest["envelope_checksum"]), + "generation_id": String(generation_id), + "archive_checksum": String(generation_manifest["archive_checksum"]), + "full_generation_validated": false, + } + + +func _read_chunk_file(generation_path: String, entry: Dictionary) -> Dictionary: + var serialized_bytes := int(entry["serialized_bytes"]) + if serialized_bytes <= 0 or serialized_bytes > MAX_CHUNK_BYTES: + return {} + var chunk_bytes := _read_bytes( + generation_path.path_join(String(entry["file_name"])), MAX_CHUNK_BYTES + ) + if ( + chunk_bytes.size() != serialized_bytes + or _bytes_checksum(chunk_bytes) != String(entry["file_checksum"]) + ): + return {} + var validation_started := Time.get_ticks_usec() + var decoded_chunk: Variant = bytes_to_var(chunk_bytes) + if not decoded_chunk is Dictionary or not _is_primitive_tree(decoded_chunk): + _add_validation_time(validation_started) + return {} + var chunk: Dictionary = decoded_chunk + if ( + String(chunk.get("chunk_id", "")) != String(entry["chunk_id"]) + or JSON.stringify(chunk).sha256_text() != String(entry["canonical_checksum"]) + ): + _add_validation_time(validation_started) + return {} + _add_validation_time(validation_started) + return chunk + + +func _generation_manifest_is_valid(manifest: Dictionary, generation_id: String) -> bool: + var fields := [ + "schema_version", + "format_id", + "generation_id", + "archive_checksum", + "envelope_checksum", + "archive_manifest", + "chunk_files", + ] + if manifest.size() != fields.size() or not manifest.has_all(fields): + return false + if ( + not manifest["schema_version"] is int + or int(manifest["schema_version"]) != SCHEMA_VERSION + or not manifest["format_id"] is String + or String(manifest["format_id"]) != FORMAT_ID + or not manifest["generation_id"] is String + or String(manifest["generation_id"]) != generation_id + or not _is_generation_id(generation_id) + or not _is_sha256(manifest["archive_checksum"]) + or not _is_sha256(manifest["envelope_checksum"]) + or not manifest["archive_manifest"] is Dictionary + or not manifest["chunk_files"] is Array + ): + return false + var archive_manifest: Dictionary = manifest["archive_manifest"] + var archive_fields := [ + "schema_version", + "format_id", + "world_id", + "envelope_checksum", + "event_segment_size", + "chunk_count", + "chunks", + ] + if ( + archive_manifest.size() != archive_fields.size() + or not archive_manifest.has_all(archive_fields) + or not archive_manifest["schema_version"] is int + or not archive_manifest["format_id"] is String + or not archive_manifest["world_id"] is String + or String(archive_manifest["world_id"]).is_empty() + or not archive_manifest["envelope_checksum"] is String + or not archive_manifest["event_segment_size"] is int + or not archive_manifest["chunks"] is Array + or not archive_manifest["chunk_count"] is int + or int(archive_manifest["chunk_count"]) <= 0 + or int(archive_manifest["chunk_count"]) > MAX_CHUNKS + or (archive_manifest["chunks"] as Array).size() != int(archive_manifest["chunk_count"]) + or (manifest["chunk_files"] as Array).size() != int(archive_manifest["chunk_count"]) + or String(archive_manifest["format_id"]) != RegionalChunkedPersistence.FORMAT_ID + or int(archive_manifest["schema_version"]) != RegionalChunkedPersistence.SCHEMA_VERSION + or String(archive_manifest["envelope_checksum"]) != String(manifest["envelope_checksum"]) + ): + return false + var ids: Dictionary = {} + var archive_descriptors: Array = archive_manifest["chunks"] + var chunk_files: Array = manifest["chunk_files"] + for index in range(archive_descriptors.size()): + var descriptor: Variant = archive_descriptors[index] + var entry: Variant = chunk_files[index] + if not descriptor is Dictionary or not entry is Dictionary: + return false + var descriptor_fields := [ + "chunk_id", + "chunk_type", + "scope_id", + "location_id", + "route_id", + "record_count", + "byte_size", + "checksum", + ] + var entry_fields := [ + "chunk_id", + "file_name", + "serialized_bytes", + "file_checksum", + "canonical_checksum", + ] + if ( + descriptor.size() != descriptor_fields.size() + or not descriptor.has_all(descriptor_fields) + or entry.size() != entry_fields.size() + or not entry.has_all(entry_fields) + ): + return false + var chunk_id := String(entry["chunk_id"]) + if ( + not descriptor["chunk_id"] is String + or not descriptor["chunk_type"] is String + or not descriptor["scope_id"] is String + or not descriptor["location_id"] is String + or not descriptor["route_id"] is String + or not descriptor["record_count"] is int + or not descriptor["byte_size"] is int + or not descriptor["checksum"] is String + or not entry["chunk_id"] is String + or not entry["file_name"] is String + or not entry["file_checksum"] is String + or not entry["canonical_checksum"] is String + or chunk_id.is_empty() + or chunk_id.length() > MAX_CHUNK_ID_LENGTH + or ids.has(chunk_id) + or String(descriptor.get("chunk_id", "")) != chunk_id + or String(entry["file_name"]) != _chunk_file_name(index) + or not _is_safe_file_name(entry["file_name"]) + or not entry["serialized_bytes"] is int + or int(entry["serialized_bytes"]) <= 0 + or int(entry["serialized_bytes"]) > MAX_CHUNK_BYTES + or not _is_sha256(entry["file_checksum"]) + or not _is_sha256(entry["canonical_checksum"]) + or String(descriptor.get("checksum", "")) != String(entry["canonical_checksum"]) + ): + return false + ids[chunk_id] = true + return true + + +func _chunk_header_is_valid(chunk: Dictionary) -> bool: + var fields := [ + "schema_version", "chunk_id", "chunk_type", "scope_id", "location_id", "route_id", "payload" + ] + return ( + chunk.size() == fields.size() + and chunk.has_all(fields) + and chunk["schema_version"] is int + and int(chunk["schema_version"]) == RegionalChunkedPersistence.SCHEMA_VERSION + and chunk["chunk_id"] is String + and chunk["chunk_type"] is String + and chunk["scope_id"] is String + and chunk["location_id"] is String + and chunk["route_id"] is String + and chunk["payload"] is Dictionary + ) + + +func _install_current_pointer(pointer: Dictionary) -> bool: + if not _pointer_is_valid(pointer): + return _fail("Regional current pointer is invalid") + var pointer_bytes := var_to_bytes(pointer) + if pointer_bytes.is_empty() or pointer_bytes.size() > MAX_POINTER_BYTES: + return _fail("Regional current pointer exceeds the size limit") + _last_telemetry["serialized_bytes"] += pointer_bytes.size() + var temporary_path := _absolute_base_directory.path_join(TEMP_POINTER_FILE_NAME) + var current_path := _absolute_base_directory.path_join(CURRENT_POINTER_FILE_NAME) + var previous_path := _absolute_base_directory.path_join(PREVIOUS_POINTER_FILE_NAME) + _remove_file_if_present(temporary_path) + if not _write_bytes(temporary_path, pointer_bytes): + return _fail("Could not write the temporary regional current pointer") + var readback_bytes := _read_bytes(temporary_path, MAX_POINTER_BYTES) + var readback: Variant = bytes_to_var(readback_bytes) if not readback_bytes.is_empty() else null + if not readback is Dictionary or readback != pointer or not _pointer_is_valid(readback): + return _fail("Regional current pointer failed readback validation") + if DirAccess.rename_absolute(temporary_path, current_path) == OK: + return true + _remove_file_if_present(previous_path) + var had_current := FileAccess.file_exists(current_path) + if had_current and DirAccess.rename_absolute(current_path, previous_path) != OK: + return _fail("Could not preserve the previous regional current pointer") + if DirAccess.rename_absolute(temporary_path, current_path) == OK: + return true + if had_current: + DirAccess.rename_absolute(previous_path, current_path) + return _fail("Could not atomically install the regional current pointer") + + +func _read_current_pointer() -> Dictionary: + var pointer_bytes := _read_bytes( + _absolute_base_directory.path_join(CURRENT_POINTER_FILE_NAME), MAX_POINTER_BYTES + ) + if pointer_bytes.is_empty(): + return {} + var validation_started := Time.get_ticks_usec() + var decoded_pointer: Variant = bytes_to_var(pointer_bytes) + if ( + not decoded_pointer is Dictionary + or not _is_primitive_tree(decoded_pointer) + or not _pointer_is_valid(decoded_pointer) + ): + _add_validation_time(validation_started) + return {} + _add_validation_time(validation_started) + return decoded_pointer + + +func _pointer_is_valid(pointer: Dictionary) -> bool: + var fields := [ + "schema_version", "format_id", "generation_id", "manifest_checksum", "archive_checksum" + ] + return ( + pointer.size() == fields.size() + and pointer.has_all(fields) + and pointer["schema_version"] is int + and int(pointer["schema_version"]) == SCHEMA_VERSION + and pointer["format_id"] is String + and String(pointer["format_id"]) == FORMAT_ID + and pointer["generation_id"] is String + and _is_generation_id(String(pointer["generation_id"])) + and _is_sha256(pointer["manifest_checksum"]) + and _is_sha256(pointer["archive_checksum"]) + ) + + +func _ensure_directories() -> bool: + var base_error := DirAccess.make_dir_recursive_absolute(_absolute_base_directory) + if base_error != OK and base_error != ERR_ALREADY_EXISTS: + return _fail("Could not create the caller-supplied regional save directory") + var generations_error := DirAccess.make_dir_recursive_absolute(_generations_path()) + if generations_error != OK and generations_error != ERR_ALREADY_EXISTS: + return _fail("Could not create the regional generations directory") + return true + + +func _base_directory_is_safe() -> bool: + if ( + _base_directory.is_empty() + or _base_directory.length() > MAX_BASE_PATH_LENGTH + or _base_directory.to_utf8_buffer().has(0) + ): + return false + var normalized := _base_directory.replace("\\", "/") + if normalized == "user://" or normalized == "res://": + return false + if not ( + normalized.begins_with("user://") + or normalized.begins_with("res://") + or normalized.is_absolute_path() + ): + return false + for segment: String in normalized.split("/", false): + if segment == "." or segment == "..": + return false + if ( + _absolute_base_directory.is_empty() + or not _absolute_base_directory.is_absolute_path() + or _absolute_base_directory == "/" + or _absolute_base_directory.get_file().is_empty() + ): + return false + return true + + +func _list_generation_ids() -> Array[String]: + var ids: Array[String] = [] + var directory := DirAccess.open(_generations_path()) + if directory == null: + return ids + for raw_name: String in directory.get_directories(): + if _is_generation_id(raw_name): + ids.append(raw_name) + ids.sort() + return ids + + +func _next_generation_id(existing_ids: Array[String]) -> String: + var next_sequence := 1 + if not existing_ids.is_empty(): + var last_id := existing_ids[-1] + var last_sequence_text := last_id.substr(GENERATION_PREFIX.length()) + var last_sequence := last_sequence_text.to_int() + if last_sequence < 0 or last_sequence == 9_223_372_036_854_775_807: + return "" + next_sequence = last_sequence + 1 + return GENERATION_PREFIX + str(next_sequence).pad_zeros(GENERATION_DIGITS) + + +func _allocate_temporary_generation_path(generation_id: String) -> String: + for attempt in range(16): + var temporary_name := ( + "temporary_%s_%016d_%02d" % [generation_id, Time.get_ticks_usec(), attempt] + ) + var candidate := _generations_path().path_join(temporary_name) + if DirAccess.open(candidate) == null and not FileAccess.file_exists(candidate): + return candidate + return "" + + +func _is_generation_id(generation_id: String) -> bool: + if generation_id.length() != GENERATION_PREFIX.length() + GENERATION_DIGITS: + return false + if not generation_id.begins_with(GENERATION_PREFIX): + return false + var sequence_text := generation_id.substr(GENERATION_PREFIX.length()) + for character: String in sequence_text: + if character < "0" or character > "9": + return false + return sequence_text.to_int() > 0 + + +func _chunk_file_name(index: int) -> String: + return "chunk_%05d.bin" % index + + +func _is_safe_file_name(file_name: Variant) -> bool: + if not file_name is String: + return false + var value := String(file_name) + if value.is_empty() or value.length() > 64 or value.get_file() != value: + return false + var allowed := "abcdefghijklmnopqrstuvwxyz0123456789_.-" + for character: String in value: + if not allowed.contains(character): + return false + return true + + +func _is_sha256(value: Variant) -> bool: + if not value is String or String(value).length() != 64: + return false + for character: String in String(value): + if not "0123456789abcdef".contains(character): + return false + return true + + +func _write_bytes(path: String, bytes: PackedByteArray) -> bool: + var started_usec := Time.get_ticks_usec() + var file := FileAccess.open(path, FileAccess.WRITE) + if file == null: + _last_telemetry["write_usec"] += Time.get_ticks_usec() - started_usec + return false + file.store_buffer(bytes) + file.flush() + var write_error := file.get_error() + file.close() + _last_telemetry["write_usec"] += Time.get_ticks_usec() - started_usec + if write_error != OK: + return false + _last_telemetry["bytes_written"] += bytes.size() + _last_telemetry["files_written"] += 1 + return true + + +func _read_bytes(path: String, maximum_bytes: int) -> PackedByteArray: + var started_usec := Time.get_ticks_usec() + var file := FileAccess.open(path, FileAccess.READ) + if file == null: + _last_telemetry["read_usec"] += Time.get_ticks_usec() - started_usec + return PackedByteArray() + var length := file.get_length() + if length <= 0 or length > maximum_bytes: + file.close() + _last_telemetry["read_usec"] += Time.get_ticks_usec() - started_usec + return PackedByteArray() + var bytes := file.get_buffer(length) + var read_error := file.get_error() + file.close() + _last_telemetry["read_usec"] += Time.get_ticks_usec() - started_usec + if read_error != OK or bytes.size() != length: + return PackedByteArray() + _last_telemetry["bytes_read"] += bytes.size() + _last_telemetry["files_read"] += 1 + return bytes + + +func _bytes_checksum(bytes: PackedByteArray) -> String: + var hashing_context := HashingContext.new() + hashing_context.start(HashingContext.HASH_SHA256) + hashing_context.update(bytes) + return hashing_context.finish().hex_encode() + + +func _generations_path() -> String: + return _absolute_base_directory.path_join(GENERATIONS_DIRECTORY) + + +func _generation_path(generation_id: String) -> String: + return _generations_path().path_join(generation_id) + + +func _remove_file_if_present(path: String) -> void: + if FileAccess.file_exists(path): + DirAccess.remove_absolute(path) + + +func _remove_valid_generation(generation_id: String, archive: RegionalChunkedPersistence) -> bool: + if not _is_generation_id(generation_id) or archive == null: + return false + var generation_path := _generation_path(generation_id) + if not generation_path.begins_with(_generations_path() + "/"): + return false + var directory := DirAccess.open(generation_path) + if directory == null: + return false + directory.include_hidden = true + if not directory.get_directories().is_empty(): + return false + var expected_files: Array[String] = [MANIFEST_FILE_NAME] + for index in range(archive.get_chunk_ids().size()): + expected_files.append(_chunk_file_name(index)) + expected_files.sort() + var actual_files: Array[String] = [] + for file_name: String in directory.get_files(): + actual_files.append(file_name) + actual_files.sort() + if actual_files != expected_files: + return false + var started_usec := Time.get_ticks_usec() + for file_name: String in actual_files: + if DirAccess.remove_absolute(generation_path.path_join(file_name)) != OK: + _last_telemetry["write_usec"] += Time.get_ticks_usec() - started_usec + return false + _last_telemetry["files_deleted"] += 1 + var remove_error := DirAccess.remove_absolute(generation_path) + _last_telemetry["write_usec"] += Time.get_ticks_usec() - started_usec + return remove_error == OK + + +func _begin_telemetry(operation: String) -> void: + _last_telemetry = { + "operation": operation, + "generation_id": "", + "serialized_bytes": 0, + "bytes_written": 0, + "bytes_read": 0, + "files_written": 0, + "files_read": 0, + "files_deleted": 0, + "write_usec": 0, + "read_usec": 0, + "validation_usec": 0, + "candidates_validated": 0, + "recovered_generation": false, + "pointer_mismatch": false, + "generations_pruned": 0, + "partial_load": false, + "succeeded": false, + } + + +func _add_validation_time(started_usec: int) -> void: + _last_telemetry["validation_usec"] += Time.get_ticks_usec() - started_usec + + +func _fail(message: String) -> bool: + _last_error = message + return false + + +static func _is_primitive_tree(value: Variant) -> bool: + match typeof(value): + TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING: + return true + TYPE_FLOAT: + return is_finite(float(value)) + TYPE_ARRAY: + for item: Variant in value: + if not _is_primitive_tree(item): + return false + return true + TYPE_DICTIONARY: + for key: Variant in value: + if not key is String or not _is_primitive_tree(value[key]): + return false + return true + return false diff --git a/simulation/regional/RegionalChunkedFileStore.gd.uid b/simulation/regional/RegionalChunkedFileStore.gd.uid new file mode 100644 index 0000000..ac47366 --- /dev/null +++ b/simulation/regional/RegionalChunkedFileStore.gd.uid @@ -0,0 +1 @@ +uid://bt8aupkjekmnt diff --git a/tests/unit/test_regional_chunked_file_store.gd b/tests/unit/test_regional_chunked_file_store.gd new file mode 100644 index 0000000..38af28b --- /dev/null +++ b/tests/unit/test_regional_chunked_file_store.gd @@ -0,0 +1,406 @@ +extends GutTest + +const WORLD_ID := &"regional_world" +const ORIGIN_LOCATION_ID := &"location_origin" +const DESTINATION_LOCATION_ID := &"location_destination" +const ORIGIN_SETTLEMENT_ID := &"settlement_origin" +const DESTINATION_SETTLEMENT_ID := &"settlement_destination" +const GROUP_ID := &"group_caravan" +const ROUTE_ID := &"route_trade" +const DEPARTURE_TICK := 10 + +var _test_directory := "" + + +func before_each() -> void: + var test_parent := ( + ProjectSettings.globalize_path("user://regional_chunked_file_store_tests").simplify_path() + ) + _remove_tree(test_parent) + _test_directory = ( + "user://regional_chunked_file_store_tests/run_%d_%d" + % [Time.get_ticks_usec(), get_instance_id()] + ) + + +func after_each() -> void: + var absolute_path := ProjectSettings.globalize_path(_test_directory).simplify_path() + var expected_parent := ( + ProjectSettings.globalize_path("user://regional_chunked_file_store_tests").simplify_path() + ) + if absolute_path.begins_with(expected_parent + "/"): + _remove_tree(absolute_path) + + +func test_atomic_generations_round_trip_exact_archive_and_report_telemetry() -> void: + var explicit_absolute_base := ProjectSettings.globalize_path(_test_directory) + var store := RegionalChunkedFileStore.new(explicit_absolute_base) + var first := _fixture_archive(false) + var second := _fixture_archive(true) + + assert_eq(store.get_base_directory(), explicit_absolute_base) + assert_true(store.save(first)) + var first_generation := store.get_generation_ids()[0] + assert_true(store.save(second)) + var generation_ids := store.get_generation_ids() + assert_eq(generation_ids.size(), 2) + assert_eq(generation_ids[0], first_generation) + assert_true( + ( + DirAccess.open( + ProjectSettings.globalize_path(store.get_generation_directory(first_generation)) + ) + != null + ) + ) + var save_telemetry := store.get_last_telemetry() + assert_true(save_telemetry["succeeded"]) + assert_gt(save_telemetry["serialized_bytes"], 0) + assert_gt(save_telemetry["bytes_written"], 0) + assert_gte(save_telemetry["write_usec"], 0) + assert_gte(save_telemetry["validation_usec"], 0) + + var latest_generation := generation_ids[-1] + var generation_path := store.get_generation_directory(latest_generation) + var generation_files := DirAccess.get_files_at(generation_path) + assert_eq(generation_files.size(), second.get_chunk_ids().size() + 1) + assert_true(RegionalChunkedFileStore.MANIFEST_FILE_NAME in generation_files) + for file_name: String in generation_files: + assert_false("/" in file_name) + assert_false("\\" in file_name) + assert_lte(file_name.length(), 64) + var decoded: Variant = _read_variant(generation_path.path_join(file_name)) + assert_true(_contains_only_primitive_values(decoded)) + + var restored := store.load_archive() + assert_not_null(restored) + assert_eq(restored.checksum(), second.checksum()) + assert_eq(restored.restore_service().checksum(), second.restore_service().checksum()) + var load_telemetry := store.get_last_telemetry() + assert_true(load_telemetry["succeeded"]) + assert_eq(load_telemetry["generation_id"], latest_generation) + assert_false(load_telemetry["recovered_generation"]) + assert_gt(load_telemetry["bytes_read"], 0) + assert_gte(load_telemetry["read_usec"], 0) + assert_gte(load_telemetry["validation_usec"], 0) + + +func test_interrupted_pointer_and_corrupt_latest_generation_recover_last_valid() -> void: + var store := RegionalChunkedFileStore.new(_test_directory) + var first := _fixture_archive(false) + var second := _fixture_archive(true) + assert_true(store.save(first)) + var first_generation := store.get_generation_ids()[0] + + var current_pointer := ProjectSettings.globalize_path(store.get_current_pointer_path()) + var previous_pointer := ProjectSettings.globalize_path( + _test_directory.path_join(RegionalChunkedFileStore.PREVIOUS_POINTER_FILE_NAME) + ) + assert_eq(DirAccess.rename_absolute(current_pointer, previous_pointer), OK) + var recovered_missing_pointer := store.load_archive() + assert_not_null(recovered_missing_pointer) + assert_eq(recovered_missing_pointer.checksum(), first.checksum()) + assert_true(store.get_last_telemetry()["recovered_generation"]) + + assert_true(store.save(second)) + var latest_generation := store.get_generation_ids()[-1] + var latest_path := store.get_generation_directory(latest_generation) + var manifest: Dictionary = _read_variant( + latest_path.path_join(RegionalChunkedFileStore.MANIFEST_FILE_NAME) + ) + var first_chunk_file := String(manifest["chunk_files"][0]["file_name"]) + _write_variant(latest_path.path_join(first_chunk_file), {"corrupt": true}) + var interrupted_path := ProjectSettings.globalize_path( + store.get_generations_directory().path_join( + "temporary_generation_00000000000000000003_interrupted" + ) + ) + assert_eq(DirAccess.make_dir_recursive_absolute(interrupted_path), OK) + _write_variant(interrupted_path.path_join("partial.bin"), {"partial": true}) + + var recovered := store.load_archive() + assert_not_null(recovered) + assert_eq(recovered.checksum(), first.checksum()) + assert_eq(store.get_last_telemetry()["generation_id"], first_generation) + assert_true(store.get_last_telemetry()["recovered_generation"]) + assert_gte(store.get_last_telemetry()["candidates_validated"], 2) + assert_true(FileAccess.file_exists(latest_path.path_join(first_chunk_file))) + + +func test_partial_location_load_reads_only_selected_chunks_and_keeps_unloaded_authority() -> void: + var store := RegionalChunkedFileStore.new(_test_directory) + var archive := _fixture_archive(false) + assert_true(store.save(archive)) + var saved_telemetry := store.get_last_telemetry() + var generation_id := store.get_generation_ids()[0] + var generation_path := store.get_generation_directory(generation_id) + var manifest: Dictionary = _read_variant( + generation_path.path_join(RegionalChunkedFileStore.MANIFEST_FILE_NAME) + ) + var scheduler_entry := _find_chunk_file_entry( + manifest, RegionalChunkedPersistence.SCHEDULER_CHUNK_ID + ) + var scheduler_path := generation_path.path_join(String(scheduler_entry["file_name"])) + _write_variant(scheduler_path, {"corrupt_unloaded_scheduler": true}) + + var partial := store.load_active_location_metadata(ORIGIN_LOCATION_ID) + assert_false(partial.is_empty()) + assert_eq(partial["generation_id"], generation_id) + assert_eq( + partial["loaded_chunk_ids"], + [ + RegionalChunkedPersistence.GLOBAL_CHUNK_ID, + "location:%s" % ORIGIN_LOCATION_ID, + ] + ) + assert_true(RegionalChunkedPersistence.SCHEDULER_CHUNK_ID in partial["unloaded_chunk_ids"]) + assert_eq(partial["mobile_group_refs"].size(), 1) + assert_false(partial["full_generation_validated"]) + assert_true(FileAccess.file_exists(scheduler_path)) + var partial_telemetry := store.get_last_telemetry() + assert_true(partial_telemetry["partial_load"]) + assert_eq(partial_telemetry["files_read"], 4) + assert_lt(partial_telemetry["bytes_read"], saved_telemetry["bytes_written"]) + assert_null(store.load_archive()) + + +func test_unsafe_base_and_traversal_pointer_cannot_escape_generation_root() -> void: + var archive := _fixture_archive(false) + var unsafe_relative := RegionalChunkedFileStore.new("../regional_escape") + assert_false(unsafe_relative.save(archive)) + assert_true("unsafe" in unsafe_relative.get_last_error()) + assert_false(RegionalChunkedFileStore.new("user://").save(archive)) + assert_false(RegionalChunkedFileStore.new("res://").save(archive)) + + var store := RegionalChunkedFileStore.new(_test_directory) + assert_true(store.save(archive)) + assert_eq(store.get_generation_directory("../outside"), "") + var malicious_pointer := { + "schema_version": RegionalChunkedFileStore.SCHEMA_VERSION, + "format_id": RegionalChunkedFileStore.FORMAT_ID, + "generation_id": "../outside", + "manifest_checksum": "0".repeat(64), + "archive_checksum": "0".repeat(64), + } + _write_variant(store.get_current_pointer_path(), malicious_pointer) + var recovered := store.load_archive() + assert_not_null(recovered) + assert_eq(recovered.checksum(), archive.checksum()) + assert_true(store.get_last_telemetry()["recovered_generation"]) + + +func test_explicit_conservative_prune_keeps_current_and_two_newest_valid_generations() -> void: + var store := RegionalChunkedFileStore.new(_test_directory) + var first := _fixture_archive(false) + var latest := _fixture_archive(true) + assert_true(store.save(first)) + assert_true(store.save(latest)) + assert_true(store.save(latest)) + var before := store.get_generation_ids() + assert_eq(before.size(), 3) + + assert_eq(store.prune_old_generations(2), 1) + assert_eq(store.get_generation_ids(), [before[1], before[2]]) + assert_eq(store.get_last_telemetry()["operation"], "prune_old_generations") + assert_eq(store.get_last_telemetry()["generations_pruned"], 1) + assert_true(store.get_last_telemetry()["succeeded"]) + assert_gt(store.get_last_telemetry()["files_deleted"], 0) + assert_eq(store.load_archive().checksum(), latest.checksum()) + + var retained := store.get_generation_ids() + assert_eq(store.prune_old_generations(1), 0) + assert_eq(store.get_generation_ids(), retained) + assert_true("at least two" in store.get_last_error()) + + +func test_partial_load_does_not_resurrect_a_location_removed_by_newer_authority() -> void: + var store := RegionalChunkedFileStore.new(_test_directory) + var older_with_destination := _fixture_archive(false) + var latest_without_destination := _single_location_archive() + assert_true(store.save(older_with_destination)) + assert_true(store.save(latest_without_destination)) + + assert_true(store.load_active_location_metadata(DESTINATION_LOCATION_ID).is_empty()) + assert_true("does not contain" in store.get_last_error()) + assert_eq(store.load_archive().checksum(), latest_without_destination.checksum()) + + +func _fixture_archive(departed: bool) -> RegionalChunkedPersistence: + var origin_location := LocationStateRecord.create( + ORIGIN_LOCATION_ID, WORLD_ID, &"settlement", "Origin", _address(ORIGIN_LOCATION_ID) + ) + var destination_location := LocationStateRecord.create( + DESTINATION_LOCATION_ID, + WORLD_ID, + &"settlement", + "Destination", + _address(DESTINATION_LOCATION_ID) + ) + var origin := SettlementStateRecord.create( + ORIGIN_SETTLEMENT_ID, + ORIGIN_LOCATION_ID, + "Origin", + &"polity_origin", + 1, + &"founders_origin", + 0, + {&"food": 20.0} + ) + var destination := SettlementStateRecord.create( + DESTINATION_SETTLEMENT_ID, + DESTINATION_LOCATION_ID, + "Destination", + &"polity_destination", + 1, + &"founders_destination", + 0, + {&"food": 4.0} + ) + var route := RouteStateRecord.create(ROUTE_ID, ORIGIN_LOCATION_ID, DESTINATION_LOCATION_ID, 5) + var group := MobileGroupStateRecord.create_at_location( + GROUP_ID, + &"caravan", + "Caravan", + &"polity_origin", + _address(ORIGIN_LOCATION_ID), + [], + {}, + {}, + 10.0 + ) + var origin_polity := PolityStateRecord.create( + &"polity_origin", "Origin polity", ORIGIN_LOCATION_ID, [ORIGIN_SETTLEMENT_ID] + ) + var destination_polity := PolityStateRecord.create( + &"polity_destination", + "Destination polity", + DESTINATION_LOCATION_ID, + [DESTINATION_SETTLEMENT_ID] + ) + var locations: Array[LocationStateRecord] = [destination_location, origin_location] + var settlements: Array[SettlementStateRecord] = [destination, origin] + var routes: Array[RouteStateRecord] = [route] + var groups: Array[MobileGroupStateRecord] = [group] + var persons: Array[PersonStateRecord] = [] + var cohorts: Array[PopulationCohortRecord] = [] + var polities: Array[PolityStateRecord] = [destination_polity, origin_polity] + var relations: Array[DiplomaticRelationRecord] = [] + var world_state := RegionalWorldState.create( + WORLD_ID, locations, settlements, routes, groups, persons, cohorts, polities, relations + ) + var service := RegionalCaravanService.create( + world_state, RegionalJobScheduler.new(), WorldEventStore.new() + ) + if departed: + assert_true( + service.depart( + GROUP_ID, + ORIGIN_SETTLEMENT_ID, + DESTINATION_SETTLEMENT_ID, + ROUTE_ID, + {&"food": 6.0}, + DEPARTURE_TICK + ) + ) + return RegionalChunkedPersistence.capture(service, 2) + + +func _single_location_archive() -> RegionalChunkedPersistence: + var location := LocationStateRecord.create( + ORIGIN_LOCATION_ID, WORLD_ID, &"settlement", "Origin", _address(ORIGIN_LOCATION_ID) + ) + var settlement := SettlementStateRecord.create( + ORIGIN_SETTLEMENT_ID, + ORIGIN_LOCATION_ID, + "Origin", + &"polity_origin", + 1, + &"founders_origin", + 0, + {&"food": 20.0} + ) + var polity := PolityStateRecord.create( + &"polity_origin", "Origin polity", ORIGIN_LOCATION_ID, [ORIGIN_SETTLEMENT_ID] + ) + var locations: Array[LocationStateRecord] = [location] + var settlements: Array[SettlementStateRecord] = [settlement] + var routes: Array[RouteStateRecord] = [] + var groups: Array[MobileGroupStateRecord] = [] + var persons: Array[PersonStateRecord] = [] + var cohorts: Array[PopulationCohortRecord] = [] + var polities: Array[PolityStateRecord] = [polity] + var relations: Array[DiplomaticRelationRecord] = [] + var world_state := RegionalWorldState.create( + WORLD_ID, locations, settlements, routes, groups, persons, cohorts, polities, relations + ) + var service := RegionalCaravanService.create( + world_state, RegionalJobScheduler.new(), WorldEventStore.new() + ) + return RegionalChunkedPersistence.capture(service, 2) + + +func _address(location_id: StringName) -> SpatialAddress: + return SpatialAddress.create(WORLD_ID, location_id, Vector3.ZERO) + + +func _find_chunk_file_entry(manifest: Dictionary, chunk_id: String) -> Dictionary: + for raw_entry: Variant in manifest["chunk_files"]: + if raw_entry is Dictionary and String(raw_entry.get("chunk_id", "")) == chunk_id: + return raw_entry + return {} + + +func _read_variant(path: String) -> Variant: + var file := FileAccess.open(path, FileAccess.READ) + if file == null: + return null + var bytes := file.get_buffer(file.get_length()) + file.close() + return bytes_to_var(bytes) if not bytes.is_empty() else null + + +func _write_variant(path: String, value: Variant) -> void: + var file := FileAccess.open(path, FileAccess.WRITE) + assert_not_null(file) + if file == null: + return + file.store_buffer(var_to_bytes(value)) + file.flush() + file.close() + + +func _contains_only_primitive_values(value: Variant) -> bool: + match typeof(value): + TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING: + return true + TYPE_FLOAT: + return is_finite(float(value)) + TYPE_ARRAY: + for item: Variant in value: + if not _contains_only_primitive_values(item): + return false + return true + TYPE_DICTIONARY: + for key: Variant in value: + if not key is String or not _contains_only_primitive_values(value[key]): + return false + return true + return false + + +func _remove_tree(absolute_path: String) -> void: + var directory := DirAccess.open(absolute_path) + if directory == null: + return + directory.include_hidden = true + directory.list_dir_begin() + var entry_name := directory.get_next() + while not entry_name.is_empty(): + var entry_path := absolute_path.path_join(entry_name) + if directory.current_is_dir(): + _remove_tree(entry_path) + else: + DirAccess.remove_absolute(entry_path) + entry_name = directory.get_next() + directory.list_dir_end() + DirAccess.remove_absolute(absolute_path) diff --git a/tests/unit/test_regional_chunked_file_store.gd.uid b/tests/unit/test_regional_chunked_file_store.gd.uid new file mode 100644 index 0000000..90464d4 --- /dev/null +++ b/tests/unit/test_regional_chunked_file_store.gd.uid @@ -0,0 +1 @@ +uid://cpesubt4wseva