From e85ee2739544b884688987d1020609e3872dba40 Mon Sep 17 00:00:00 2001 From: Rijad Zuzo Date: Wed, 12 Aug 2026 22:02:40 +0200 Subject: [PATCH] feat: chunk regional simulation state --- .../regional/RegionalChunkedPersistence.gd | 947 ++++++++++++++++++ .../RegionalChunkedPersistence.gd.uid | 1 + .../unit/test_regional_chunked_persistence.gd | 281 ++++++ .../test_regional_chunked_persistence.gd.uid | 1 + 4 files changed, 1230 insertions(+) create mode 100644 simulation/regional/RegionalChunkedPersistence.gd create mode 100644 simulation/regional/RegionalChunkedPersistence.gd.uid create mode 100644 tests/unit/test_regional_chunked_persistence.gd create mode 100644 tests/unit/test_regional_chunked_persistence.gd.uid diff --git a/simulation/regional/RegionalChunkedPersistence.gd b/simulation/regional/RegionalChunkedPersistence.gd new file mode 100644 index 0000000..3d57a2a --- /dev/null +++ b/simulation/regional/RegionalChunkedPersistence.gd @@ -0,0 +1,947 @@ +class_name RegionalChunkedPersistence +extends RefCounted + +const SCHEMA_VERSION := 1 +const FORMAT_ID := "regional_caravan_chunks" +const DEFAULT_EVENT_SEGMENT_SIZE := 64 +const MAX_EVENT_SEGMENT_SIZE := 1024 +const CHUNK_GLOBAL := "global_index" +const CHUNK_LOCATION := "location" +const CHUNK_MOBILE_GROUP := "mobile_group" +const CHUNK_SCHEDULER := "scheduler" +const CHUNK_EVENTS := "event_segment" +const GLOBAL_CHUNK_ID := "global:index" +const SCHEDULER_CHUNK_ID := "scheduler:regional" + +var _manifest: Dictionary = {} +var _chunks_by_id: Dictionary = {} +var _ordered_chunk_ids: Array[String] = [] + + +static func capture( + service: RegionalCaravanService, event_segment_size: int = DEFAULT_EVENT_SEGMENT_SIZE +) -> RegionalChunkedPersistence: + if ( + service == null + or not service.is_valid() + or event_segment_size <= 0 + or event_segment_size > MAX_EVENT_SEGMENT_SIZE + ): + return null + var envelope := service.to_dictionary() + if not _is_primitive_tree(envelope): + return null + var chunks := _capture_chunks(service, event_segment_size) + if chunks.is_empty(): + return null + chunks.sort_custom(_chunk_precedes) + var descriptors: Array[Dictionary] = [] + for chunk: Dictionary in chunks: + descriptors.append(_descriptor_for_chunk(chunk)) + var manifest := { + "schema_version": SCHEMA_VERSION, + "format_id": FORMAT_ID, + "world_id": String(service.get_world_state().get_world_id()), + "envelope_checksum": service.checksum(), + "event_segment_size": event_segment_size, + "chunk_count": chunks.size(), + "chunks": descriptors, + } + var bundle := { + "schema_version": SCHEMA_VERSION, + "manifest": manifest, + "chunks": chunks, + } + return from_dictionary(bundle) + + +static func from_dictionary(bundle: Dictionary) -> RegionalChunkedPersistence: + if not _is_primitive_tree(bundle): + return null + if ( + not _has_exact_fields(bundle, ["schema_version", "manifest", "chunks"]) + or not bundle.get("schema_version") is int + or int(bundle["schema_version"]) != SCHEMA_VERSION + or not bundle.get("manifest") is Dictionary + or not bundle.get("chunks") is Array + ): + return null + var manifest: Dictionary = bundle["manifest"] + var raw_chunks: Array = bundle["chunks"] + if not _manifest_header_is_valid(manifest, raw_chunks.size()): + return null + var descriptors: Array = manifest["chunks"] + var chunks: Array[Dictionary] = [] + for raw_chunk: Variant in raw_chunks: + if not raw_chunk is Dictionary: + return null + chunks.append(raw_chunk) + if not _canonical_order_is_valid(descriptors, chunks): + return null + var chunks_by_id: Dictionary = {} + var descriptors_by_id: Dictionary = {} + for index in range(chunks.size()): + var chunk: Dictionary = chunks[index] + var descriptor: Dictionary = descriptors[index] + if not _chunk_and_descriptor_are_valid(chunk, descriptor): + return null + var chunk_id := String(chunk["chunk_id"]) + if chunks_by_id.has(chunk_id) or descriptors_by_id.has(chunk_id): + return null + chunks_by_id[chunk_id] = chunk.duplicate(true) + descriptors_by_id[chunk_id] = descriptor.duplicate(true) + if not _chunk_topology_is_valid(manifest, chunks, chunks_by_id): + return null + var archive := RegionalChunkedPersistence.new() + archive._manifest = manifest.duplicate(true) + archive._chunks_by_id = chunks_by_id + for chunk: Dictionary in chunks: + archive._ordered_chunk_ids.append(String(chunk["chunk_id"])) + var restored := archive.restore_service() + if restored == null or restored.checksum() != String(manifest["envelope_checksum"]): + return null + return archive + + +func get_manifest() -> Dictionary: + return _manifest.duplicate(true) + + +func get_chunk_ids() -> Array[String]: + return _ordered_chunk_ids.duplicate() + + +func get_chunk(chunk_id: StringName) -> Dictionary: + var chunk: Dictionary = _chunks_by_id.get(String(chunk_id), {} as Dictionary) + return chunk.duplicate(true) + + +func get_chunk_ids_for_type(chunk_type: StringName) -> Array[String]: + var ids: Array[String] = [] + for chunk_id: String in _ordered_chunk_ids: + var chunk: Dictionary = _chunks_by_id[chunk_id] + if String(chunk["chunk_type"]) == String(chunk_type): + ids.append(chunk_id) + return ids + + +func to_dictionary() -> Dictionary: + var chunks: Array[Dictionary] = [] + for chunk_id: String in _ordered_chunk_ids: + var chunk: Dictionary = _chunks_by_id[chunk_id] + chunks.append(chunk.duplicate(true)) + return { + "schema_version": SCHEMA_VERSION, + "manifest": _manifest.duplicate(true), + "chunks": chunks, + } + + +func checksum() -> String: + return JSON.stringify(to_dictionary()).sha256_text() + + +func restore_service() -> RegionalCaravanService: + var global_chunk := get_chunk(StringName(GLOBAL_CHUNK_ID)) + var scheduler_chunk := get_chunk(StringName(SCHEDULER_CHUNK_ID)) + if global_chunk.is_empty() or scheduler_chunk.is_empty(): + return null + var global_payload: Dictionary = global_chunk["payload"] + var scheduler_payload: Dictionary = scheduler_chunk["payload"] + var locations: Array[Dictionary] = [] + var settlements: Array[Dictionary] = [] + var mobile_groups: Array[Dictionary] = [] + var persons: Array[Dictionary] = [] + var cohorts: Array[Dictionary] = [] + var event_records: Array[Dictionary] = [] + for chunk_id: String in _ordered_chunk_ids: + var chunk: Dictionary = _chunks_by_id[chunk_id] + var payload: Dictionary = chunk["payload"] + match String(chunk["chunk_type"]): + CHUNK_LOCATION: + locations.append(payload["location"].duplicate(true)) + _append_dictionary_records(settlements, payload["settlements"]) + _append_dictionary_records(persons, payload["persons"]) + _append_dictionary_records(cohorts, payload["population_cohorts"]) + CHUNK_MOBILE_GROUP: + mobile_groups.append(payload["mobile_group"].duplicate(true)) + _append_dictionary_records(persons, payload["persons"]) + _append_dictionary_records(cohorts, payload["population_cohorts"]) + CHUNK_EVENTS: + _append_dictionary_records(event_records, payload["events"]) + var world_state := { + "schema_version": int(global_payload["world_state_schema_version"]), + "world_id": String(global_payload["world_id"]), + "locations": locations, + "settlements": settlements, + "routes": global_payload["routes"].duplicate(true), + "mobile_groups": mobile_groups, + "persons": persons, + "population_cohorts": cohorts, + "polities": global_payload["polities"].duplicate(true), + "diplomatic_relations": global_payload["diplomatic_relations"].duplicate(true), + } + var envelope := { + "schema_version": int(global_payload["service_schema_version"]), + "next_event_id": int(global_payload["next_event_id"]), + "world_state": world_state, + "scheduler": scheduler_payload["scheduler"].duplicate(true), + "event_store": + { + "schema_version": int(global_payload["event_store_schema_version"]), + "events": event_records, + }, + } + var service := RegionalCaravanService.from_dictionary(envelope) + if service == null or service.checksum() != String(_manifest["envelope_checksum"]): + return null + return service + + +func load_active_location_metadata(location_id: StringName) -> Dictionary: + var location_chunk_id := _location_chunk_id(location_id) + var location_chunk := get_chunk(StringName(location_chunk_id)) + var global_chunk := get_chunk(StringName(GLOBAL_CHUNK_ID)) + if location_chunk.is_empty() or global_chunk.is_empty(): + return {} + var loaded_chunk_ids: Array[String] = [GLOBAL_CHUNK_ID, location_chunk_id] + var unloaded_chunk_ids: Array[String] = [] + var mobile_group_refs: Array[Dictionary] = [] + var descriptors: Array = _manifest["chunks"] + for descriptor: Dictionary in descriptors: + var chunk_id := String(descriptor["chunk_id"]) + if chunk_id not in loaded_chunk_ids: + unloaded_chunk_ids.append(chunk_id) + if ( + String(descriptor["chunk_type"]) == CHUNK_MOBILE_GROUP + and String(descriptor["location_id"]) == String(location_id) + ): + ( + mobile_group_refs + . append( + { + "group_id": String(descriptor["scope_id"]), + "chunk_id": chunk_id, + } + ) + ) + return { + "schema_version": SCHEMA_VERSION, + "world_id": String(_manifest["world_id"]), + "active_location_id": String(location_id), + "loaded_chunk_ids": loaded_chunk_ids, + "unloaded_chunk_ids": unloaded_chunk_ids, + "global_index": (global_chunk["payload"] as Dictionary).duplicate(true), + "active_location": (location_chunk["payload"] as Dictionary).duplicate(true), + "mobile_group_refs": mobile_group_refs, + "envelope_checksum": String(_manifest["envelope_checksum"]), + } + + +static func _capture_chunks( + service: RegionalCaravanService, event_segment_size: int +) -> Array[Dictionary]: + var chunks: Array[Dictionary] = [] + var state := service.get_world_state() + var location_ids: Array[String] = [] + for location: LocationStateRecord in state.get_all_locations(): + location_ids.append(String(location.get_location_id())) + var mobile_group_ids: Array[String] = [] + for group: MobileGroupStateRecord in state.get_all_mobile_groups(): + mobile_group_ids.append(String(group.get_group_id())) + var global_payload := { + "service_schema_version": RegionalCaravanService.SCHEMA_VERSION, + "world_state_schema_version": RegionalWorldState.SCHEMA_VERSION, + "scheduler_schema_version": RegionalJobScheduler.SCHEMA_VERSION, + "event_store_schema_version": WorldEventStore.SCHEMA_VERSION, + "world_id": String(state.get_world_id()), + "next_event_id": service.get_next_event_id(), + "location_ids": location_ids, + "mobile_group_ids": mobile_group_ids, + "routes": _records_to_dictionaries(state.get_all_routes()), + "polities": _records_to_dictionaries(state.get_all_polities()), + "diplomatic_relations": _records_to_dictionaries(state.get_all_diplomatic_relations()), + } + chunks.append(_make_chunk(GLOBAL_CHUNK_ID, CHUNK_GLOBAL, state.get_world_id(), global_payload)) + for location: LocationStateRecord in state.get_all_locations(): + var location_id := location.get_location_id() + var payload := { + "world_id": String(state.get_world_id()), + "location_id": String(location_id), + "location": location.to_dictionary(), + "settlements": _records_to_dictionaries(state.get_settlements_at_location(location_id)), + "persons": _records_to_dictionaries(state.get_persons_at_location(location_id)), + "population_cohorts": + _records_to_dictionaries(state.get_cohorts_at_location(location_id)), + } + chunks.append( + _make_chunk(_location_chunk_id(location_id), CHUNK_LOCATION, location_id, payload) + ) + for group: MobileGroupStateRecord in state.get_all_mobile_groups(): + var group_id := group.get_group_id() + var payload := { + "world_id": String(state.get_world_id()), + "group_id": String(group_id), + "mobile_group": group.to_dictionary(), + "persons": _records_to_dictionaries(state.get_persons_in_group(group_id)), + "population_cohorts": _records_to_dictionaries(state.get_cohorts_in_group(group_id)), + } + var location_id := group.get_current_location_id() if group.is_at_location() else &"" + var route_id := group.get_route_id() if group.is_on_route() else &"" + chunks.append( + _make_chunk( + _mobile_group_chunk_id(group_id), + CHUNK_MOBILE_GROUP, + group_id, + payload, + location_id, + route_id + ) + ) + chunks.append( + _make_chunk( + SCHEDULER_CHUNK_ID, + CHUNK_SCHEDULER, + &"regional", + {"scheduler": service.get_scheduler().to_dictionary()} + ) + ) + var event_data := service.get_event_store().to_dictionary() + var events: Array = event_data["events"] + var segment_count := maxi(1, ceili(float(events.size()) / float(event_segment_size))) + for segment_index in range(segment_count): + var first := segment_index * event_segment_size + var last := mini(first + event_segment_size, events.size()) + var segment_events: Array[Dictionary] = [] + for event_index in range(first, last): + var event_record: Dictionary = events[event_index] + segment_events.append(event_record.duplicate(true)) + ( + chunks + . append( + _make_chunk( + _event_chunk_id(segment_index), + CHUNK_EVENTS, + StringName(str(segment_index)), + { + "segment_index": segment_index, + "events": segment_events, + } + ) + ) + ) + return chunks + + +static func _make_chunk( + chunk_id: String, + chunk_type: String, + scope_id: StringName, + payload: Dictionary, + location_id: StringName = &"", + route_id: StringName = &"" +) -> Dictionary: + return { + "schema_version": SCHEMA_VERSION, + "chunk_id": chunk_id, + "chunk_type": chunk_type, + "scope_id": String(scope_id), + "location_id": String(location_id), + "route_id": String(route_id), + "payload": payload.duplicate(true), + } + + +static func _descriptor_for_chunk(chunk: Dictionary) -> Dictionary: + var serialized := JSON.stringify(chunk) + return { + "chunk_id": String(chunk["chunk_id"]), + "chunk_type": String(chunk["chunk_type"]), + "scope_id": String(chunk["scope_id"]), + "location_id": String(chunk["location_id"]), + "route_id": String(chunk["route_id"]), + "record_count": _chunk_record_count(chunk), + "byte_size": serialized.to_utf8_buffer().size(), + "checksum": serialized.sha256_text(), + } + + +static func _manifest_header_is_valid(manifest: Dictionary, raw_chunk_count: int) -> bool: + if not _has_exact_fields( + manifest, + [ + "schema_version", + "format_id", + "world_id", + "envelope_checksum", + "event_segment_size", + "chunk_count", + "chunks", + ] + ): + return false + return ( + manifest["schema_version"] is int + and int(manifest["schema_version"]) == SCHEMA_VERSION + and manifest["format_id"] is String + and String(manifest["format_id"]) == FORMAT_ID + and manifest["world_id"] is String + and not String(manifest["world_id"]).is_empty() + and manifest["envelope_checksum"] is String + and not String(manifest["envelope_checksum"]).is_empty() + and manifest["event_segment_size"] is int + and int(manifest["event_segment_size"]) > 0 + and int(manifest["event_segment_size"]) <= MAX_EVENT_SEGMENT_SIZE + and manifest["chunk_count"] is int + and int(manifest["chunk_count"]) == raw_chunk_count + and manifest["chunks"] is Array + and (manifest["chunks"] as Array).size() == raw_chunk_count + ) + + +static func _canonical_order_is_valid(descriptors: Array, chunks: Array[Dictionary]) -> bool: + var typed_descriptors: Array[Dictionary] = [] + for raw_descriptor: Variant in descriptors: + if not raw_descriptor is Dictionary: + return false + typed_descriptors.append(raw_descriptor) + var sorted_chunks := chunks.duplicate(true) + sorted_chunks.sort_custom(_chunk_precedes) + var sorted_descriptors := typed_descriptors.duplicate(true) + sorted_descriptors.sort_custom(_descriptor_precedes) + for index in range(chunks.size()): + if ( + ( + String(chunks[index].get("chunk_id", "")) + != String(sorted_chunks[index].get("chunk_id", "")) + ) + or ( + String(typed_descriptors[index].get("chunk_id", "")) + != String(sorted_descriptors[index].get("chunk_id", "")) + ) + or ( + String(chunks[index].get("chunk_id", "")) + != String(typed_descriptors[index].get("chunk_id", "")) + ) + ): + return false + return true + + +static func _chunk_and_descriptor_are_valid(chunk: Dictionary, descriptor: Dictionary) -> bool: + var chunk_fields := [ + "schema_version", "chunk_id", "chunk_type", "scope_id", "location_id", "route_id", "payload" + ] + var descriptor_fields := [ + "chunk_id", + "chunk_type", + "scope_id", + "location_id", + "route_id", + "record_count", + "byte_size", + "checksum", + ] + if ( + not _has_exact_fields(chunk, chunk_fields) + or not _has_exact_fields(descriptor, descriptor_fields) + ): + return false + if ( + not chunk["schema_version"] is int + or int(chunk["schema_version"]) != SCHEMA_VERSION + or not chunk["chunk_id"] is String + or String(chunk["chunk_id"]).is_empty() + or not chunk["chunk_type"] is String + or not chunk["scope_id"] is String + or not chunk["location_id"] is String + or not chunk["route_id"] is String + or not chunk["payload"] is Dictionary + or not descriptor["record_count"] is int + or not descriptor["byte_size"] is int + or not descriptor["checksum"] is String + ): + return false + for field: String in ["chunk_id", "chunk_type", "scope_id", "location_id", "route_id"]: + if not descriptor[field] is String or String(descriptor[field]) != String(chunk[field]): + return false + var serialized := JSON.stringify(chunk) + var record_count := _chunk_record_count(chunk) + return ( + record_count >= 0 + and int(descriptor["record_count"]) == record_count + and int(descriptor["byte_size"]) == serialized.to_utf8_buffer().size() + and String(descriptor["checksum"]) == serialized.sha256_text() + ) + + +static func _chunk_topology_is_valid( + manifest: Dictionary, chunks: Array[Dictionary], chunks_by_id: Dictionary +) -> bool: + var allowed_types := { + CHUNK_GLOBAL: true, + CHUNK_LOCATION: true, + CHUNK_MOBILE_GROUP: true, + CHUNK_SCHEDULER: true, + CHUNK_EVENTS: true, + } + for chunk: Dictionary in chunks: + if not allowed_types.has(String(chunk["chunk_type"])): + return false + var global_chunks := _chunks_for_type(chunks, CHUNK_GLOBAL) + var scheduler_chunks := _chunks_for_type(chunks, CHUNK_SCHEDULER) + var location_chunks := _chunks_for_type(chunks, CHUNK_LOCATION) + var group_chunks := _chunks_for_type(chunks, CHUNK_MOBILE_GROUP) + var event_chunks := _chunks_for_type(chunks, CHUNK_EVENTS) + if global_chunks.size() != 1 or scheduler_chunks.size() != 1 or event_chunks.is_empty(): + return false + if ( + String(global_chunks[0]["chunk_id"]) != GLOBAL_CHUNK_ID + or String(scheduler_chunks[0]["chunk_id"]) != SCHEDULER_CHUNK_ID + or String(global_chunks[0]["scope_id"]) != String(manifest["world_id"]) + or not String(global_chunks[0]["location_id"]).is_empty() + or not String(global_chunks[0]["route_id"]).is_empty() + or String(scheduler_chunks[0]["scope_id"]) != "regional" + or not String(scheduler_chunks[0]["location_id"]).is_empty() + or not String(scheduler_chunks[0]["route_id"]).is_empty() + ): + return false + var global_payload: Dictionary = global_chunks[0]["payload"] + if not _global_payload_is_valid(global_payload, manifest): + return false + if not _location_chunks_are_valid(location_chunks, global_payload): + return false + if not _mobile_group_chunks_are_valid(group_chunks, global_payload): + return false + if not _scheduler_chunk_is_valid(scheduler_chunks[0]): + return false + if not _event_chunks_are_valid(event_chunks, int(manifest["event_segment_size"])): + return false + for location_id: Variant in global_payload["location_ids"]: + if not chunks_by_id.has(_location_chunk_id(StringName(location_id))): + return false + for group_id: Variant in global_payload["mobile_group_ids"]: + if not chunks_by_id.has(_mobile_group_chunk_id(StringName(group_id))): + return false + return true + + +static func _global_payload_is_valid(payload: Dictionary, manifest: Dictionary) -> bool: + if not _has_exact_fields( + payload, + [ + "service_schema_version", + "world_state_schema_version", + "scheduler_schema_version", + "event_store_schema_version", + "world_id", + "next_event_id", + "location_ids", + "mobile_group_ids", + "routes", + "polities", + "diplomatic_relations", + ] + ): + return false + if ( + not payload["service_schema_version"] is int + or int(payload["service_schema_version"]) != RegionalCaravanService.SCHEMA_VERSION + or not payload["world_state_schema_version"] is int + or int(payload["world_state_schema_version"]) != RegionalWorldState.SCHEMA_VERSION + or not payload["scheduler_schema_version"] is int + or int(payload["scheduler_schema_version"]) != RegionalJobScheduler.SCHEMA_VERSION + or not payload["event_store_schema_version"] is int + or int(payload["event_store_schema_version"]) != WorldEventStore.SCHEMA_VERSION + or not payload["world_id"] is String + or String(payload["world_id"]) != String(manifest["world_id"]) + or not payload["next_event_id"] is int + or not payload["location_ids"] is Array + or not payload["mobile_group_ids"] is Array + or not payload["routes"] is Array + or not payload["polities"] is Array + or not payload["diplomatic_relations"] is Array + ): + return false + var normalized_locations: Variant = RegionalStateSupport.normalize_id_array( + payload["location_ids"] + ) + var normalized_groups: Variant = RegionalStateSupport.normalize_id_array( + payload["mobile_group_ids"] + ) + if ( + normalized_locations == null + or normalized_locations != payload["location_ids"] + or normalized_groups == null + or normalized_groups != payload["mobile_group_ids"] + ): + return false + return ( + _dictionary_records_are_canonical( + payload["routes"], "route_id", RouteStateRecord.from_dictionary + ) + and _dictionary_records_are_canonical( + payload["polities"], "polity_id", PolityStateRecord.from_dictionary + ) + and _dictionary_records_are_canonical( + payload["diplomatic_relations"], "relation_id", DiplomaticRelationRecord.from_dictionary + ) + ) + + +static func _location_chunks_are_valid( + chunks: Array[Dictionary], global_payload: Dictionary +) -> bool: + var ids: Array[String] = [] + for chunk: Dictionary in chunks: + var payload: Dictionary = chunk["payload"] + if not _has_exact_fields( + payload, + [ + "world_id", + "location_id", + "location", + "settlements", + "persons", + "population_cohorts", + ] + ): + return false + if ( + not payload["world_id"] is String + or String(payload["world_id"]) != String(global_payload["world_id"]) + or not payload["location_id"] is String + or String(payload["location_id"]) != String(chunk["scope_id"]) + or String(chunk["chunk_id"]) != _location_chunk_id(StringName(chunk["scope_id"])) + or not payload["location"] is Dictionary + or not payload["settlements"] is Array + or not payload["persons"] is Array + or not payload["population_cohorts"] is Array + ): + return false + var location := LocationStateRecord.from_dictionary(payload["location"]) + if ( + location == null + or location.to_dictionary() != payload["location"] + or location.get_location_id() != StringName(chunk["scope_id"]) + or not String(chunk["location_id"]).is_empty() + or not String(chunk["route_id"]).is_empty() + ): + return false + if not _records_belong_to_location(payload, StringName(chunk["scope_id"])): + return false + ids.append(String(chunk["scope_id"])) + ids.sort() + return ids == global_payload["location_ids"] + + +static func _mobile_group_chunks_are_valid( + chunks: Array[Dictionary], global_payload: Dictionary +) -> bool: + var ids: Array[String] = [] + for chunk: Dictionary in chunks: + var payload: Dictionary = chunk["payload"] + if not _has_exact_fields( + payload, ["world_id", "group_id", "mobile_group", "persons", "population_cohorts"] + ): + return false + if ( + not payload["world_id"] is String + or String(payload["world_id"]) != String(global_payload["world_id"]) + or not payload["group_id"] is String + or String(payload["group_id"]) != String(chunk["scope_id"]) + or String(chunk["chunk_id"]) != _mobile_group_chunk_id(StringName(chunk["scope_id"])) + or not payload["mobile_group"] is Dictionary + or not payload["persons"] is Array + or not payload["population_cohorts"] is Array + ): + return false + var group := MobileGroupStateRecord.from_dictionary(payload["mobile_group"]) + if ( + group == null + or group.to_dictionary() != payload["mobile_group"] + or group.get_group_id() != StringName(chunk["scope_id"]) + ): + return false + var expected_location := group.get_current_location_id() if group.is_at_location() else &"" + var expected_route := group.get_route_id() if group.is_on_route() else &"" + if ( + String(chunk["location_id"]) != String(expected_location) + or String(chunk["route_id"]) != String(expected_route) + ): + return false + if not _records_belong_to_group(payload, group.get_group_id()): + return false + ids.append(String(chunk["scope_id"])) + ids.sort() + return ids == global_payload["mobile_group_ids"] + + +static func _scheduler_chunk_is_valid(chunk: Dictionary) -> bool: + var payload: Dictionary = chunk["payload"] + if not _has_exact_fields(payload, ["scheduler"]) or not payload["scheduler"] is Dictionary: + return false + var scheduler := RegionalJobScheduler.from_dictionary(payload["scheduler"]) + return scheduler != null and scheduler.to_dictionary() == payload["scheduler"] + + +static func _event_chunks_are_valid(chunks: Array[Dictionary], segment_size: int) -> bool: + var all_events: Array[Dictionary] = [] + for index in range(chunks.size()): + var chunk: Dictionary = chunks[index] + var payload: Dictionary = chunk["payload"] + if ( + not _has_exact_fields(payload, ["segment_index", "events"]) + or not payload["segment_index"] is int + or int(payload["segment_index"]) != index + or String(chunk["chunk_id"]) != _event_chunk_id(index) + or String(chunk["scope_id"]) != str(index) + or not String(chunk["location_id"]).is_empty() + or not String(chunk["route_id"]).is_empty() + or not payload["events"] is Array + ): + return false + var events: Array = payload["events"] + if events.size() > segment_size: + return false + if index < chunks.size() - 1 and events.size() != segment_size: + return false + if chunks.size() > 1 and index == chunks.size() - 1 and events.is_empty(): + return false + for event: Variant in events: + if not event is Dictionary: + return false + all_events.append(event) + var store := WorldEventStore.from_dictionary( + {"schema_version": WorldEventStore.SCHEMA_VERSION, "events": all_events} + ) + return store != null and store.to_dictionary()["events"] == all_events + + +static func _chunk_record_count(chunk: Dictionary) -> int: + if not chunk.get("payload") is Dictionary or not chunk.get("chunk_type") is String: + return -1 + var payload: Dictionary = chunk["payload"] + match String(chunk["chunk_type"]): + CHUNK_GLOBAL: + if not _payload_has_array_fields( + payload, ["routes", "polities", "diplomatic_relations"] + ): + return -1 + return ( + (payload["routes"] as Array).size() + + (payload["polities"] as Array).size() + + (payload["diplomatic_relations"] as Array).size() + ) + CHUNK_LOCATION: + if ( + not payload.get("location") is Dictionary + or not _payload_has_array_fields( + payload, ["settlements", "persons", "population_cohorts"] + ) + ): + return -1 + return ( + 1 + + (payload["settlements"] as Array).size() + + (payload["persons"] as Array).size() + + (payload["population_cohorts"] as Array).size() + ) + CHUNK_MOBILE_GROUP: + if ( + not payload.get("mobile_group") is Dictionary + or not _payload_has_array_fields(payload, ["persons", "population_cohorts"]) + ): + return -1 + return ( + 1 + + (payload["persons"] as Array).size() + + (payload["population_cohorts"] as Array).size() + ) + CHUNK_SCHEDULER: + if ( + not payload.get("scheduler") is Dictionary + or not (payload["scheduler"] as Dictionary).get("jobs") is Array + ): + return -1 + return ((payload["scheduler"] as Dictionary)["jobs"] as Array).size() + CHUNK_EVENTS: + if not payload.get("events") is Array: + return -1 + return (payload["events"] as Array).size() + return -1 + + +static func _payload_has_array_fields(payload: Dictionary, fields: Array[String]) -> bool: + for field: String in fields: + if not payload.get(field) is Array: + return false + return true + + +static func _records_belong_to_location(payload: Dictionary, location_id: StringName) -> bool: + if ( + not _dictionary_records_are_canonical( + payload["settlements"], "settlement_id", SettlementStateRecord.from_dictionary + ) + or not _dictionary_records_are_canonical( + payload["persons"], "person_id", PersonStateRecord.from_dictionary + ) + or not _dictionary_records_are_canonical( + payload["population_cohorts"], "cohort_id", PopulationCohortRecord.from_dictionary + ) + ): + return false + for raw_record: Variant in payload["settlements"]: + var settlement := SettlementStateRecord.from_dictionary(raw_record) + if settlement == null or settlement.get_location_id() != location_id: + return false + for raw_record: Variant in payload["persons"]: + var person := PersonStateRecord.from_dictionary(raw_record) + if ( + person == null + or not person.is_at_location() + or person.get_location_address().get_location_id() != location_id + ): + return false + for raw_record: Variant in payload["population_cohorts"]: + var cohort := PopulationCohortRecord.from_dictionary(raw_record) + if cohort == null or not cohort.is_at_location() or cohort.get_location_id() != location_id: + return false + return true + + +static func _records_belong_to_group(payload: Dictionary, group_id: StringName) -> bool: + if ( + not _dictionary_records_are_canonical( + payload["persons"], "person_id", PersonStateRecord.from_dictionary + ) + or not _dictionary_records_are_canonical( + payload["population_cohorts"], "cohort_id", PopulationCohortRecord.from_dictionary + ) + ): + return false + for raw_record: Variant in payload["persons"]: + var person := PersonStateRecord.from_dictionary(raw_record) + if ( + person == null + or not person.is_in_mobile_group() + or person.get_mobile_group_id() != group_id + ): + return false + for raw_record: Variant in payload["population_cohorts"]: + var cohort := PopulationCohortRecord.from_dictionary(raw_record) + if ( + cohort == null + or not cohort.is_in_mobile_group() + or cohort.get_mobile_group_id() != group_id + ): + return false + return true + + +static func _dictionary_records_are_canonical( + records: Array, id_field: String, parser: Callable +) -> bool: + var previous_id := "" + for index in range(records.size()): + var raw_record: Variant = records[index] + if not raw_record is Dictionary: + return false + var record: Dictionary = raw_record + if not record.get(id_field) is String or String(record[id_field]).is_empty(): + return false + var parsed_record: Variant = parser.call(record) + if ( + parsed_record == null + or not parsed_record.has_method("to_dictionary") + or parsed_record.call("to_dictionary") != record + ): + return false + var record_id := String(record[id_field]) + if index > 0 and previous_id >= record_id: + return false + previous_id = record_id + return true + + +static func _has_exact_fields(value: Dictionary, fields: Array) -> bool: + return value.size() == fields.size() and value.has_all(fields) + + +static func _chunks_for_type(chunks: Array[Dictionary], chunk_type: String) -> Array[Dictionary]: + var filtered: Array[Dictionary] = [] + for chunk: Dictionary in chunks: + if String(chunk["chunk_type"]) == chunk_type: + filtered.append(chunk) + return filtered + + +static func _records_to_dictionaries(records: Array) -> Array[Dictionary]: + var serialized: Array[Dictionary] = [] + for record: Variant in records: + var record_data: Dictionary = record.call("to_dictionary") + serialized.append(record_data) + return serialized + + +static func _append_dictionary_records(destination: Array[Dictionary], source: Array) -> void: + for raw_record: Variant in source: + if raw_record is Dictionary: + destination.append((raw_record as Dictionary).duplicate(true)) + + +static func _chunk_precedes(first: Dictionary, second: Dictionary) -> bool: + return _chunk_sort_key(first) < _chunk_sort_key(second) + + +static func _descriptor_precedes(first: Dictionary, second: Dictionary) -> bool: + return _chunk_sort_key(first) < _chunk_sort_key(second) + + +static func _chunk_sort_key(value: Dictionary) -> String: + var chunk_type := String(value.get("chunk_type", "")) + var rank := int( + ( + { + CHUNK_GLOBAL: 0, + CHUNK_LOCATION: 1, + CHUNK_MOBILE_GROUP: 2, + CHUNK_SCHEDULER: 3, + CHUNK_EVENTS: 4, + } + . get(chunk_type, 99) + ) + ) + return "%02d:%s" % [rank, String(value.get("chunk_id", ""))] + + +static func _location_chunk_id(location_id: StringName) -> String: + return "location:%s" % location_id + + +static func _mobile_group_chunk_id(group_id: StringName) -> String: + return "mobile_group:%s" % group_id + + +static func _event_chunk_id(segment_index: int) -> String: + return "events:%08d" % segment_index + + +static func _is_primitive_tree(value: Variant) -> bool: + match typeof(value): + TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING: + return true + TYPE_FLOAT: + return is_finite(float(value)) + TYPE_ARRAY: + for item: Variant in value: + if not _is_primitive_tree(item): + return false + return true + TYPE_DICTIONARY: + for key: Variant in value: + if not key is String or not _is_primitive_tree(value[key]): + return false + return true + return false diff --git a/simulation/regional/RegionalChunkedPersistence.gd.uid b/simulation/regional/RegionalChunkedPersistence.gd.uid new file mode 100644 index 0000000..575c966 --- /dev/null +++ b/simulation/regional/RegionalChunkedPersistence.gd.uid @@ -0,0 +1 @@ +uid://chuvpslmcnw66 diff --git a/tests/unit/test_regional_chunked_persistence.gd b/tests/unit/test_regional_chunked_persistence.gd new file mode 100644 index 0000000..744005e --- /dev/null +++ b/tests/unit/test_regional_chunked_persistence.gd @@ -0,0 +1,281 @@ +extends GutTest + +const WORLD_ID := &"regional_world" +const ORIGIN_LOCATION_ID := &"location_origin" +const DESTINATION_LOCATION_ID := &"location_destination" +const ORIGIN_SETTLEMENT_ID := &"settlement_origin" +const DESTINATION_SETTLEMENT_ID := &"settlement_destination" +const ROUTE_ID := &"route_trade" +const DEPARTURE_TICK := 10 +const ARRIVAL_TICK := 15 + + +func test_mid_route_round_trip_preserves_exact_envelope_and_continuation() -> void: + var service := _fixture_service(1) + assert_true(_depart_group(service, 0)) + var expected_envelope := service.to_dictionary() + var archive := RegionalChunkedPersistence.capture(service, 1) + + assert_not_null(archive) + assert_eq(archive.get_manifest()["envelope_checksum"], service.checksum()) + assert_eq(archive.get_chunk_ids_for_type(RegionalChunkedPersistence.CHUNK_EVENTS).size(), 2) + var restored := archive.restore_service() + assert_not_null(restored) + assert_eq(restored.to_dictionary(), expected_envelope) + assert_eq(restored.checksum(), service.checksum()) + + assert_eq(service.process_due(ARRIVAL_TICK, 1).size(), 1) + assert_eq(restored.process_due(ARRIVAL_TICK, 1).size(), 1) + assert_eq(restored.to_dictionary(), service.to_dictionary()) + + +func test_active_location_metadata_does_not_hydrate_or_discard_unloaded_authority() -> void: + var service := _fixture_service(2) + var archive := RegionalChunkedPersistence.capture(service) + assert_not_null(archive) + var archive_before := archive.to_dictionary() + + var partial := archive.load_active_location_metadata(ORIGIN_LOCATION_ID) + assert_false(partial.is_empty()) + assert_eq( + partial["loaded_chunk_ids"], + [ + RegionalChunkedPersistence.GLOBAL_CHUNK_ID, + "location:%s" % ORIGIN_LOCATION_ID, + ] + ) + assert_eq(partial["mobile_group_refs"].size(), 2) + assert_true("scheduler:regional" in partial["unloaded_chunk_ids"]) + assert_true("mobile_group:group_00" in partial["unloaded_chunk_ids"]) + assert_false((partial["global_index"] as Dictionary).has("mobile_groups")) + + (partial["active_location"] as Dictionary)["settlements"] = [] + (partial["unloaded_chunk_ids"] as Array).clear() + assert_eq(archive.to_dictionary(), archive_before) + assert_eq(archive.restore_service().checksum(), service.checksum()) + + +func test_missing_duplicate_tampered_and_noncanonical_chunks_are_rejected() -> void: + var archive := RegionalChunkedPersistence.capture(_fixture_service(1)) + assert_not_null(archive) + + var missing := archive.to_dictionary() + (missing["chunks"] as Array).pop_back() + assert_null(RegionalChunkedPersistence.from_dictionary(missing)) + + var duplicate := archive.to_dictionary() + var duplicated_chunk: Dictionary = (duplicate["chunks"] as Array)[-1].duplicate(true) + var duplicated_descriptor: Dictionary = ( + ((duplicate["manifest"] as Dictionary)["chunks"] as Array)[-1].duplicate(true) + ) + (duplicate["chunks"] as Array).append(duplicated_chunk) + ((duplicate["manifest"] as Dictionary)["chunks"] as Array).append(duplicated_descriptor) + (duplicate["manifest"] as Dictionary)["chunk_count"] = ((duplicate["chunks"] as Array).size()) + assert_null(RegionalChunkedPersistence.from_dictionary(duplicate)) + + var tampered_payload := archive.to_dictionary() + var origin_chunk := _find_chunk(tampered_payload, "location:%s" % ORIGIN_LOCATION_ID) + (origin_chunk["payload"] as Dictionary)["location_id"] = "location_forged" + assert_null(RegionalChunkedPersistence.from_dictionary(tampered_payload)) + + var tampered_checksum := archive.to_dictionary() + var descriptors: Array = (tampered_checksum["manifest"] as Dictionary)["chunks"] + descriptors[0]["checksum"] = "forged" + assert_null(RegionalChunkedPersistence.from_dictionary(tampered_checksum)) + + var self_consistent_but_noncanonical := archive.to_dictionary() + var noncanonical_chunk := _find_chunk( + self_consistent_but_noncanonical, "location:%s" % ORIGIN_LOCATION_ID + ) + (noncanonical_chunk["payload"] as Dictionary)["ignored_authority"] = true + _refresh_chunk_descriptor(self_consistent_but_noncanonical, noncanonical_chunk) + assert_null(RegionalChunkedPersistence.from_dictionary(self_consistent_but_noncanonical)) + + var noncanonical := archive.to_dictionary() + var chunks: Array = noncanonical["chunks"] + var noncanonical_descriptors: Array = (noncanonical["manifest"] as Dictionary)["chunks"] + var first_chunk: Variant = chunks[0] + chunks[0] = chunks[1] + chunks[1] = first_chunk + var first_descriptor: Variant = noncanonical_descriptors[0] + noncanonical_descriptors[0] = noncanonical_descriptors[1] + noncanonical_descriptors[1] = first_descriptor + assert_null(RegionalChunkedPersistence.from_dictionary(noncanonical)) + + +func test_payload_contract_rejects_nodes_resources_and_node_paths() -> void: + var archive := RegionalChunkedPersistence.capture(_fixture_service(1)) + assert_not_null(archive) + assert_true(_contains_only_primitive_values(archive.to_dictionary())) + + var node_bundle := archive.to_dictionary() + var node := Node.new() + (_first_chunk_payload(node_bundle))["forbidden"] = node + assert_null(RegionalChunkedPersistence.from_dictionary(node_bundle)) + node.free() + + var resource_bundle := archive.to_dictionary() + (_first_chunk_payload(resource_bundle))["forbidden"] = Resource.new() + assert_null(RegionalChunkedPersistence.from_dictionary(resource_bundle)) + + var path_bundle := archive.to_dictionary() + (_first_chunk_payload(path_bundle))["forbidden"] = NodePath("World/Caravan") + assert_null(RegionalChunkedPersistence.from_dictionary(path_bundle)) + + +func test_twenty_caravans_remain_structurally_chunked_and_bounded() -> void: + var service := _fixture_service(20) + for index in range(20): + assert_true(_depart_group(service, index)) + var archive := RegionalChunkedPersistence.capture(service, 8) + + assert_not_null(archive) + assert_eq( + archive.get_chunk_ids_for_type(RegionalChunkedPersistence.CHUNK_MOBILE_GROUP).size(), 20 + ) + assert_eq(archive.get_chunk_ids_for_type(RegionalChunkedPersistence.CHUNK_EVENTS).size(), 5) + assert_eq(archive.get_manifest()["chunk_count"], 29) + for event_chunk_id: String in archive.get_chunk_ids_for_type( + RegionalChunkedPersistence.CHUNK_EVENTS + ): + assert_true( + ( + (archive.get_chunk(StringName(event_chunk_id))["payload"]["events"] as Array).size() + <= 8 + ) + ) + var encoded_size := JSON.stringify(archive.to_dictionary()).to_utf8_buffer().size() + assert_lt(encoded_size, 250_000) + assert_eq(archive.restore_service().checksum(), service.checksum()) + + +func _fixture_service(group_count: int) -> RegionalCaravanService: + var origin_location := LocationStateRecord.create( + ORIGIN_LOCATION_ID, WORLD_ID, &"settlement", "Origin", _address(ORIGIN_LOCATION_ID) + ) + var destination_location := LocationStateRecord.create( + DESTINATION_LOCATION_ID, + WORLD_ID, + &"settlement", + "Destination", + _address(DESTINATION_LOCATION_ID) + ) + var origin := SettlementStateRecord.create( + ORIGIN_SETTLEMENT_ID, + ORIGIN_LOCATION_ID, + "Origin", + &"polity_origin", + 1, + &"founders_origin", + 0, + {&"food": float(group_count * 2 + 10)} + ) + var destination := SettlementStateRecord.create( + DESTINATION_SETTLEMENT_ID, + DESTINATION_LOCATION_ID, + "Destination", + &"polity_destination", + 1, + &"founders_destination", + 0, + {&"food": 4.0} + ) + var route := RouteStateRecord.create(ROUTE_ID, ORIGIN_LOCATION_ID, DESTINATION_LOCATION_ID, 5) + var groups: Array[MobileGroupStateRecord] = [] + for index in range(group_count): + groups.append( + MobileGroupStateRecord.create_at_location( + _group_id(index), + &"caravan", + "Caravan %02d" % index, + &"polity_origin", + _address(ORIGIN_LOCATION_ID), + [], + {}, + {}, + 2.0 + ) + ) + var origin_polity := PolityStateRecord.create( + &"polity_origin", "Origin polity", ORIGIN_LOCATION_ID, [ORIGIN_SETTLEMENT_ID] + ) + var destination_polity := PolityStateRecord.create( + &"polity_destination", + "Destination polity", + DESTINATION_LOCATION_ID, + [DESTINATION_SETTLEMENT_ID] + ) + var locations: Array[LocationStateRecord] = [destination_location, origin_location] + var settlements: Array[SettlementStateRecord] = [destination, origin] + var routes: Array[RouteStateRecord] = [route] + var persons: Array[PersonStateRecord] = [] + var cohorts: Array[PopulationCohortRecord] = [] + var polities: Array[PolityStateRecord] = [destination_polity, origin_polity] + var relations: Array[DiplomaticRelationRecord] = [] + var world_state := RegionalWorldState.create( + WORLD_ID, locations, settlements, routes, groups, persons, cohorts, polities, relations + ) + return RegionalCaravanService.create( + world_state, RegionalJobScheduler.new(), WorldEventStore.new() + ) + + +func _depart_group(service: RegionalCaravanService, index: int) -> bool: + return service.depart( + _group_id(index), + ORIGIN_SETTLEMENT_ID, + DESTINATION_SETTLEMENT_ID, + ROUTE_ID, + {&"food": 1.0}, + DEPARTURE_TICK + ) + + +func _group_id(index: int) -> StringName: + return StringName("group_%02d" % index) + + +func _address(location_id: StringName) -> SpatialAddress: + return SpatialAddress.create(WORLD_ID, location_id, Vector3.ZERO) + + +func _find_chunk(bundle: Dictionary, chunk_id: String) -> Dictionary: + for raw_chunk: Variant in bundle["chunks"]: + if raw_chunk is Dictionary and String(raw_chunk.get("chunk_id", "")) == chunk_id: + return raw_chunk + return {} + + +func _first_chunk_payload(bundle: Dictionary) -> Dictionary: + return ((bundle["chunks"] as Array)[0] as Dictionary)["payload"] + + +func _refresh_chunk_descriptor(bundle: Dictionary, chunk: Dictionary) -> void: + var serialized := JSON.stringify(chunk) + for raw_descriptor: Variant in (bundle["manifest"] as Dictionary)["chunks"]: + if ( + raw_descriptor is Dictionary + and String(raw_descriptor["chunk_id"]) == String(chunk["chunk_id"]) + ): + raw_descriptor["byte_size"] = serialized.to_utf8_buffer().size() + raw_descriptor["checksum"] = serialized.sha256_text() + return + + +func _contains_only_primitive_values(value: Variant) -> bool: + match typeof(value): + TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING: + return true + TYPE_FLOAT: + return is_finite(float(value)) + TYPE_ARRAY: + for item: Variant in value: + if not _contains_only_primitive_values(item): + return false + return true + TYPE_DICTIONARY: + for key: Variant in value: + if not key is String or not _contains_only_primitive_values(value[key]): + return false + return true + return false diff --git a/tests/unit/test_regional_chunked_persistence.gd.uid b/tests/unit/test_regional_chunked_persistence.gd.uid new file mode 100644 index 0000000..1ac4f45 --- /dev/null +++ b/tests/unit/test_regional_chunked_persistence.gd.uid @@ -0,0 +1 @@ +uid://bmxqfpf0jl4um