feat: chunk regional simulation state

simulation/regional/RegionalChunkedPersistence.gd

@@ -0,0 +1,947 @@
+class_name RegionalChunkedPersistence
+extends RefCounted
+
+const SCHEMA_VERSION := 1
+const FORMAT_ID := "regional_caravan_chunks"
+const DEFAULT_EVENT_SEGMENT_SIZE := 64
+const MAX_EVENT_SEGMENT_SIZE := 1024
+const CHUNK_GLOBAL := "global_index"
+const CHUNK_LOCATION := "location"
+const CHUNK_MOBILE_GROUP := "mobile_group"
+const CHUNK_SCHEDULER := "scheduler"
+const CHUNK_EVENTS := "event_segment"
+const GLOBAL_CHUNK_ID := "global:index"
+const SCHEDULER_CHUNK_ID := "scheduler:regional"
+
+var _manifest: Dictionary = {}
+var _chunks_by_id: Dictionary = {}
+var _ordered_chunk_ids: Array[String] = []
+
+
+static func capture(
+	service: RegionalCaravanService, event_segment_size: int = DEFAULT_EVENT_SEGMENT_SIZE
+) -> RegionalChunkedPersistence:
+	if (
+		service == null
+		or not service.is_valid()
+		or event_segment_size <= 0
+		or event_segment_size > MAX_EVENT_SEGMENT_SIZE
+	):
+		return null
+	var envelope := service.to_dictionary()
+	if not _is_primitive_tree(envelope):
+		return null
+	var chunks := _capture_chunks(service, event_segment_size)
+	if chunks.is_empty():
+		return null
+	chunks.sort_custom(_chunk_precedes)
+	var descriptors: Array[Dictionary] = []
+	for chunk: Dictionary in chunks:
+		descriptors.append(_descriptor_for_chunk(chunk))
+	var manifest := {
+		"schema_version": SCHEMA_VERSION,
+		"format_id": FORMAT_ID,
+		"world_id": String(service.get_world_state().get_world_id()),
+		"envelope_checksum": service.checksum(),
+		"event_segment_size": event_segment_size,
+		"chunk_count": chunks.size(),
+		"chunks": descriptors,
+	}
+	var bundle := {
+		"schema_version": SCHEMA_VERSION,
+		"manifest": manifest,
+		"chunks": chunks,
+	}
+	return from_dictionary(bundle)
+
+
+static func from_dictionary(bundle: Dictionary) -> RegionalChunkedPersistence:
+	if not _is_primitive_tree(bundle):
+		return null
+	if (
+		not _has_exact_fields(bundle, ["schema_version", "manifest", "chunks"])
+		or not bundle.get("schema_version") is int
+		or int(bundle["schema_version"]) != SCHEMA_VERSION
+		or not bundle.get("manifest") is Dictionary
+		or not bundle.get("chunks") is Array
+	):
+		return null
+	var manifest: Dictionary = bundle["manifest"]
+	var raw_chunks: Array = bundle["chunks"]
+	if not _manifest_header_is_valid(manifest, raw_chunks.size()):
+		return null
+	var descriptors: Array = manifest["chunks"]
+	var chunks: Array[Dictionary] = []
+	for raw_chunk: Variant in raw_chunks:
+		if not raw_chunk is Dictionary:
+			return null
+		chunks.append(raw_chunk)
+	if not _canonical_order_is_valid(descriptors, chunks):
+		return null
+	var chunks_by_id: Dictionary = {}
+	var descriptors_by_id: Dictionary = {}
+	for index in range(chunks.size()):
+		var chunk: Dictionary = chunks[index]
+		var descriptor: Dictionary = descriptors[index]
+		if not _chunk_and_descriptor_are_valid(chunk, descriptor):
+			return null
+		var chunk_id := String(chunk["chunk_id"])
+		if chunks_by_id.has(chunk_id) or descriptors_by_id.has(chunk_id):
+			return null
+		chunks_by_id[chunk_id] = chunk.duplicate(true)
+		descriptors_by_id[chunk_id] = descriptor.duplicate(true)
+	if not _chunk_topology_is_valid(manifest, chunks, chunks_by_id):
+		return null
+	var archive := RegionalChunkedPersistence.new()
+	archive._manifest = manifest.duplicate(true)
+	archive._chunks_by_id = chunks_by_id
+	for chunk: Dictionary in chunks:
+		archive._ordered_chunk_ids.append(String(chunk["chunk_id"]))
+	var restored := archive.restore_service()
+	if restored == null or restored.checksum() != String(manifest["envelope_checksum"]):
+		return null
+	return archive
+
+
+func get_manifest() -> Dictionary:
+	return _manifest.duplicate(true)
+
+
+func get_chunk_ids() -> Array[String]:
+	return _ordered_chunk_ids.duplicate()
+
+
+func get_chunk(chunk_id: StringName) -> Dictionary:
+	var chunk: Dictionary = _chunks_by_id.get(String(chunk_id), {} as Dictionary)
+	return chunk.duplicate(true)
+
+
+func get_chunk_ids_for_type(chunk_type: StringName) -> Array[String]:
+	var ids: Array[String] = []
+	for chunk_id: String in _ordered_chunk_ids:
+		var chunk: Dictionary = _chunks_by_id[chunk_id]
+		if String(chunk["chunk_type"]) == String(chunk_type):
+			ids.append(chunk_id)
+	return ids
+
+
+func to_dictionary() -> Dictionary:
+	var chunks: Array[Dictionary] = []
+	for chunk_id: String in _ordered_chunk_ids:
+		var chunk: Dictionary = _chunks_by_id[chunk_id]
+		chunks.append(chunk.duplicate(true))
+	return {
+		"schema_version": SCHEMA_VERSION,
+		"manifest": _manifest.duplicate(true),
+		"chunks": chunks,
+	}
+
+
+func checksum() -> String:
+	return JSON.stringify(to_dictionary()).sha256_text()
+
+
+func restore_service() -> RegionalCaravanService:
+	var global_chunk := get_chunk(StringName(GLOBAL_CHUNK_ID))
+	var scheduler_chunk := get_chunk(StringName(SCHEDULER_CHUNK_ID))
+	if global_chunk.is_empty() or scheduler_chunk.is_empty():
+		return null
+	var global_payload: Dictionary = global_chunk["payload"]
+	var scheduler_payload: Dictionary = scheduler_chunk["payload"]
+	var locations: Array[Dictionary] = []
+	var settlements: Array[Dictionary] = []
+	var mobile_groups: Array[Dictionary] = []
+	var persons: Array[Dictionary] = []
+	var cohorts: Array[Dictionary] = []
+	var event_records: Array[Dictionary] = []
+	for chunk_id: String in _ordered_chunk_ids:
+		var chunk: Dictionary = _chunks_by_id[chunk_id]
+		var payload: Dictionary = chunk["payload"]
+		match String(chunk["chunk_type"]):
+			CHUNK_LOCATION:
+				locations.append(payload["location"].duplicate(true))
+				_append_dictionary_records(settlements, payload["settlements"])
+				_append_dictionary_records(persons, payload["persons"])
+				_append_dictionary_records(cohorts, payload["population_cohorts"])
+			CHUNK_MOBILE_GROUP:
+				mobile_groups.append(payload["mobile_group"].duplicate(true))
+				_append_dictionary_records(persons, payload["persons"])
+				_append_dictionary_records(cohorts, payload["population_cohorts"])
+			CHUNK_EVENTS:
+				_append_dictionary_records(event_records, payload["events"])
+	var world_state := {
+		"schema_version": int(global_payload["world_state_schema_version"]),
+		"world_id": String(global_payload["world_id"]),
+		"locations": locations,
+		"settlements": settlements,
+		"routes": global_payload["routes"].duplicate(true),
+		"mobile_groups": mobile_groups,
+		"persons": persons,
+		"population_cohorts": cohorts,
+		"polities": global_payload["polities"].duplicate(true),
+		"diplomatic_relations": global_payload["diplomatic_relations"].duplicate(true),
+	}
+	var envelope := {
+		"schema_version": int(global_payload["service_schema_version"]),
+		"next_event_id": int(global_payload["next_event_id"]),
+		"world_state": world_state,
+		"scheduler": scheduler_payload["scheduler"].duplicate(true),
+		"event_store":
+		{
+			"schema_version": int(global_payload["event_store_schema_version"]),
+			"events": event_records,
+		},
+	}
+	var service := RegionalCaravanService.from_dictionary(envelope)
+	if service == null or service.checksum() != String(_manifest["envelope_checksum"]):
+		return null
+	return service
+
+
+func load_active_location_metadata(location_id: StringName) -> Dictionary:
+	var location_chunk_id := _location_chunk_id(location_id)
+	var location_chunk := get_chunk(StringName(location_chunk_id))
+	var global_chunk := get_chunk(StringName(GLOBAL_CHUNK_ID))
+	if location_chunk.is_empty() or global_chunk.is_empty():
+		return {}
+	var loaded_chunk_ids: Array[String] = [GLOBAL_CHUNK_ID, location_chunk_id]
+	var unloaded_chunk_ids: Array[String] = []
+	var mobile_group_refs: Array[Dictionary] = []
+	var descriptors: Array = _manifest["chunks"]
+	for descriptor: Dictionary in descriptors:
+		var chunk_id := String(descriptor["chunk_id"])
+		if chunk_id not in loaded_chunk_ids:
+			unloaded_chunk_ids.append(chunk_id)
+		if (
+			String(descriptor["chunk_type"]) == CHUNK_MOBILE_GROUP
+			and String(descriptor["location_id"]) == String(location_id)
+		):
+			(
+				mobile_group_refs
+				. append(
+					{
+						"group_id": String(descriptor["scope_id"]),
+						"chunk_id": chunk_id,
+					}
+				)
+			)
+	return {
+		"schema_version": SCHEMA_VERSION,
+		"world_id": String(_manifest["world_id"]),
+		"active_location_id": String(location_id),
+		"loaded_chunk_ids": loaded_chunk_ids,
+		"unloaded_chunk_ids": unloaded_chunk_ids,
+		"global_index": (global_chunk["payload"] as Dictionary).duplicate(true),
+		"active_location": (location_chunk["payload"] as Dictionary).duplicate(true),
+		"mobile_group_refs": mobile_group_refs,
+		"envelope_checksum": String(_manifest["envelope_checksum"]),
+	}
+
+
+static func _capture_chunks(
+	service: RegionalCaravanService, event_segment_size: int
+) -> Array[Dictionary]:
+	var chunks: Array[Dictionary] = []
+	var state := service.get_world_state()
+	var location_ids: Array[String] = []
+	for location: LocationStateRecord in state.get_all_locations():
+		location_ids.append(String(location.get_location_id()))
+	var mobile_group_ids: Array[String] = []
+	for group: MobileGroupStateRecord in state.get_all_mobile_groups():
+		mobile_group_ids.append(String(group.get_group_id()))
+	var global_payload := {
+		"service_schema_version": RegionalCaravanService.SCHEMA_VERSION,
+		"world_state_schema_version": RegionalWorldState.SCHEMA_VERSION,
+		"scheduler_schema_version": RegionalJobScheduler.SCHEMA_VERSION,
+		"event_store_schema_version": WorldEventStore.SCHEMA_VERSION,
+		"world_id": String(state.get_world_id()),
+		"next_event_id": service.get_next_event_id(),
+		"location_ids": location_ids,
+		"mobile_group_ids": mobile_group_ids,
+		"routes": _records_to_dictionaries(state.get_all_routes()),
+		"polities": _records_to_dictionaries(state.get_all_polities()),
+		"diplomatic_relations": _records_to_dictionaries(state.get_all_diplomatic_relations()),
+	}
+	chunks.append(_make_chunk(GLOBAL_CHUNK_ID, CHUNK_GLOBAL, state.get_world_id(), global_payload))
+	for location: LocationStateRecord in state.get_all_locations():
+		var location_id := location.get_location_id()
+		var payload := {
+			"world_id": String(state.get_world_id()),
+			"location_id": String(location_id),
+			"location": location.to_dictionary(),
+			"settlements": _records_to_dictionaries(state.get_settlements_at_location(location_id)),
+			"persons": _records_to_dictionaries(state.get_persons_at_location(location_id)),
+			"population_cohorts":
+			_records_to_dictionaries(state.get_cohorts_at_location(location_id)),
+		}
+		chunks.append(
+			_make_chunk(_location_chunk_id(location_id), CHUNK_LOCATION, location_id, payload)
+		)
+	for group: MobileGroupStateRecord in state.get_all_mobile_groups():
+		var group_id := group.get_group_id()
+		var payload := {
+			"world_id": String(state.get_world_id()),
+			"group_id": String(group_id),
+			"mobile_group": group.to_dictionary(),
+			"persons": _records_to_dictionaries(state.get_persons_in_group(group_id)),
+			"population_cohorts": _records_to_dictionaries(state.get_cohorts_in_group(group_id)),
+		}
+		var location_id := group.get_current_location_id() if group.is_at_location() else &""
+		var route_id := group.get_route_id() if group.is_on_route() else &""
+		chunks.append(
+			_make_chunk(
+				_mobile_group_chunk_id(group_id),
+				CHUNK_MOBILE_GROUP,
+				group_id,
+				payload,
+				location_id,
+				route_id
+			)
+		)
+	chunks.append(
+		_make_chunk(
+			SCHEDULER_CHUNK_ID,
+			CHUNK_SCHEDULER,
+			&"regional",
+			{"scheduler": service.get_scheduler().to_dictionary()}
+		)
+	)
+	var event_data := service.get_event_store().to_dictionary()
+	var events: Array = event_data["events"]
+	var segment_count := maxi(1, ceili(float(events.size()) / float(event_segment_size)))
+	for segment_index in range(segment_count):
+		var first := segment_index * event_segment_size
+		var last := mini(first + event_segment_size, events.size())
+		var segment_events: Array[Dictionary] = []
+		for event_index in range(first, last):
+			var event_record: Dictionary = events[event_index]
+			segment_events.append(event_record.duplicate(true))
+		(
+			chunks
+			. append(
+				_make_chunk(
+					_event_chunk_id(segment_index),
+					CHUNK_EVENTS,
+					StringName(str(segment_index)),
+					{
+						"segment_index": segment_index,
+						"events": segment_events,
+					}
+				)
+			)
+		)
+	return chunks
+
+
+static func _make_chunk(
+	chunk_id: String,
+	chunk_type: String,
+	scope_id: StringName,
+	payload: Dictionary,
+	location_id: StringName = &"",
+	route_id: StringName = &""
+) -> Dictionary:
+	return {
+		"schema_version": SCHEMA_VERSION,
+		"chunk_id": chunk_id,
+		"chunk_type": chunk_type,
+		"scope_id": String(scope_id),
+		"location_id": String(location_id),
+		"route_id": String(route_id),
+		"payload": payload.duplicate(true),
+	}
+
+
+static func _descriptor_for_chunk(chunk: Dictionary) -> Dictionary:
+	var serialized := JSON.stringify(chunk)
+	return {
+		"chunk_id": String(chunk["chunk_id"]),
+		"chunk_type": String(chunk["chunk_type"]),
+		"scope_id": String(chunk["scope_id"]),
+		"location_id": String(chunk["location_id"]),
+		"route_id": String(chunk["route_id"]),
+		"record_count": _chunk_record_count(chunk),
+		"byte_size": serialized.to_utf8_buffer().size(),
+		"checksum": serialized.sha256_text(),
+	}
+
+
+static func _manifest_header_is_valid(manifest: Dictionary, raw_chunk_count: int) -> bool:
+	if not _has_exact_fields(
+		manifest,
+		[
+			"schema_version",
+			"format_id",
+			"world_id",
+			"envelope_checksum",
+			"event_segment_size",
+			"chunk_count",
+			"chunks",
+		]
+	):
+		return false
+	return (
+		manifest["schema_version"] is int
+		and int(manifest["schema_version"]) == SCHEMA_VERSION
+		and manifest["format_id"] is String
+		and String(manifest["format_id"]) == FORMAT_ID
+		and manifest["world_id"] is String
+		and not String(manifest["world_id"]).is_empty()
+		and manifest["envelope_checksum"] is String
+		and not String(manifest["envelope_checksum"]).is_empty()
+		and manifest["event_segment_size"] is int
+		and int(manifest["event_segment_size"]) > 0
+		and int(manifest["event_segment_size"]) <= MAX_EVENT_SEGMENT_SIZE
+		and manifest["chunk_count"] is int
+		and int(manifest["chunk_count"]) == raw_chunk_count
+		and manifest["chunks"] is Array
+		and (manifest["chunks"] as Array).size() == raw_chunk_count
+	)
+
+
+static func _canonical_order_is_valid(descriptors: Array, chunks: Array[Dictionary]) -> bool:
+	var typed_descriptors: Array[Dictionary] = []
+	for raw_descriptor: Variant in descriptors:
+		if not raw_descriptor is Dictionary:
+			return false
+		typed_descriptors.append(raw_descriptor)
+	var sorted_chunks := chunks.duplicate(true)
+	sorted_chunks.sort_custom(_chunk_precedes)
+	var sorted_descriptors := typed_descriptors.duplicate(true)
+	sorted_descriptors.sort_custom(_descriptor_precedes)
+	for index in range(chunks.size()):
+		if (
+			(
+				String(chunks[index].get("chunk_id", ""))
+				!= String(sorted_chunks[index].get("chunk_id", ""))
+			)
+			or (
+				String(typed_descriptors[index].get("chunk_id", ""))
+				!= String(sorted_descriptors[index].get("chunk_id", ""))
+			)
+			or (
+				String(chunks[index].get("chunk_id", ""))
+				!= String(typed_descriptors[index].get("chunk_id", ""))
+			)
+		):
+			return false
+	return true
+
+
+static func _chunk_and_descriptor_are_valid(chunk: Dictionary, descriptor: Dictionary) -> bool:
+	var chunk_fields := [
+		"schema_version", "chunk_id", "chunk_type", "scope_id", "location_id", "route_id", "payload"
+	]
+	var descriptor_fields := [
+		"chunk_id",
+		"chunk_type",
+		"scope_id",
+		"location_id",
+		"route_id",
+		"record_count",
+		"byte_size",
+		"checksum",
+	]
+	if (
+		not _has_exact_fields(chunk, chunk_fields)
+		or not _has_exact_fields(descriptor, descriptor_fields)
+	):
+		return false
+	if (
+		not chunk["schema_version"] is int
+		or int(chunk["schema_version"]) != SCHEMA_VERSION
+		or not chunk["chunk_id"] is String
+		or String(chunk["chunk_id"]).is_empty()
+		or not chunk["chunk_type"] is String
+		or not chunk["scope_id"] is String
+		or not chunk["location_id"] is String
+		or not chunk["route_id"] is String
+		or not chunk["payload"] is Dictionary
+		or not descriptor["record_count"] is int
+		or not descriptor["byte_size"] is int
+		or not descriptor["checksum"] is String
+	):
+		return false
+	for field: String in ["chunk_id", "chunk_type", "scope_id", "location_id", "route_id"]:
+		if not descriptor[field] is String or String(descriptor[field]) != String(chunk[field]):
+			return false
+	var serialized := JSON.stringify(chunk)
+	var record_count := _chunk_record_count(chunk)
+	return (
+		record_count >= 0
+		and int(descriptor["record_count"]) == record_count
+		and int(descriptor["byte_size"]) == serialized.to_utf8_buffer().size()
+		and String(descriptor["checksum"]) == serialized.sha256_text()
+	)
+
+
+static func _chunk_topology_is_valid(
+	manifest: Dictionary, chunks: Array[Dictionary], chunks_by_id: Dictionary
+) -> bool:
+	var allowed_types := {
+		CHUNK_GLOBAL: true,
+		CHUNK_LOCATION: true,
+		CHUNK_MOBILE_GROUP: true,
+		CHUNK_SCHEDULER: true,
+		CHUNK_EVENTS: true,
+	}
+	for chunk: Dictionary in chunks:
+		if not allowed_types.has(String(chunk["chunk_type"])):
+			return false
+	var global_chunks := _chunks_for_type(chunks, CHUNK_GLOBAL)
+	var scheduler_chunks := _chunks_for_type(chunks, CHUNK_SCHEDULER)
+	var location_chunks := _chunks_for_type(chunks, CHUNK_LOCATION)
+	var group_chunks := _chunks_for_type(chunks, CHUNK_MOBILE_GROUP)
+	var event_chunks := _chunks_for_type(chunks, CHUNK_EVENTS)
+	if global_chunks.size() != 1 or scheduler_chunks.size() != 1 or event_chunks.is_empty():
+		return false
+	if (
+		String(global_chunks[0]["chunk_id"]) != GLOBAL_CHUNK_ID
+		or String(scheduler_chunks[0]["chunk_id"]) != SCHEDULER_CHUNK_ID
+		or String(global_chunks[0]["scope_id"]) != String(manifest["world_id"])
+		or not String(global_chunks[0]["location_id"]).is_empty()
+		or not String(global_chunks[0]["route_id"]).is_empty()
+		or String(scheduler_chunks[0]["scope_id"]) != "regional"
+		or not String(scheduler_chunks[0]["location_id"]).is_empty()
+		or not String(scheduler_chunks[0]["route_id"]).is_empty()
+	):
+		return false
+	var global_payload: Dictionary = global_chunks[0]["payload"]
+	if not _global_payload_is_valid(global_payload, manifest):
+		return false
+	if not _location_chunks_are_valid(location_chunks, global_payload):
+		return false
+	if not _mobile_group_chunks_are_valid(group_chunks, global_payload):
+		return false
+	if not _scheduler_chunk_is_valid(scheduler_chunks[0]):
+		return false
+	if not _event_chunks_are_valid(event_chunks, int(manifest["event_segment_size"])):
+		return false
+	for location_id: Variant in global_payload["location_ids"]:
+		if not chunks_by_id.has(_location_chunk_id(StringName(location_id))):
+			return false
+	for group_id: Variant in global_payload["mobile_group_ids"]:
+		if not chunks_by_id.has(_mobile_group_chunk_id(StringName(group_id))):
+			return false
+	return true
+
+
+static func _global_payload_is_valid(payload: Dictionary, manifest: Dictionary) -> bool:
+	if not _has_exact_fields(
+		payload,
+		[
+			"service_schema_version",
+			"world_state_schema_version",
+			"scheduler_schema_version",
+			"event_store_schema_version",
+			"world_id",
+			"next_event_id",
+			"location_ids",
+			"mobile_group_ids",
+			"routes",
+			"polities",
+			"diplomatic_relations",
+		]
+	):
+		return false
+	if (
+		not payload["service_schema_version"] is int
+		or int(payload["service_schema_version"]) != RegionalCaravanService.SCHEMA_VERSION
+		or not payload["world_state_schema_version"] is int
+		or int(payload["world_state_schema_version"]) != RegionalWorldState.SCHEMA_VERSION
+		or not payload["scheduler_schema_version"] is int
+		or int(payload["scheduler_schema_version"]) != RegionalJobScheduler.SCHEMA_VERSION
+		or not payload["event_store_schema_version"] is int
+		or int(payload["event_store_schema_version"]) != WorldEventStore.SCHEMA_VERSION
+		or not payload["world_id"] is String
+		or String(payload["world_id"]) != String(manifest["world_id"])
+		or not payload["next_event_id"] is int
+		or not payload["location_ids"] is Array
+		or not payload["mobile_group_ids"] is Array
+		or not payload["routes"] is Array
+		or not payload["polities"] is Array
+		or not payload["diplomatic_relations"] is Array
+	):
+		return false
+	var normalized_locations: Variant = RegionalStateSupport.normalize_id_array(
+		payload["location_ids"]
+	)
+	var normalized_groups: Variant = RegionalStateSupport.normalize_id_array(
+		payload["mobile_group_ids"]
+	)
+	if (
+		normalized_locations == null
+		or normalized_locations != payload["location_ids"]
+		or normalized_groups == null
+		or normalized_groups != payload["mobile_group_ids"]
+	):
+		return false
+	return (
+		_dictionary_records_are_canonical(
+			payload["routes"], "route_id", RouteStateRecord.from_dictionary
+		)
+		and _dictionary_records_are_canonical(
+			payload["polities"], "polity_id", PolityStateRecord.from_dictionary
+		)
+		and _dictionary_records_are_canonical(
+			payload["diplomatic_relations"], "relation_id", DiplomaticRelationRecord.from_dictionary
+		)
+	)
+
+
+static func _location_chunks_are_valid(
+	chunks: Array[Dictionary], global_payload: Dictionary
+) -> bool:
+	var ids: Array[String] = []
+	for chunk: Dictionary in chunks:
+		var payload: Dictionary = chunk["payload"]
+		if not _has_exact_fields(
+			payload,
+			[
+				"world_id",
+				"location_id",
+				"location",
+				"settlements",
+				"persons",
+				"population_cohorts",
+			]
+		):
+			return false
+		if (
+			not payload["world_id"] is String
+			or String(payload["world_id"]) != String(global_payload["world_id"])
+			or not payload["location_id"] is String
+			or String(payload["location_id"]) != String(chunk["scope_id"])
+			or String(chunk["chunk_id"]) != _location_chunk_id(StringName(chunk["scope_id"]))
+			or not payload["location"] is Dictionary
+			or not payload["settlements"] is Array
+			or not payload["persons"] is Array
+			or not payload["population_cohorts"] is Array
+		):
+			return false
+		var location := LocationStateRecord.from_dictionary(payload["location"])
+		if (
+			location == null
+			or location.to_dictionary() != payload["location"]
+			or location.get_location_id() != StringName(chunk["scope_id"])
+			or not String(chunk["location_id"]).is_empty()
+			or not String(chunk["route_id"]).is_empty()
+		):
+			return false
+		if not _records_belong_to_location(payload, StringName(chunk["scope_id"])):
+			return false
+		ids.append(String(chunk["scope_id"]))
+	ids.sort()
+	return ids == global_payload["location_ids"]
+
+
+static func _mobile_group_chunks_are_valid(
+	chunks: Array[Dictionary], global_payload: Dictionary
+) -> bool:
+	var ids: Array[String] = []
+	for chunk: Dictionary in chunks:
+		var payload: Dictionary = chunk["payload"]
+		if not _has_exact_fields(
+			payload, ["world_id", "group_id", "mobile_group", "persons", "population_cohorts"]
+		):
+			return false
+		if (
+			not payload["world_id"] is String
+			or String(payload["world_id"]) != String(global_payload["world_id"])
+			or not payload["group_id"] is String
+			or String(payload["group_id"]) != String(chunk["scope_id"])
+			or String(chunk["chunk_id"]) != _mobile_group_chunk_id(StringName(chunk["scope_id"]))
+			or not payload["mobile_group"] is Dictionary
+			or not payload["persons"] is Array
+			or not payload["population_cohorts"] is Array
+		):
+			return false
+		var group := MobileGroupStateRecord.from_dictionary(payload["mobile_group"])
+		if (
+			group == null
+			or group.to_dictionary() != payload["mobile_group"]
+			or group.get_group_id() != StringName(chunk["scope_id"])
+		):
+			return false
+		var expected_location := group.get_current_location_id() if group.is_at_location() else &""
+		var expected_route := group.get_route_id() if group.is_on_route() else &""
+		if (
+			String(chunk["location_id"]) != String(expected_location)
+			or String(chunk["route_id"]) != String(expected_route)
+		):
+			return false
+		if not _records_belong_to_group(payload, group.get_group_id()):
+			return false
+		ids.append(String(chunk["scope_id"]))
+	ids.sort()
+	return ids == global_payload["mobile_group_ids"]
+
+
+static func _scheduler_chunk_is_valid(chunk: Dictionary) -> bool:
+	var payload: Dictionary = chunk["payload"]
+	if not _has_exact_fields(payload, ["scheduler"]) or not payload["scheduler"] is Dictionary:
+		return false
+	var scheduler := RegionalJobScheduler.from_dictionary(payload["scheduler"])
+	return scheduler != null and scheduler.to_dictionary() == payload["scheduler"]
+
+
+static func _event_chunks_are_valid(chunks: Array[Dictionary], segment_size: int) -> bool:
+	var all_events: Array[Dictionary] = []
+	for index in range(chunks.size()):
+		var chunk: Dictionary = chunks[index]
+		var payload: Dictionary = chunk["payload"]
+		if (
+			not _has_exact_fields(payload, ["segment_index", "events"])
+			or not payload["segment_index"] is int
+			or int(payload["segment_index"]) != index
+			or String(chunk["chunk_id"]) != _event_chunk_id(index)
+			or String(chunk["scope_id"]) != str(index)
+			or not String(chunk["location_id"]).is_empty()
+			or not String(chunk["route_id"]).is_empty()
+			or not payload["events"] is Array
+		):
+			return false
+		var events: Array = payload["events"]
+		if events.size() > segment_size:
+			return false
+		if index < chunks.size() - 1 and events.size() != segment_size:
+			return false
+		if chunks.size() > 1 and index == chunks.size() - 1 and events.is_empty():
+			return false
+		for event: Variant in events:
+			if not event is Dictionary:
+				return false
+			all_events.append(event)
+	var store := WorldEventStore.from_dictionary(
+		{"schema_version": WorldEventStore.SCHEMA_VERSION, "events": all_events}
+	)
+	return store != null and store.to_dictionary()["events"] == all_events
+
+
+static func _chunk_record_count(chunk: Dictionary) -> int:
+	if not chunk.get("payload") is Dictionary or not chunk.get("chunk_type") is String:
+		return -1
+	var payload: Dictionary = chunk["payload"]
+	match String(chunk["chunk_type"]):
+		CHUNK_GLOBAL:
+			if not _payload_has_array_fields(
+				payload, ["routes", "polities", "diplomatic_relations"]
+			):
+				return -1
+			return (
+				(payload["routes"] as Array).size()
+				+ (payload["polities"] as Array).size()
+				+ (payload["diplomatic_relations"] as Array).size()
+			)
+		CHUNK_LOCATION:
+			if (
+				not payload.get("location") is Dictionary
+				or not _payload_has_array_fields(
+					payload, ["settlements", "persons", "population_cohorts"]
+				)
+			):
+				return -1
+			return (
+				1
+				+ (payload["settlements"] as Array).size()
+				+ (payload["persons"] as Array).size()
+				+ (payload["population_cohorts"] as Array).size()
+			)
+		CHUNK_MOBILE_GROUP:
+			if (
+				not payload.get("mobile_group") is Dictionary
+				or not _payload_has_array_fields(payload, ["persons", "population_cohorts"])
+			):
+				return -1
+			return (
+				1
+				+ (payload["persons"] as Array).size()
+				+ (payload["population_cohorts"] as Array).size()
+			)
+		CHUNK_SCHEDULER:
+			if (
+				not payload.get("scheduler") is Dictionary
+				or not (payload["scheduler"] as Dictionary).get("jobs") is Array
+			):
+				return -1
+			return ((payload["scheduler"] as Dictionary)["jobs"] as Array).size()
+		CHUNK_EVENTS:
+			if not payload.get("events") is Array:
+				return -1
+			return (payload["events"] as Array).size()
+	return -1
+
+
+static func _payload_has_array_fields(payload: Dictionary, fields: Array[String]) -> bool:
+	for field: String in fields:
+		if not payload.get(field) is Array:
+			return false
+	return true
+
+
+static func _records_belong_to_location(payload: Dictionary, location_id: StringName) -> bool:
+	if (
+		not _dictionary_records_are_canonical(
+			payload["settlements"], "settlement_id", SettlementStateRecord.from_dictionary
+		)
+		or not _dictionary_records_are_canonical(
+			payload["persons"], "person_id", PersonStateRecord.from_dictionary
+		)
+		or not _dictionary_records_are_canonical(
+			payload["population_cohorts"], "cohort_id", PopulationCohortRecord.from_dictionary
+		)
+	):
+		return false
+	for raw_record: Variant in payload["settlements"]:
+		var settlement := SettlementStateRecord.from_dictionary(raw_record)
+		if settlement == null or settlement.get_location_id() != location_id:
+			return false
+	for raw_record: Variant in payload["persons"]:
+		var person := PersonStateRecord.from_dictionary(raw_record)
+		if (
+			person == null
+			or not person.is_at_location()
+			or person.get_location_address().get_location_id() != location_id
+		):
+			return false
+	for raw_record: Variant in payload["population_cohorts"]:
+		var cohort := PopulationCohortRecord.from_dictionary(raw_record)
+		if cohort == null or not cohort.is_at_location() or cohort.get_location_id() != location_id:
+			return false
+	return true
+
+
+static func _records_belong_to_group(payload: Dictionary, group_id: StringName) -> bool:
+	if (
+		not _dictionary_records_are_canonical(
+			payload["persons"], "person_id", PersonStateRecord.from_dictionary
+		)
+		or not _dictionary_records_are_canonical(
+			payload["population_cohorts"], "cohort_id", PopulationCohortRecord.from_dictionary
+		)
+	):
+		return false
+	for raw_record: Variant in payload["persons"]:
+		var person := PersonStateRecord.from_dictionary(raw_record)
+		if (
+			person == null
+			or not person.is_in_mobile_group()
+			or person.get_mobile_group_id() != group_id
+		):
+			return false
+	for raw_record: Variant in payload["population_cohorts"]:
+		var cohort := PopulationCohortRecord.from_dictionary(raw_record)
+		if (
+			cohort == null
+			or not cohort.is_in_mobile_group()
+			or cohort.get_mobile_group_id() != group_id
+		):
+			return false
+	return true
+
+
+static func _dictionary_records_are_canonical(
+	records: Array, id_field: String, parser: Callable
+) -> bool:
+	var previous_id := ""
+	for index in range(records.size()):
+		var raw_record: Variant = records[index]
+		if not raw_record is Dictionary:
+			return false
+		var record: Dictionary = raw_record
+		if not record.get(id_field) is String or String(record[id_field]).is_empty():
+			return false
+		var parsed_record: Variant = parser.call(record)
+		if (
+			parsed_record == null
+			or not parsed_record.has_method("to_dictionary")
+			or parsed_record.call("to_dictionary") != record
+		):
+			return false
+		var record_id := String(record[id_field])
+		if index > 0 and previous_id >= record_id:
+			return false
+		previous_id = record_id
+	return true
+
+
+static func _has_exact_fields(value: Dictionary, fields: Array) -> bool:
+	return value.size() == fields.size() and value.has_all(fields)
+
+
+static func _chunks_for_type(chunks: Array[Dictionary], chunk_type: String) -> Array[Dictionary]:
+	var filtered: Array[Dictionary] = []
+	for chunk: Dictionary in chunks:
+		if String(chunk["chunk_type"]) == chunk_type:
+			filtered.append(chunk)
+	return filtered
+
+
+static func _records_to_dictionaries(records: Array) -> Array[Dictionary]:
+	var serialized: Array[Dictionary] = []
+	for record: Variant in records:
+		var record_data: Dictionary = record.call("to_dictionary")
+		serialized.append(record_data)
+	return serialized
+
+
+static func _append_dictionary_records(destination: Array[Dictionary], source: Array) -> void:
+	for raw_record: Variant in source:
+		if raw_record is Dictionary:
+			destination.append((raw_record as Dictionary).duplicate(true))
+
+
+static func _chunk_precedes(first: Dictionary, second: Dictionary) -> bool:
+	return _chunk_sort_key(first) < _chunk_sort_key(second)
+
+
+static func _descriptor_precedes(first: Dictionary, second: Dictionary) -> bool:
+	return _chunk_sort_key(first) < _chunk_sort_key(second)
+
+
+static func _chunk_sort_key(value: Dictionary) -> String:
+	var chunk_type := String(value.get("chunk_type", ""))
+	var rank := int(
+		(
+			{
+				CHUNK_GLOBAL: 0,
+				CHUNK_LOCATION: 1,
+				CHUNK_MOBILE_GROUP: 2,
+				CHUNK_SCHEDULER: 3,
+				CHUNK_EVENTS: 4,
+			}
+			. get(chunk_type, 99)
+		)
+	)
+	return "%02d:%s" % [rank, String(value.get("chunk_id", ""))]
+
+
+static func _location_chunk_id(location_id: StringName) -> String:
+	return "location:%s" % location_id
+
+
+static func _mobile_group_chunk_id(group_id: StringName) -> String:
+	return "mobile_group:%s" % group_id
+
+
+static func _event_chunk_id(segment_index: int) -> String:
+	return "events:%08d" % segment_index
+
+
+static func _is_primitive_tree(value: Variant) -> bool:
+	match typeof(value):
+		TYPE_NIL, TYPE_BOOL, TYPE_INT, TYPE_STRING:
+			return true
+		TYPE_FLOAT:
+			return is_finite(float(value))
+		TYPE_ARRAY:
+			for item: Variant in value:
+				if not _is_primitive_tree(item):
+					return false
+			return true
+		TYPE_DICTIONARY:
+			for key: Variant in value:
+				if not key is String or not _is_primitive_tree(value[key]):
+					return false
+			return true
+	return false

simulation/regional/RegionalChunkedPersistence.gd.uid

@@ -0,0 +1 @@
+uid://chuvpslmcnw66