feat: plan bounded event history rollups
This commit is contained in:
Rijad Zuzo
@@ -0,0 +1,435 @@
|
||||
class_name WorldEventRetentionPlanner
|
||||
extends RefCounted
|
||||
|
||||
const REASON_PINNED_FACT := "pinned_fact"
|
||||
const REASON_EXPLICIT_CAUSAL_FACT := "explicit_causal_fact"
|
||||
const REASON_CAUSAL_ANCESTOR := "causal_ancestor"
|
||||
const REASON_WITHIN_RAW_RETENTION := "within_raw_retention"
|
||||
const REASON_NOT_ROLLUP_ELIGIBLE := "not_rollup_eligible"
|
||||
const REASON_ROLLUP_PAYLOAD_INVALID := "rollup_payload_invalid"
|
||||
const REASON_ROLLED_UP_DAILY := "rolled_up_daily"
|
||||
|
||||
var _last_error := ""
|
||||
var _last_telemetry: Dictionary = {}
|
||||
|
||||
|
||||
func plan(
|
||||
source: WorldEventStore,
|
||||
current_tick: int,
|
||||
ticks_per_day: int,
|
||||
raw_retention_days: int,
|
||||
high_volume_event_types: Array,
|
||||
pinned_fact_ids: Array = [],
|
||||
causal_fact_ids: Array = []
|
||||
) -> WorldEventRetentionPlan:
|
||||
_last_error = ""
|
||||
_last_telemetry = {
|
||||
"succeeded": false,
|
||||
"input_event_count": source.size() if source != null else 0,
|
||||
"error": "",
|
||||
}
|
||||
if source == null:
|
||||
return _fail("source WorldEventStore is null")
|
||||
if current_tick < 0 or ticks_per_day <= 0 or raw_retention_days < 0:
|
||||
return _fail("retention time configuration is invalid")
|
||||
var normalized_types: Variant = _normalize_event_types(high_volume_event_types)
|
||||
var normalized_pins: Variant = _normalize_event_ids(pinned_fact_ids)
|
||||
var normalized_causal_facts: Variant = _normalize_event_ids(causal_fact_ids)
|
||||
if normalized_types == null:
|
||||
return _fail("high-volume event types are invalid or duplicated")
|
||||
if normalized_pins == null:
|
||||
return _fail("pinned fact IDs are invalid or duplicated")
|
||||
if normalized_causal_facts == null:
|
||||
return _fail("causal fact IDs are invalid or duplicated")
|
||||
for event_id: int in normalized_pins:
|
||||
if not source.has_event(event_id):
|
||||
return _fail("pinned fact ID %d is missing from the source store" % event_id)
|
||||
for event_id: int in normalized_causal_facts:
|
||||
if not source.has_event(event_id):
|
||||
return _fail("causal fact ID %d is missing from the source store" % event_id)
|
||||
var source_serialized := JSON.stringify(source.to_dictionary())
|
||||
var source_checksum := source_serialized.sha256_text()
|
||||
var events := source.get_all()
|
||||
var events_by_id: Dictionary = {}
|
||||
var order_by_id: Dictionary = {}
|
||||
var traces_by_id: Dictionary = {}
|
||||
for event_index in range(events.size()):
|
||||
var event := events[event_index] as WorldEventRecord
|
||||
if event == null or event.get_tick() > current_tick:
|
||||
return _fail("source contains a null or future event")
|
||||
var event_id := event.get_event_id()
|
||||
events_by_id[event_id] = event
|
||||
order_by_id[event_id] = event_index
|
||||
var cause_ids: Variant = _extract_cause_event_ids(event.get_payload())
|
||||
if cause_ids == null:
|
||||
return _fail("event %d has an invalid causal fact contract" % event_id)
|
||||
traces_by_id[event_id] = {
|
||||
"event_id": event_id,
|
||||
"decision": "",
|
||||
"reasons": [],
|
||||
"cause_event_ids": cause_ids,
|
||||
"required_by_event_ids": [],
|
||||
"rollup_keys": [],
|
||||
}
|
||||
for event: WorldEventRecord in events:
|
||||
var event_id := event.get_event_id()
|
||||
var trace: Dictionary = traces_by_id[event_id]
|
||||
for cause_event_id: int in trace["cause_event_ids"]:
|
||||
if not events_by_id.has(cause_event_id):
|
||||
return _fail(
|
||||
"event %d references missing causal fact %d" % [event_id, cause_event_id]
|
||||
)
|
||||
if int(order_by_id[cause_event_id]) >= int(order_by_id[event_id]):
|
||||
return _fail(
|
||||
"event %d references non-prior causal fact %d" % [event_id, cause_event_id]
|
||||
)
|
||||
var high_volume_type_set: Dictionary = {}
|
||||
for event_type: StringName in normalized_types:
|
||||
high_volume_type_set[String(event_type)] = true
|
||||
var current_day_index := int(current_tick / ticks_per_day)
|
||||
var first_raw_day_index := maxi(current_day_index - raw_retention_days, 0)
|
||||
var retained: Dictionary = {}
|
||||
var contributions_by_event_id: Dictionary = {}
|
||||
for event: WorldEventRecord in events:
|
||||
var event_id := event.get_event_id()
|
||||
var event_day_index := int(event.get_tick() / ticks_per_day)
|
||||
if event_day_index >= first_raw_day_index:
|
||||
_mark_retained(retained, traces_by_id, event_id, REASON_WITHIN_RAW_RETENTION)
|
||||
continue
|
||||
if not high_volume_type_set.has(String(event.get_event_type())):
|
||||
_mark_retained(retained, traces_by_id, event_id, REASON_NOT_ROLLUP_ELIGIBLE)
|
||||
continue
|
||||
var contributions: Variant = _extract_rollup_contributions(event, event_day_index)
|
||||
if contributions == null:
|
||||
_mark_retained(retained, traces_by_id, event_id, REASON_ROLLUP_PAYLOAD_INVALID)
|
||||
continue
|
||||
contributions_by_event_id[event_id] = contributions
|
||||
for event_id: int in normalized_pins:
|
||||
_mark_retained(retained, traces_by_id, event_id, REASON_PINNED_FACT)
|
||||
for event_id: int in normalized_causal_facts:
|
||||
_mark_retained(retained, traces_by_id, event_id, REASON_EXPLICIT_CAUSAL_FACT)
|
||||
for event: WorldEventRecord in events:
|
||||
var required_by_id := event.get_event_id()
|
||||
var child_trace: Dictionary = traces_by_id[required_by_id]
|
||||
for cause_event_id: int in child_trace["cause_event_ids"]:
|
||||
_mark_retained(retained, traces_by_id, cause_event_id, REASON_CAUSAL_ANCESTOR)
|
||||
var cause_trace: Dictionary = traces_by_id[cause_event_id]
|
||||
var required_by_ids: Array = cause_trace["required_by_event_ids"]
|
||||
if required_by_id not in required_by_ids:
|
||||
required_by_ids.append(required_by_id)
|
||||
var accumulators: Dictionary = {}
|
||||
var rolled_up: Dictionary = {}
|
||||
for event: WorldEventRecord in events:
|
||||
var event_id := event.get_event_id()
|
||||
if retained.has(event_id):
|
||||
continue
|
||||
if not contributions_by_event_id.has(event_id):
|
||||
return _fail("an event selected for rollup has no validated contribution")
|
||||
var trace: Dictionary = traces_by_id[event_id]
|
||||
_add_reason(trace, REASON_ROLLED_UP_DAILY)
|
||||
for contribution: Dictionary in contributions_by_event_id[event_id]:
|
||||
var rollup_key := WorldEventDailyRollupRecord.index_key_for(
|
||||
int(contribution["day_index"]),
|
||||
StringName(contribution["world_id"]),
|
||||
StringName(contribution["location_id"]),
|
||||
StringName(contribution["event_type"]),
|
||||
StringName(contribution["item_id"])
|
||||
)
|
||||
if not accumulators.has(rollup_key):
|
||||
accumulators[rollup_key] = {
|
||||
"day_index": int(contribution["day_index"]),
|
||||
"world_id": String(contribution["world_id"]),
|
||||
"location_id": String(contribution["location_id"]),
|
||||
"event_type": String(contribution["event_type"]),
|
||||
"item_id": String(contribution["item_id"]),
|
||||
"source_event_ids": [],
|
||||
"total_amount": 0.0,
|
||||
"minimum_tick": event.get_tick(),
|
||||
"maximum_tick": event.get_tick(),
|
||||
}
|
||||
var accumulator: Dictionary = accumulators[rollup_key]
|
||||
var source_event_ids: Array = accumulator["source_event_ids"]
|
||||
source_event_ids.append(event_id)
|
||||
accumulator["total_amount"] = (
|
||||
float(accumulator["total_amount"]) + float(contribution["amount"])
|
||||
)
|
||||
accumulator["minimum_tick"] = mini(int(accumulator["minimum_tick"]), event.get_tick())
|
||||
accumulator["maximum_tick"] = maxi(int(accumulator["maximum_tick"]), event.get_tick())
|
||||
var trace_rollup_keys: Array = trace["rollup_keys"]
|
||||
trace_rollup_keys.append(rollup_key)
|
||||
rolled_up[event_id] = true
|
||||
var rollup_store := WorldEventDailyRollupStore.new()
|
||||
var accumulator_keys: Array = accumulators.keys()
|
||||
accumulator_keys.sort()
|
||||
for rollup_key: String in accumulator_keys:
|
||||
var accumulator: Dictionary = accumulators[rollup_key]
|
||||
var record := WorldEventDailyRollupRecord.create(
|
||||
int(accumulator["day_index"]),
|
||||
StringName(accumulator["world_id"]),
|
||||
StringName(accumulator["location_id"]),
|
||||
StringName(accumulator["event_type"]),
|
||||
StringName(accumulator["item_id"]),
|
||||
_copy_int_array(accumulator["source_event_ids"]),
|
||||
float(accumulator["total_amount"]),
|
||||
int(accumulator["minimum_tick"]),
|
||||
int(accumulator["maximum_tick"])
|
||||
)
|
||||
if record == null or not rollup_store.append(record):
|
||||
return _fail("daily rollup record construction failed")
|
||||
var retained_ids := _sorted_int_keys(retained)
|
||||
var rolled_up_ids := _sorted_int_keys(rolled_up)
|
||||
var trace_event_ids := _sorted_int_keys(traces_by_id)
|
||||
var reason_traces: Array[Dictionary] = []
|
||||
for event_id: int in trace_event_ids:
|
||||
var trace: Dictionary = traces_by_id[event_id]
|
||||
var reasons: Array = trace["reasons"]
|
||||
var required_by_ids: Array = trace["required_by_event_ids"]
|
||||
var rollup_keys: Array = trace["rollup_keys"]
|
||||
reasons.sort()
|
||||
required_by_ids.sort()
|
||||
rollup_keys.sort()
|
||||
trace["decision"] = (
|
||||
WorldEventRetentionPlan.DECISION_RETAIN_RAW
|
||||
if retained.has(event_id)
|
||||
else WorldEventRetentionPlan.DECISION_ROLLED_UP
|
||||
)
|
||||
reason_traces.append(trace.duplicate(true))
|
||||
var source_unchanged := JSON.stringify(source.to_dictionary()) == source_serialized
|
||||
if not source_unchanged:
|
||||
return _fail("source WorldEventStore changed while planning")
|
||||
var telemetry := {
|
||||
"succeeded": true,
|
||||
"input_event_count": events.size(),
|
||||
"retained_event_count": retained_ids.size(),
|
||||
"rolled_up_event_count": rolled_up_ids.size(),
|
||||
"rollup_record_count": rollup_store.size(),
|
||||
"rollup_contribution_count": _rollup_contribution_count(rollup_store),
|
||||
"pinned_fact_count": normalized_pins.size(),
|
||||
"causal_fact_count": normalized_causal_facts.size(),
|
||||
"causal_ancestor_count": _reason_count(reason_traces, REASON_CAUSAL_ANCESTOR),
|
||||
"recent_event_count": _reason_count(reason_traces, REASON_WITHIN_RAW_RETENTION),
|
||||
"non_rollup_event_count": _reason_count(reason_traces, REASON_NOT_ROLLUP_ELIGIBLE),
|
||||
"invalid_rollup_event_count": _reason_count(reason_traces, REASON_ROLLUP_PAYLOAD_INVALID),
|
||||
"first_raw_day_index": first_raw_day_index,
|
||||
"source_checksum": source_checksum,
|
||||
"source_unchanged": source_unchanged,
|
||||
}
|
||||
var result := WorldEventRetentionPlan.create(
|
||||
current_tick,
|
||||
ticks_per_day,
|
||||
raw_retention_days,
|
||||
first_raw_day_index,
|
||||
source_checksum,
|
||||
_copy_string_name_array(normalized_types),
|
||||
_copy_int_array(normalized_pins),
|
||||
_copy_int_array(normalized_causal_facts),
|
||||
retained_ids,
|
||||
rolled_up_ids,
|
||||
rollup_store,
|
||||
reason_traces,
|
||||
telemetry
|
||||
)
|
||||
if result == null:
|
||||
return _fail("retention plan validation failed")
|
||||
_last_telemetry = telemetry.duplicate(true)
|
||||
return result
|
||||
|
||||
|
||||
func get_last_error() -> String:
|
||||
return _last_error
|
||||
|
||||
|
||||
func get_last_telemetry() -> Dictionary:
|
||||
return _last_telemetry.duplicate(true)
|
||||
|
||||
|
||||
func _fail(message: String) -> WorldEventRetentionPlan:
|
||||
_last_error = message
|
||||
_last_telemetry["error"] = message
|
||||
_last_telemetry["succeeded"] = false
|
||||
return null
|
||||
|
||||
|
||||
static func _mark_retained(
|
||||
retained: Dictionary, traces_by_id: Dictionary, event_id: int, reason: String
|
||||
) -> void:
|
||||
retained[event_id] = true
|
||||
var trace: Dictionary = traces_by_id[event_id]
|
||||
_add_reason(trace, reason)
|
||||
|
||||
|
||||
static func _add_reason(trace: Dictionary, reason: String) -> void:
|
||||
var reasons: Array = trace["reasons"]
|
||||
if reason not in reasons:
|
||||
reasons.append(reason)
|
||||
|
||||
|
||||
static func _extract_cause_event_ids(payload: Dictionary) -> Variant:
|
||||
var seen: Dictionary = {}
|
||||
var event_ids: Array[int] = []
|
||||
if payload.has("cause_event_id"):
|
||||
var single_cause: Variant = payload["cause_event_id"]
|
||||
if not single_cause is int or int(single_cause) < -1:
|
||||
return null
|
||||
if int(single_cause) >= 0:
|
||||
seen[int(single_cause)] = true
|
||||
event_ids.append(int(single_cause))
|
||||
if payload.has("cause_event_ids"):
|
||||
var multiple_causes: Variant = payload["cause_event_ids"]
|
||||
if not multiple_causes is Array:
|
||||
return null
|
||||
for raw_cause: Variant in multiple_causes:
|
||||
if not raw_cause is int or int(raw_cause) < 0:
|
||||
return null
|
||||
if seen.has(int(raw_cause)):
|
||||
continue
|
||||
seen[int(raw_cause)] = true
|
||||
event_ids.append(int(raw_cause))
|
||||
event_ids.sort()
|
||||
return event_ids
|
||||
|
||||
|
||||
static func _extract_rollup_contributions(event: WorldEventRecord, day_index: int) -> Variant:
|
||||
if event == null or not event.has_location():
|
||||
return null
|
||||
var location := event.get_location()
|
||||
if (
|
||||
location == null
|
||||
or location.get_world_id().is_empty()
|
||||
or location.get_location_id().is_empty()
|
||||
):
|
||||
return null
|
||||
var payload := event.get_payload()
|
||||
var has_item_amount := payload.has("item_id") or payload.has("amount")
|
||||
var has_cargo_ledger := payload.has("cargo_ledger")
|
||||
if has_item_amount and has_cargo_ledger:
|
||||
return null
|
||||
var contributions: Array[Dictionary] = []
|
||||
if has_cargo_ledger:
|
||||
if not payload["cargo_ledger"] is Dictionary:
|
||||
return null
|
||||
var ledger: Dictionary = payload["cargo_ledger"]
|
||||
var source_keys: Dictionary = {}
|
||||
for raw_item_id: Variant in ledger:
|
||||
if not raw_item_id is String and not raw_item_id is StringName:
|
||||
return null
|
||||
var item_id := String(raw_item_id)
|
||||
if item_id.is_empty() or source_keys.has(item_id):
|
||||
return null
|
||||
var raw_amount: Variant = ledger[raw_item_id]
|
||||
if (
|
||||
not raw_amount is int and not raw_amount is float
|
||||
or not is_finite(float(raw_amount))
|
||||
or float(raw_amount) <= 0.0
|
||||
):
|
||||
return null
|
||||
source_keys[item_id] = raw_item_id
|
||||
var item_ids: Array = source_keys.keys()
|
||||
item_ids.sort()
|
||||
for item_id: String in item_ids:
|
||||
contributions.append(
|
||||
_contribution(
|
||||
event, location, day_index, item_id, float(ledger[source_keys[item_id]])
|
||||
)
|
||||
)
|
||||
elif payload.has("item_id") and payload.has("amount"):
|
||||
var raw_item_id: Variant = payload["item_id"]
|
||||
var raw_amount: Variant = payload["amount"]
|
||||
if (
|
||||
(not raw_item_id is String and not raw_item_id is StringName)
|
||||
or String(raw_item_id).is_empty()
|
||||
or (not raw_amount is int and not raw_amount is float)
|
||||
or not is_finite(float(raw_amount))
|
||||
or float(raw_amount) <= 0.0
|
||||
):
|
||||
return null
|
||||
contributions.append(
|
||||
_contribution(event, location, day_index, String(raw_item_id), float(raw_amount))
|
||||
)
|
||||
else:
|
||||
return null
|
||||
return contributions if not contributions.is_empty() else null
|
||||
|
||||
|
||||
static func _contribution(
|
||||
event: WorldEventRecord,
|
||||
location: SpatialAddress,
|
||||
day_index: int,
|
||||
item_id: String,
|
||||
amount: float
|
||||
) -> Dictionary:
|
||||
return {
|
||||
"day_index": day_index,
|
||||
"world_id": String(location.get_world_id()),
|
||||
"location_id": String(location.get_location_id()),
|
||||
"event_type": String(event.get_event_type()),
|
||||
"item_id": item_id,
|
||||
"amount": amount,
|
||||
}
|
||||
|
||||
|
||||
static func _normalize_event_types(values: Array) -> Variant:
|
||||
var seen: Dictionary = {}
|
||||
var normalized: Array[StringName] = []
|
||||
for value: Variant in values:
|
||||
if (not value is String and not value is StringName) or StringName(value).is_empty():
|
||||
return null
|
||||
var event_type := StringName(value)
|
||||
if seen.has(String(event_type)):
|
||||
return null
|
||||
seen[String(event_type)] = true
|
||||
normalized.append(event_type)
|
||||
normalized.sort_custom(
|
||||
func(first: StringName, second: StringName) -> bool: return String(first) < String(second)
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
static func _normalize_event_ids(values: Array) -> Variant:
|
||||
var seen: Dictionary = {}
|
||||
var normalized: Array[int] = []
|
||||
for value: Variant in values:
|
||||
if not value is int or int(value) < 0 or seen.has(int(value)):
|
||||
return null
|
||||
seen[int(value)] = true
|
||||
normalized.append(int(value))
|
||||
normalized.sort()
|
||||
return normalized
|
||||
|
||||
|
||||
static func _sorted_int_keys(values: Dictionary) -> Array[int]:
|
||||
var result: Array[int] = []
|
||||
for value: Variant in values:
|
||||
result.append(int(value))
|
||||
result.sort()
|
||||
return result
|
||||
|
||||
|
||||
static func _copy_int_array(values: Array) -> Array[int]:
|
||||
var result: Array[int] = []
|
||||
for value: Variant in values:
|
||||
result.append(int(value))
|
||||
return result
|
||||
|
||||
|
||||
static func _copy_string_name_array(values: Array) -> Array[StringName]:
|
||||
var result: Array[StringName] = []
|
||||
for value: Variant in values:
|
||||
result.append(StringName(value))
|
||||
return result
|
||||
|
||||
|
||||
static func _reason_count(traces: Array[Dictionary], reason: String) -> int:
|
||||
var count := 0
|
||||
for trace: Dictionary in traces:
|
||||
var reasons: Array = trace["reasons"]
|
||||
if reason in reasons:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
static func _rollup_contribution_count(store: WorldEventDailyRollupStore) -> int:
|
||||
var count := 0
|
||||
for record: WorldEventDailyRollupRecord in store.get_all():
|
||||
count += record.get_event_count()
|
||||
return count
|
||||
Reference in New Issue
Block a user