feat: establish simulation scaling baseline
This commit is contained in:
Rijad Zuzo
@@ -0,0 +1,222 @@
|
||||
extends SceneTree
|
||||
|
||||
const SimulationManagerScript := preload("res://simulation/SimulationManager.gd")
|
||||
const BenchmarkScript := preload("res://simulation/benchmark/SimulationScalingBenchmark.gd")
|
||||
const DEFAULT_OUTPUT_PATH := "user://simulation_scaling_latest.json"
|
||||
const DEFAULT_HOST_LABEL := "unspecified"
|
||||
const BENCHMARK_SEED := 8088
|
||||
const WARMUP_TICKS := 10
|
||||
const MEASURED_TICKS := 200
|
||||
const SAMPLE_COUNT := 3
|
||||
const CASES := [
|
||||
{"case_id": "population_006", "population": 6, "history_events": 0},
|
||||
{"case_id": "population_060", "population": 60, "history_events": 0},
|
||||
{"case_id": "population_600", "population": 600, "history_events": 0},
|
||||
{"case_id": "history_000600", "population": 60, "history_events": 600},
|
||||
{"case_id": "history_006000", "population": 60, "history_events": 6000},
|
||||
]
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
call_deferred("_run")
|
||||
|
||||
|
||||
func _run() -> void:
|
||||
var benchmark := BenchmarkScript.new()
|
||||
var case_results: Array[Dictionary] = []
|
||||
for case_config in CASES:
|
||||
var samples: Array[Dictionary] = []
|
||||
for _sample_index in SAMPLE_COUNT:
|
||||
var setup_started_usec := Time.get_ticks_usec()
|
||||
var manager := SimulationManagerScript.new()
|
||||
manager.simulation_seed = BENCHMARK_SEED
|
||||
manager.debug_logs = false
|
||||
manager.set_process(false)
|
||||
root.add_child(manager)
|
||||
var prepared := benchmark.prepare_manager(
|
||||
manager,
|
||||
int(case_config["population"]),
|
||||
int(case_config["history_events"]),
|
||||
BENCHMARK_SEED
|
||||
)
|
||||
var fixture_valid := (
|
||||
prepared and SimulationStateRecord.from_json(manager.serialize_state()) != null
|
||||
)
|
||||
var setup_usec := Time.get_ticks_usec() - setup_started_usec
|
||||
if not fixture_valid:
|
||||
push_error(
|
||||
"Scaling benchmark prepared invalid state for %s" % case_config["case_id"]
|
||||
)
|
||||
manager.free()
|
||||
quit(1)
|
||||
return
|
||||
var sample := benchmark.measure_manager(
|
||||
manager,
|
||||
int(case_config["population"]),
|
||||
int(case_config["history_events"]),
|
||||
WARMUP_TICKS,
|
||||
MEASURED_TICKS
|
||||
)
|
||||
if sample.is_empty():
|
||||
push_error("Scaling benchmark could not measure %s" % case_config["case_id"])
|
||||
manager.free()
|
||||
quit(1)
|
||||
return
|
||||
sample["setup_usec"] = maxi(setup_usec, 1)
|
||||
samples.append(sample)
|
||||
manager.free()
|
||||
var summarized := _summarize_samples(String(case_config["case_id"]), samples)
|
||||
if summarized.is_empty():
|
||||
quit(1)
|
||||
return
|
||||
case_results.append(summarized)
|
||||
_print_case(summarized)
|
||||
|
||||
var report := {
|
||||
"schema_version": SimulationScalingBenchmark.SCHEMA_VERSION,
|
||||
"captured_utc": Time.get_datetime_string_from_system(true) + "Z",
|
||||
"engine_version": String(Engine.get_version_info().get("string", "unknown")),
|
||||
"platform": OS.get_name(),
|
||||
"processor_count": OS.get_processor_count(),
|
||||
"host_label": _get_argument_value("--host-label=", DEFAULT_HOST_LABEL),
|
||||
"benchmark_seed": BENCHMARK_SEED,
|
||||
"workload_id": String(SimulationScalingBenchmark.WORKLOAD_ID),
|
||||
"workload":
|
||||
(
|
||||
"All NPCs receive full per-tick needs/task updates and ordinary action decisions; "
|
||||
+ "travel resolves through the deterministic immediate-arrival headless convention."
|
||||
),
|
||||
"timed_phases": ["simulation_tick", "headless_arrival_completion"],
|
||||
"exclusions":
|
||||
[
|
||||
"manager_and_fixture_setup",
|
||||
"state_serialization",
|
||||
"world_scene",
|
||||
"rendering",
|
||||
"navigation",
|
||||
"npc_visual",
|
||||
],
|
||||
"warmup_ticks": WARMUP_TICKS,
|
||||
"measured_ticks": MEASURED_TICKS,
|
||||
"sample_count": SAMPLE_COUNT,
|
||||
"cases": case_results,
|
||||
}
|
||||
var output_path := _get_output_path()
|
||||
var output := FileAccess.open(output_path, FileAccess.WRITE)
|
||||
if output == null:
|
||||
push_error("Could not write scaling benchmark report to %s" % output_path)
|
||||
quit(1)
|
||||
return
|
||||
output.store_string(JSON.stringify(report, "\t") + "\n")
|
||||
output.close()
|
||||
print("[BENCH] Report: %s" % ProjectSettings.globalize_path(output_path))
|
||||
quit(0)
|
||||
|
||||
|
||||
func _summarize_samples(case_id: String, samples: Array[Dictionary]) -> Dictionary:
|
||||
if samples.size() != SAMPLE_COUNT or samples[0].is_empty():
|
||||
push_error("Scaling benchmark case %s returned incomplete samples" % case_id)
|
||||
return {}
|
||||
var checksum := String(samples[0]["final_checksum"])
|
||||
var deterministic_keys := [
|
||||
"start_state_bytes",
|
||||
"end_state_bytes",
|
||||
"state_growth_bytes",
|
||||
"start_event_count",
|
||||
"end_event_count",
|
||||
"events_recorded",
|
||||
"start_known_reference_count",
|
||||
"end_known_reference_count",
|
||||
"arrivals_processed",
|
||||
"warmup_arrivals",
|
||||
"npc_updates",
|
||||
"start_tick",
|
||||
"end_tick",
|
||||
]
|
||||
for sample in samples:
|
||||
if String(sample["final_checksum"]) != checksum:
|
||||
push_error("Scaling benchmark case %s diverged across checksums" % case_id)
|
||||
return {}
|
||||
for key in deterministic_keys:
|
||||
if sample[key] != samples[0][key]:
|
||||
push_error("Scaling benchmark case %s diverged at %s" % [case_id, key])
|
||||
return {}
|
||||
var elapsed_samples: Array[int] = []
|
||||
var setup_samples: Array[int] = []
|
||||
var simulation_samples: Array[int] = []
|
||||
var arrival_samples: Array[int] = []
|
||||
for sample in samples:
|
||||
elapsed_samples.append(int(sample["elapsed_usec"]))
|
||||
setup_samples.append(int(sample["setup_usec"]))
|
||||
simulation_samples.append(int(sample["simulation_usec"]))
|
||||
arrival_samples.append(int(sample["arrival_usec"]))
|
||||
elapsed_samples.sort()
|
||||
setup_samples.sort()
|
||||
simulation_samples.sort()
|
||||
arrival_samples.sort()
|
||||
var elapsed_median := elapsed_samples[elapsed_samples.size() / 2]
|
||||
var simulation_median := simulation_samples[simulation_samples.size() / 2]
|
||||
var arrival_median := arrival_samples[arrival_samples.size() / 2]
|
||||
var result := samples[0].duplicate(true)
|
||||
for transient_key in [
|
||||
"setup_usec",
|
||||
"elapsed_usec",
|
||||
"simulation_usec",
|
||||
"arrival_usec",
|
||||
"usec_per_tick",
|
||||
"ticks_per_second",
|
||||
"realtime_factor",
|
||||
]:
|
||||
result.erase(transient_key)
|
||||
result["case_id"] = case_id
|
||||
result["sample_count"] = SAMPLE_COUNT
|
||||
result["setup_usec_samples"] = setup_samples
|
||||
result["setup_usec_median"] = setup_samples[setup_samples.size() / 2]
|
||||
result["elapsed_usec_samples"] = elapsed_samples
|
||||
result["elapsed_usec_min"] = elapsed_samples[0]
|
||||
result["elapsed_usec_median"] = elapsed_median
|
||||
result["elapsed_usec_max"] = elapsed_samples[-1]
|
||||
result["simulation_usec_samples"] = simulation_samples
|
||||
result["simulation_usec_median"] = simulation_median
|
||||
result["arrival_usec_samples"] = arrival_samples
|
||||
result["arrival_usec_median"] = arrival_median
|
||||
result["arrival_share_percent"] = float(arrival_median) / float(elapsed_median) * 100.0
|
||||
result["usec_per_tick_median"] = float(elapsed_median) / float(MEASURED_TICKS)
|
||||
result["ticks_per_second_median"] = (float(MEASURED_TICKS) * 1000000.0 / float(elapsed_median))
|
||||
result["realtime_factor_median"] = (
|
||||
float(result["ticks_per_second_median"]) * float(result["tick_interval"])
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
func _print_case(result: Dictionary) -> void:
|
||||
print(
|
||||
(
|
||||
(
|
||||
"[BENCH] %s | NPC %d | seeded events %d | %.1f us/tick | "
|
||||
+ "arrival %.1f%% | %.1fx realtime | %.2f MiB"
|
||||
)
|
||||
% [
|
||||
result["case_id"],
|
||||
result["population"],
|
||||
result["history_seed_events"],
|
||||
result["usec_per_tick_median"],
|
||||
result["arrival_share_percent"],
|
||||
result["realtime_factor_median"],
|
||||
float(result["end_state_bytes"]) / (1024.0 * 1024.0),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
func _get_output_path() -> String:
|
||||
return _get_argument_value("--output=", DEFAULT_OUTPUT_PATH)
|
||||
|
||||
|
||||
func _get_argument_value(prefix: String, default_value: String) -> String:
|
||||
for argument in OS.get_cmdline_user_args():
|
||||
if argument.begins_with(prefix):
|
||||
var value := argument.trim_prefix(prefix)
|
||||
if not value.is_empty():
|
||||
return value
|
||||
return default_value
|
||||
Reference in New Issue
Block a user