perf: benchmark regional simulation scale

tools/benchmark_regional_scale.gd

@@ -0,0 +1,167 @@
+extends SceneTree
+
+const BenchmarkScript := preload("res://simulation/benchmark/RegionalScaleBenchmark.gd")
+const DEFAULT_OUTPUT_PATH := "user://regional_scale_latest.json"
+const DEFAULT_HOST_LABEL := "unspecified"
+const SAMPLE_COUNT := 3
+const DEFAULT_BUDGET := 8
+
+
+func _initialize() -> void:
+	call_deferred("_run")
+
+
+func _run() -> void:
+	var benchmark := BenchmarkScript.new()
+	var samples: Array[Dictionary] = []
+	for _sample_index in SAMPLE_COUNT:
+		var sample := benchmark.run(DEFAULT_BUDGET)
+		if not _valid_sample(sample):
+			push_error("Regional scale benchmark produced an invalid default-budget sample")
+			quit(1)
+			return
+		samples.append(sample)
+	var budget_one := benchmark.run(1)
+	var unlimited := benchmark.run(RegionalJobScheduler.UNLIMITED_BUDGET)
+	if not _valid_sample(budget_one) or not _valid_sample(unlimited):
+		push_error("Regional scale benchmark produced an invalid budget-parity sample")
+		quit(1)
+		return
+	if not _deterministic_samples(samples + [budget_one, unlimited]):
+		push_error("Regional scale benchmark diverged across samples or execution budgets")
+		quit(1)
+		return
+
+	var report := {
+		"schema_version": BenchmarkScript.SCHEMA_VERSION,
+		"captured_utc": Time.get_datetime_string_from_system(true) + "Z",
+		"engine_version": String(Engine.get_version_info().get("string", "unknown")),
+		"platform": OS.get_name(),
+		"processor_count": OS.get_processor_count(),
+		"host_label": _get_argument_value("--host-label=", DEFAULT_HOST_LABEL),
+		"workload_id": String(BenchmarkScript.WORKLOAD_ID),
+		"benchmark_scope": "regional_data_and_scheduler_only",
+		"rendered_throughput_measured": false,
+		"full_economy_throughput_measured": false,
+		"sample_count": SAMPLE_COUNT,
+		"default_execution_budget": DEFAULT_BUDGET,
+		"fixture": _fixture_summary(samples[0]),
+		"default_budget": _summarize_timings(samples),
+		"budget_parity": {
+			"budget_one_final_checksum": budget_one["final_checksum"],
+			"budget_one_drain_batches": budget_one["drain_batches"],
+			"unlimited_final_checksum": unlimited["final_checksum"],
+			"unlimited_drain_batches": unlimited["drain_batches"],
+			"identical": true,
+		},
+	}
+	var output_path := _get_argument_value("--output=", DEFAULT_OUTPUT_PATH)
+	var output := FileAccess.open(output_path, FileAccess.WRITE)
+	if output == null:
+		push_error("Could not write regional scale benchmark report to %s" % output_path)
+		quit(1)
+		return
+	output.store_string(JSON.stringify(report, "\t") + "\n")
+	output.close()
+	print(
+		(
+			"[BENCH] regional | %d named + %d aggregate | %d caravans | "
+			+ "%d jobs | median %.2f ms total | %.2f MiB state"
+		)
+		% [
+			report["fixture"]["named_person_count"],
+			report["fixture"]["cohort_resident_count"],
+			report["fixture"]["caravan_count"],
+			report["fixture"]["initial_job_count"],
+			float(report["default_budget"]["elapsed_usec_median"]) / 1000.0,
+			float(report["fixture"]["regional_state_bytes"]) / (1024.0 * 1024.0),
+		]
+	)
+	print("[BENCH] Report: %s" % ProjectSettings.globalize_path(output_path))
+	quit(0)
+
+
+func _valid_sample(sample: Dictionary) -> bool:
+	return (
+		not sample.is_empty()
+		and bool(sample.get("fixture_valid", false))
+		and bool(sample.get("roundtrip_valid", false))
+		and bool(sample.get("headcount_conserved", false))
+		and bool(sample.get("cargo_conserved", false))
+		and int(sample.get("backlog_final", -1)) == 0
+	)
+
+
+func _deterministic_samples(samples: Array) -> bool:
+	if samples.is_empty():
+		return false
+	var expected: Dictionary = samples[0]
+	for sample: Dictionary in samples:
+		for key in [
+			"fixture_checksum",
+			"execution_order_checksum",
+			"final_checksum",
+			"initial_job_count",
+			"executed_job_count",
+			"regional_state_bytes",
+			"scheduler_state_bytes",
+		]:
+			if sample.get(key) != expected.get(key):
+				return false
+	return true
+
+
+func _fixture_summary(sample: Dictionary) -> Dictionary:
+	var result := {}
+	for key in [
+		"seed",
+		"location_count",
+		"settlement_count",
+		"caravan_count",
+		"detailed_members_per_caravan",
+		"detailed_caravan_member_count",
+		"named_person_count",
+		"cohort_resident_count",
+		"total_headcount",
+		"max_active_visuals",
+		"cargo_total",
+		"initial_job_count",
+		"regional_state_bytes",
+		"scheduler_state_bytes",
+		"fixture_checksum",
+		"execution_order_checksum",
+		"final_checksum",
+	]:
+		result[key] = sample[key]
+	return result
+
+
+func _summarize_timings(samples: Array[Dictionary]) -> Dictionary:
+	var result := {}
+	for key in [
+		"fixture_build_usec",
+		"roundtrip_usec",
+		"scheduler_build_usec",
+		"execution_usec",
+		"elapsed_usec",
+	]:
+		var values: Array[int] = []
+		for sample in samples:
+			values.append(int(sample[key]))
+		values.sort()
+		result[key + "_samples"] = values
+		result[key + "_median"] = values[values.size() / 2]
+	result["drain_batches"] = samples[0]["drain_batches"]
+	result["backlog_initial"] = samples[0]["backlog_initial"]
+	result["backlog_peak"] = samples[0]["backlog_peak"]
+	result["backlog_final"] = samples[0]["backlog_final"]
+	return result
+
+
+func _get_argument_value(prefix: String, default_value: String) -> String:
+	for argument in OS.get_cmdline_user_args():
+		if argument.begins_with(prefix):
+			var value := argument.trim_prefix(prefix)
+			if not value.is_empty():
+				return value
+	return default_value

tools/benchmark_regional_scale.gd.uid

@@ -0,0 +1 @@
+uid://br2cgdiw4s6an