perf: benchmark regional simulation scale
This commit is contained in:
Rijad Zuzo
@@ -24,8 +24,24 @@ Loaded-resource discovery has its own bounded runner:
|
|||||||
--script res://tools/benchmark_loaded_resource_discovery.gd
|
--script res://tools/benchmark_loaded_resource_discovery.gd
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The regional record/scheduler workload has a separate honest runner:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/Applications/Godot.app/Contents/MacOS/Godot \
|
||||||
|
--headless --path "$PWD" \
|
||||||
|
--script res://tools/benchmark_regional_scale.gd
|
||||||
|
```
|
||||||
|
|
||||||
|
It constructs five settlements, 20 caravans, 2,000 named people, 50,000
|
||||||
|
aggregate residents, and 2,025 deterministic due jobs. It measures record
|
||||||
|
construction, canonical roundtrip, scheduler construction, and job draining;
|
||||||
|
it does not claim rendered or full-economy throughput.
|
||||||
|
|
||||||
Reviewed captures:
|
Reviewed captures:
|
||||||
|
|
||||||
|
- [Regional scale baseline 01](REGIONAL_SCALE_BASELINE_01.md) records the
|
||||||
|
primitive regional state and deterministic scheduler workload at five
|
||||||
|
settlements, 20 caravans, 2,000 named people, and 50,000 aggregate residents.
|
||||||
- [Simulation scaling baseline 01](SIMULATION_SCALING_BASELINE_01.md) records
|
- [Simulation scaling baseline 01](SIMULATION_SCALING_BASELINE_01.md) records
|
||||||
the first full-fidelity population/history measurements and the bounded next
|
the first full-fidelity population/history measurements and the bounded next
|
||||||
optimization.
|
optimization.
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Regional scale baseline 01
|
||||||
|
|
||||||
|
This is the first reviewed capture of workload schema 1,
|
||||||
|
`regional_records_scheduler_v1`. It is a structural headless baseline for the
|
||||||
|
regional record and deterministic due-job contracts—not a claim about rendered
|
||||||
|
gameplay or the complete economy.
|
||||||
|
|
||||||
|
The machine-readable report is in
|
||||||
|
[`regional_scale_baseline_01.json`](regional_scale_baseline_01.json). It was
|
||||||
|
captured with Godot 4.7 on the `Apple_M1_Max_64_GB` development host, seed
|
||||||
|
424242, three default-budget samples, and parity runs at budgets one and
|
||||||
|
unlimited.
|
||||||
|
|
||||||
|
## Fixed workload
|
||||||
|
|
||||||
|
- five locations, settlements, polities, and ring routes;
|
||||||
|
- 20 caravans with 12 named members each;
|
||||||
|
- 2,000 persistent named people;
|
||||||
|
- five aggregate cohorts totaling 50,000 residents;
|
||||||
|
- 2,025 keyed deterministic due jobs;
|
||||||
|
- maximum active-presentation metadata of 40;
|
||||||
|
- 736,115 bytes of canonical regional state and 540,942 bytes of initial
|
||||||
|
scheduler state.
|
||||||
|
|
||||||
|
All records round-tripped through their primitive-only schemas. The fixture
|
||||||
|
contained 52,000 people and 2,734 cargo units without duplicate global IDs or
|
||||||
|
membership errors. Budget one, budget eight, and unlimited execution drained
|
||||||
|
the backlog to zero in exactly the same order and produced final checksum
|
||||||
|
`57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25`.
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Phase | Median | Samples |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| Fixture construction | 273.79 ms | 272.37, 273.79, 282.07 ms |
|
||||||
|
| Canonical roundtrip | 255.86 ms | 255.33, 255.86, 266.03 ms |
|
||||||
|
| Scheduler construction | 74.10 ms | 73.66, 74.10, 76.50 ms |
|
||||||
|
| Drain 2,025 jobs (budget 8) | 57.47 ms | 55.52, 57.47, 59.51 ms |
|
||||||
|
| Complete measured run | 785.65 ms | 776.58, 785.65, 808.22 ms |
|
||||||
|
|
||||||
|
Budget eight drained the queue in 254 batches. Budget one used 2,025 batches;
|
||||||
|
unlimited used one. Their execution-order, fixture, and final checksums were
|
||||||
|
identical.
|
||||||
|
|
||||||
|
## Interpretation
|
||||||
|
|
||||||
|
The baseline proves that the target record counts fit in a small canonical
|
||||||
|
payload and that job-budget changes do not alter outcomes. It intentionally
|
||||||
|
constructs only 2,025 event-boundary jobs, so the 57.47 ms drain is not a
|
||||||
|
30-day simulation-time throughput result. It also excludes scene loading,
|
||||||
|
navigation, rendering, dialogue/situation queries, market transactions,
|
||||||
|
conflict, event rollups, chunked disk I/O, and live caravan transfers.
|
||||||
|
|
||||||
|
The next benchmark must run scheduled settlement and caravan domain handlers
|
||||||
|
over 30 simulated days. Separately, a recorded weak-PC active-world capture
|
||||||
|
must measure the Low presentation tier with at most 40 detailed visuals and two
|
||||||
|
new navigation requests per frame. These measurements cannot be inferred from
|
||||||
|
this development-host structural result.
|
||||||
|
|
||||||
|
## Capture command
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/Applications/Godot.app/Contents/MacOS/Godot \
|
||||||
|
--headless --path "$PWD" \
|
||||||
|
--script res://tools/benchmark_regional_scale.gd -- \
|
||||||
|
--host-label=Apple_M1_Max_64_GB \
|
||||||
|
--output=res://docs/benchmarks/regional_scale_baseline_01.json
|
||||||
|
```
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
{
|
||||||
|
"benchmark_scope": "regional_data_and_scheduler_only",
|
||||||
|
"budget_parity": {
|
||||||
|
"budget_one_drain_batches": 2025,
|
||||||
|
"budget_one_final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25",
|
||||||
|
"identical": true,
|
||||||
|
"unlimited_drain_batches": 1,
|
||||||
|
"unlimited_final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25"
|
||||||
|
},
|
||||||
|
"captured_utc": "2026-08-12T19:36:00Z",
|
||||||
|
"default_budget": {
|
||||||
|
"backlog_final": 0,
|
||||||
|
"backlog_initial": 2025,
|
||||||
|
"backlog_peak": 2025,
|
||||||
|
"drain_batches": 254,
|
||||||
|
"elapsed_usec_median": 785655,
|
||||||
|
"elapsed_usec_samples": [776580, 785655, 808216],
|
||||||
|
"execution_usec_median": 57471,
|
||||||
|
"execution_usec_samples": [55516, 57471, 59508],
|
||||||
|
"fixture_build_usec_median": 273789,
|
||||||
|
"fixture_build_usec_samples": [272365, 273789, 282068],
|
||||||
|
"roundtrip_usec_median": 255860,
|
||||||
|
"roundtrip_usec_samples": [255326, 255860, 266034],
|
||||||
|
"scheduler_build_usec_median": 74100,
|
||||||
|
"scheduler_build_usec_samples": [73657, 74100, 76495]
|
||||||
|
},
|
||||||
|
"default_execution_budget": 8,
|
||||||
|
"engine_version": "4.7-stable (official)",
|
||||||
|
"fixture": {
|
||||||
|
"caravan_count": 20,
|
||||||
|
"cargo_total": 2734.0,
|
||||||
|
"cohort_resident_count": 50000,
|
||||||
|
"detailed_caravan_member_count": 240,
|
||||||
|
"detailed_members_per_caravan": 12,
|
||||||
|
"execution_order_checksum": "38815c49987cf66ec0d78b82d332789f708742959bab83c9feee631d8bbf8174",
|
||||||
|
"final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25",
|
||||||
|
"fixture_checksum": "5ee2dcda7067b34e1ffc22861fbcf1e6d1897f87523a88fbf216fc9ea6a19ed5",
|
||||||
|
"initial_job_count": 2025,
|
||||||
|
"location_count": 5,
|
||||||
|
"max_active_visuals": 40,
|
||||||
|
"named_person_count": 2000,
|
||||||
|
"regional_state_bytes": 736115,
|
||||||
|
"scheduler_state_bytes": 540942,
|
||||||
|
"seed": 424242,
|
||||||
|
"settlement_count": 5,
|
||||||
|
"total_headcount": 52000
|
||||||
|
},
|
||||||
|
"full_economy_throughput_measured": false,
|
||||||
|
"host_label": "Apple_M1_Max_64_GB",
|
||||||
|
"platform": "macOS",
|
||||||
|
"processor_count": 10,
|
||||||
|
"rendered_throughput_measured": false,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"workload_id": "regional_records_scheduler_v1"
|
||||||
|
}
|
||||||
@@ -0,0 +1,429 @@
|
|||||||
|
class_name RegionalScaleBenchmark
|
||||||
|
extends RefCounted
|
||||||
|
|
||||||
|
# This workload measures primitive regional records and deterministic scheduler
|
||||||
|
# draining only. It does not load scenes or exercise the complete economy.
|
||||||
|
const SCHEMA_VERSION := 1
|
||||||
|
const WORKLOAD_ID := &"regional_records_scheduler_v1"
|
||||||
|
const WORLD_ID := &"regional_benchmark"
|
||||||
|
const FIXED_SEED := 424242
|
||||||
|
const LOCATION_COUNT := 5
|
||||||
|
const SETTLEMENT_COUNT := 5
|
||||||
|
const CARAVAN_COUNT := 20
|
||||||
|
const DETAILED_MEMBERS_PER_CARAVAN := 12
|
||||||
|
const NAMED_PERSON_COUNT := 2000
|
||||||
|
const COHORT_RESIDENT_COUNT := 50000
|
||||||
|
const MAX_ACTIVE_VISUALS := 40
|
||||||
|
const MAX_DUE_TICK := 256
|
||||||
|
const ROLE_IDS: Array[StringName] = [&"artisan", &"farmer", &"guard", &"merchant", &"scholar"]
|
||||||
|
|
||||||
|
|
||||||
|
func build_fixture(seed_value: int = FIXED_SEED) -> RegionalWorldState:
|
||||||
|
var locations: Array[LocationStateRecord] = []
|
||||||
|
var settlements: Array[SettlementStateRecord] = []
|
||||||
|
var routes: Array[RouteStateRecord] = []
|
||||||
|
var groups: Array[MobileGroupStateRecord] = []
|
||||||
|
var persons: Array[PersonStateRecord] = []
|
||||||
|
var cohorts: Array[PopulationCohortRecord] = []
|
||||||
|
var polities: Array[PolityStateRecord] = []
|
||||||
|
var relations: Array[DiplomaticRelationRecord] = []
|
||||||
|
|
||||||
|
for index in LOCATION_COUNT:
|
||||||
|
var location_id := _location_id(index)
|
||||||
|
var location := LocationStateRecord.create(
|
||||||
|
location_id,
|
||||||
|
WORLD_ID,
|
||||||
|
&"settlement",
|
||||||
|
"Regional Settlement %02d" % index,
|
||||||
|
_address(location_id, Vector3(float(index) * 100.0, 0.0, 0.0))
|
||||||
|
)
|
||||||
|
var settlement := SettlementStateRecord.create(
|
||||||
|
_settlement_id(index),
|
||||||
|
location_id,
|
||||||
|
"Regional Settlement %02d" % index,
|
||||||
|
_polity_id(index),
|
||||||
|
2,
|
||||||
|
_group_id(index),
|
||||||
|
0,
|
||||||
|
{&"food": 10000.0 + index * 100.0, &"wood": 5000.0 + index * 50.0}
|
||||||
|
)
|
||||||
|
var polity := PolityStateRecord.create(
|
||||||
|
_polity_id(index), "Regional Polity %02d" % index, location_id, [_settlement_id(index)]
|
||||||
|
)
|
||||||
|
if location == null or settlement == null or polity == null:
|
||||||
|
return null
|
||||||
|
locations.append(location)
|
||||||
|
settlements.append(settlement)
|
||||||
|
polities.append(polity)
|
||||||
|
|
||||||
|
for index in LOCATION_COUNT:
|
||||||
|
var route := RouteStateRecord.create(
|
||||||
|
_route_id(index),
|
||||||
|
_location_id(index),
|
||||||
|
_location_id((index + 1) % LOCATION_COUNT),
|
||||||
|
20 + index,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
if route == null:
|
||||||
|
return null
|
||||||
|
routes.append(route)
|
||||||
|
|
||||||
|
for group_index in CARAVAN_COUNT:
|
||||||
|
var route_index := group_index % LOCATION_COUNT
|
||||||
|
var member_ids: Array[StringName] = []
|
||||||
|
for member_index in DETAILED_MEMBERS_PER_CARAVAN:
|
||||||
|
member_ids.append(_person_id(group_index * DETAILED_MEMBERS_PER_CARAVAN + member_index))
|
||||||
|
var departure_tick := group_index * 3
|
||||||
|
var cargo := {
|
||||||
|
&"food":
|
||||||
|
float(
|
||||||
|
KeyedRandom.range_int(seed_value, &"cargo_food", _group_id(group_index), 0, 50, 99)
|
||||||
|
),
|
||||||
|
&"herb":
|
||||||
|
float(
|
||||||
|
KeyedRandom.range_int(seed_value, &"cargo_herb", _group_id(group_index), 0, 10, 29)
|
||||||
|
),
|
||||||
|
&"wood":
|
||||||
|
float(
|
||||||
|
KeyedRandom.range_int(seed_value, &"cargo_wood", _group_id(group_index), 0, 20, 49)
|
||||||
|
),
|
||||||
|
}
|
||||||
|
var group := MobileGroupStateRecord.create_on_route(
|
||||||
|
_group_id(group_index),
|
||||||
|
&"caravan",
|
||||||
|
"Regional Caravan %02d" % group_index,
|
||||||
|
_polity_id(route_index),
|
||||||
|
_route_id(route_index),
|
||||||
|
_location_id(route_index),
|
||||||
|
_location_id((route_index + 1) % LOCATION_COUNT),
|
||||||
|
departure_tick,
|
||||||
|
departure_tick + 20 + route_index,
|
||||||
|
member_ids,
|
||||||
|
{},
|
||||||
|
cargo,
|
||||||
|
500.0
|
||||||
|
)
|
||||||
|
if group == null:
|
||||||
|
return null
|
||||||
|
groups.append(group)
|
||||||
|
|
||||||
|
for person_index in NAMED_PERSON_COUNT:
|
||||||
|
var person: PersonStateRecord
|
||||||
|
if person_index < CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN:
|
||||||
|
var group_index := person_index / DETAILED_MEMBERS_PER_CARAVAN
|
||||||
|
var home_index := group_index % LOCATION_COUNT
|
||||||
|
person = PersonStateRecord.create_in_group(
|
||||||
|
_person_id(person_index),
|
||||||
|
"Regional Person %04d" % person_index,
|
||||||
|
_role_id(seed_value, person_index),
|
||||||
|
_polity_id(home_index),
|
||||||
|
_location_id(home_index),
|
||||||
|
_group_id(group_index)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
var location_index := (
|
||||||
|
(person_index - CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN) % LOCATION_COUNT
|
||||||
|
)
|
||||||
|
var location_id := _location_id(location_index)
|
||||||
|
person = PersonStateRecord.create_at_location(
|
||||||
|
_person_id(person_index),
|
||||||
|
"Regional Person %04d" % person_index,
|
||||||
|
_role_id(seed_value, person_index),
|
||||||
|
_polity_id(location_index),
|
||||||
|
location_id,
|
||||||
|
_address(
|
||||||
|
location_id,
|
||||||
|
Vector3(float(person_index % 64), 0.0, float((person_index / 64) % 64))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if person == null:
|
||||||
|
return null
|
||||||
|
persons.append(person)
|
||||||
|
|
||||||
|
var residents_per_location := COHORT_RESIDENT_COUNT / LOCATION_COUNT
|
||||||
|
for index in LOCATION_COUNT:
|
||||||
|
var cohort := PopulationCohortRecord.create_at_location(
|
||||||
|
_cohort_id(index),
|
||||||
|
&"aggregate_residents",
|
||||||
|
_polity_id(index),
|
||||||
|
residents_per_location,
|
||||||
|
_location_id(index)
|
||||||
|
)
|
||||||
|
if cohort == null:
|
||||||
|
return null
|
||||||
|
cohorts.append(cohort)
|
||||||
|
|
||||||
|
return RegionalWorldState.create(
|
||||||
|
WORLD_ID, locations, settlements, routes, groups, persons, cohorts, polities, relations
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
func run(execution_budget: int = 8, seed_value: int = FIXED_SEED) -> Dictionary:
|
||||||
|
if execution_budget not in [1, 8, RegionalJobScheduler.UNLIMITED_BUDGET]:
|
||||||
|
return {}
|
||||||
|
var total_started_usec := Time.get_ticks_usec()
|
||||||
|
var fixture_started_usec := Time.get_ticks_usec()
|
||||||
|
var state := build_fixture(seed_value)
|
||||||
|
var fixture_build_usec := maxi(Time.get_ticks_usec() - fixture_started_usec, 1)
|
||||||
|
if state == null:
|
||||||
|
return {}
|
||||||
|
var fixture_audit := audit_fixture(state)
|
||||||
|
if not fixture_audit["valid"]:
|
||||||
|
return {}
|
||||||
|
var state_dictionary := state.to_dictionary()
|
||||||
|
var state_json := JSON.stringify(state_dictionary)
|
||||||
|
var state_bytes := state_json.to_utf8_buffer().size()
|
||||||
|
var fixture_checksum := state.checksum()
|
||||||
|
|
||||||
|
var roundtrip_started_usec := Time.get_ticks_usec()
|
||||||
|
var restored := RegionalWorldState.from_dictionary(state_dictionary)
|
||||||
|
var roundtrip_usec := maxi(Time.get_ticks_usec() - roundtrip_started_usec, 1)
|
||||||
|
if (
|
||||||
|
restored == null
|
||||||
|
or restored.to_dictionary() != state_dictionary
|
||||||
|
or restored.checksum() != fixture_checksum
|
||||||
|
):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
var scheduler_started_usec := Time.get_ticks_usec()
|
||||||
|
var scheduler := _build_scheduler(state, seed_value)
|
||||||
|
var scheduler_build_usec := maxi(Time.get_ticks_usec() - scheduler_started_usec, 1)
|
||||||
|
if scheduler == null:
|
||||||
|
return {}
|
||||||
|
var initial_jobs := scheduler.size()
|
||||||
|
var scheduler_state_bytes := JSON.stringify(scheduler.to_dictionary()).to_utf8_buffer().size()
|
||||||
|
var execution_order: Array[StringName] = []
|
||||||
|
var processed_named_people := 0
|
||||||
|
var processed_cohort_residents := 0
|
||||||
|
var processed_cargo := 0.0
|
||||||
|
var backlog_peak := scheduler.size()
|
||||||
|
var drain_batches := 0
|
||||||
|
var max_batch_size := 0
|
||||||
|
var execution_started_usec := Time.get_ticks_usec()
|
||||||
|
while not scheduler.is_empty():
|
||||||
|
backlog_peak = maxi(backlog_peak, scheduler.size())
|
||||||
|
var batch := scheduler.take_due(MAX_DUE_TICK, execution_budget)
|
||||||
|
if batch.is_empty():
|
||||||
|
return {}
|
||||||
|
drain_batches += 1
|
||||||
|
max_batch_size = maxi(max_batch_size, batch.size())
|
||||||
|
for job in batch:
|
||||||
|
execution_order.append(job.get_job_id())
|
||||||
|
var payload := job.get_payload()
|
||||||
|
processed_named_people += int(payload.get("named_people", 0))
|
||||||
|
processed_cohort_residents += int(payload.get("cohort_residents", 0))
|
||||||
|
processed_cargo += float(payload.get("cargo_total", 0.0))
|
||||||
|
var execution_usec := maxi(Time.get_ticks_usec() - execution_started_usec, 1)
|
||||||
|
var final_audit := audit_fixture(state)
|
||||||
|
var headcount_conserved: bool = (
|
||||||
|
final_audit["named_person_count"] == fixture_audit["named_person_count"]
|
||||||
|
and final_audit["cohort_resident_count"] == fixture_audit["cohort_resident_count"]
|
||||||
|
and processed_named_people == NAMED_PERSON_COUNT
|
||||||
|
and processed_cohort_residents == COHORT_RESIDENT_COUNT
|
||||||
|
)
|
||||||
|
var cargo_conserved: bool = (
|
||||||
|
is_equal_approx(float(final_audit["cargo_total"]), float(fixture_audit["cargo_total"]))
|
||||||
|
and is_equal_approx(processed_cargo, float(fixture_audit["cargo_total"]))
|
||||||
|
)
|
||||||
|
var order_checksum := JSON.stringify(execution_order).sha256_text()
|
||||||
|
var final_checksum := (
|
||||||
|
JSON
|
||||||
|
. stringify(
|
||||||
|
{
|
||||||
|
"fixture_checksum": fixture_checksum,
|
||||||
|
"execution_order": execution_order,
|
||||||
|
"processed_cargo": processed_cargo,
|
||||||
|
"processed_cohort_residents": processed_cohort_residents,
|
||||||
|
"processed_named_people": processed_named_people,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
. sha256_text()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": SCHEMA_VERSION,
|
||||||
|
"workload_id": String(WORKLOAD_ID),
|
||||||
|
"benchmark_scope": "regional_data_and_scheduler_only",
|
||||||
|
"rendered_throughput_measured": false,
|
||||||
|
"full_economy_throughput_measured": false,
|
||||||
|
"seed": seed_value,
|
||||||
|
"location_count": LOCATION_COUNT,
|
||||||
|
"settlement_count": SETTLEMENT_COUNT,
|
||||||
|
"caravan_count": CARAVAN_COUNT,
|
||||||
|
"detailed_members_per_caravan": DETAILED_MEMBERS_PER_CARAVAN,
|
||||||
|
"detailed_caravan_member_count": fixture_audit["detailed_caravan_member_count"],
|
||||||
|
"named_person_count": fixture_audit["named_person_count"],
|
||||||
|
"cohort_resident_count": fixture_audit["cohort_resident_count"],
|
||||||
|
"total_headcount": fixture_audit["total_headcount"],
|
||||||
|
"max_active_visuals": MAX_ACTIVE_VISUALS,
|
||||||
|
"fixture_valid": fixture_audit["valid"],
|
||||||
|
"roundtrip_valid": true,
|
||||||
|
"headcount_conserved": headcount_conserved,
|
||||||
|
"cargo_conserved": cargo_conserved,
|
||||||
|
"cargo_total": fixture_audit["cargo_total"],
|
||||||
|
"processed_cargo_total": processed_cargo,
|
||||||
|
"fixture_checksum": fixture_checksum,
|
||||||
|
"execution_order_checksum": order_checksum,
|
||||||
|
"final_checksum": final_checksum,
|
||||||
|
"execution_order": execution_order,
|
||||||
|
"initial_job_count": initial_jobs,
|
||||||
|
"executed_job_count": execution_order.size(),
|
||||||
|
"execution_budget": execution_budget,
|
||||||
|
"backlog_initial": initial_jobs,
|
||||||
|
"backlog_peak": backlog_peak,
|
||||||
|
"backlog_final": scheduler.size(),
|
||||||
|
"drain_batches": drain_batches,
|
||||||
|
"max_batch_size": max_batch_size,
|
||||||
|
"regional_state_bytes": state_bytes,
|
||||||
|
"scheduler_state_bytes": scheduler_state_bytes,
|
||||||
|
"fixture_build_usec": fixture_build_usec,
|
||||||
|
"roundtrip_usec": roundtrip_usec,
|
||||||
|
"scheduler_build_usec": scheduler_build_usec,
|
||||||
|
"execution_usec": execution_usec,
|
||||||
|
"elapsed_usec": maxi(Time.get_ticks_usec() - total_started_usec, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func audit_fixture(state: RegionalWorldState) -> Dictionary:
|
||||||
|
if state == null:
|
||||||
|
return {"valid": false}
|
||||||
|
var valid := (
|
||||||
|
state.get_world_id() == WORLD_ID
|
||||||
|
and state.get_all_locations().size() == LOCATION_COUNT
|
||||||
|
and state.get_all_settlements().size() == SETTLEMENT_COUNT
|
||||||
|
and state.get_all_routes().size() == LOCATION_COUNT
|
||||||
|
and state.get_all_mobile_groups().size() == CARAVAN_COUNT
|
||||||
|
and state.get_all_persons().size() == NAMED_PERSON_COUNT
|
||||||
|
and state.get_all_population_cohorts().size() == LOCATION_COUNT
|
||||||
|
and state.get_all_polities().size() == LOCATION_COUNT
|
||||||
|
)
|
||||||
|
var detailed_members := 0
|
||||||
|
var cargo_total := 0.0
|
||||||
|
for group in state.get_all_mobile_groups():
|
||||||
|
var member_ids := group.get_detailed_member_ids()
|
||||||
|
detailed_members += member_ids.size()
|
||||||
|
cargo_total += RegionalStateSupport.amount_ledger_total(group.get_cargo_ledger())
|
||||||
|
valid = (
|
||||||
|
valid
|
||||||
|
and member_ids.size() == DETAILED_MEMBERS_PER_CARAVAN
|
||||||
|
and (
|
||||||
|
state.get_persons_in_group(group.get_group_id()).size()
|
||||||
|
== DETAILED_MEMBERS_PER_CARAVAN
|
||||||
|
)
|
||||||
|
and group.is_on_route()
|
||||||
|
)
|
||||||
|
for person_id in member_ids:
|
||||||
|
valid = valid and state.get_entity_type(person_id) == &"person"
|
||||||
|
var cohort_residents := 0
|
||||||
|
for cohort in state.get_all_population_cohorts():
|
||||||
|
cohort_residents += cohort.get_count()
|
||||||
|
valid = valid and cohort.is_at_location()
|
||||||
|
for index in LOCATION_COUNT:
|
||||||
|
valid = (
|
||||||
|
valid
|
||||||
|
and state.get_settlements_at_location(_location_id(index)).size() == 1
|
||||||
|
and state.get_entity_type(_location_id(index)) == &"location"
|
||||||
|
and state.get_entity_type(_settlement_id(index)) == &"settlement"
|
||||||
|
and state.get_entity_type(_polity_id(index)) == &"polity"
|
||||||
|
)
|
||||||
|
valid = (
|
||||||
|
valid
|
||||||
|
and detailed_members == CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN
|
||||||
|
and cohort_residents == COHORT_RESIDENT_COUNT
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"valid": valid,
|
||||||
|
"named_person_count": state.get_all_persons().size(),
|
||||||
|
"detailed_caravan_member_count": detailed_members,
|
||||||
|
"cohort_resident_count": cohort_residents,
|
||||||
|
"total_headcount": state.get_all_persons().size() + cohort_residents,
|
||||||
|
"cargo_total": cargo_total,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func _build_scheduler(state: RegionalWorldState, seed_value: int) -> RegionalJobScheduler:
|
||||||
|
var scheduler := RegionalJobScheduler.new()
|
||||||
|
for person in state.get_all_persons():
|
||||||
|
if (
|
||||||
|
scheduler.schedule_new(
|
||||||
|
&"regional_person:%s" % person.get_person_id(),
|
||||||
|
&"regional_person_review",
|
||||||
|
KeyedRandom.range_int(
|
||||||
|
seed_value, &"regional_person_job", person.get_person_id(), 0, 1, MAX_DUE_TICK
|
||||||
|
),
|
||||||
|
0,
|
||||||
|
person.get_person_id(),
|
||||||
|
{"named_people": 1}
|
||||||
|
)
|
||||||
|
== null
|
||||||
|
):
|
||||||
|
return null
|
||||||
|
for cohort in state.get_all_population_cohorts():
|
||||||
|
if (
|
||||||
|
scheduler.schedule_new(
|
||||||
|
&"regional_cohort:%s" % cohort.get_cohort_id(),
|
||||||
|
&"regional_cohort_review",
|
||||||
|
KeyedRandom.range_int(
|
||||||
|
seed_value, &"regional_cohort_job", cohort.get_cohort_id(), 0, 1, MAX_DUE_TICK
|
||||||
|
),
|
||||||
|
1,
|
||||||
|
cohort.get_cohort_id(),
|
||||||
|
{"cohort_residents": cohort.get_count()}
|
||||||
|
)
|
||||||
|
== null
|
||||||
|
):
|
||||||
|
return null
|
||||||
|
for group in state.get_all_mobile_groups():
|
||||||
|
if (
|
||||||
|
scheduler.schedule_new(
|
||||||
|
&"regional_caravan:%s" % group.get_group_id(),
|
||||||
|
&"regional_caravan_review",
|
||||||
|
KeyedRandom.range_int(
|
||||||
|
seed_value, &"regional_caravan_job", group.get_group_id(), 0, 1, MAX_DUE_TICK
|
||||||
|
),
|
||||||
|
2,
|
||||||
|
group.get_group_id(),
|
||||||
|
{"cargo_total": RegionalStateSupport.amount_ledger_total(group.get_cargo_ledger())}
|
||||||
|
)
|
||||||
|
== null
|
||||||
|
):
|
||||||
|
return null
|
||||||
|
return scheduler
|
||||||
|
|
||||||
|
|
||||||
|
func _role_id(seed_value: int, person_index: int) -> StringName:
|
||||||
|
var role_index := KeyedRandom.range_int(
|
||||||
|
seed_value, &"person_role", _person_id(person_index), 0, 0, ROLE_IDS.size() - 1
|
||||||
|
)
|
||||||
|
return ROLE_IDS[role_index]
|
||||||
|
|
||||||
|
|
||||||
|
func _address(location_id: StringName, position: Vector3) -> SpatialAddress:
|
||||||
|
return SpatialAddress.create(WORLD_ID, location_id, position)
|
||||||
|
|
||||||
|
|
||||||
|
func _location_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_location_%02d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _settlement_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_settlement_%02d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _route_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_route_%02d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _group_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_caravan_%02d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _person_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_person_%04d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _cohort_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_residents_%02d" % index)
|
||||||
|
|
||||||
|
|
||||||
|
func _polity_id(index: int) -> StringName:
|
||||||
|
return StringName("regional_polity_%02d" % index)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://bbh75scb18pak
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
extends GutTest
|
||||||
|
|
||||||
|
const BenchmarkScript := preload("res://simulation/benchmark/RegionalScaleBenchmark.gd")
|
||||||
|
|
||||||
|
|
||||||
|
func test_exact_regional_workload_round_trips_and_conserves_membership() -> void:
|
||||||
|
var benchmark := BenchmarkScript.new()
|
||||||
|
var state := benchmark.build_fixture(BenchmarkScript.FIXED_SEED)
|
||||||
|
assert_not_null(state)
|
||||||
|
var audit := benchmark.audit_fixture(state)
|
||||||
|
assert_true(audit["valid"])
|
||||||
|
assert_eq(state.get_all_locations().size(), 5)
|
||||||
|
assert_eq(state.get_all_settlements().size(), 5)
|
||||||
|
assert_eq(state.get_all_mobile_groups().size(), 20)
|
||||||
|
assert_eq(audit["detailed_caravan_member_count"], 240)
|
||||||
|
assert_eq(audit["named_person_count"], 2000)
|
||||||
|
assert_eq(audit["cohort_resident_count"], 50000)
|
||||||
|
assert_eq(audit["total_headcount"], 52000)
|
||||||
|
for group in state.get_all_mobile_groups():
|
||||||
|
assert_eq(group.get_detailed_member_ids().size(), 12)
|
||||||
|
assert_eq(state.get_persons_in_group(group.get_group_id()).size(), 12)
|
||||||
|
var serialized := state.to_dictionary()
|
||||||
|
var restored := RegionalWorldState.from_dictionary(serialized)
|
||||||
|
assert_not_null(restored)
|
||||||
|
assert_eq(restored.to_dictionary(), serialized)
|
||||||
|
assert_eq(restored.checksum(), state.checksum())
|
||||||
|
assert_almost_eq(
|
||||||
|
float(benchmark.audit_fixture(restored)["cargo_total"]), float(audit["cargo_total"]), 0.0001
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
func test_scheduler_budgets_and_three_repeats_preserve_order_and_checksum() -> void:
|
||||||
|
var benchmark := BenchmarkScript.new()
|
||||||
|
var budget_one := benchmark.run(1)
|
||||||
|
var budget_eight := benchmark.run(8)
|
||||||
|
var budget_eight_repeat_a := benchmark.run(8)
|
||||||
|
var budget_eight_repeat_b := benchmark.run(8)
|
||||||
|
var unlimited := benchmark.run(RegionalJobScheduler.UNLIMITED_BUDGET)
|
||||||
|
for result in [
|
||||||
|
budget_one, budget_eight, budget_eight_repeat_a, budget_eight_repeat_b, unlimited
|
||||||
|
]:
|
||||||
|
_assert_valid_result(result)
|
||||||
|
|
||||||
|
for result in [budget_one, unlimited, budget_eight_repeat_a, budget_eight_repeat_b]:
|
||||||
|
assert_eq(result["execution_order"], budget_eight["execution_order"])
|
||||||
|
assert_eq(result["execution_order_checksum"], budget_eight["execution_order_checksum"])
|
||||||
|
assert_eq(result["final_checksum"], budget_eight["final_checksum"])
|
||||||
|
assert_eq(result["fixture_checksum"], budget_eight["fixture_checksum"])
|
||||||
|
assert_eq(result["regional_state_bytes"], budget_eight["regional_state_bytes"])
|
||||||
|
assert_eq(budget_one["drain_batches"], budget_one["initial_job_count"])
|
||||||
|
assert_eq(budget_one["max_batch_size"], 1)
|
||||||
|
assert_eq(budget_eight["drain_batches"], ceili(float(budget_eight["initial_job_count"]) / 8.0))
|
||||||
|
assert_eq(budget_eight["max_batch_size"], 8)
|
||||||
|
assert_eq(unlimited["drain_batches"], 1)
|
||||||
|
assert_eq(unlimited["max_batch_size"], unlimited["initial_job_count"])
|
||||||
|
|
||||||
|
|
||||||
|
func _assert_valid_result(result: Dictionary) -> void:
|
||||||
|
assert_false(result.is_empty())
|
||||||
|
if result.is_empty():
|
||||||
|
return
|
||||||
|
assert_eq(result["schema_version"], 1)
|
||||||
|
assert_eq(StringName(result["workload_id"]), &"regional_records_scheduler_v1")
|
||||||
|
assert_eq(result["benchmark_scope"], "regional_data_and_scheduler_only")
|
||||||
|
assert_false(result["rendered_throughput_measured"])
|
||||||
|
assert_false(result["full_economy_throughput_measured"])
|
||||||
|
assert_eq(result["seed"], 424242)
|
||||||
|
assert_eq(result["location_count"], 5)
|
||||||
|
assert_eq(result["settlement_count"], 5)
|
||||||
|
assert_eq(result["caravan_count"], 20)
|
||||||
|
assert_eq(result["detailed_members_per_caravan"], 12)
|
||||||
|
assert_eq(result["detailed_caravan_member_count"], 240)
|
||||||
|
assert_eq(result["named_person_count"], 2000)
|
||||||
|
assert_eq(result["cohort_resident_count"], 50000)
|
||||||
|
assert_eq(result["total_headcount"], 52000)
|
||||||
|
assert_eq(result["max_active_visuals"], 40)
|
||||||
|
assert_true(result["fixture_valid"])
|
||||||
|
assert_true(result["roundtrip_valid"])
|
||||||
|
assert_true(result["headcount_conserved"])
|
||||||
|
assert_true(result["cargo_conserved"])
|
||||||
|
assert_eq(result["initial_job_count"], 2025)
|
||||||
|
assert_eq(result["executed_job_count"], 2025)
|
||||||
|
assert_eq(result["backlog_initial"], 2025)
|
||||||
|
assert_eq(result["backlog_peak"], 2025)
|
||||||
|
assert_eq(result["backlog_final"], 0)
|
||||||
|
assert_gt(result["regional_state_bytes"], 0)
|
||||||
|
assert_gt(result["scheduler_state_bytes"], 0)
|
||||||
|
for timing_key in [
|
||||||
|
"fixture_build_usec",
|
||||||
|
"roundtrip_usec",
|
||||||
|
"scheduler_build_usec",
|
||||||
|
"execution_usec",
|
||||||
|
"elapsed_usec",
|
||||||
|
]:
|
||||||
|
assert_gt(result[timing_key], 0)
|
||||||
|
assert_eq(String(result["fixture_checksum"]).length(), 64)
|
||||||
|
assert_eq(String(result["execution_order_checksum"]).length(), 64)
|
||||||
|
assert_eq(String(result["final_checksum"]).length(), 64)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://crgxogf0ldlki
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
extends SceneTree
|
||||||
|
|
||||||
|
const BenchmarkScript := preload("res://simulation/benchmark/RegionalScaleBenchmark.gd")
|
||||||
|
const DEFAULT_OUTPUT_PATH := "user://regional_scale_latest.json"
|
||||||
|
const DEFAULT_HOST_LABEL := "unspecified"
|
||||||
|
const SAMPLE_COUNT := 3
|
||||||
|
const DEFAULT_BUDGET := 8
|
||||||
|
|
||||||
|
|
||||||
|
func _initialize() -> void:
|
||||||
|
call_deferred("_run")
|
||||||
|
|
||||||
|
|
||||||
|
func _run() -> void:
|
||||||
|
var benchmark := BenchmarkScript.new()
|
||||||
|
var samples: Array[Dictionary] = []
|
||||||
|
for _sample_index in SAMPLE_COUNT:
|
||||||
|
var sample := benchmark.run(DEFAULT_BUDGET)
|
||||||
|
if not _valid_sample(sample):
|
||||||
|
push_error("Regional scale benchmark produced an invalid default-budget sample")
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
samples.append(sample)
|
||||||
|
var budget_one := benchmark.run(1)
|
||||||
|
var unlimited := benchmark.run(RegionalJobScheduler.UNLIMITED_BUDGET)
|
||||||
|
if not _valid_sample(budget_one) or not _valid_sample(unlimited):
|
||||||
|
push_error("Regional scale benchmark produced an invalid budget-parity sample")
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
if not _deterministic_samples(samples + [budget_one, unlimited]):
|
||||||
|
push_error("Regional scale benchmark diverged across samples or execution budgets")
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
|
||||||
|
var report := {
|
||||||
|
"schema_version": BenchmarkScript.SCHEMA_VERSION,
|
||||||
|
"captured_utc": Time.get_datetime_string_from_system(true) + "Z",
|
||||||
|
"engine_version": String(Engine.get_version_info().get("string", "unknown")),
|
||||||
|
"platform": OS.get_name(),
|
||||||
|
"processor_count": OS.get_processor_count(),
|
||||||
|
"host_label": _get_argument_value("--host-label=", DEFAULT_HOST_LABEL),
|
||||||
|
"workload_id": String(BenchmarkScript.WORKLOAD_ID),
|
||||||
|
"benchmark_scope": "regional_data_and_scheduler_only",
|
||||||
|
"rendered_throughput_measured": false,
|
||||||
|
"full_economy_throughput_measured": false,
|
||||||
|
"sample_count": SAMPLE_COUNT,
|
||||||
|
"default_execution_budget": DEFAULT_BUDGET,
|
||||||
|
"fixture": _fixture_summary(samples[0]),
|
||||||
|
"default_budget": _summarize_timings(samples),
|
||||||
|
"budget_parity": {
|
||||||
|
"budget_one_final_checksum": budget_one["final_checksum"],
|
||||||
|
"budget_one_drain_batches": budget_one["drain_batches"],
|
||||||
|
"unlimited_final_checksum": unlimited["final_checksum"],
|
||||||
|
"unlimited_drain_batches": unlimited["drain_batches"],
|
||||||
|
"identical": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
var output_path := _get_argument_value("--output=", DEFAULT_OUTPUT_PATH)
|
||||||
|
var output := FileAccess.open(output_path, FileAccess.WRITE)
|
||||||
|
if output == null:
|
||||||
|
push_error("Could not write regional scale benchmark report to %s" % output_path)
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
output.store_string(JSON.stringify(report, "\t") + "\n")
|
||||||
|
output.close()
|
||||||
|
print(
|
||||||
|
(
|
||||||
|
"[BENCH] regional | %d named + %d aggregate | %d caravans | "
|
||||||
|
+ "%d jobs | median %.2f ms total | %.2f MiB state"
|
||||||
|
)
|
||||||
|
% [
|
||||||
|
report["fixture"]["named_person_count"],
|
||||||
|
report["fixture"]["cohort_resident_count"],
|
||||||
|
report["fixture"]["caravan_count"],
|
||||||
|
report["fixture"]["initial_job_count"],
|
||||||
|
float(report["default_budget"]["elapsed_usec_median"]) / 1000.0,
|
||||||
|
float(report["fixture"]["regional_state_bytes"]) / (1024.0 * 1024.0),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
print("[BENCH] Report: %s" % ProjectSettings.globalize_path(output_path))
|
||||||
|
quit(0)
|
||||||
|
|
||||||
|
|
||||||
|
func _valid_sample(sample: Dictionary) -> bool:
|
||||||
|
return (
|
||||||
|
not sample.is_empty()
|
||||||
|
and bool(sample.get("fixture_valid", false))
|
||||||
|
and bool(sample.get("roundtrip_valid", false))
|
||||||
|
and bool(sample.get("headcount_conserved", false))
|
||||||
|
and bool(sample.get("cargo_conserved", false))
|
||||||
|
and int(sample.get("backlog_final", -1)) == 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
func _deterministic_samples(samples: Array) -> bool:
|
||||||
|
if samples.is_empty():
|
||||||
|
return false
|
||||||
|
var expected: Dictionary = samples[0]
|
||||||
|
for sample: Dictionary in samples:
|
||||||
|
for key in [
|
||||||
|
"fixture_checksum",
|
||||||
|
"execution_order_checksum",
|
||||||
|
"final_checksum",
|
||||||
|
"initial_job_count",
|
||||||
|
"executed_job_count",
|
||||||
|
"regional_state_bytes",
|
||||||
|
"scheduler_state_bytes",
|
||||||
|
]:
|
||||||
|
if sample.get(key) != expected.get(key):
|
||||||
|
return false
|
||||||
|
return true
|
||||||
|
|
||||||
|
|
||||||
|
func _fixture_summary(sample: Dictionary) -> Dictionary:
|
||||||
|
var result := {}
|
||||||
|
for key in [
|
||||||
|
"seed",
|
||||||
|
"location_count",
|
||||||
|
"settlement_count",
|
||||||
|
"caravan_count",
|
||||||
|
"detailed_members_per_caravan",
|
||||||
|
"detailed_caravan_member_count",
|
||||||
|
"named_person_count",
|
||||||
|
"cohort_resident_count",
|
||||||
|
"total_headcount",
|
||||||
|
"max_active_visuals",
|
||||||
|
"cargo_total",
|
||||||
|
"initial_job_count",
|
||||||
|
"regional_state_bytes",
|
||||||
|
"scheduler_state_bytes",
|
||||||
|
"fixture_checksum",
|
||||||
|
"execution_order_checksum",
|
||||||
|
"final_checksum",
|
||||||
|
]:
|
||||||
|
result[key] = sample[key]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
func _summarize_timings(samples: Array[Dictionary]) -> Dictionary:
|
||||||
|
var result := {}
|
||||||
|
for key in [
|
||||||
|
"fixture_build_usec",
|
||||||
|
"roundtrip_usec",
|
||||||
|
"scheduler_build_usec",
|
||||||
|
"execution_usec",
|
||||||
|
"elapsed_usec",
|
||||||
|
]:
|
||||||
|
var values: Array[int] = []
|
||||||
|
for sample in samples:
|
||||||
|
values.append(int(sample[key]))
|
||||||
|
values.sort()
|
||||||
|
result[key + "_samples"] = values
|
||||||
|
result[key + "_median"] = values[values.size() / 2]
|
||||||
|
result["drain_batches"] = samples[0]["drain_batches"]
|
||||||
|
result["backlog_initial"] = samples[0]["backlog_initial"]
|
||||||
|
result["backlog_peak"] = samples[0]["backlog_peak"]
|
||||||
|
result["backlog_final"] = samples[0]["backlog_final"]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
func _get_argument_value(prefix: String, default_value: String) -> String:
|
||||||
|
for argument in OS.get_cmdline_user_args():
|
||||||
|
if argument.begins_with(prefix):
|
||||||
|
var value := argument.trim_prefix(prefix)
|
||||||
|
if not value.is_empty():
|
||||||
|
return value
|
||||||
|
return default_value
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://br2cgdiw4s6an
|
||||||
Reference in New Issue
Block a user