diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 29f745a..20f267c 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -24,8 +24,24 @@ Loaded-resource discovery has its own bounded runner: --script res://tools/benchmark_loaded_resource_discovery.gd ``` +The regional record/scheduler workload has a separate honest runner: + +```bash +/Applications/Godot.app/Contents/MacOS/Godot \ + --headless --path "$PWD" \ + --script res://tools/benchmark_regional_scale.gd +``` + +It constructs five settlements, 20 caravans, 2,000 named people, 50,000 +aggregate residents, and 2,025 deterministic due jobs. It measures record +construction, canonical roundtrip, scheduler construction, and job draining; +it does not claim rendered or full-economy throughput. + Reviewed captures: +- [Regional scale baseline 01](REGIONAL_SCALE_BASELINE_01.md) records the + primitive regional state and deterministic scheduler workload at five + settlements, 20 caravans, 2,000 named people, and 50,000 aggregate residents. - [Simulation scaling baseline 01](SIMULATION_SCALING_BASELINE_01.md) records the first full-fidelity population/history measurements and the bounded next optimization. diff --git a/docs/benchmarks/REGIONAL_SCALE_BASELINE_01.md b/docs/benchmarks/REGIONAL_SCALE_BASELINE_01.md new file mode 100644 index 0000000..d606da7 --- /dev/null +++ b/docs/benchmarks/REGIONAL_SCALE_BASELINE_01.md @@ -0,0 +1,68 @@ +# Regional scale baseline 01 + +This is the first reviewed capture of workload schema 1, +`regional_records_scheduler_v1`. It is a structural headless baseline for the +regional record and deterministic due-job contracts—not a claim about rendered +gameplay or the complete economy. + +The machine-readable report is in +[`regional_scale_baseline_01.json`](regional_scale_baseline_01.json). It was +captured with Godot 4.7 on the `Apple_M1_Max_64_GB` development host, seed +424242, three default-budget samples, and parity runs at budgets one and +unlimited. + +## Fixed workload + +- five locations, settlements, polities, and ring routes; +- 20 caravans with 12 named members each; +- 2,000 persistent named people; +- five aggregate cohorts totaling 50,000 residents; +- 2,025 keyed deterministic due jobs; +- maximum active-presentation metadata of 40; +- 736,115 bytes of canonical regional state and 540,942 bytes of initial + scheduler state. + +All records round-tripped through their primitive-only schemas. The fixture +contained 52,000 people and 2,734 cargo units without duplicate global IDs or +membership errors. Budget one, budget eight, and unlimited execution drained +the backlog to zero in exactly the same order and produced final checksum +`57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25`. + +## Results + +| Phase | Median | Samples | +| --- | ---: | --- | +| Fixture construction | 273.79 ms | 272.37, 273.79, 282.07 ms | +| Canonical roundtrip | 255.86 ms | 255.33, 255.86, 266.03 ms | +| Scheduler construction | 74.10 ms | 73.66, 74.10, 76.50 ms | +| Drain 2,025 jobs (budget 8) | 57.47 ms | 55.52, 57.47, 59.51 ms | +| Complete measured run | 785.65 ms | 776.58, 785.65, 808.22 ms | + +Budget eight drained the queue in 254 batches. Budget one used 2,025 batches; +unlimited used one. Their execution-order, fixture, and final checksums were +identical. + +## Interpretation + +The baseline proves that the target record counts fit in a small canonical +payload and that job-budget changes do not alter outcomes. It intentionally +constructs only 2,025 event-boundary jobs, so the 57.47 ms drain is not a +30-day simulation-time throughput result. It also excludes scene loading, +navigation, rendering, dialogue/situation queries, market transactions, +conflict, event rollups, chunked disk I/O, and live caravan transfers. + +The next benchmark must run scheduled settlement and caravan domain handlers +over 30 simulated days. Separately, a recorded weak-PC active-world capture +must measure the Low presentation tier with at most 40 detailed visuals and two +new navigation requests per frame. These measurements cannot be inferred from +this development-host structural result. + +## Capture command + +```bash +/Applications/Godot.app/Contents/MacOS/Godot \ + --headless --path "$PWD" \ + --script res://tools/benchmark_regional_scale.gd -- \ + --host-label=Apple_M1_Max_64_GB \ + --output=res://docs/benchmarks/regional_scale_baseline_01.json +``` diff --git a/docs/benchmarks/regional_scale_baseline_01.json b/docs/benchmarks/regional_scale_baseline_01.json new file mode 100644 index 0000000..16b219a --- /dev/null +++ b/docs/benchmarks/regional_scale_baseline_01.json @@ -0,0 +1,56 @@ +{ + "benchmark_scope": "regional_data_and_scheduler_only", + "budget_parity": { + "budget_one_drain_batches": 2025, + "budget_one_final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25", + "identical": true, + "unlimited_drain_batches": 1, + "unlimited_final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25" + }, + "captured_utc": "2026-08-12T19:36:00Z", + "default_budget": { + "backlog_final": 0, + "backlog_initial": 2025, + "backlog_peak": 2025, + "drain_batches": 254, + "elapsed_usec_median": 785655, + "elapsed_usec_samples": [776580, 785655, 808216], + "execution_usec_median": 57471, + "execution_usec_samples": [55516, 57471, 59508], + "fixture_build_usec_median": 273789, + "fixture_build_usec_samples": [272365, 273789, 282068], + "roundtrip_usec_median": 255860, + "roundtrip_usec_samples": [255326, 255860, 266034], + "scheduler_build_usec_median": 74100, + "scheduler_build_usec_samples": [73657, 74100, 76495] + }, + "default_execution_budget": 8, + "engine_version": "4.7-stable (official)", + "fixture": { + "caravan_count": 20, + "cargo_total": 2734.0, + "cohort_resident_count": 50000, + "detailed_caravan_member_count": 240, + "detailed_members_per_caravan": 12, + "execution_order_checksum": "38815c49987cf66ec0d78b82d332789f708742959bab83c9feee631d8bbf8174", + "final_checksum": "57c8a8eb9dd48bbea309cdb5654ce508f6e4965b07228217aa55fc43de699a25", + "fixture_checksum": "5ee2dcda7067b34e1ffc22861fbcf1e6d1897f87523a88fbf216fc9ea6a19ed5", + "initial_job_count": 2025, + "location_count": 5, + "max_active_visuals": 40, + "named_person_count": 2000, + "regional_state_bytes": 736115, + "scheduler_state_bytes": 540942, + "seed": 424242, + "settlement_count": 5, + "total_headcount": 52000 + }, + "full_economy_throughput_measured": false, + "host_label": "Apple_M1_Max_64_GB", + "platform": "macOS", + "processor_count": 10, + "rendered_throughput_measured": false, + "sample_count": 3, + "schema_version": 1, + "workload_id": "regional_records_scheduler_v1" +} diff --git a/simulation/benchmark/RegionalScaleBenchmark.gd b/simulation/benchmark/RegionalScaleBenchmark.gd new file mode 100644 index 0000000..e151078 --- /dev/null +++ b/simulation/benchmark/RegionalScaleBenchmark.gd @@ -0,0 +1,429 @@ +class_name RegionalScaleBenchmark +extends RefCounted + +# This workload measures primitive regional records and deterministic scheduler +# draining only. It does not load scenes or exercise the complete economy. +const SCHEMA_VERSION := 1 +const WORKLOAD_ID := &"regional_records_scheduler_v1" +const WORLD_ID := &"regional_benchmark" +const FIXED_SEED := 424242 +const LOCATION_COUNT := 5 +const SETTLEMENT_COUNT := 5 +const CARAVAN_COUNT := 20 +const DETAILED_MEMBERS_PER_CARAVAN := 12 +const NAMED_PERSON_COUNT := 2000 +const COHORT_RESIDENT_COUNT := 50000 +const MAX_ACTIVE_VISUALS := 40 +const MAX_DUE_TICK := 256 +const ROLE_IDS: Array[StringName] = [&"artisan", &"farmer", &"guard", &"merchant", &"scholar"] + + +func build_fixture(seed_value: int = FIXED_SEED) -> RegionalWorldState: + var locations: Array[LocationStateRecord] = [] + var settlements: Array[SettlementStateRecord] = [] + var routes: Array[RouteStateRecord] = [] + var groups: Array[MobileGroupStateRecord] = [] + var persons: Array[PersonStateRecord] = [] + var cohorts: Array[PopulationCohortRecord] = [] + var polities: Array[PolityStateRecord] = [] + var relations: Array[DiplomaticRelationRecord] = [] + + for index in LOCATION_COUNT: + var location_id := _location_id(index) + var location := LocationStateRecord.create( + location_id, + WORLD_ID, + &"settlement", + "Regional Settlement %02d" % index, + _address(location_id, Vector3(float(index) * 100.0, 0.0, 0.0)) + ) + var settlement := SettlementStateRecord.create( + _settlement_id(index), + location_id, + "Regional Settlement %02d" % index, + _polity_id(index), + 2, + _group_id(index), + 0, + {&"food": 10000.0 + index * 100.0, &"wood": 5000.0 + index * 50.0} + ) + var polity := PolityStateRecord.create( + _polity_id(index), "Regional Polity %02d" % index, location_id, [_settlement_id(index)] + ) + if location == null or settlement == null or polity == null: + return null + locations.append(location) + settlements.append(settlement) + polities.append(polity) + + for index in LOCATION_COUNT: + var route := RouteStateRecord.create( + _route_id(index), + _location_id(index), + _location_id((index + 1) % LOCATION_COUNT), + 20 + index, + true + ) + if route == null: + return null + routes.append(route) + + for group_index in CARAVAN_COUNT: + var route_index := group_index % LOCATION_COUNT + var member_ids: Array[StringName] = [] + for member_index in DETAILED_MEMBERS_PER_CARAVAN: + member_ids.append(_person_id(group_index * DETAILED_MEMBERS_PER_CARAVAN + member_index)) + var departure_tick := group_index * 3 + var cargo := { + &"food": + float( + KeyedRandom.range_int(seed_value, &"cargo_food", _group_id(group_index), 0, 50, 99) + ), + &"herb": + float( + KeyedRandom.range_int(seed_value, &"cargo_herb", _group_id(group_index), 0, 10, 29) + ), + &"wood": + float( + KeyedRandom.range_int(seed_value, &"cargo_wood", _group_id(group_index), 0, 20, 49) + ), + } + var group := MobileGroupStateRecord.create_on_route( + _group_id(group_index), + &"caravan", + "Regional Caravan %02d" % group_index, + _polity_id(route_index), + _route_id(route_index), + _location_id(route_index), + _location_id((route_index + 1) % LOCATION_COUNT), + departure_tick, + departure_tick + 20 + route_index, + member_ids, + {}, + cargo, + 500.0 + ) + if group == null: + return null + groups.append(group) + + for person_index in NAMED_PERSON_COUNT: + var person: PersonStateRecord + if person_index < CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN: + var group_index := person_index / DETAILED_MEMBERS_PER_CARAVAN + var home_index := group_index % LOCATION_COUNT + person = PersonStateRecord.create_in_group( + _person_id(person_index), + "Regional Person %04d" % person_index, + _role_id(seed_value, person_index), + _polity_id(home_index), + _location_id(home_index), + _group_id(group_index) + ) + else: + var location_index := ( + (person_index - CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN) % LOCATION_COUNT + ) + var location_id := _location_id(location_index) + person = PersonStateRecord.create_at_location( + _person_id(person_index), + "Regional Person %04d" % person_index, + _role_id(seed_value, person_index), + _polity_id(location_index), + location_id, + _address( + location_id, + Vector3(float(person_index % 64), 0.0, float((person_index / 64) % 64)) + ) + ) + if person == null: + return null + persons.append(person) + + var residents_per_location := COHORT_RESIDENT_COUNT / LOCATION_COUNT + for index in LOCATION_COUNT: + var cohort := PopulationCohortRecord.create_at_location( + _cohort_id(index), + &"aggregate_residents", + _polity_id(index), + residents_per_location, + _location_id(index) + ) + if cohort == null: + return null + cohorts.append(cohort) + + return RegionalWorldState.create( + WORLD_ID, locations, settlements, routes, groups, persons, cohorts, polities, relations + ) + + +func run(execution_budget: int = 8, seed_value: int = FIXED_SEED) -> Dictionary: + if execution_budget not in [1, 8, RegionalJobScheduler.UNLIMITED_BUDGET]: + return {} + var total_started_usec := Time.get_ticks_usec() + var fixture_started_usec := Time.get_ticks_usec() + var state := build_fixture(seed_value) + var fixture_build_usec := maxi(Time.get_ticks_usec() - fixture_started_usec, 1) + if state == null: + return {} + var fixture_audit := audit_fixture(state) + if not fixture_audit["valid"]: + return {} + var state_dictionary := state.to_dictionary() + var state_json := JSON.stringify(state_dictionary) + var state_bytes := state_json.to_utf8_buffer().size() + var fixture_checksum := state.checksum() + + var roundtrip_started_usec := Time.get_ticks_usec() + var restored := RegionalWorldState.from_dictionary(state_dictionary) + var roundtrip_usec := maxi(Time.get_ticks_usec() - roundtrip_started_usec, 1) + if ( + restored == null + or restored.to_dictionary() != state_dictionary + or restored.checksum() != fixture_checksum + ): + return {} + + var scheduler_started_usec := Time.get_ticks_usec() + var scheduler := _build_scheduler(state, seed_value) + var scheduler_build_usec := maxi(Time.get_ticks_usec() - scheduler_started_usec, 1) + if scheduler == null: + return {} + var initial_jobs := scheduler.size() + var scheduler_state_bytes := JSON.stringify(scheduler.to_dictionary()).to_utf8_buffer().size() + var execution_order: Array[StringName] = [] + var processed_named_people := 0 + var processed_cohort_residents := 0 + var processed_cargo := 0.0 + var backlog_peak := scheduler.size() + var drain_batches := 0 + var max_batch_size := 0 + var execution_started_usec := Time.get_ticks_usec() + while not scheduler.is_empty(): + backlog_peak = maxi(backlog_peak, scheduler.size()) + var batch := scheduler.take_due(MAX_DUE_TICK, execution_budget) + if batch.is_empty(): + return {} + drain_batches += 1 + max_batch_size = maxi(max_batch_size, batch.size()) + for job in batch: + execution_order.append(job.get_job_id()) + var payload := job.get_payload() + processed_named_people += int(payload.get("named_people", 0)) + processed_cohort_residents += int(payload.get("cohort_residents", 0)) + processed_cargo += float(payload.get("cargo_total", 0.0)) + var execution_usec := maxi(Time.get_ticks_usec() - execution_started_usec, 1) + var final_audit := audit_fixture(state) + var headcount_conserved: bool = ( + final_audit["named_person_count"] == fixture_audit["named_person_count"] + and final_audit["cohort_resident_count"] == fixture_audit["cohort_resident_count"] + and processed_named_people == NAMED_PERSON_COUNT + and processed_cohort_residents == COHORT_RESIDENT_COUNT + ) + var cargo_conserved: bool = ( + is_equal_approx(float(final_audit["cargo_total"]), float(fixture_audit["cargo_total"])) + and is_equal_approx(processed_cargo, float(fixture_audit["cargo_total"])) + ) + var order_checksum := JSON.stringify(execution_order).sha256_text() + var final_checksum := ( + JSON + . stringify( + { + "fixture_checksum": fixture_checksum, + "execution_order": execution_order, + "processed_cargo": processed_cargo, + "processed_cohort_residents": processed_cohort_residents, + "processed_named_people": processed_named_people, + } + ) + . sha256_text() + ) + return { + "schema_version": SCHEMA_VERSION, + "workload_id": String(WORKLOAD_ID), + "benchmark_scope": "regional_data_and_scheduler_only", + "rendered_throughput_measured": false, + "full_economy_throughput_measured": false, + "seed": seed_value, + "location_count": LOCATION_COUNT, + "settlement_count": SETTLEMENT_COUNT, + "caravan_count": CARAVAN_COUNT, + "detailed_members_per_caravan": DETAILED_MEMBERS_PER_CARAVAN, + "detailed_caravan_member_count": fixture_audit["detailed_caravan_member_count"], + "named_person_count": fixture_audit["named_person_count"], + "cohort_resident_count": fixture_audit["cohort_resident_count"], + "total_headcount": fixture_audit["total_headcount"], + "max_active_visuals": MAX_ACTIVE_VISUALS, + "fixture_valid": fixture_audit["valid"], + "roundtrip_valid": true, + "headcount_conserved": headcount_conserved, + "cargo_conserved": cargo_conserved, + "cargo_total": fixture_audit["cargo_total"], + "processed_cargo_total": processed_cargo, + "fixture_checksum": fixture_checksum, + "execution_order_checksum": order_checksum, + "final_checksum": final_checksum, + "execution_order": execution_order, + "initial_job_count": initial_jobs, + "executed_job_count": execution_order.size(), + "execution_budget": execution_budget, + "backlog_initial": initial_jobs, + "backlog_peak": backlog_peak, + "backlog_final": scheduler.size(), + "drain_batches": drain_batches, + "max_batch_size": max_batch_size, + "regional_state_bytes": state_bytes, + "scheduler_state_bytes": scheduler_state_bytes, + "fixture_build_usec": fixture_build_usec, + "roundtrip_usec": roundtrip_usec, + "scheduler_build_usec": scheduler_build_usec, + "execution_usec": execution_usec, + "elapsed_usec": maxi(Time.get_ticks_usec() - total_started_usec, 1), + } + + +func audit_fixture(state: RegionalWorldState) -> Dictionary: + if state == null: + return {"valid": false} + var valid := ( + state.get_world_id() == WORLD_ID + and state.get_all_locations().size() == LOCATION_COUNT + and state.get_all_settlements().size() == SETTLEMENT_COUNT + and state.get_all_routes().size() == LOCATION_COUNT + and state.get_all_mobile_groups().size() == CARAVAN_COUNT + and state.get_all_persons().size() == NAMED_PERSON_COUNT + and state.get_all_population_cohorts().size() == LOCATION_COUNT + and state.get_all_polities().size() == LOCATION_COUNT + ) + var detailed_members := 0 + var cargo_total := 0.0 + for group in state.get_all_mobile_groups(): + var member_ids := group.get_detailed_member_ids() + detailed_members += member_ids.size() + cargo_total += RegionalStateSupport.amount_ledger_total(group.get_cargo_ledger()) + valid = ( + valid + and member_ids.size() == DETAILED_MEMBERS_PER_CARAVAN + and ( + state.get_persons_in_group(group.get_group_id()).size() + == DETAILED_MEMBERS_PER_CARAVAN + ) + and group.is_on_route() + ) + for person_id in member_ids: + valid = valid and state.get_entity_type(person_id) == &"person" + var cohort_residents := 0 + for cohort in state.get_all_population_cohorts(): + cohort_residents += cohort.get_count() + valid = valid and cohort.is_at_location() + for index in LOCATION_COUNT: + valid = ( + valid + and state.get_settlements_at_location(_location_id(index)).size() == 1 + and state.get_entity_type(_location_id(index)) == &"location" + and state.get_entity_type(_settlement_id(index)) == &"settlement" + and state.get_entity_type(_polity_id(index)) == &"polity" + ) + valid = ( + valid + and detailed_members == CARAVAN_COUNT * DETAILED_MEMBERS_PER_CARAVAN + and cohort_residents == COHORT_RESIDENT_COUNT + ) + return { + "valid": valid, + "named_person_count": state.get_all_persons().size(), + "detailed_caravan_member_count": detailed_members, + "cohort_resident_count": cohort_residents, + "total_headcount": state.get_all_persons().size() + cohort_residents, + "cargo_total": cargo_total, + } + + +func _build_scheduler(state: RegionalWorldState, seed_value: int) -> RegionalJobScheduler: + var scheduler := RegionalJobScheduler.new() + for person in state.get_all_persons(): + if ( + scheduler.schedule_new( + &"regional_person:%s" % person.get_person_id(), + &"regional_person_review", + KeyedRandom.range_int( + seed_value, &"regional_person_job", person.get_person_id(), 0, 1, MAX_DUE_TICK + ), + 0, + person.get_person_id(), + {"named_people": 1} + ) + == null + ): + return null + for cohort in state.get_all_population_cohorts(): + if ( + scheduler.schedule_new( + &"regional_cohort:%s" % cohort.get_cohort_id(), + &"regional_cohort_review", + KeyedRandom.range_int( + seed_value, &"regional_cohort_job", cohort.get_cohort_id(), 0, 1, MAX_DUE_TICK + ), + 1, + cohort.get_cohort_id(), + {"cohort_residents": cohort.get_count()} + ) + == null + ): + return null + for group in state.get_all_mobile_groups(): + if ( + scheduler.schedule_new( + &"regional_caravan:%s" % group.get_group_id(), + &"regional_caravan_review", + KeyedRandom.range_int( + seed_value, &"regional_caravan_job", group.get_group_id(), 0, 1, MAX_DUE_TICK + ), + 2, + group.get_group_id(), + {"cargo_total": RegionalStateSupport.amount_ledger_total(group.get_cargo_ledger())} + ) + == null + ): + return null + return scheduler + + +func _role_id(seed_value: int, person_index: int) -> StringName: + var role_index := KeyedRandom.range_int( + seed_value, &"person_role", _person_id(person_index), 0, 0, ROLE_IDS.size() - 1 + ) + return ROLE_IDS[role_index] + + +func _address(location_id: StringName, position: Vector3) -> SpatialAddress: + return SpatialAddress.create(WORLD_ID, location_id, position) + + +func _location_id(index: int) -> StringName: + return StringName("regional_location_%02d" % index) + + +func _settlement_id(index: int) -> StringName: + return StringName("regional_settlement_%02d" % index) + + +func _route_id(index: int) -> StringName: + return StringName("regional_route_%02d" % index) + + +func _group_id(index: int) -> StringName: + return StringName("regional_caravan_%02d" % index) + + +func _person_id(index: int) -> StringName: + return StringName("regional_person_%04d" % index) + + +func _cohort_id(index: int) -> StringName: + return StringName("regional_residents_%02d" % index) + + +func _polity_id(index: int) -> StringName: + return StringName("regional_polity_%02d" % index) diff --git a/simulation/benchmark/RegionalScaleBenchmark.gd.uid b/simulation/benchmark/RegionalScaleBenchmark.gd.uid new file mode 100644 index 0000000..ffb5989 --- /dev/null +++ b/simulation/benchmark/RegionalScaleBenchmark.gd.uid @@ -0,0 +1 @@ +uid://bbh75scb18pak diff --git a/tests/unit/test_regional_scale_benchmark.gd b/tests/unit/test_regional_scale_benchmark.gd new file mode 100644 index 0000000..b4318d6 --- /dev/null +++ b/tests/unit/test_regional_scale_benchmark.gd @@ -0,0 +1,98 @@ +extends GutTest + +const BenchmarkScript := preload("res://simulation/benchmark/RegionalScaleBenchmark.gd") + + +func test_exact_regional_workload_round_trips_and_conserves_membership() -> void: + var benchmark := BenchmarkScript.new() + var state := benchmark.build_fixture(BenchmarkScript.FIXED_SEED) + assert_not_null(state) + var audit := benchmark.audit_fixture(state) + assert_true(audit["valid"]) + assert_eq(state.get_all_locations().size(), 5) + assert_eq(state.get_all_settlements().size(), 5) + assert_eq(state.get_all_mobile_groups().size(), 20) + assert_eq(audit["detailed_caravan_member_count"], 240) + assert_eq(audit["named_person_count"], 2000) + assert_eq(audit["cohort_resident_count"], 50000) + assert_eq(audit["total_headcount"], 52000) + for group in state.get_all_mobile_groups(): + assert_eq(group.get_detailed_member_ids().size(), 12) + assert_eq(state.get_persons_in_group(group.get_group_id()).size(), 12) + var serialized := state.to_dictionary() + var restored := RegionalWorldState.from_dictionary(serialized) + assert_not_null(restored) + assert_eq(restored.to_dictionary(), serialized) + assert_eq(restored.checksum(), state.checksum()) + assert_almost_eq( + float(benchmark.audit_fixture(restored)["cargo_total"]), float(audit["cargo_total"]), 0.0001 + ) + + +func test_scheduler_budgets_and_three_repeats_preserve_order_and_checksum() -> void: + var benchmark := BenchmarkScript.new() + var budget_one := benchmark.run(1) + var budget_eight := benchmark.run(8) + var budget_eight_repeat_a := benchmark.run(8) + var budget_eight_repeat_b := benchmark.run(8) + var unlimited := benchmark.run(RegionalJobScheduler.UNLIMITED_BUDGET) + for result in [ + budget_one, budget_eight, budget_eight_repeat_a, budget_eight_repeat_b, unlimited + ]: + _assert_valid_result(result) + + for result in [budget_one, unlimited, budget_eight_repeat_a, budget_eight_repeat_b]: + assert_eq(result["execution_order"], budget_eight["execution_order"]) + assert_eq(result["execution_order_checksum"], budget_eight["execution_order_checksum"]) + assert_eq(result["final_checksum"], budget_eight["final_checksum"]) + assert_eq(result["fixture_checksum"], budget_eight["fixture_checksum"]) + assert_eq(result["regional_state_bytes"], budget_eight["regional_state_bytes"]) + assert_eq(budget_one["drain_batches"], budget_one["initial_job_count"]) + assert_eq(budget_one["max_batch_size"], 1) + assert_eq(budget_eight["drain_batches"], ceili(float(budget_eight["initial_job_count"]) / 8.0)) + assert_eq(budget_eight["max_batch_size"], 8) + assert_eq(unlimited["drain_batches"], 1) + assert_eq(unlimited["max_batch_size"], unlimited["initial_job_count"]) + + +func _assert_valid_result(result: Dictionary) -> void: + assert_false(result.is_empty()) + if result.is_empty(): + return + assert_eq(result["schema_version"], 1) + assert_eq(StringName(result["workload_id"]), &"regional_records_scheduler_v1") + assert_eq(result["benchmark_scope"], "regional_data_and_scheduler_only") + assert_false(result["rendered_throughput_measured"]) + assert_false(result["full_economy_throughput_measured"]) + assert_eq(result["seed"], 424242) + assert_eq(result["location_count"], 5) + assert_eq(result["settlement_count"], 5) + assert_eq(result["caravan_count"], 20) + assert_eq(result["detailed_members_per_caravan"], 12) + assert_eq(result["detailed_caravan_member_count"], 240) + assert_eq(result["named_person_count"], 2000) + assert_eq(result["cohort_resident_count"], 50000) + assert_eq(result["total_headcount"], 52000) + assert_eq(result["max_active_visuals"], 40) + assert_true(result["fixture_valid"]) + assert_true(result["roundtrip_valid"]) + assert_true(result["headcount_conserved"]) + assert_true(result["cargo_conserved"]) + assert_eq(result["initial_job_count"], 2025) + assert_eq(result["executed_job_count"], 2025) + assert_eq(result["backlog_initial"], 2025) + assert_eq(result["backlog_peak"], 2025) + assert_eq(result["backlog_final"], 0) + assert_gt(result["regional_state_bytes"], 0) + assert_gt(result["scheduler_state_bytes"], 0) + for timing_key in [ + "fixture_build_usec", + "roundtrip_usec", + "scheduler_build_usec", + "execution_usec", + "elapsed_usec", + ]: + assert_gt(result[timing_key], 0) + assert_eq(String(result["fixture_checksum"]).length(), 64) + assert_eq(String(result["execution_order_checksum"]).length(), 64) + assert_eq(String(result["final_checksum"]).length(), 64) diff --git a/tests/unit/test_regional_scale_benchmark.gd.uid b/tests/unit/test_regional_scale_benchmark.gd.uid new file mode 100644 index 0000000..9b9dcb7 --- /dev/null +++ b/tests/unit/test_regional_scale_benchmark.gd.uid @@ -0,0 +1 @@ +uid://crgxogf0ldlki diff --git a/tools/benchmark_regional_scale.gd b/tools/benchmark_regional_scale.gd new file mode 100644 index 0000000..76281e0 --- /dev/null +++ b/tools/benchmark_regional_scale.gd @@ -0,0 +1,167 @@ +extends SceneTree + +const BenchmarkScript := preload("res://simulation/benchmark/RegionalScaleBenchmark.gd") +const DEFAULT_OUTPUT_PATH := "user://regional_scale_latest.json" +const DEFAULT_HOST_LABEL := "unspecified" +const SAMPLE_COUNT := 3 +const DEFAULT_BUDGET := 8 + + +func _initialize() -> void: + call_deferred("_run") + + +func _run() -> void: + var benchmark := BenchmarkScript.new() + var samples: Array[Dictionary] = [] + for _sample_index in SAMPLE_COUNT: + var sample := benchmark.run(DEFAULT_BUDGET) + if not _valid_sample(sample): + push_error("Regional scale benchmark produced an invalid default-budget sample") + quit(1) + return + samples.append(sample) + var budget_one := benchmark.run(1) + var unlimited := benchmark.run(RegionalJobScheduler.UNLIMITED_BUDGET) + if not _valid_sample(budget_one) or not _valid_sample(unlimited): + push_error("Regional scale benchmark produced an invalid budget-parity sample") + quit(1) + return + if not _deterministic_samples(samples + [budget_one, unlimited]): + push_error("Regional scale benchmark diverged across samples or execution budgets") + quit(1) + return + + var report := { + "schema_version": BenchmarkScript.SCHEMA_VERSION, + "captured_utc": Time.get_datetime_string_from_system(true) + "Z", + "engine_version": String(Engine.get_version_info().get("string", "unknown")), + "platform": OS.get_name(), + "processor_count": OS.get_processor_count(), + "host_label": _get_argument_value("--host-label=", DEFAULT_HOST_LABEL), + "workload_id": String(BenchmarkScript.WORKLOAD_ID), + "benchmark_scope": "regional_data_and_scheduler_only", + "rendered_throughput_measured": false, + "full_economy_throughput_measured": false, + "sample_count": SAMPLE_COUNT, + "default_execution_budget": DEFAULT_BUDGET, + "fixture": _fixture_summary(samples[0]), + "default_budget": _summarize_timings(samples), + "budget_parity": { + "budget_one_final_checksum": budget_one["final_checksum"], + "budget_one_drain_batches": budget_one["drain_batches"], + "unlimited_final_checksum": unlimited["final_checksum"], + "unlimited_drain_batches": unlimited["drain_batches"], + "identical": true, + }, + } + var output_path := _get_argument_value("--output=", DEFAULT_OUTPUT_PATH) + var output := FileAccess.open(output_path, FileAccess.WRITE) + if output == null: + push_error("Could not write regional scale benchmark report to %s" % output_path) + quit(1) + return + output.store_string(JSON.stringify(report, "\t") + "\n") + output.close() + print( + ( + "[BENCH] regional | %d named + %d aggregate | %d caravans | " + + "%d jobs | median %.2f ms total | %.2f MiB state" + ) + % [ + report["fixture"]["named_person_count"], + report["fixture"]["cohort_resident_count"], + report["fixture"]["caravan_count"], + report["fixture"]["initial_job_count"], + float(report["default_budget"]["elapsed_usec_median"]) / 1000.0, + float(report["fixture"]["regional_state_bytes"]) / (1024.0 * 1024.0), + ] + ) + print("[BENCH] Report: %s" % ProjectSettings.globalize_path(output_path)) + quit(0) + + +func _valid_sample(sample: Dictionary) -> bool: + return ( + not sample.is_empty() + and bool(sample.get("fixture_valid", false)) + and bool(sample.get("roundtrip_valid", false)) + and bool(sample.get("headcount_conserved", false)) + and bool(sample.get("cargo_conserved", false)) + and int(sample.get("backlog_final", -1)) == 0 + ) + + +func _deterministic_samples(samples: Array) -> bool: + if samples.is_empty(): + return false + var expected: Dictionary = samples[0] + for sample: Dictionary in samples: + for key in [ + "fixture_checksum", + "execution_order_checksum", + "final_checksum", + "initial_job_count", + "executed_job_count", + "regional_state_bytes", + "scheduler_state_bytes", + ]: + if sample.get(key) != expected.get(key): + return false + return true + + +func _fixture_summary(sample: Dictionary) -> Dictionary: + var result := {} + for key in [ + "seed", + "location_count", + "settlement_count", + "caravan_count", + "detailed_members_per_caravan", + "detailed_caravan_member_count", + "named_person_count", + "cohort_resident_count", + "total_headcount", + "max_active_visuals", + "cargo_total", + "initial_job_count", + "regional_state_bytes", + "scheduler_state_bytes", + "fixture_checksum", + "execution_order_checksum", + "final_checksum", + ]: + result[key] = sample[key] + return result + + +func _summarize_timings(samples: Array[Dictionary]) -> Dictionary: + var result := {} + for key in [ + "fixture_build_usec", + "roundtrip_usec", + "scheduler_build_usec", + "execution_usec", + "elapsed_usec", + ]: + var values: Array[int] = [] + for sample in samples: + values.append(int(sample[key])) + values.sort() + result[key + "_samples"] = values + result[key + "_median"] = values[values.size() / 2] + result["drain_batches"] = samples[0]["drain_batches"] + result["backlog_initial"] = samples[0]["backlog_initial"] + result["backlog_peak"] = samples[0]["backlog_peak"] + result["backlog_final"] = samples[0]["backlog_final"] + return result + + +func _get_argument_value(prefix: String, default_value: String) -> String: + for argument in OS.get_cmdline_user_args(): + if argument.begins_with(prefix): + var value := argument.trim_prefix(prefix) + if not value.is_empty(): + return value + return default_value diff --git a/tools/benchmark_regional_scale.gd.uid b/tools/benchmark_regional_scale.gd.uid new file mode 100644 index 0000000..2b101cd --- /dev/null +++ b/tools/benchmark_regional_scale.gd.uid @@ -0,0 +1 @@ +uid://br2cgdiw4s6an