feat: establish simulation scaling baseline
This commit is contained in:
Rijad Zuzo
@@ -57,6 +57,10 @@ would otherwise obscure that lifecycle:
|
|||||||
- `simulation/definitions/` owns stable IDs and immutable action/profession
|
- `simulation/definitions/` owns stable IDs and immutable action/profession
|
||||||
definitions.
|
definitions.
|
||||||
|
|
||||||
|
Outside the runtime lifecycle, `simulation/benchmark/` owns reusable,
|
||||||
|
schema-valid workload fixtures. CLI tools and headless scenarios consume those
|
||||||
|
fixtures; production simulation does not depend on benchmark code.
|
||||||
|
|
||||||
The manager deliberately remains a façade instead of being split into a
|
The manager deliberately remains a façade instead of being split into a
|
||||||
collection of scene-tree manager nodes. A new collaborator is justified when
|
collection of scene-tree manager nodes. A new collaborator is justified when
|
||||||
one cohesive rule set has several real consumers or makes the tick lifecycle
|
one cohesive rule set has several real consumers or makes the tick lifecycle
|
||||||
@@ -76,6 +80,7 @@ hard to read.
|
|||||||
| `simulation/state/` | Versioned, serializable mutable records |
|
| `simulation/state/` | Versioned, serializable mutable records |
|
||||||
| `simulation/definitions/` | Stable IDs and immutable gameplay definitions |
|
| `simulation/definitions/` | Stable IDs and immutable gameplay definitions |
|
||||||
| `simulation/persistence/` | Validated local save-file storage |
|
| `simulation/persistence/` | Validated local save-file storage |
|
||||||
|
| `simulation/benchmark/` | Reproducible full-fidelity headless workloads and metrics |
|
||||||
| `world/` | Loaded-world interaction geometry and presentation adapters |
|
| `world/` | Loaded-world interaction geometry and presentation adapters |
|
||||||
| `world/resource_nodes/` | Finite resource presentation bound by stable ID |
|
| `world/resource_nodes/` | Finite resource presentation bound by stable ID |
|
||||||
| `world/storage/` | Storage interaction geometry, never stored quantities |
|
| `world/storage/` | Storage interaction geometry, never stored quantities |
|
||||||
@@ -110,6 +115,9 @@ improving ownership.
|
|||||||
- New mutable features define serialization and deterministic continuation at
|
- New mutable features define serialization and deterministic continuation at
|
||||||
the same time as their first gameplay use. Cross-record causes use stable
|
the same time as their first gameplay use. Cross-record causes use stable
|
||||||
event IDs rather than object references or prose.
|
event IDs rather than object references or prose.
|
||||||
|
- Benchmark fixtures use ordinary simulation records and must round-trip
|
||||||
|
through the current state schema. They may control workload setup, but must
|
||||||
|
not add benchmark-only fields or branches to production saves and ticks.
|
||||||
- Opportunity records reference stable NPC, storage, resource, trigger-event,
|
- Opportunity records reference stable NPC, storage, resource, trigger-event,
|
||||||
and resolution-event IDs. Their generator may observe authoritative state
|
and resolution-event IDs. Their generator may observe authoritative state
|
||||||
and history, but it does not mutate the economy or command NPC behavior. The
|
and history, but it does not mutate the economy or command NPC behavior. The
|
||||||
|
|||||||
@@ -750,14 +750,19 @@ Completed:
|
|||||||
derives one concise ordinary harvest route from current player-usable finite
|
derives one concise ordinary harvest route from current player-usable finite
|
||||||
resources and the real pantry or woodpile. It clears when help emerges or
|
resources and the real pantry or woodpile. It clears when help emerges or
|
||||||
the need closes, and adds no quest, waypoint, tracker, or saved UI state.
|
the need closes, and adds no quest, waypoint, tracker, or saved UI state.
|
||||||
|
32. Deterministic scaling baseline: one schema-valid full-fidelity fixture now
|
||||||
|
drives a repeatable CLI ledger and fast headless regression across rising
|
||||||
|
NPC and event counts. The first reviewed capture sets a local 600-NPC target,
|
||||||
|
records identical checksums across samples, and exposes repeated scarce-food
|
||||||
|
population scans plus objective-event growth as the next measured work.
|
||||||
|
|
||||||
Next:
|
Next:
|
||||||
|
|
||||||
1. Begin Milestone 8 with a deterministic headless scaling baseline over
|
1. Reuse one stable per-tick population view in the existing
|
||||||
increasing NPC and event counts. Measure current tick throughput,
|
trusted-starving-subject query, preserve exact action/checksum behavior, and
|
||||||
serialized-state growth, and retained history before choosing a target
|
rerun simulation scaling baseline 01. Let the resulting profile determine
|
||||||
population or adding spatial partitions and simulation LOD, as sequenced in
|
whether the following slice needs another bounded query optimization, a
|
||||||
`LEARNING_ROADMAP.md`.
|
spatial index, or the first active/abstract LOD contract.
|
||||||
|
|
||||||
Do not start with GIS data, a full city, a large asset pack, or more NPC
|
Do not start with GIS data, a full city, a large asset pack, or more NPC
|
||||||
mechanics. The next proof is a beautiful stage for the systems that already
|
mechanics. The next proof is a beautiful stage for the systems that already
|
||||||
|
|||||||
@@ -467,7 +467,10 @@ a smaller active population remains fully represented. Moving an NPC between
|
|||||||
fidelity levels preserves identity, inventory, task, relationships, and
|
fidelity levels preserves identity, inventory, task, relationships, and
|
||||||
important history.
|
important history.
|
||||||
|
|
||||||
Set the target population only after collecting baseline measurements.
|
[Simulation scaling baseline 01](benchmarks/SIMULATION_SCALING_BASELINE_01.md)
|
||||||
|
sets the first local target at 600 data-only full-fidelity NPCs while preserving
|
||||||
|
deterministic state. Revisit that target when world presentation or LOD enters
|
||||||
|
the measured workload.
|
||||||
|
|
||||||
## Milestone 9 — Player interaction and social agency
|
## Milestone 9 — Player interaction and social agency
|
||||||
|
|
||||||
@@ -849,14 +852,29 @@ cared-about shortage now produces an inspectable opportunity whose NPC and
|
|||||||
player responses both use the originating resource, relationship, knowledge,
|
player responses both use the originating resource, relationship, knowledge,
|
||||||
and event systems rather than quest-only duplicates.
|
and event systems rather than quest-only duplicates.
|
||||||
|
|
||||||
The immediate next slice should begin Milestone 8 with a deterministic headless
|
The first Milestone 8 measurement slice is complete. A reusable schema-valid
|
||||||
scaling baseline: measure tick throughput, serialized-state growth, and retained
|
headless fixture and CLI runner now measure three fresh deterministic samples
|
||||||
history across increasing NPC/event counts using the current full-fidelity
|
across 6, 60, and 600 NPCs plus 600 and 6,000 seeded event histories. The
|
||||||
simulation. Set an evidence-backed target before adding spatial partitions,
|
reviewed Apple M1 Max baseline reaches 65.84 ticks per second at 600 NPCs, but
|
||||||
batching, or active/abstract LOD transitions.
|
10x population from 60 to 600 costs about 24.4x per tick and creates 20,950
|
||||||
|
objective events over 200 measured ticks. The machine-readable samples,
|
||||||
|
workload exclusions, checksums, and local 50-ticks-per-second reference target
|
||||||
|
live in
|
||||||
|
[Simulation scaling baseline 01](benchmarks/SIMULATION_SCALING_BASELINE_01.md).
|
||||||
|
|
||||||
|
The immediate next slice should build one bounded per-tick population view for
|
||||||
|
the existing trusted-starving-subject query. That scarce-food path currently
|
||||||
|
rebuilds an all-NPC lookup for each applicable idle decision, making it the
|
||||||
|
first code-level candidate consistent with the measured superlinear growth.
|
||||||
|
Preserve exact selection/checksum behavior and rerun the same ledger before
|
||||||
|
choosing a spatial index or active/abstract LOD design.
|
||||||
|
|
||||||
Recently completed:
|
Recently completed:
|
||||||
|
|
||||||
|
- Deterministic simulation scaling baseline: schema-valid full-fidelity
|
||||||
|
fixtures, a repeatable CLI runner, and a fast headless regression now record
|
||||||
|
population throughput, phase timing, serialized-state growth, history
|
||||||
|
retention, event growth, and deterministic checksums before optimization.
|
||||||
- Simulation responsibility cleanup: storage/inventory transactions now live
|
- Simulation responsibility cleanup: storage/inventory transactions now live
|
||||||
in `VillageEconomy`, ordered history and rate queries live in
|
in `VillageEconomy`, ordered history and rate queries live in
|
||||||
`SimulationEventLog`, and `SimulationManager` exposes a shorter tick
|
`SimulationEventLog`, and `SimulationManager` exposes a shorter tick
|
||||||
|
|||||||
+15
-4
@@ -943,10 +943,21 @@ the need closes. The result and HUD add no quest acceptance, waypoint, tracker,
|
|||||||
save field, RNG draw, or simulation mutation. This completes the Milestone 7
|
save field, RNG draw, or simulation mutation. This completes the Milestone 7
|
||||||
simulation-garden exit proof.
|
simulation-garden exit proof.
|
||||||
|
|
||||||
The next slice should start Milestone 8 with a deterministic headless scaling
|
The first Milestone 8 measurement slice is complete. A reusable schema-valid
|
||||||
baseline over increasing NPC and event counts. Measure tick throughput,
|
headless fixture and CLI runner now measure full-fidelity simulation throughput,
|
||||||
serialized-state growth, and retained history before selecting a target
|
phase timing, serialized-state growth, retained knowledge, objective event
|
||||||
population or implementing spatial partitions and active/abstract LOD.
|
growth, and deterministic checksums across 6, 60, and 600 NPCs plus 600 and
|
||||||
|
6,000 seeded histories. The reviewed Apple M1 Max baseline reaches 65.84 ticks
|
||||||
|
per second for 600 data-only NPCs, above the initial local 50-ticks-per-second
|
||||||
|
reference target, while exposing superlinear population cost and rapid
|
||||||
|
objective-log growth. Workload details and raw samples live in
|
||||||
|
[`docs/benchmarks/`](benchmarks/SIMULATION_SCALING_BASELINE_01.md).
|
||||||
|
|
||||||
|
The next slice should build one stable per-tick population view for the
|
||||||
|
existing trusted-starving-subject query, which currently rebuilds an all-NPC
|
||||||
|
map for each applicable scarce-food decision. Preserve exact selection and
|
||||||
|
continuation checksums, rerun the same benchmark, and use the remaining measured
|
||||||
|
cost before committing to spatial partitions or active/abstract LOD.
|
||||||
|
|
||||||
The remaining simulation-garden target still aims for:
|
The remaining simulation-garden target still aims for:
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,9 @@ sources of truth.
|
|||||||
current data contracts.
|
current data contracts.
|
||||||
7. [`decisions/`](decisions/) contains durable architectural decisions,
|
7. [`decisions/`](decisions/) contains durable architectural decisions,
|
||||||
including consequences and revisit conditions.
|
including consequences and revisit conditions.
|
||||||
|
8. [`benchmarks/`](benchmarks/) contains reviewed, workload-specific
|
||||||
|
performance ledgers and their machine-readable samples. Measurements are
|
||||||
|
local evidence, not portable CI limits.
|
||||||
|
|
||||||
When documents disagree:
|
When documents disagree:
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Simulation benchmark ledger
|
||||||
|
|
||||||
|
This directory stores reviewed benchmark snapshots. Each snapshot must name the
|
||||||
|
workload, seed, Godot version, hardware context, sample count, simulated tick
|
||||||
|
count, and important exclusions so later comparisons remain honest.
|
||||||
|
|
||||||
|
Run the current scaling harness from the project root with Godot 4.7:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/Applications/Godot.app/Contents/MacOS/Godot \
|
||||||
|
--headless --path "$PWD" \
|
||||||
|
--script res://tools/benchmark_simulation_scaling.gd
|
||||||
|
```
|
||||||
|
|
||||||
|
The default report is written under `user://`. Pass
|
||||||
|
`-- --host-label="<hardware>" --output=res://docs/benchmarks/<name>.json` only
|
||||||
|
when intentionally capturing a reviewed project baseline.
|
||||||
|
|
||||||
|
Reviewed captures:
|
||||||
|
|
||||||
|
- [Simulation scaling baseline 01](SIMULATION_SCALING_BASELINE_01.md) records
|
||||||
|
the first full-fidelity population/history measurements and the bounded next
|
||||||
|
optimization.
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
# Simulation scaling baseline 01
|
||||||
|
|
||||||
|
This is the first reviewed Milestone 8 measurement of the current
|
||||||
|
full-fidelity, data-only simulation. It establishes a reproducible reference
|
||||||
|
before introducing batching, spatial partitions, or simulation LOD.
|
||||||
|
|
||||||
|
The machine-readable samples and checksums are stored beside this note in
|
||||||
|
[`simulation_scaling_baseline_01.json`](simulation_scaling_baseline_01.json).
|
||||||
|
|
||||||
|
## Capture context
|
||||||
|
|
||||||
|
- captured: 2026-07-16;
|
||||||
|
- engine: Godot 4.7 stable;
|
||||||
|
- host: Apple M1 Max with 64 GB memory on macOS, with 10 processors reported
|
||||||
|
by Godot;
|
||||||
|
- benchmark seed: `8088`;
|
||||||
|
- workload: `full_fidelity_headless_arrival`;
|
||||||
|
- samples: three fresh managers per case, using the median;
|
||||||
|
- cadence: 10 warmup ticks, then 200 measured ticks at 1.2 simulated seconds
|
||||||
|
per tick.
|
||||||
|
|
||||||
|
Run the reviewed capture from the project root:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/Applications/Godot.app/Contents/MacOS/Godot \
|
||||||
|
--headless --path "$PWD" \
|
||||||
|
--script res://tools/benchmark_simulation_scaling.gd -- \
|
||||||
|
--host-label="Apple M1 Max, 64 GB" \
|
||||||
|
--output=res://docs/benchmarks/simulation_scaling_baseline_01.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Workload contract
|
||||||
|
|
||||||
|
Every NPC remains a named `SimNPC` and receives the normal needs, schedule,
|
||||||
|
task, opportunity, and action-selection work each tick. The pantry starts
|
||||||
|
empty, so ordinary food-supply decisions exercise the current population-wide
|
||||||
|
queries. Travel completes through the same deterministic immediate-arrival
|
||||||
|
convention used by existing headless continuation scenarios. Seeded history is
|
||||||
|
made of valid immutable storage-deposit events plus at most three known-event
|
||||||
|
references per NPC, and every prepared fixture must parse through the normal
|
||||||
|
`SimulationStateRecord` schema.
|
||||||
|
|
||||||
|
The timed section excludes manager construction, fixture preparation,
|
||||||
|
serialization, world scenes, rendering, navigation, and `NpcVisual`. This is a
|
||||||
|
simulation-throughput baseline, not a complete frame-time or memory profile.
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Case | NPCs | Seeded history | Median us/tick | Ticks/s | Simulated realtime | Arrival share | End JSON | Events added |
|
||||||
|
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||||
|
| Population 6 | 6 | 0 | 55.74 | 17,942.05 | 21,530.5x | 5.2% | 0.06 MiB | 209 |
|
||||||
|
| Population 60 | 60 | 0 | 621.87 | 1,608.07 | 1,929.7x | 6.4% | 0.56 MiB | 2,090 |
|
||||||
|
| Population 600 | 600 | 0 | 15,187.74 | 65.84 | 79.0x | 14.0% | 5.67 MiB | 20,950 |
|
||||||
|
| History 600 | 60 | 600 | 677.12 | 1,476.85 | 1,772.2x | 5.6% | 0.71 MiB | 2,090 |
|
||||||
|
| History 6,000 | 60 | 6,000 | 1,421.19 | 703.64 | 844.4x | 2.7% | 2.04 MiB | 2,090 |
|
||||||
|
|
||||||
|
All three samples in each case produced the same deterministic checksum and
|
||||||
|
state counters. The benchmark runner rejects the report if they diverge.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
Population scaling is the first measured knee. Raising the fixture from 60 to
|
||||||
|
600 NPCs increases median tick cost by about 24.4x for 10x the population.
|
||||||
|
Immediate-arrival completion accounts for only 14.0% of the 600-NPC timed
|
||||||
|
section, so most of that growth remains inside the ordinary simulation tick.
|
||||||
|
|
||||||
|
The benchmark is not a function profiler, but code inspection identifies a
|
||||||
|
bounded first candidate: while food is scarce, each applicable idle decision
|
||||||
|
calls `RelationshipSystem.get_trusted_starving_subject()`, which rebuilds a map
|
||||||
|
by scanning every NPC. Building the same stable-ID population view once per
|
||||||
|
tick should remove repeated work without changing action semantics or tie
|
||||||
|
breaks. The benchmark and checksum give that change a concrete comparison.
|
||||||
|
|
||||||
|
History also has a visible but less urgent cost. At a fixed 60 NPCs, increasing
|
||||||
|
seeded objective events from 600 to 6,000 raises tick cost about 2.1x and final
|
||||||
|
serialized state from 0.71 MiB to 2.04 MiB. The 180 seeded known-event
|
||||||
|
references age out during the measured window, while the objective event log
|
||||||
|
remains complete. Event-log indexing or archiving therefore remains a later,
|
||||||
|
separate decision.
|
||||||
|
|
||||||
|
Organic event growth matters independently of CPU time. The 600-NPC case adds
|
||||||
|
20,950 objective events and grows serialized state by 5,235,086 bytes over only
|
||||||
|
200 measured ticks. Future long-session work should measure event retention and
|
||||||
|
save cost explicitly rather than treating tick throughput as the whole scale
|
||||||
|
problem.
|
||||||
|
|
||||||
|
## Reference target and next slice
|
||||||
|
|
||||||
|
The first local reference target is at least **50 measured ticks per second for
|
||||||
|
600 data-only full-fidelity NPCs** on this Apple M1 Max workload, with identical
|
||||||
|
deterministic state and checksum. Baseline 01 reaches 65.84 ticks per second.
|
||||||
|
This is a local comparison target, not a cross-machine CI timing assertion or a
|
||||||
|
claim about 600 rendered and navigating characters.
|
||||||
|
|
||||||
|
The next slice should build one reusable per-tick population view for the
|
||||||
|
existing trusted-starving-subject consumer, preserve exact selection and
|
||||||
|
continuation results, and rerun this ledger. Spatial partitioning and
|
||||||
|
active/abstract LOD should wait until that bounded change shows what cost
|
||||||
|
remains.
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
{
|
||||||
|
"benchmark_seed": 8088,
|
||||||
|
"captured_utc": "2026-07-16T11:05:19Z",
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"arrival_share_percent": 5.23010675518077,
|
||||||
|
"arrival_usec_median": 583,
|
||||||
|
"arrival_usec_samples": [
|
||||||
|
578,
|
||||||
|
583,
|
||||||
|
587
|
||||||
|
],
|
||||||
|
"arrivals_processed": 209,
|
||||||
|
"case_id": "population_006",
|
||||||
|
"elapsed_usec_max": 11309,
|
||||||
|
"elapsed_usec_median": 11147,
|
||||||
|
"elapsed_usec_min": 10991,
|
||||||
|
"elapsed_usec_samples": [
|
||||||
|
10991,
|
||||||
|
11147,
|
||||||
|
11309
|
||||||
|
],
|
||||||
|
"end_event_count": 221,
|
||||||
|
"end_known_reference_count": 0,
|
||||||
|
"end_state_bytes": 58576,
|
||||||
|
"end_tick": 210,
|
||||||
|
"events_recorded": 209,
|
||||||
|
"final_checksum": "100dd5dc1e0c22ea23f35dd876d6f56fd5c33c58b1586b6fa9aec47ee1536511",
|
||||||
|
"history_seed_events": 0,
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"npc_updates": 1200,
|
||||||
|
"population": 6,
|
||||||
|
"realtime_factor_median": 21530.4566251009,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"seed": 8088,
|
||||||
|
"setup_usec_median": 772,
|
||||||
|
"setup_usec_samples": [
|
||||||
|
535,
|
||||||
|
772,
|
||||||
|
3251
|
||||||
|
],
|
||||||
|
"simulation_usec_median": 10544,
|
||||||
|
"simulation_usec_samples": [
|
||||||
|
10387,
|
||||||
|
10544,
|
||||||
|
10700
|
||||||
|
],
|
||||||
|
"start_event_count": 12,
|
||||||
|
"start_known_reference_count": 0,
|
||||||
|
"start_state_bytes": 7632,
|
||||||
|
"start_tick": 10,
|
||||||
|
"state_growth_bytes": 50944,
|
||||||
|
"tick_interval": 1.2,
|
||||||
|
"ticks_per_second_median": 17942.0471875841,
|
||||||
|
"usec_per_tick_median": 55.735,
|
||||||
|
"warmup_arrivals": 12,
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"arrival_share_percent": 6.39769081713877,
|
||||||
|
"arrival_usec_median": 7957,
|
||||||
|
"arrival_usec_samples": [
|
||||||
|
7747,
|
||||||
|
7957,
|
||||||
|
8499
|
||||||
|
],
|
||||||
|
"arrivals_processed": 2090,
|
||||||
|
"case_id": "population_060",
|
||||||
|
"elapsed_usec_max": 142845,
|
||||||
|
"elapsed_usec_median": 124373,
|
||||||
|
"elapsed_usec_min": 123769,
|
||||||
|
"elapsed_usec_samples": [
|
||||||
|
123769,
|
||||||
|
124373,
|
||||||
|
142845
|
||||||
|
],
|
||||||
|
"end_event_count": 2210,
|
||||||
|
"end_known_reference_count": 0,
|
||||||
|
"end_state_bytes": 587324,
|
||||||
|
"end_tick": 210,
|
||||||
|
"events_recorded": 2090,
|
||||||
|
"final_checksum": "58c7f96fc5df7f5399fd82d1a9c212b3c53067e807059e88ffa484ba0eb41e16",
|
||||||
|
"history_seed_events": 0,
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"npc_updates": 12000,
|
||||||
|
"population": 60,
|
||||||
|
"realtime_factor_median": 1929.67927122446,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"seed": 8088,
|
||||||
|
"setup_usec_median": 3734,
|
||||||
|
"setup_usec_samples": [
|
||||||
|
3603,
|
||||||
|
3734,
|
||||||
|
5045
|
||||||
|
],
|
||||||
|
"simulation_usec_median": 116595,
|
||||||
|
"simulation_usec_samples": [
|
||||||
|
115787,
|
||||||
|
116595,
|
||||||
|
134317
|
||||||
|
],
|
||||||
|
"start_event_count": 120,
|
||||||
|
"start_known_reference_count": 0,
|
||||||
|
"start_state_bytes": 71278,
|
||||||
|
"start_tick": 10,
|
||||||
|
"state_growth_bytes": 516046,
|
||||||
|
"tick_interval": 1.2,
|
||||||
|
"ticks_per_second_median": 1608.06605935372,
|
||||||
|
"usec_per_tick_median": 621.865,
|
||||||
|
"warmup_arrivals": 120,
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"arrival_share_percent": 14.0226590657991,
|
||||||
|
"arrival_usec_median": 425945,
|
||||||
|
"arrival_usec_samples": [
|
||||||
|
425585,
|
||||||
|
425945,
|
||||||
|
427652
|
||||||
|
],
|
||||||
|
"arrivals_processed": 20950,
|
||||||
|
"case_id": "population_600",
|
||||||
|
"elapsed_usec_max": 3040453,
|
||||||
|
"elapsed_usec_median": 3037548,
|
||||||
|
"elapsed_usec_min": 3028975,
|
||||||
|
"elapsed_usec_samples": [
|
||||||
|
3028975,
|
||||||
|
3037548,
|
||||||
|
3040453
|
||||||
|
],
|
||||||
|
"end_event_count": 22150,
|
||||||
|
"end_known_reference_count": 0,
|
||||||
|
"end_state_bytes": 5948447,
|
||||||
|
"end_tick": 210,
|
||||||
|
"events_recorded": 20950,
|
||||||
|
"final_checksum": "fa4efac63fde63f92eb28141bebd1922f665e85ffe7b7ac2b9702dae6e4c24b7",
|
||||||
|
"history_seed_events": 0,
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"npc_updates": 120000,
|
||||||
|
"population": 600,
|
||||||
|
"realtime_factor_median": 79.0110971085889,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"seed": 8088,
|
||||||
|
"setup_usec_median": 35480,
|
||||||
|
"setup_usec_samples": [
|
||||||
|
35329,
|
||||||
|
35480,
|
||||||
|
35664
|
||||||
|
],
|
||||||
|
"simulation_usec_median": 2609839,
|
||||||
|
"simulation_usec_samples": [
|
||||||
|
2603329,
|
||||||
|
2609839,
|
||||||
|
2614455
|
||||||
|
],
|
||||||
|
"start_event_count": 1200,
|
||||||
|
"start_known_reference_count": 0,
|
||||||
|
"start_state_bytes": 713361,
|
||||||
|
"start_tick": 10,
|
||||||
|
"state_growth_bytes": 5235086,
|
||||||
|
"tick_interval": 1.2,
|
||||||
|
"ticks_per_second_median": 65.8425809238241,
|
||||||
|
"usec_per_tick_median": 15187.74,
|
||||||
|
"warmup_arrivals": 1200,
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"arrival_share_percent": 5.63567488535921,
|
||||||
|
"arrival_usec_median": 7632,
|
||||||
|
"arrival_usec_samples": [
|
||||||
|
7621,
|
||||||
|
7632,
|
||||||
|
7830
|
||||||
|
],
|
||||||
|
"arrivals_processed": 2090,
|
||||||
|
"case_id": "history_000600",
|
||||||
|
"elapsed_usec_max": 136012,
|
||||||
|
"elapsed_usec_median": 135423,
|
||||||
|
"elapsed_usec_min": 134887,
|
||||||
|
"elapsed_usec_samples": [
|
||||||
|
134887,
|
||||||
|
135423,
|
||||||
|
136012
|
||||||
|
],
|
||||||
|
"end_event_count": 2810,
|
||||||
|
"end_known_reference_count": 0,
|
||||||
|
"end_state_bytes": 743004,
|
||||||
|
"end_tick": 810,
|
||||||
|
"events_recorded": 2090,
|
||||||
|
"final_checksum": "7ae86044f7683359d6bfbc8753e2656f3af90f23d040039154c6869dca48cf65",
|
||||||
|
"history_seed_events": 600,
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"npc_updates": 12000,
|
||||||
|
"population": 60,
|
||||||
|
"realtime_factor_median": 1772.22480671673,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"seed": 8088,
|
||||||
|
"setup_usec_median": 22795,
|
||||||
|
"setup_usec_samples": [
|
||||||
|
22782,
|
||||||
|
22795,
|
||||||
|
22881
|
||||||
|
],
|
||||||
|
"simulation_usec_median": 127766,
|
||||||
|
"simulation_usec_samples": [
|
||||||
|
127030,
|
||||||
|
127766,
|
||||||
|
128361
|
||||||
|
],
|
||||||
|
"start_event_count": 720,
|
||||||
|
"start_known_reference_count": 180,
|
||||||
|
"start_state_bytes": 253098,
|
||||||
|
"start_tick": 610,
|
||||||
|
"state_growth_bytes": 489906,
|
||||||
|
"tick_interval": 1.2,
|
||||||
|
"ticks_per_second_median": 1476.85400559728,
|
||||||
|
"usec_per_tick_median": 677.115,
|
||||||
|
"warmup_arrivals": 120,
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"arrival_share_percent": 2.72202422626189,
|
||||||
|
"arrival_usec_median": 7737,
|
||||||
|
"arrival_usec_samples": [
|
||||||
|
7650,
|
||||||
|
7737,
|
||||||
|
7809
|
||||||
|
],
|
||||||
|
"arrivals_processed": 2090,
|
||||||
|
"case_id": "history_006000",
|
||||||
|
"elapsed_usec_max": 287265,
|
||||||
|
"elapsed_usec_median": 284237,
|
||||||
|
"elapsed_usec_min": 280605,
|
||||||
|
"elapsed_usec_samples": [
|
||||||
|
280605,
|
||||||
|
284237,
|
||||||
|
287265
|
||||||
|
],
|
||||||
|
"end_event_count": 8210,
|
||||||
|
"end_known_reference_count": 0,
|
||||||
|
"end_state_bytes": 2141615,
|
||||||
|
"end_tick": 6210,
|
||||||
|
"events_recorded": 2090,
|
||||||
|
"final_checksum": "b4df2bac21ed5ce6dcd92736e210db5f3114d959dcb15ae7796493362911b6d7",
|
||||||
|
"history_seed_events": 6000,
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"npc_updates": 12000,
|
||||||
|
"population": 60,
|
||||||
|
"realtime_factor_median": 844.365793334436,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"seed": 8088,
|
||||||
|
"setup_usec_median": 154433,
|
||||||
|
"setup_usec_samples": [
|
||||||
|
153698,
|
||||||
|
154433,
|
||||||
|
155153
|
||||||
|
],
|
||||||
|
"simulation_usec_median": 276401,
|
||||||
|
"simulation_usec_samples": [
|
||||||
|
272849,
|
||||||
|
276401,
|
||||||
|
279590
|
||||||
|
],
|
||||||
|
"start_event_count": 6120,
|
||||||
|
"start_known_reference_count": 180,
|
||||||
|
"start_state_bytes": 1649700,
|
||||||
|
"start_tick": 6010,
|
||||||
|
"state_growth_bytes": 491915,
|
||||||
|
"tick_interval": 1.2,
|
||||||
|
"ticks_per_second_median": 703.63816111203,
|
||||||
|
"usec_per_tick_median": 1421.185,
|
||||||
|
"warmup_arrivals": 120,
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"engine_version": "4.7-stable (official)",
|
||||||
|
"exclusions": [
|
||||||
|
"manager_and_fixture_setup",
|
||||||
|
"state_serialization",
|
||||||
|
"world_scene",
|
||||||
|
"rendering",
|
||||||
|
"navigation",
|
||||||
|
"npc_visual"
|
||||||
|
],
|
||||||
|
"host_label": "Apple M1 Max, 64 GB",
|
||||||
|
"measured_ticks": 200,
|
||||||
|
"platform": "macOS",
|
||||||
|
"processor_count": 10,
|
||||||
|
"sample_count": 3,
|
||||||
|
"schema_version": 1,
|
||||||
|
"timed_phases": [
|
||||||
|
"simulation_tick",
|
||||||
|
"headless_arrival_completion"
|
||||||
|
],
|
||||||
|
"warmup_ticks": 10,
|
||||||
|
"workload": "All NPCs receive full per-tick needs/task updates and ordinary action decisions; travel resolves through the deterministic immediate-arrival headless convention.",
|
||||||
|
"workload_id": "full_fidelity_headless_arrival"
|
||||||
|
}
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
class_name SimulationScalingBenchmark
|
||||||
|
extends RefCounted
|
||||||
|
|
||||||
|
const SCHEMA_VERSION := 1
|
||||||
|
const WORKLOAD_ID := &"full_fidelity_headless_arrival"
|
||||||
|
const RECENT_FACTS_PER_NPC := 3
|
||||||
|
const STARTING_CLOCK_TICK := 50
|
||||||
|
|
||||||
|
|
||||||
|
func prepare_manager(
|
||||||
|
manager: Node, population: int, history_event_count: int, seed_value: int
|
||||||
|
) -> bool:
|
||||||
|
if manager == null or manager.clock == null or population <= 0 or history_event_count < 0:
|
||||||
|
return false
|
||||||
|
manager.set_process(false)
|
||||||
|
manager.simulation_seed = seed_value
|
||||||
|
manager.tick_count = history_event_count
|
||||||
|
manager.clock.accumulator = 0.0
|
||||||
|
manager.clock.elapsed_ticks = STARTING_CLOCK_TICK
|
||||||
|
manager.npcs.clear()
|
||||||
|
manager.wander_random_sources.clear()
|
||||||
|
manager.latest_decisions.clear()
|
||||||
|
manager.resource_states.clear()
|
||||||
|
var no_relationships: Array[RelationshipStateRecord] = []
|
||||||
|
var no_opportunities: Array[OpportunityStateRecord] = []
|
||||||
|
manager.relationship_system.restore(no_relationships)
|
||||||
|
manager.opportunity_system.restore(no_opportunities, 0)
|
||||||
|
|
||||||
|
for npc_id in population:
|
||||||
|
var npc_random := _create_random_source(seed_value, npc_id, 0)
|
||||||
|
var npc := SimNPC.new(
|
||||||
|
npc_id,
|
||||||
|
"Benchmark %04d" % npc_id,
|
||||||
|
SimulationIds.PROFESSION_FARMER,
|
||||||
|
npc_random.randf_range(1.0, 10.0),
|
||||||
|
npc_random.randf_range(1.0, 10.0),
|
||||||
|
npc_random
|
||||||
|
)
|
||||||
|
npc.debug_logs = false
|
||||||
|
npc.hunger = 35.0 + float(npc_id % 10) * 0.25
|
||||||
|
npc.energy = 90.0
|
||||||
|
npc.position = _grid_position(npc_id)
|
||||||
|
npc.home_position = npc.position
|
||||||
|
manager.npcs.append(npc)
|
||||||
|
manager.wander_random_sources[npc_id] = _create_random_source(seed_value, npc_id, 1)
|
||||||
|
|
||||||
|
var pantry: StorageStateRecord = manager.get_pantry()
|
||||||
|
pantry.withdraw(SimulationIds.RESOURCE_FOOD, pantry.get_amount(SimulationIds.RESOURCE_FOOD))
|
||||||
|
manager.economy.sync_resource(SimulationIds.RESOURCE_FOOD)
|
||||||
|
manager.village.update_modifiers()
|
||||||
|
manager.village.update_priorities()
|
||||||
|
var history := _build_history(population, history_event_count)
|
||||||
|
manager.event_log.restore(history["events"], history_event_count)
|
||||||
|
manager.event_knowledge_system.restore(history["knowledge"])
|
||||||
|
return true
|
||||||
|
|
||||||
|
|
||||||
|
func measure_manager(
|
||||||
|
manager: Node, population: int, history_event_count: int, warmup_ticks: int, measured_ticks: int
|
||||||
|
) -> Dictionary:
|
||||||
|
if (
|
||||||
|
manager == null
|
||||||
|
or manager.npcs.size() != population
|
||||||
|
or warmup_ticks < 0
|
||||||
|
or measured_ticks <= 0
|
||||||
|
):
|
||||||
|
return {}
|
||||||
|
var warmup_arrivals := 0
|
||||||
|
for _tick in warmup_ticks:
|
||||||
|
warmup_arrivals += _advance_headless_tick(manager)
|
||||||
|
|
||||||
|
var start_json: String = manager.serialize_state()
|
||||||
|
var start_event_count: int = manager.economic_events.size()
|
||||||
|
var start_known_count: int = manager.event_knowledge_system.get_all_sorted().size()
|
||||||
|
var start_tick: int = manager.tick_count
|
||||||
|
var measured_arrivals := 0
|
||||||
|
var simulation_usec := 0
|
||||||
|
var arrival_usec := 0
|
||||||
|
var started_usec := Time.get_ticks_usec()
|
||||||
|
for _tick in measured_ticks:
|
||||||
|
var phase_started_usec := Time.get_ticks_usec()
|
||||||
|
manager.simulate_tick()
|
||||||
|
simulation_usec += Time.get_ticks_usec() - phase_started_usec
|
||||||
|
phase_started_usec = Time.get_ticks_usec()
|
||||||
|
measured_arrivals += _complete_headless_arrivals(manager)
|
||||||
|
arrival_usec += Time.get_ticks_usec() - phase_started_usec
|
||||||
|
var elapsed_usec := maxi(Time.get_ticks_usec() - started_usec, 1)
|
||||||
|
var end_json: String = manager.serialize_state()
|
||||||
|
var start_state_bytes := start_json.to_utf8_buffer().size()
|
||||||
|
var end_state_bytes := end_json.to_utf8_buffer().size()
|
||||||
|
var ticks_per_second := float(measured_ticks) * 1000000.0 / float(elapsed_usec)
|
||||||
|
return {
|
||||||
|
"schema_version": SCHEMA_VERSION,
|
||||||
|
"workload_id": String(WORKLOAD_ID),
|
||||||
|
"seed": manager.simulation_seed,
|
||||||
|
"population": population,
|
||||||
|
"history_seed_events": history_event_count,
|
||||||
|
"warmup_ticks": warmup_ticks,
|
||||||
|
"measured_ticks": measured_ticks,
|
||||||
|
"npc_updates": population * measured_ticks,
|
||||||
|
"warmup_arrivals": warmup_arrivals,
|
||||||
|
"arrivals_processed": measured_arrivals,
|
||||||
|
"elapsed_usec": elapsed_usec,
|
||||||
|
"simulation_usec": maxi(simulation_usec, 1),
|
||||||
|
"arrival_usec": maxi(arrival_usec, 1),
|
||||||
|
"usec_per_tick": float(elapsed_usec) / float(measured_ticks),
|
||||||
|
"ticks_per_second": ticks_per_second,
|
||||||
|
"tick_interval": manager.tick_interval,
|
||||||
|
"realtime_factor": ticks_per_second * manager.tick_interval,
|
||||||
|
"start_tick": start_tick,
|
||||||
|
"end_tick": manager.tick_count,
|
||||||
|
"start_state_bytes": start_state_bytes,
|
||||||
|
"end_state_bytes": end_state_bytes,
|
||||||
|
"state_growth_bytes": end_state_bytes - start_state_bytes,
|
||||||
|
"start_event_count": start_event_count,
|
||||||
|
"end_event_count": manager.economic_events.size(),
|
||||||
|
"events_recorded": manager.economic_events.size() - start_event_count,
|
||||||
|
"start_known_reference_count": start_known_count,
|
||||||
|
"end_known_reference_count": manager.event_knowledge_system.get_all_sorted().size(),
|
||||||
|
"final_checksum": end_json.sha256_text(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func _advance_headless_tick(manager: Node) -> int:
|
||||||
|
manager.simulate_tick()
|
||||||
|
return _complete_headless_arrivals(manager)
|
||||||
|
|
||||||
|
|
||||||
|
func _complete_headless_arrivals(manager: Node) -> int:
|
||||||
|
var arrival_ids: Array[int] = []
|
||||||
|
for npc in manager.npcs:
|
||||||
|
if npc.task_state == SimNPC.TASK_STATE_TRAVELING:
|
||||||
|
arrival_ids.append(npc.id)
|
||||||
|
for npc_id in arrival_ids:
|
||||||
|
manager.notify_npc_arrived(npc_id)
|
||||||
|
return arrival_ids.size()
|
||||||
|
|
||||||
|
|
||||||
|
func _build_history(population: int, history_event_count: int) -> Dictionary:
|
||||||
|
var events: Array[EconomicEventRecord] = []
|
||||||
|
for event_id in history_event_count:
|
||||||
|
var actor_id := event_id % population
|
||||||
|
events.append(
|
||||||
|
EconomicEventRecord.create(
|
||||||
|
event_id,
|
||||||
|
SimulationIds.EVENT_STORAGE_DEPOSITED,
|
||||||
|
event_id,
|
||||||
|
actor_id,
|
||||||
|
SimulationIds.npc_inventory_id(actor_id),
|
||||||
|
SimulationIds.STORAGE_VILLAGE_PANTRY,
|
||||||
|
SimulationIds.RESOURCE_FOOD,
|
||||||
|
1.0
|
||||||
|
)
|
||||||
|
)
|
||||||
|
var knowledge: Array[KnownEventStateRecord] = []
|
||||||
|
var retained_by_actor := {}
|
||||||
|
for event_id in range(history_event_count - 1, -1, -1):
|
||||||
|
var actor_id := event_id % population
|
||||||
|
var retained_count := int(retained_by_actor.get(actor_id, 0))
|
||||||
|
if retained_count >= RECENT_FACTS_PER_NPC:
|
||||||
|
continue
|
||||||
|
knowledge.append(
|
||||||
|
KnownEventStateRecord.create(
|
||||||
|
actor_id,
|
||||||
|
event_id,
|
||||||
|
SimulationIds.KNOWLEDGE_ACQUISITION_PERFORMED,
|
||||||
|
KnownEventStateRecord.NO_SOURCE_NPC_ID,
|
||||||
|
event_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
retained_by_actor[actor_id] = retained_count + 1
|
||||||
|
if knowledge.size() >= mini(history_event_count, population * RECENT_FACTS_PER_NPC):
|
||||||
|
break
|
||||||
|
return {"events": events, "knowledge": knowledge}
|
||||||
|
|
||||||
|
|
||||||
|
func _create_random_source(seed_value: int, npc_id: int, stream_id: int) -> RandomNumberGenerator:
|
||||||
|
var source := RandomNumberGenerator.new()
|
||||||
|
source.seed = seed_value + (npc_id + 1) * 1000003 + stream_id * 7919
|
||||||
|
return source
|
||||||
|
|
||||||
|
|
||||||
|
func _grid_position(npc_id: int) -> Vector3:
|
||||||
|
var row := npc_id / 32
|
||||||
|
var column := npc_id % 32
|
||||||
|
return Vector3(float(column) * 12.0, 0.0, float(row) * 12.0)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://dpwely8ykc1sp
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
extends SceneTree
|
||||||
|
|
||||||
|
const SimulationManagerScript := preload("res://simulation/SimulationManager.gd")
|
||||||
|
const BenchmarkScript := preload("res://simulation/benchmark/SimulationScalingBenchmark.gd")
|
||||||
|
|
||||||
|
var failures: Array[String] = []
|
||||||
|
|
||||||
|
|
||||||
|
func _initialize() -> void:
|
||||||
|
call_deferred("_run")
|
||||||
|
|
||||||
|
|
||||||
|
func _run() -> void:
|
||||||
|
var first := _run_case(9001)
|
||||||
|
var repeated := _run_case(9001)
|
||||||
|
var different_seed := _run_case(9002)
|
||||||
|
_check(not first.is_empty(), "The benchmark fixture should produce one result")
|
||||||
|
if first.is_empty():
|
||||||
|
_finish()
|
||||||
|
return
|
||||||
|
_check(first["fixture_valid"], "The prepared benchmark state should pass schema validation")
|
||||||
|
_check(
|
||||||
|
(
|
||||||
|
int(first["schema_version"]) == SimulationScalingBenchmark.SCHEMA_VERSION
|
||||||
|
and StringName(first["workload_id"]) == SimulationScalingBenchmark.WORKLOAD_ID
|
||||||
|
and int(first["population"]) == 12
|
||||||
|
and int(first["history_seed_events"]) == 24
|
||||||
|
and int(first["measured_ticks"]) == 8
|
||||||
|
and int(first["npc_updates"]) == 96
|
||||||
|
and is_equal_approx(float(first["tick_interval"]), 1.2)
|
||||||
|
and int(first["end_tick"]) - int(first["start_tick"]) == 8
|
||||||
|
),
|
||||||
|
"The result should identify the exact workload and measured update count"
|
||||||
|
)
|
||||||
|
_check(
|
||||||
|
(
|
||||||
|
int(first["elapsed_usec"]) > 0
|
||||||
|
and int(first["simulation_usec"]) > 0
|
||||||
|
and int(first["arrival_usec"]) > 0
|
||||||
|
and int(first["start_state_bytes"]) > 0
|
||||||
|
and int(first["end_state_bytes"]) >= int(first["start_state_bytes"])
|
||||||
|
and int(first["arrivals_processed"]) > 0
|
||||||
|
and int(first["events_recorded"]) > 0
|
||||||
|
and int(first["end_event_count"]) >= 24
|
||||||
|
and String(first["final_checksum"]).length() == 64
|
||||||
|
),
|
||||||
|
"The benchmark should report timing, state growth, updates, history, and a checksum"
|
||||||
|
)
|
||||||
|
for deterministic_key in [
|
||||||
|
"final_checksum",
|
||||||
|
"start_state_bytes",
|
||||||
|
"end_state_bytes",
|
||||||
|
"state_growth_bytes",
|
||||||
|
"start_event_count",
|
||||||
|
"end_event_count",
|
||||||
|
"events_recorded",
|
||||||
|
"start_known_reference_count",
|
||||||
|
"end_known_reference_count",
|
||||||
|
"arrivals_processed",
|
||||||
|
"warmup_arrivals",
|
||||||
|
"tick_interval",
|
||||||
|
]:
|
||||||
|
_check(
|
||||||
|
first[deterministic_key] == repeated[deterministic_key],
|
||||||
|
"Repeated benchmark runs should preserve %s" % deterministic_key
|
||||||
|
)
|
||||||
|
_check(
|
||||||
|
first["final_checksum"] != different_seed["final_checksum"],
|
||||||
|
"A different fixture seed should produce a different deterministic checksum"
|
||||||
|
)
|
||||||
|
_finish()
|
||||||
|
|
||||||
|
|
||||||
|
func _run_case(seed_value: int) -> Dictionary:
|
||||||
|
var manager := SimulationManagerScript.new()
|
||||||
|
manager.simulation_seed = seed_value
|
||||||
|
manager.debug_logs = false
|
||||||
|
manager.set_process(false)
|
||||||
|
root.add_child(manager)
|
||||||
|
var benchmark := BenchmarkScript.new()
|
||||||
|
if not benchmark.prepare_manager(manager, 12, 24, seed_value):
|
||||||
|
manager.free()
|
||||||
|
return {}
|
||||||
|
var fixture_valid := SimulationStateRecord.from_json(manager.serialize_state()) != null
|
||||||
|
var result: Dictionary = benchmark.measure_manager(manager, 12, 24, 2, 8)
|
||||||
|
result["fixture_valid"] = fixture_valid
|
||||||
|
manager.free()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
func _check(condition: bool, message: String) -> void:
|
||||||
|
if not condition:
|
||||||
|
failures.append(message)
|
||||||
|
|
||||||
|
|
||||||
|
func _finish() -> void:
|
||||||
|
if failures.is_empty():
|
||||||
|
print("[TEST] Simulation scaling benchmark passed: valid fixture -> stable metrics")
|
||||||
|
quit(0)
|
||||||
|
return
|
||||||
|
for failure in failures:
|
||||||
|
push_error("[TEST] " + failure)
|
||||||
|
quit(1)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://sn24dfw1g3pd
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
extends SceneTree
|
||||||
|
|
||||||
|
const SimulationManagerScript := preload("res://simulation/SimulationManager.gd")
|
||||||
|
const BenchmarkScript := preload("res://simulation/benchmark/SimulationScalingBenchmark.gd")
|
||||||
|
const DEFAULT_OUTPUT_PATH := "user://simulation_scaling_latest.json"
|
||||||
|
const DEFAULT_HOST_LABEL := "unspecified"
|
||||||
|
const BENCHMARK_SEED := 8088
|
||||||
|
const WARMUP_TICKS := 10
|
||||||
|
const MEASURED_TICKS := 200
|
||||||
|
const SAMPLE_COUNT := 3
|
||||||
|
const CASES := [
|
||||||
|
{"case_id": "population_006", "population": 6, "history_events": 0},
|
||||||
|
{"case_id": "population_060", "population": 60, "history_events": 0},
|
||||||
|
{"case_id": "population_600", "population": 600, "history_events": 0},
|
||||||
|
{"case_id": "history_000600", "population": 60, "history_events": 600},
|
||||||
|
{"case_id": "history_006000", "population": 60, "history_events": 6000},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
func _initialize() -> void:
|
||||||
|
call_deferred("_run")
|
||||||
|
|
||||||
|
|
||||||
|
func _run() -> void:
|
||||||
|
var benchmark := BenchmarkScript.new()
|
||||||
|
var case_results: Array[Dictionary] = []
|
||||||
|
for case_config in CASES:
|
||||||
|
var samples: Array[Dictionary] = []
|
||||||
|
for _sample_index in SAMPLE_COUNT:
|
||||||
|
var setup_started_usec := Time.get_ticks_usec()
|
||||||
|
var manager := SimulationManagerScript.new()
|
||||||
|
manager.simulation_seed = BENCHMARK_SEED
|
||||||
|
manager.debug_logs = false
|
||||||
|
manager.set_process(false)
|
||||||
|
root.add_child(manager)
|
||||||
|
var prepared := benchmark.prepare_manager(
|
||||||
|
manager,
|
||||||
|
int(case_config["population"]),
|
||||||
|
int(case_config["history_events"]),
|
||||||
|
BENCHMARK_SEED
|
||||||
|
)
|
||||||
|
var fixture_valid := (
|
||||||
|
prepared and SimulationStateRecord.from_json(manager.serialize_state()) != null
|
||||||
|
)
|
||||||
|
var setup_usec := Time.get_ticks_usec() - setup_started_usec
|
||||||
|
if not fixture_valid:
|
||||||
|
push_error(
|
||||||
|
"Scaling benchmark prepared invalid state for %s" % case_config["case_id"]
|
||||||
|
)
|
||||||
|
manager.free()
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
var sample := benchmark.measure_manager(
|
||||||
|
manager,
|
||||||
|
int(case_config["population"]),
|
||||||
|
int(case_config["history_events"]),
|
||||||
|
WARMUP_TICKS,
|
||||||
|
MEASURED_TICKS
|
||||||
|
)
|
||||||
|
if sample.is_empty():
|
||||||
|
push_error("Scaling benchmark could not measure %s" % case_config["case_id"])
|
||||||
|
manager.free()
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
sample["setup_usec"] = maxi(setup_usec, 1)
|
||||||
|
samples.append(sample)
|
||||||
|
manager.free()
|
||||||
|
var summarized := _summarize_samples(String(case_config["case_id"]), samples)
|
||||||
|
if summarized.is_empty():
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
case_results.append(summarized)
|
||||||
|
_print_case(summarized)
|
||||||
|
|
||||||
|
var report := {
|
||||||
|
"schema_version": SimulationScalingBenchmark.SCHEMA_VERSION,
|
||||||
|
"captured_utc": Time.get_datetime_string_from_system(true) + "Z",
|
||||||
|
"engine_version": String(Engine.get_version_info().get("string", "unknown")),
|
||||||
|
"platform": OS.get_name(),
|
||||||
|
"processor_count": OS.get_processor_count(),
|
||||||
|
"host_label": _get_argument_value("--host-label=", DEFAULT_HOST_LABEL),
|
||||||
|
"benchmark_seed": BENCHMARK_SEED,
|
||||||
|
"workload_id": String(SimulationScalingBenchmark.WORKLOAD_ID),
|
||||||
|
"workload":
|
||||||
|
(
|
||||||
|
"All NPCs receive full per-tick needs/task updates and ordinary action decisions; "
|
||||||
|
+ "travel resolves through the deterministic immediate-arrival headless convention."
|
||||||
|
),
|
||||||
|
"timed_phases": ["simulation_tick", "headless_arrival_completion"],
|
||||||
|
"exclusions":
|
||||||
|
[
|
||||||
|
"manager_and_fixture_setup",
|
||||||
|
"state_serialization",
|
||||||
|
"world_scene",
|
||||||
|
"rendering",
|
||||||
|
"navigation",
|
||||||
|
"npc_visual",
|
||||||
|
],
|
||||||
|
"warmup_ticks": WARMUP_TICKS,
|
||||||
|
"measured_ticks": MEASURED_TICKS,
|
||||||
|
"sample_count": SAMPLE_COUNT,
|
||||||
|
"cases": case_results,
|
||||||
|
}
|
||||||
|
var output_path := _get_output_path()
|
||||||
|
var output := FileAccess.open(output_path, FileAccess.WRITE)
|
||||||
|
if output == null:
|
||||||
|
push_error("Could not write scaling benchmark report to %s" % output_path)
|
||||||
|
quit(1)
|
||||||
|
return
|
||||||
|
output.store_string(JSON.stringify(report, "\t") + "\n")
|
||||||
|
output.close()
|
||||||
|
print("[BENCH] Report: %s" % ProjectSettings.globalize_path(output_path))
|
||||||
|
quit(0)
|
||||||
|
|
||||||
|
|
||||||
|
func _summarize_samples(case_id: String, samples: Array[Dictionary]) -> Dictionary:
|
||||||
|
if samples.size() != SAMPLE_COUNT or samples[0].is_empty():
|
||||||
|
push_error("Scaling benchmark case %s returned incomplete samples" % case_id)
|
||||||
|
return {}
|
||||||
|
var checksum := String(samples[0]["final_checksum"])
|
||||||
|
var deterministic_keys := [
|
||||||
|
"start_state_bytes",
|
||||||
|
"end_state_bytes",
|
||||||
|
"state_growth_bytes",
|
||||||
|
"start_event_count",
|
||||||
|
"end_event_count",
|
||||||
|
"events_recorded",
|
||||||
|
"start_known_reference_count",
|
||||||
|
"end_known_reference_count",
|
||||||
|
"arrivals_processed",
|
||||||
|
"warmup_arrivals",
|
||||||
|
"npc_updates",
|
||||||
|
"start_tick",
|
||||||
|
"end_tick",
|
||||||
|
]
|
||||||
|
for sample in samples:
|
||||||
|
if String(sample["final_checksum"]) != checksum:
|
||||||
|
push_error("Scaling benchmark case %s diverged across checksums" % case_id)
|
||||||
|
return {}
|
||||||
|
for key in deterministic_keys:
|
||||||
|
if sample[key] != samples[0][key]:
|
||||||
|
push_error("Scaling benchmark case %s diverged at %s" % [case_id, key])
|
||||||
|
return {}
|
||||||
|
var elapsed_samples: Array[int] = []
|
||||||
|
var setup_samples: Array[int] = []
|
||||||
|
var simulation_samples: Array[int] = []
|
||||||
|
var arrival_samples: Array[int] = []
|
||||||
|
for sample in samples:
|
||||||
|
elapsed_samples.append(int(sample["elapsed_usec"]))
|
||||||
|
setup_samples.append(int(sample["setup_usec"]))
|
||||||
|
simulation_samples.append(int(sample["simulation_usec"]))
|
||||||
|
arrival_samples.append(int(sample["arrival_usec"]))
|
||||||
|
elapsed_samples.sort()
|
||||||
|
setup_samples.sort()
|
||||||
|
simulation_samples.sort()
|
||||||
|
arrival_samples.sort()
|
||||||
|
var elapsed_median := elapsed_samples[elapsed_samples.size() / 2]
|
||||||
|
var simulation_median := simulation_samples[simulation_samples.size() / 2]
|
||||||
|
var arrival_median := arrival_samples[arrival_samples.size() / 2]
|
||||||
|
var result := samples[0].duplicate(true)
|
||||||
|
for transient_key in [
|
||||||
|
"setup_usec",
|
||||||
|
"elapsed_usec",
|
||||||
|
"simulation_usec",
|
||||||
|
"arrival_usec",
|
||||||
|
"usec_per_tick",
|
||||||
|
"ticks_per_second",
|
||||||
|
"realtime_factor",
|
||||||
|
]:
|
||||||
|
result.erase(transient_key)
|
||||||
|
result["case_id"] = case_id
|
||||||
|
result["sample_count"] = SAMPLE_COUNT
|
||||||
|
result["setup_usec_samples"] = setup_samples
|
||||||
|
result["setup_usec_median"] = setup_samples[setup_samples.size() / 2]
|
||||||
|
result["elapsed_usec_samples"] = elapsed_samples
|
||||||
|
result["elapsed_usec_min"] = elapsed_samples[0]
|
||||||
|
result["elapsed_usec_median"] = elapsed_median
|
||||||
|
result["elapsed_usec_max"] = elapsed_samples[-1]
|
||||||
|
result["simulation_usec_samples"] = simulation_samples
|
||||||
|
result["simulation_usec_median"] = simulation_median
|
||||||
|
result["arrival_usec_samples"] = arrival_samples
|
||||||
|
result["arrival_usec_median"] = arrival_median
|
||||||
|
result["arrival_share_percent"] = float(arrival_median) / float(elapsed_median) * 100.0
|
||||||
|
result["usec_per_tick_median"] = float(elapsed_median) / float(MEASURED_TICKS)
|
||||||
|
result["ticks_per_second_median"] = (float(MEASURED_TICKS) * 1000000.0 / float(elapsed_median))
|
||||||
|
result["realtime_factor_median"] = (
|
||||||
|
float(result["ticks_per_second_median"]) * float(result["tick_interval"])
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
func _print_case(result: Dictionary) -> void:
|
||||||
|
print(
|
||||||
|
(
|
||||||
|
(
|
||||||
|
"[BENCH] %s | NPC %d | seeded events %d | %.1f us/tick | "
|
||||||
|
+ "arrival %.1f%% | %.1fx realtime | %.2f MiB"
|
||||||
|
)
|
||||||
|
% [
|
||||||
|
result["case_id"],
|
||||||
|
result["population"],
|
||||||
|
result["history_seed_events"],
|
||||||
|
result["usec_per_tick_median"],
|
||||||
|
result["arrival_share_percent"],
|
||||||
|
result["realtime_factor_median"],
|
||||||
|
float(result["end_state_bytes"]) / (1024.0 * 1024.0),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
func _get_output_path() -> String:
|
||||||
|
return _get_argument_value("--output=", DEFAULT_OUTPUT_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
func _get_argument_value(prefix: String, default_value: String) -> String:
|
||||||
|
for argument in OS.get_cmdline_user_args():
|
||||||
|
if argument.begins_with(prefix):
|
||||||
|
var value := argument.trim_prefix(prefix)
|
||||||
|
if not value.is_empty():
|
||||||
|
return value
|
||||||
|
return default_value
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://dyo8sn71hxchp
|
||||||
Reference in New Issue
Block a user