1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
/* Copyright (C) 2025 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef INCLUDED_RLINTERFACE
#define INCLUDED_RLINTERFACE
#include "lib/code_annotation.h"
#include "simulation2/helpers/Player.h"
#include "third_party/mongoose/mongoose.h"
#include <condition_variable>
#include <exception>
#include <mutex>
#include <string>
#include <vector>
struct mg_context;
namespace RL
{
struct ScenarioConfig
{
bool saveReplay;
player_id_t playerID;
std::string content;
};
struct GameCommand
{
int playerID;
std::string json_cmd;
};
enum class GameMessageType
{
None,
Reset,
Commands,
Evaluate,
};
/**
* Holds messages from the RL client to the game.
*/
struct GameMessage
{
GameMessageType type;
std::vector<GameCommand> commands;
};
struct SetupError : std::exception
{
using std::exception::exception;
};
/**
* Implements an interface providing fundamental capabilities required for reinforcement
* learning (over HTTP).
*
* This consists of enabling an external script to configure the scenario (via Reset) and
* then step the game engine manually and apply player actions (via Step). The interface
* also supports querying unit templates to provide information about max health and other
* potentially relevant game state information.
*
* See source/tools/rlclient/ for the external client code.
*
* The HTTP server is threaded.
* Flow of data (with the interface active):
* 0. The game/main thread calls TryApplyMessage()
* - If no messages are pending, GOTO 0 (the simulation is not advanced).
* 1. TryApplyMessage locks m_MsgLock, pulls the message, processes it, advances the simulation, and sets m_ReturnValue.
* 2. TryApplyMessage notifies the RL thread that it can carry on and unlocks m_MsgLock. The main thread carries on frame rendering and goes back to 0.
* 3. The RL thread locks m_MsgLock, reads m_ReturnValue, unlocks m_MsgLock, and sends the gamestate as HTTP Response to the RL client.
* 4. The client processes the response and ultimately sends a new HTTP message to the RL Interface.
* 5. The RL thread locks m_MsgLock, pushes the message, and starts waiting on the game/main thread to notify it (step 2).
* - GOTO 0.
*/
class Interface
{
NONCOPYABLE(Interface);
public:
Interface(const char* server_address);
~Interface();
/**
* Non-blocking call to process any pending messages from the RL client.
* Updates m_ReturnValue to the gamestate after messages have been processed.
*/
void TryApplyMessage();
private:
static void* MgCallback(mg_event event, struct mg_connection *conn, const struct mg_request_info *request_info);
static std::string GetRequestContent(struct mg_connection *conn);
/**
* Process commands, update the simulation by one turn.
* @return the gamestate after processing commands.
*/
std::string Step(std::vector<GameCommand>&& commands);
/**
* Reset the game state according to scenario, cleaning up existing games if required.
* @return the gamestate after resetting.
*/
std::string Reset(ScenarioConfig&& scenario);
/**
* Evaluate JS code in the engine such as applying arbitrary modifiers.
* @return the gamestate after script evaluation.
*/
std::string Evaluate(std::string&& code);
/**
* @return template data for all templates of @param names.
*/
std::vector<std::string> GetTemplates(const std::vector<std::string>& names) const;
/**
* @return true if a game is currently running.
*/
bool IsGameRunning() const;
/**
* Internal helper. Move @param msg into m_GameMessage, wait until it has been processed by the main thread,
* and @return the gamestate after that message is processed.
* It is invalid to call this if m_GameMessage is not currently empty.
*/
std::string SendGameMessage(GameMessage&& msg);
/**
* Internal helper.
* @return true if m_GameMessage is not empty, and updates @param msg, false otherwise (msg is then unchanged).
*/
bool TryGetGameMessage(GameMessage& msg);
/**
* Process any pending messages from the RL client.
* Updates m_ReturnValue to the gamestate after messages have been processed.
*/
void ApplyMessage(const GameMessage& msg);
/**
* @return the full gamestate as a JSON strong.
* This uses the AI representation since it is readily available in the JS Engine.
*/
std::string GetGameState() const;
private:
GameMessage m_GameMessage{GameMessageType::None};
ScenarioConfig m_ScenarioConfig;
std::string m_ReturnValue;
bool m_NeedsGameState = false;
mutable std::mutex m_Lock;
std::mutex m_MsgLock;
std::condition_variable m_MsgApplied;
std::string m_Code;
mg_context* m_Context;
};
}
#endif // INCLUDED_RLINTERFACE
|
