Skip to content

Commit 4d08704

Browse files
iagaponenkoiagaponenko
committed
The HTTP-based backend of the Replication worker services
1 parent d990faf commit 4d08704

33 files changed

+5589
-30
lines changed

‎src/replica/apps/WorkerApp.cc

+4-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
#include "replica/services/ServiceProvider.h"
4040
#include "replica/util/FileUtils.h"
4141
#include "replica/worker/FileServer.h"
42-
#include "replica/worker/WorkerProcessor.h"
42+
#include "replica/worker/WorkerHttpSvc.h"
4343
#include "replica/worker/WorkerServer.h"
4444

4545
// LSST headers
@@ -113,6 +113,9 @@ int WorkerApp::runImpl() {
113113
auto const reqProcSvr = WorkerServer::create(serviceProvider(), worker);
114114
thread reqProcSvrThread([reqProcSvr]() { reqProcSvr->run(); });
115115

116+
auto const reqProcHttpSvr = WorkerHttpSvc::create(serviceProvider(), worker);
117+
thread reqProcHttpSvrThread([reqProcHttpSvr]() { reqProcHttpSvr->run(); });
118+
116119
auto const fileSvr = FileServer::create(serviceProvider(), worker);
117120
thread fileSvrThread([fileSvr]() { fileSvr->run(); });
118121

‎src/replica/proto/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ add_library(replica_proto OBJECT)
44
target_sources(replica_proto PRIVATE
55
${REPLICA_PB_SRCS}
66
${REPLICA_PB_HDRS}
7+
Protocol.cc
78
)

‎src/replica/proto/Protocol.cc

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* LSST Data Management System
3+
*
4+
* This product includes software developed by the
5+
* LSST Project (http://www.lsst.org/).
6+
*
7+
* This program is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the LSST License Statement and
18+
* the GNU General Public License along with this program. If not,
19+
* see <http://www.lsstcorp.org/LegalNotices/>.
20+
*/
21+
22+
// Class header
23+
#include "replica/proto/Protocol.h"
24+
25+
// System headers
26+
#include <stdexcept>
27+
28+
using namespace std;
29+
30+
namespace lsst::qserv::replica::protocol {
31+
32+
string toString(SqlRequestType status) {
33+
switch (status) {
34+
case SqlRequestType::QUERY:
35+
return "QUERY";
36+
case SqlRequestType::CREATE_DATABASE:
37+
return "CREATE_DATABASE";
38+
case SqlRequestType::DROP_DATABASE:
39+
return "DROP_DATABASE";
40+
case SqlRequestType::ENABLE_DATABASE:
41+
return "ENABLE_DATABASE";
42+
case SqlRequestType::DISABLE_DATABASE:
43+
return "DISABLE_DATABASE";
44+
case SqlRequestType::GRANT_ACCESS:
45+
return "GRANT_ACCESS";
46+
case SqlRequestType::CREATE_TABLE:
47+
return "CREATE_TABLE";
48+
case SqlRequestType::DROP_TABLE:
49+
return "DROP_TABLE";
50+
case SqlRequestType::REMOVE_TABLE_PARTITIONING:
51+
return "REMOVE_TABLE_PARTITIONING";
52+
case SqlRequestType::DROP_TABLE_PARTITION:
53+
return "DROP_TABLE_PARTITION";
54+
case SqlRequestType::GET_TABLE_INDEX:
55+
return "GET_TABLE_INDEX";
56+
case SqlRequestType::CREATE_TABLE_INDEX:
57+
return "CREATE_TABLE_INDEX";
58+
case SqlRequestType::DROP_TABLE_INDEX:
59+
return "DROP_TABLE_INDEX";
60+
case SqlRequestType::ALTER_TABLE:
61+
return "ALTER_TABLE";
62+
case SqlRequestType::TABLE_ROW_STATS:
63+
return "TABLE_ROW_STATS";
64+
default:
65+
throw logic_error("Unhandled SQL request type: " + to_string(static_cast<int>(status)));
66+
}
67+
}
68+
69+
string toString(Status status) {
70+
switch (status) {
71+
case Status::CREATED:
72+
return "CREATED";
73+
case Status::SUCCESS:
74+
return "SUCCESS";
75+
case Status::QUEUED:
76+
return "QUEUED";
77+
case Status::IN_PROGRESS:
78+
return "IN_PROGRESS";
79+
case Status::IS_CANCELLING:
80+
return "IS_CANCELLING";
81+
case Status::BAD:
82+
return "BAD";
83+
case Status::FAILED:
84+
return "FAILED";
85+
case Status::CANCELLED:
86+
return "CANCELLED";
87+
default:
88+
throw logic_error("Unhandled status: " + to_string(static_cast<int>(status)));
89+
}
90+
}
91+
92+
string toString(StatusExt extendedStatus) {
93+
switch (extendedStatus) {
94+
case StatusExt::NONE:
95+
return "NONE";
96+
case StatusExt::INVALID_PARAM:
97+
return "INVALID_PARAM";
98+
case StatusExt::INVALID_ID:
99+
return "INVALID_ID";
100+
case StatusExt::FOLDER_STAT:
101+
return "FOLDER_STAT";
102+
case StatusExt::FOLDER_CREATE:
103+
return "FOLDER_CREATE";
104+
case StatusExt::FILE_STAT:
105+
return "FILE_STAT";
106+
case StatusExt::FILE_SIZE:
107+
return "FILE_SIZE";
108+
case StatusExt::FOLDER_READ:
109+
return "FOLDER_READ";
110+
case StatusExt::FILE_READ:
111+
return "FILE_READ";
112+
case StatusExt::FILE_ROPEN:
113+
return "FILE_ROPEN";
114+
case StatusExt::FILE_CREATE:
115+
return "FILE_CREATE";
116+
case StatusExt::FILE_OPEN:
117+
return "FILE_OPEN";
118+
case StatusExt::FILE_RESIZE:
119+
return "FILE_RESIZE";
120+
case StatusExt::FILE_WRITE:
121+
return "FILE_WRITE";
122+
case StatusExt::FILE_COPY:
123+
return "FILE_COPY";
124+
case StatusExt::FILE_DELETE:
125+
return "FILE_DELETE";
126+
case StatusExt::FILE_RENAME:
127+
return "FILE_RENAME";
128+
case StatusExt::FILE_EXISTS:
129+
return "FILE_EXISTS";
130+
case StatusExt::SPACE_REQ:
131+
return "SPACE_REQ";
132+
case StatusExt::NO_FOLDER:
133+
return "NO_FOLDER";
134+
case StatusExt::NO_FILE:
135+
return "NO_FILE";
136+
case StatusExt::NO_ACCESS:
137+
return "NO_ACCESS";
138+
case StatusExt::NO_SPACE:
139+
return "NO_SPACE";
140+
case StatusExt::FILE_MTIME:
141+
return "FILE_MTIME";
142+
case StatusExt::MYSQL_ERROR:
143+
return "MYSQL_ERROR";
144+
case StatusExt::LARGE_RESULT:
145+
return "LARGE_RESULT";
146+
case StatusExt::NO_SUCH_TABLE:
147+
return "NO_SUCH_TABLE";
148+
case StatusExt::NOT_PARTITIONED_TABLE:
149+
return "NOT_PARTITIONED_TABLE";
150+
case StatusExt::NO_SUCH_PARTITION:
151+
return "NO_SUCH_PARTITION";
152+
case StatusExt::MULTIPLE:
153+
return "MULTIPLE";
154+
case StatusExt::OTHER_EXCEPTION:
155+
return "OTHER_EXCEPTION";
156+
case StatusExt::FOREIGN_INSTANCE:
157+
return "FOREIGN_INSTANCE";
158+
case StatusExt::DUPLICATE_KEY:
159+
return "DUPLICATE_KEY";
160+
case StatusExt::CANT_DROP_KEY:
161+
return "CANT_DROP_KEY";
162+
default:
163+
throw logic_error("Unhandled extended status: " + to_string(static_cast<int>(extendedStatus)));
164+
}
165+
}
166+
167+
string toString(Status status, StatusExt extendedStatus) {
168+
return toString(status) + "::" + toString(extendedStatus);
169+
}
170+
171+
string toString(ServiceState state) {
172+
switch (state) {
173+
case ServiceState::SUSPEND_IN_PROGRESS:
174+
return "SUSPEND_IN_PROGRESS";
175+
case ServiceState::SUSPENDED:
176+
return "SUSPENDED";
177+
case ServiceState::RUNNING:
178+
return "RUNNING";
179+
default:
180+
throw logic_error("Unhandled service state: " + to_string(static_cast<int>(state)));
181+
}
182+
}
183+
184+
} // namespace lsst::qserv::replica::protocol

‎src/replica/proto/Protocol.h

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* LSST Data Management System
3+
*
4+
* This product includes software developed by the
5+
* LSST Project (http://www.lsst.org/).
6+
*
7+
* This program is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the LSST License Statement and
18+
* the GNU General Public License along with this program. If not,
19+
* see <http://www.lsstcorp.org/LegalNotices/>.
20+
*/
21+
#ifndef LSST_QSERV_REPLICA_PROTOCOL_H
22+
#define LSST_QSERV_REPLICA_PROTOCOL_H
23+
24+
// System headers
25+
#include <string>
26+
27+
// Third party headers
28+
#include "nlohmann/json.hpp"
29+
30+
// This header declarations
31+
namespace lsst::qserv::replica::protocol {
32+
33+
/// Subtypes of the SQL requests.
34+
enum class SqlRequestType : int {
35+
36+
QUERY = 0,
37+
CREATE_DATABASE = 1,
38+
DROP_DATABASE = 2,
39+
ENABLE_DATABASE = 3, ///< in Qserv
40+
DISABLE_DATABASE = 4, ///< in Qserv
41+
GRANT_ACCESS = 5,
42+
CREATE_TABLE = 6,
43+
DROP_TABLE = 7,
44+
REMOVE_TABLE_PARTITIONING = 8,
45+
DROP_TABLE_PARTITION = 9,
46+
GET_TABLE_INDEX = 10,
47+
CREATE_TABLE_INDEX = 11,
48+
DROP_TABLE_INDEX = 12,
49+
ALTER_TABLE = 13,
50+
TABLE_ROW_STATS = 14
51+
};
52+
53+
/// @return the string representation of the SQL request type
54+
std::string toString(SqlRequestType status);
55+
56+
/// Types of the table indexes specified in the index management requests requests.
57+
enum class SqlIndexSpec : int { DEFAULT = 1, UNIQUE = 2, FULLTEXT = 3, SPATIAL = 4 };
58+
59+
/// Status values returned by all request related to operations with
60+
/// replicas. Request management operations always return messages whose types
61+
/// match the return types of the corresponding (original) replica-related requests.
62+
/// Service management requests have their own set of status values.
63+
///
64+
enum class Status : int {
65+
CREATED = 0,
66+
SUCCESS = 1,
67+
QUEUED = 2,
68+
IN_PROGRESS = 3,
69+
IS_CANCELLING = 4,
70+
BAD = 5,
71+
FAILED = 6,
72+
CANCELLED = 7
73+
};
74+
75+
enum class StatusExt : int {
76+
NONE = 0, ///< Unspecified problem.
77+
INVALID_PARAM = 1, ///< Invalid parameter(s) of a request.
78+
INVALID_ID = 2, ///< An invalid request identifier.
79+
FOLDER_STAT = 4, ///< Failed to obtain fstat() for a folder.
80+
FOLDER_CREATE = 5, ///< Failed to create a folder.
81+
FILE_STAT = 6, ///< Failed to obtain fstat() for a file.
82+
FILE_SIZE = 7, ///< Failed to obtain a size of a file.
83+
FOLDER_READ = 8, ///< Failed to read the contents of a folder.
84+
FILE_READ = 9, ///< Failed to read the contents of a file.
85+
FILE_ROPEN = 10, ///< Failed to open a remote file.
86+
FILE_CREATE = 11, ///< Failed to create a file.
87+
FILE_OPEN = 12, ///< Failed to open a file.
88+
FILE_RESIZE = 13, ///< Failed to resize a file.
89+
FILE_WRITE = 14, ///< Failed to write into a file.
90+
FILE_COPY = 15, ///< Failed to copy a file.
91+
FILE_DELETE = 16, ///< Failed to delete a file.
92+
FILE_RENAME = 17, ///< Failed to rename a file.
93+
FILE_EXISTS = 18, ///< File already exists.
94+
SPACE_REQ = 19, ///< Space availability check failed.
95+
NO_FOLDER = 20, ///< Folder doesn't exist.
96+
NO_FILE = 21, ///< File doesn't exist.
97+
NO_ACCESS = 22, ///< No access to a file or a folder.
98+
NO_SPACE = 23, ///< No space left on a device as required by an operation.
99+
FILE_MTIME = 24, ///< Get/set 'mtime' operation failed.
100+
MYSQL_ERROR = 25, ///< General MySQL error (other than any specific ones listed here).
101+
LARGE_RESULT = 26, ///< Result exceeds a limit set in a request.
102+
NO_SUCH_TABLE = 27, ///< No table found while performing a MySQL operation.
103+
NOT_PARTITIONED_TABLE = 28, ///< The table is not MySQL partitioned as it was expected.
104+
NO_SUCH_PARTITION = 29, ///< No MySQL partition found in a table as it was expected.
105+
MULTIPLE = 30, ///< Multiple unspecified errors encountered when processing a request.
106+
OTHER_EXCEPTION = 31, ///< Other exception not listed here.
107+
FOREIGN_INSTANCE = 32, ///< Detected a request from a Controller serving an unrelated Qserv.
108+
DUPLICATE_KEY = 33, ///< Duplicate key found when creating an index or altering a table schema.
109+
CANT_DROP_KEY = 34 ///< Can't drop a field or a key which doesn't exist.
110+
};
111+
112+
/// @return the string representation of the status
113+
std::string toString(Status status);
114+
115+
/// @return the string representation of the extended status
116+
std::string toString(StatusExt extendedStatus);
117+
118+
/// @return the string representation of the full status
119+
std::string toString(Status status, StatusExt extendedStatus);
120+
121+
/// Status of a service.
122+
enum class ServiceState : int { SUSPEND_IN_PROGRESS = 0, SUSPENDED = 1, RUNNING = 2 };
123+
124+
/// @return the string representation of the service state
125+
std::string toString(ServiceState state);
126+
127+
/// The header to be sent with the requests processed through the worker's queueing system.
128+
struct QueuedRequestHdr {
129+
std::string id;
130+
int priority;
131+
unsigned int timeout;
132+
QueuedRequestHdr(std::string const& id_, int priority_, unsigned int timeout_)
133+
: id(id_), priority(priority_), timeout(timeout_) {}
134+
nlohmann::json toJson() const { return {{"id", id}, {"priority", priority}, {"timeout", timeout}}; };
135+
};
136+
137+
} // namespace lsst::qserv::replica::protocol
138+
139+
#endif // LSST_QSERV_REPLICA_PROTOCOL_H

0 commit comments

Comments
 (0)