Skip to content

Commit c81adb3

Browse files
haiodohaiodo
committed
UBERF-9747
Signed-off-by: Andrey Sobolev <[email protected]>
1 parent 8452c1f commit c81adb3

File tree

198 files changed

+4748
-2505
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

198 files changed

+4748
-2505
lines changed

‎.vscode/launch.json

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@
103103
"UPLOAD_URL": "/files",
104104
"AI_BOT_URL": "http://localhost:4010",
105105
"STATS_URL": "http://huly.local:4900",
106-
"QUEUE_CONFIG": "localhost:19092"
106+
"QUEUE_CONFIG": "localhost:19092",
107+
"RATE_LIMIT_MAX": "25000"
107108
},
108109
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
109110
"runtimeVersion": "20",
@@ -120,9 +121,9 @@
120121
"args": ["src/__start.ts"],
121122
"env": {
122123
"FULLTEXT_URL": "http://localhost:4710",
123-
// "DB_URL": "mongodb://localhost:27018",
124+
"DB_URL": "mongodb://localhost:27018",
124125
// "DB_URL": "postgresql://postgres:example@localhost:5432",
125-
"DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
126+
// "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
126127
// "GREEN_URL": "http://huly.local:6767?token=secret",
127128
"SERVER_PORT": "3334",
128129
"METRICS_CONSOLE": "false",
@@ -183,7 +184,7 @@
183184
"request": "launch",
184185
"args": ["src/__start.ts"],
185186
"env": {
186-
"MONGO_URL": "mongodb://localhost:27017",
187+
// "MONGO_URL": "mongodb://localhost:27017",
187188
// "DB_URL": "mongodb://localhost:27017",
188189
"DB_URL": "postgresql://[email protected]:26257/defaultdb?sslmode=disable",
189190
"SERVER_SECRET": "secret",
@@ -217,12 +218,13 @@
217218
"args": ["src/__start.ts"],
218219
"env": {
219220
// "MONGO_URL": "mongodb://localhost:27018",
220-
// "DB_URL": "mongodb://localhost:27018",
221+
"DB_URL": "mongodb://localhost:27018",
221222
// "DB_URL": "postgresql://postgres:example@localhost:5432",
222-
"DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
223+
// "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
223224
"SERVER_SECRET": "secret",
224-
"REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
225-
"TRANSACTOR_URL": "ws://transactor:3334;ws://localhost:3334",
225+
"QUEUE_CONFIG": "huly.local:19092",
226+
"REGION_INFO": "|America;europe|Europe",
227+
"TRANSACTOR_URL": "ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe",
226228
"ACCOUNTS_URL": "http://localhost:3003",
227229
"ACCOUNT_PORT": "3003",
228230
"FRONT_URL": "http://localhost:8083",

‎common/config/rush/command-line.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
*/
55
{
66
"$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
7-
87
"phases": [
98
{
109
"name": "_phase:build",

‎common/scripts/rush_docker.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
export RUSH_ALLOW_UNSUPPORTED_NODEJS=1
2+
rush docker:build -p 20 \
3+
--to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-inbound-mail --to @hcengineering/pod-export --to @hcengineering/pod-msg2file

‎dev/doc-import-tool/src/import.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ export default async function importExtractedFile (
4747

4848
try {
4949
console.log(`Connected to ${transactorUrl}`)
50-
const txops = new TxOperations(connection, core.account.System)
50+
const txops = new TxOperations(connection, core.account.System, workspaceId)
5151

5252
try {
5353
const docId = await createDocument(txops, extractedFile, config)

‎dev/import-tool/src/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ export function importTool (): void {
9595

9696
console.log('Connecting to Transactor URL: ', selectedWs.endpoint)
9797
const connection = await createClient(selectedWs.endpoint, selectedWs.token)
98-
const client = new TxOperations(connection, socialId)
98+
const client = new TxOperations(connection, socialId, selectedWs.workspace)
9999
const fileUploader = new FrontFileUploader(
100100
getFrontUrl(),
101101
selectedWs.workspace,

‎dev/prod/src/platform.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ import { uploaderId } from '@hcengineering/uploader'
6868
import { mediaId } from '@hcengineering/media'
6969
import recorder, { recorderId } from '@hcengineering/recorder'
7070
import { viewId } from '@hcengineering/view'
71-
import workbench, { workbenchId } from '@hcengineering/workbench'
71+
import workbench, { workbenchId, workbenchAppsId } from '@hcengineering/workbench'
7272
import { mailId } from '@hcengineering/mail'
7373
import { chatId } from '@hcengineering/chat'
7474
import github, { githubId } from '@hcengineering/github'
@@ -148,6 +148,7 @@ import { preferenceId } from '@hcengineering/preference'
148148
import { uiId } from '@hcengineering/ui/src/plugin'
149149
import { configureAnalytics } from './analytics'
150150

151+
151152
export interface Config {
152153
ACCOUNTS_URL: string
153154
UPLOAD_URL: string
@@ -487,6 +488,7 @@ export async function configurePlatform() {
487488
uiPlugin.metadata.Routes,
488489
new Map([
489490
[workbenchId, workbench.component.WorkbenchApp],
491+
[workbenchAppsId, workbench.component.WorkbenchApps],
490492
[loginId, login.component.LoginApp],
491493
[onboardId, onboard.component.OnboardApp],
492494
[githubId, github.component.ConnectApp],

‎dev/tool/src/benchmark.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@ export function benchmarkWorker (): void {
410410

411411
if (msg.options.mode === 'find-all') {
412412
const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId
413-
const opt = new TxOperations(connection, benchmarkPersonId)
413+
const opt = new TxOperations(connection, benchmarkPersonId, msg.workspaceId)
414414
parentPort?.postMessage({
415415
type: 'operate',
416416
workId: msg.workId
@@ -551,7 +551,7 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
551551
export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise<void> {
552552
const connection = await connect(endpoint, workspace, account)
553553
try {
554-
const client = new TxOperations(connection, core.account.System)
554+
const client = new TxOperations(connection, core.account.System, workspace)
555555
const start = platformNow()
556556
const res = await client.findAll(
557557
recruit.class.Applicant,
@@ -576,7 +576,7 @@ export async function generateWorkspaceData (
576576
email: string
577577
): Promise<void> {
578578
const connection = await connect(endpoint, workspace)
579-
const client = new TxOperations(connection, core.account.System)
579+
const client = new TxOperations(connection, core.account.System, workspace)
580580
try {
581581
const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
582582
const person = await getPersonBySocialKey(client, emailSocialString)

‎dev/tool/src/clean.ts

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,10 @@ export async function cleanWorkspace (
8686
opt: { recruit: boolean, tracker: boolean, removedTx: boolean }
8787
): Promise<void> {
8888
const connection = (await connect(transactorUrl, workspaceId, undefined, {
89-
mode: 'backup',
90-
model: 'upgrade'
89+
mode: 'backup'
9190
})) as unknown as CoreClient & BackupClient
9291
try {
93-
const ops = new TxOperations(connection, core.account.System)
92+
const ops = new TxOperations(connection, core.account.System, workspaceId)
9493

9594
const hierarchy = ops.getHierarchy()
9695

@@ -164,6 +163,7 @@ export async function cleanWorkspace (
164163
client.close()
165164
}
166165
} catch (err: any) {
166+
// TODO: Add force-close
167167
console.trace(err)
168168
} finally {
169169
await connection.close()
@@ -214,6 +214,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
214214
objectId: { $in: removedDocs.map((it) => it.objectId) }
215215
})
216216
await connection.clean(
217+
workspaceId,
217218
DOMAIN_TX,
218219
toRemove.map((it) => it._id)
219220
)
@@ -232,8 +233,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
232233

233234
export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise<void> {
234235
const connection = (await connect(transactorUrl, workspaceId, undefined, {
235-
mode: 'backup',
236-
model: 'upgrade'
236+
mode: 'backup'
237237
})) as unknown as CoreClient & BackupClient
238238
try {
239239
let count = 0
@@ -289,7 +289,7 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
289289
}
290290
}
291291

292-
await connection.clean(DOMAIN_TX, toRemove)
292+
await connection.clean(workspaceId, DOMAIN_TX, toRemove)
293293

294294
count += toRemove.length
295295
console.log('processed', count)
@@ -298,6 +298,8 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
298298
} catch (err: any) {
299299
console.trace(err)
300300
} finally {
301+
// TODO: Add force-close
302+
await connection.sendForceClose(workspaceId)
301303
await connection.close()
302304
}
303305
}
@@ -307,7 +309,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transacto
307309
})) as unknown as CoreClient & BackupClient
308310
try {
309311
const count = 0
310-
const ops = new TxOperations(connection, core.account.System)
312+
const ops = new TxOperations(connection, core.account.System, workspaceId)
311313
while (true) {
312314
const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 })
313315
if (spaces.length === 0) {
@@ -370,7 +372,7 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
370372
// We have found duplicate one, let's rename it.
371373
const doc = TxProcessor.createDoc2Doc<ChatMessage>(c as unknown as TxCreateDoc<ChatMessage>)
372374
if (doc.message !== '' && doc.message.trim() !== '<p></p>') {
373-
await connection.clean(DOMAIN_TX, [c._id])
375+
await connection.clean(workspaceId, DOMAIN_TX, [c._id])
374376
if (oldValue.get(cid) === doc.message.trim()) {
375377
console.log('delete tx', cid, doc.message)
376378
} else {
@@ -379,9 +381,9 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
379381
// Remove previous transaction.
380382
c.objectId = generateId()
381383
doc._id = c.objectId as Ref<ChatMessage>
382-
await connection.upload(DOMAIN_TX, [c])
384+
await connection.upload(workspaceId, DOMAIN_TX, [c])
383385
// Also we need to create snapsot
384-
await connection.upload(DOMAIN_ACTIVITY, [doc])
386+
await connection.upload(workspaceId, DOMAIN_ACTIVITY, [doc])
385387
}
386388
}
387389
}
@@ -475,7 +477,7 @@ export async function fixSkills (
475477
// fix skills with + and -
476478
if (step === '3') {
477479
console.log('STEP 3')
478-
const ops = new TxOperations(connection, core.account.System)
480+
const ops = new TxOperations(connection, core.account.System, workspaceId)
479481
const regex = /\S+(?:[-+]\S+)+/g
480482
const tagsToClean = (await connection.findAll(tags.class.TagElement, {
481483
category: {
@@ -532,7 +534,7 @@ export async function fixSkills (
532534
}
533535
})) as TagElement[]
534536
goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2)
535-
const ops = new TxOperations(connection, core.account.System)
537+
const ops = new TxOperations(connection, core.account.System, workspaceId)
536538
const tagsToClean = (await connection.findAll(tags.class.TagElement, {
537539
category: {
538540
$in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref<TagCategory>[]
@@ -669,8 +671,7 @@ export async function restoreRecruitingTaskTypes (
669671
transactorUrl: string
670672
): Promise<void> {
671673
const connection = (await connect(transactorUrl, workspaceId, undefined, {
672-
mode: 'backup',
673-
model: 'upgrade'
674+
mode: 'backup'
674675
})) as unknown as CoreClient & BackupClient
675676
const client = getMongoClient(mongoUrl)
676677
try {
@@ -772,6 +773,7 @@ export async function restoreRecruitingTaskTypes (
772773
statusCategories.sort(compareCategories)
773774

774775
const createTxNew: TxCreateDoc<TaskType> = {
776+
_uuid: workspaceId,
775777
_id: generateId(),
776778
_class: core.class.TxCreateDoc,
777779
space: core.space.Tx,
@@ -822,7 +824,9 @@ export async function restoreRecruitingTaskTypes (
822824
} catch (err: any) {
823825
console.trace(err)
824826
} finally {
827+
// TODO: Add force-close
825828
client.close()
829+
await connection.sendForceClose(workspaceId)
826830
await connection.close()
827831
}
828832
}
@@ -833,8 +837,7 @@ export async function restoreHrTaskTypesFromUpdates (
833837
transactorUrl: string
834838
): Promise<void> {
835839
const connection = (await connect(transactorUrl, workspaceId, undefined, {
836-
mode: 'backup',
837-
model: 'upgrade'
840+
mode: 'backup'
838841
})) as unknown as CoreClient & BackupClient
839842
const client = getMongoClient(mongoUrl)
840843
try {
@@ -927,6 +930,7 @@ export async function restoreHrTaskTypesFromUpdates (
927930
const ofClassClass = hierarchy.getClass(recruit.class.Applicant)
928931

929932
await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
933+
_uuid: workspaceId,
930934
_id: generateId(),
931935
_class: core.class.TxCreateDoc,
932936
space: core.space.Tx,
@@ -946,6 +950,7 @@ export async function restoreHrTaskTypesFromUpdates (
946950
})
947951

948952
createTaskTypeTx = {
953+
_uuid: workspaceId,
949954
_id: generateId(),
950955
_class: core.class.TxCreateDoc,
951956
space: core.space.Tx,
@@ -980,6 +985,7 @@ export async function restoreHrTaskTypesFromUpdates (
980985
const ofClassClass = hierarchy.getClass(recruit.class.Vacancy)
981986

982987
await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
988+
_uuid: workspaceId,
983989
_id: generateId(),
984990
_class: core.class.TxCreateDoc,
985991
space: core.space.Tx,
@@ -999,6 +1005,7 @@ export async function restoreHrTaskTypesFromUpdates (
9991005
})
10001006

10011007
const createProjectTypeTx: TxCreateDoc<ProjectType> = {
1008+
_uuid: workspaceId,
10021009
_id: generateId(),
10031010
_class: core.class.TxCreateDoc,
10041011
space: core.space.Tx,
@@ -1031,6 +1038,7 @@ export async function restoreHrTaskTypesFromUpdates (
10311038
console.trace(err)
10321039
} finally {
10331040
client.close()
1041+
await connection.sendForceClose(workspaceId)
10341042
await connection.close()
10351043
}
10361044
}
@@ -1127,7 +1135,7 @@ export async function removeDuplicateIds (
11271135
// await updateId(ctx, wsClient, db, storageAdapter, wsDataId, doc)
11281136
// }
11291137
// }
1130-
// await wsClient.sendForceClose()
1138+
// await wsClient.sendForceClose(workspaceId)
11311139
// await wsClient.close()
11321140
// await db.collection<MigrationState>(DOMAIN_MIGRATION).insertOne({
11331141
// _id: generateId(),

‎dev/tool/src/configuration.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export async function changeConfiguration (
4141
}
4242
const enable = (cmd.enable ?? '').trim().split(',')
4343
console.log('enable', enable)
44-
const ops = new TxFactory(core.account.ConfigUser)
44+
const ops = new TxFactory(core.account.ConfigUser, workspaceId)
4545
if (enable.length > 0) {
4646
const p = config.filter((it) => enable.includes(it.pluginId) || enable.includes('*'))
4747
for (const pp of p) {

‎dev/tool/src/db.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ async function moveWorkspace (
9696
const token = generateToken(systemAccountUuid, wsId, { service: 'tool' })
9797
const endpoint = await getTransactorEndpoint(token, 'external')
9898
const connection = (await connect(endpoint, wsId, undefined, {
99-
model: 'upgrade'
99+
mode: 'backup'
100100
})) as unknown as Client & BackupClient
101101
for (const collection of collections) {
102102
const domain = translateDomain(collection.collectionName)
@@ -157,7 +157,7 @@ async function moveWorkspace (
157157
}
158158
// TODO: FIXME
159159
// await updateWorkspace(accountDb, ws, { region })
160-
await connection.sendForceClose()
160+
await connection.sendForceClose(wsId)
161161
await connection.close()
162162
} catch (err) {
163163
console.log('Error when move workspace', ws.name ?? ws.url, err)

‎dev/tool/src/github.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,15 +94,15 @@ export async function performGithubAccountMigrations (db: Db, region: string | n
9494
let idx: number | undefined
9595

9696
while (true) {
97-
const info = await client.loadChunk(DOMAIN_MODEL_TX, idx)
97+
const info = await client.loadChunk(ws.uuid, DOMAIN_MODEL_TX, idx)
9898
idx = info.idx
9999
const ids = Array.from(info.docs.map((it) => it.id as Ref<Doc>))
100-
const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) =>
100+
const docs = (await client.loadDocs(ws.uuid, DOMAIN_MODEL_TX, ids)).filter((it) =>
101101
TxProcessor.isExtendsCUD(it._class)
102102
) as TxCUD<Doc>[]
103103
accountsTxes.push(...docs)
104104
if (info.finished && idx !== undefined) {
105-
await client.closeChunk(info.idx)
105+
await client.closeChunk(ws.uuid, info.idx)
106106
break
107107
}
108108
}

‎dev/tool/src/gmail.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -274,15 +274,15 @@ export async function loadAccounts (ws: WorkspaceInfoWithStatus): Promise<(Doc &
274274
let idx: number | undefined
275275

276276
while (true) {
277-
const info = await client.loadChunk(DOMAIN_MODEL_TX, idx)
277+
const info = await client.loadChunk(ws.uuid, DOMAIN_MODEL_TX, idx)
278278
idx = info.idx
279279
const ids = Array.from(info.docs.map((it) => it.id as Ref<Doc>))
280-
const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) =>
280+
const docs = (await client.loadDocs(ws.uuid, DOMAIN_MODEL_TX, ids)).filter((it) =>
281281
TxProcessor.isExtendsCUD(it._class)
282282
) as TxCUD<Doc>[]
283283
accountsTxes.push(...docs)
284284
if (info.finished && idx !== undefined) {
285-
await client.closeChunk(info.idx)
285+
await client.closeChunk(ws.uuid, info.idx)
286286
break
287287
}
288288
}

0 commit comments

Comments
 (0)