Skip to content

Commit fc3da71

Browse files
marcolinkmarcolink
committed
feat: add more logging to the task system [NONE]
1 parent a546583 commit fc3da71

File tree

7 files changed

+82
-36
lines changed

7 files changed

+82
-36
lines changed

‎src/engine/client/index.ts‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { ContentTypeCollection, Entry, EntryCollection } from 'contentful'
33
import { EntryProps } from 'contentful-management'
44
import { createHttpClient, getUserAgentHeader } from 'contentful-sdk-core'
55
import { pickBy } from 'lodash'
6-
import { ClientLogHandler } from '../logger/types'
6+
import { ClientLogHandler, ILogger, LogLevel } from '../logger/types'
77
import { stripSys } from '../utils/strip-sys'
88

99
import ContentfulSdkCorePKGJson from 'contentful-sdk-core/package.json'
@@ -89,30 +89,30 @@ export const createClient = ({
8989
cma: {
9090
requestCounts: () => count.cma,
9191
entries: {
92-
getMany: async ({ environment, query }: GetEntriesParams) => {
92+
getMany: async ({ environment, query }: GetEntriesParams, logger?: ILogger) => {
9393
count.cma++
9494
const result = await cmaClient.get(`${environment}/entries`, {
9595
params: { ...cleanQuery(query) },
9696
})
9797
return result.data as EntryCollection<any>
9898
},
99-
get: async ({ environment, entryId, query }: GetEntryParams) => {
99+
get: async ({ environment, entryId, query }: GetEntryParams, logger?: ILogger) => {
100100
count.cma++
101101
const result = await cmaClient.get(`${environment}/entries/${entryId}`, {
102102
params: { ...cleanQuery(query) },
103103
})
104104
return result.data as EntryProps<any>
105105
},
106-
unpublish: async ({ environment, entryId }: DeleteEntryParams) => {
106+
unpublish: async ({ environment, entryId }: DeleteEntryParams, logger?: ILogger) => {
107107
count.cma++
108108
const result = await cmaClient.delete(`${environment}/entries/${entryId}/published`)
109109
return result.data as EntryProps<any>
110110
},
111-
delete: async ({ environment, entryId }: DeleteEntryParams) => {
111+
delete: async ({ environment, entryId }: DeleteEntryParams, logger?: ILogger) => {
112112
count.cma++
113113
await cmaClient.delete(`${environment}/entries/${entryId}`)
114114
},
115-
create: async ({ environment, entry, entryId, contentType }: CreateEntryParams) => {
115+
create: async ({ environment, entry, entryId, contentType }: CreateEntryParams, logger?: ILogger) => {
116116
count.cma++
117117
const result = await cmaClient.put(`${environment}/entries/${entryId}`, stripSys(entry), {
118118
headers: {
@@ -121,7 +121,7 @@ export const createClient = ({
121121
})
122122
return result.data as EntryProps<any>
123123
},
124-
update: async ({ environment, entry, entryId }: UpdateEntryParams) => {
124+
update: async ({ environment, entry, entryId }: UpdateEntryParams, logger?: ILogger) => {
125125
count.cma++
126126
const result = await cmaClient.put(`${environment}/entries/${entryId}`, entry, {
127127
headers: {
@@ -130,7 +130,7 @@ export const createClient = ({
130130
})
131131
return result.data as EntryProps<any>
132132
},
133-
publish: async ({ environment, entry, entryId }: PublishEntryParams) => {
133+
publish: async ({ environment, entry, entryId }: PublishEntryParams, logger?: ILogger) => {
134134
count.cma++
135135
return cmaClient.put(`${environment}/entries/${entryId}/published`, entry, {
136136
headers: {
@@ -143,24 +143,27 @@ export const createClient = ({
143143
cda: {
144144
requestCounts: () => count.cda,
145145
entries: {
146-
getMany: async ({ environment, query }: GetEntriesParams) => {
146+
getMany: async ({ environment, query }: GetEntriesParams, logger?: ILogger) => {
147147
count.cda++
148+
logger?.log(LogLevel.INFO, `"GET /${space}/${`${environment}/entries`} ${JSON.stringify(query)}"`)
148149
const result = await cdaClient.get(`${environment}/entries`, {
149150
params: { ...cleanQuery(query) },
150151
})
151152
return result.data as EntryCollection<any>
152153
},
153-
get: async ({ environment, entryId, query }: GetEntryParams) => {
154+
get: async ({ environment, entryId, query }: GetEntryParams, logger?: ILogger) => {
154155
count.cda++
156+
logger?.log(LogLevel.INFO, `"GET /${space}/${environment}/entries/${entryId} ${JSON.stringify(query)}"`)
155157
const result = await cdaClient.get(`${environment}/entries/${entryId}`, {
156158
params: { ...cleanQuery(query) },
157159
})
158160
return result.data as Entry<any>
159161
},
160162
},
161163
contentTypes: {
162-
getMany: async ({ environment, query }: GetEntriesParams) => {
164+
getMany: async ({ environment, query }: GetEntriesParams, logger?: ILogger) => {
163165
count.cda++
166+
logger?.log(LogLevel.INFO, `"GET ${`/${space}/${environment}/content_types`} ${JSON.stringify(query)}"`)
164167
const result = await cdaClient.get(`${environment}/content_types`, {
165168
params: { ...cleanQuery(query) },
166169
})

‎src/engine/create-changeset/tasks/create-affected-content-types-diverged-task.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ import { CreateChangesetContext } from '../types'
22
import { ListrTask } from 'listr2'
33
import { LogLevel } from '../../logger/types'
44
import { affectedEntitiesIds } from '../../utils/affected-entities-ids'
5+
import { createScopedLogger } from '../../logger/create-scoped-logger'
56

67
export function createAffectedContentTypesDivergedTask(): ListrTask {
78
return {
89
title: `Checking for diverged content types`,
910
task: async (context: CreateChangesetContext) => {
10-
context.logger.log(LogLevel.INFO, `Start createAffectedContentTypesDivergedTask`)
11-
11+
const logger = createScopedLogger(context.logger, `CreateAffectedContentTypesDivergedTask`)
12+
logger.startScope()
1213
const affectedContentTypeIds = affectedEntitiesIds(context.affectedEntities.contentTypes, [
1314
'added',
1415
'removed',
@@ -29,7 +30,7 @@ export function createAffectedContentTypesDivergedTask(): ListrTask {
2930
if (contentModelDiverged.length) {
3031
context.contentModelDiverged = true
3132
}
32-
33+
logger.stopScope()
3334
return context
3435
},
3536
}

‎src/engine/create-changeset/tasks/create-compute-ids-task.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { LogLevel } from '../../logger/types'
33
import { CreateChangesetContext, EnvironmentScope } from '../types'
44
import { doesExceedLimits } from '../../utils/exceeds-limits'
55
import { EntityType } from '../../types'
6+
import { createScopedLogger } from '../../logger/create-scoped-logger'
67
type ComputeIdsTaskProps = {
78
entityType: EntityType
89
}
@@ -11,8 +12,9 @@ export const createComputeIdsTask = ({ entityType }: ComputeIdsTaskProps): Listr
1112
return {
1213
title: `Counting number of ${entityType} changes between environments`,
1314
task: async (context: CreateChangesetContext) => {
14-
const { sourceData, targetData, logger } = context
15-
logger.log(LogLevel.INFO, `Start computeIdsTask ${[entityType]}`)
15+
const { sourceData, targetData } = context
16+
const logger = createScopedLogger(context.logger, `CreateComputeIdsTask '${entityType}'`)
17+
logger.startScope()
1618

1719
const added = new Set(sourceData[entityType].ids.filter((item) => !targetData[entityType].ids.includes(item)))
1820
const removed = new Set(targetData[entityType].ids.filter((item) => !sourceData[entityType].ids.includes(item)))
@@ -33,7 +35,7 @@ export const createComputeIdsTask = ({ entityType }: ComputeIdsTaskProps): Listr
3335

3436
const exceedsLimits = doesExceedLimits(context, entityType)
3537
context.exceedsLimits = exceedsLimits
36-
38+
logger.stopScope()
3739
return Promise.resolve(context)
3840
},
3941
}

‎src/engine/create-changeset/tasks/create-entities-task.ts‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import type { ListrTaskWrapper } from 'listr2'
22
import { ListrTask } from 'listr2'
33
import pLimit from 'p-limit'
4-
import { LogLevel } from '../../logger/types'
4+
import { ILogger, LogLevel } from '../../logger/types'
55
import { Comparable, CreateChangesetContext, EntityData, EnvironmentScope } from '../types'
66
import { EntityType } from '../../types'
7+
import { createScopedLogger } from '../../logger/create-scoped-logger'
78

89
const LIMIT = 1000
910

@@ -14,9 +15,10 @@ type ExecuteParams = {
1415
environmentId: string
1516
entityType: EntityType
1617
additionalFields: string[]
18+
logger: ILogger
1719
}
1820

19-
const execute = async ({ context, environmentId, task, entityType, additionalFields = [] }: ExecuteParams) => {
21+
const execute = async ({ context, logger, environmentId, task, entityType, additionalFields = [] }: ExecuteParams) => {
2022
const {
2123
client: { cda },
2224
} = context
@@ -28,7 +30,7 @@ const execute = async ({ context, environmentId, task, entityType, additionalFie
2830

2931
const api = cda[entityType]
3032

31-
const { total } = await api.getMany({ environment: environmentId, query: { limit: 0 } })
33+
const { total } = await api.getMany({ environment: environmentId, query: { limit: 0 } }, logger)
3234

3335
const promises = []
3436
let requestsDone = 0
@@ -40,10 +42,14 @@ const execute = async ({ context, environmentId, task, entityType, additionalFie
4042
for (let i = 0; i < iterations; i++) {
4143
promises.push(
4244
limiter(async () => {
43-
const response = await api.getMany({
44-
environment: environmentId,
45-
query: { select: ['sys.id', 'sys.updatedAt', ...additionalFields], limit: LIMIT, skip: LIMIT * i },
46-
})
45+
const query = { select: ['sys.id', 'sys.updatedAt', ...additionalFields], limit: LIMIT, skip: LIMIT * i }
46+
const response = await api.getMany(
47+
{
48+
environment: environmentId,
49+
query,
50+
},
51+
logger
52+
)
4753
requestsDone++
4854
task.output = `Fetching ${requestsDone * LIMIT}/${total} ${entityType}`
4955
result.push(...response.items)
@@ -69,10 +75,19 @@ export function createEntitiesTask({ scope, environmentId, entityType }: Entitie
6975
return {
7076
title: `Reading the ${scope} environment "${environmentId}"`,
7177
task: async (context: CreateChangesetContext, task) => {
72-
context.logger.log(LogLevel.INFO, `Start createEntitiesTask for ${entityType}`)
78+
const logger = createScopedLogger(context.logger, `CreateEntitiesTask '${entityType}'`)
79+
logger.startScope()
7380
const additionalFields = entityType === 'entries' ? ['sys.contentType.sys.id'] : []
74-
const result = await execute({ context, task, scope, environmentId, entityType, additionalFields })
75-
context[`${scope}Data`][entityType] = result
81+
context[`${scope}Data`][entityType] = await execute({
82+
context,
83+
task,
84+
scope,
85+
environmentId,
86+
entityType,
87+
additionalFields,
88+
logger,
89+
})
90+
logger.stopScope()
7691
return context
7792
},
7893
}

‎src/engine/create-changeset/tasks/create-fetch-added-entities-task.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { LogLevel } from '../../logger/types'
55
import { CreateChangesetContext, SkipHandler } from '../types'
66
import { pluralizeEntry } from '../../utils/pluralize'
77
import { EntityType } from '../../types'
8+
import { createScopedLogger } from '../../logger/create-scoped-logger'
89

910
export function cleanEntity(entry: Entry<any>): any {
1011
return { ...entry, sys: pick(entry.sys, ['id', 'type', 'contentType']) }
@@ -20,8 +21,9 @@ export function createFetchAddedEntitiesTask({ entityType, skipHandler }: FetchA
2021
title: 'Fetching full payload for added entries',
2122
skip: skipHandler,
2223
task: async (context: CreateChangesetContext, task) => {
23-
const { client, affectedEntities, sourceEnvironmentId, changeset, limit, logger } = context
24-
logger.log(LogLevel.INFO, 'Start createFetchAddedEntitiesTask')
24+
const { client, affectedEntities, sourceEnvironmentId, changeset, limit } = context
25+
const logger = createScopedLogger(context.logger, `CreateFetchAddedEntitiesTask '${entityType}'`)
26+
logger.startScope()
2527

2628
const {
2729
[entityType]: { added },
@@ -38,10 +40,13 @@ export function createFetchAddedEntitiesTask({ entityType, skipHandler }: FetchA
3840
const query = { 'sys.id[in]': chunk.join(','), locale: '*', limit }
3941
// eslint-disable-next-line no-await-in-loop
4042
const entries = await client.cda.entries
41-
.getMany({
42-
environment: sourceEnvironmentId,
43-
query,
44-
})
43+
.getMany(
44+
{
45+
environment: sourceEnvironmentId,
46+
query,
47+
},
48+
logger
49+
)
4550
.then((response) => response.items)
4651

4752
for (const entry of entries) {
@@ -51,6 +56,8 @@ export function createFetchAddedEntitiesTask({ entityType, skipHandler }: FetchA
5156
}
5257
}
5358
}
59+
logger.startScope()
60+
return context
5461
},
5562
}
5663
}

‎src/engine/create-changeset/tasks/create-fetch-changed-tasks.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@ import type { CreateChangesetContext } from '../types'
66
import { createPatch } from '../../utils/create-patch'
77
import { pluralizeEntry } from '../../utils/pluralize'
88
import { SkipHandler } from '../types'
9+
import { createScopedLogger } from '../../logger/create-scoped-logger'
10+
import { ILogger } from '../../logger/types'
911

1012
type GetEntryPatchParams = {
1113
context: BaseContext
1214
source: string
1315
target: string
1416
entryIds: string[]
1517
entityType: EntityType
18+
logger: ILogger
1619
}
1720

1821
const EntityTypeMap: Record<EntityType, 'Entry' | 'ContentType'> = {
@@ -26,6 +29,7 @@ async function getEntityPatches({
2629
target,
2730
entryIds,
2831
entityType,
32+
logger,
2933
}: GetEntryPatchParams): Promise<ChangedChangesetItem[]> {
3034
const {
3135
client: { cda },
@@ -34,8 +38,8 @@ async function getEntityPatches({
3438

3539
const api = cda[entityType]
3640

37-
const sourceEntries = await api.getMany({ environment: source, query }).then((response) => response.items)
38-
const targetEntries = await api.getMany({ environment: target, query }).then((response) => response.items)
41+
const sourceEntries = await api.getMany({ environment: source, query }, logger).then((response) => response.items)
42+
const targetEntries = await api.getMany({ environment: target, query }, logger).then((response) => response.items)
3943

4044
const result: ChangedChangesetItem[] = []
4145

@@ -70,7 +74,8 @@ export const createFetchChangedTasks = ({ entityType, skipHandler }: FetchChange
7074
skip: skipHandler,
7175
task: async (context: CreateChangesetContext, task) => {
7276
const { sourceEnvironmentId, affectedEntities, targetEnvironmentId, statistics, limit, changeset } = context
73-
77+
const logger = createScopedLogger(context.logger, `CreateFetchChangedTasks '${entityType}'`)
78+
logger.startScope()
7479
const entityTypeStatistics = statistics[entityType]
7580

7681
const {
@@ -99,6 +104,7 @@ export const createFetchChangedTasks = ({ entityType, skipHandler }: FetchChange
99104
target: targetEnvironmentId,
100105
entryIds: chunk,
101106
entityType,
107+
logger,
102108
})
103109

104110
const withChange = changedObjects.filter((o) => o.patch.length > 0)
@@ -117,6 +123,7 @@ export const createFetchChangedTasks = ({ entityType, skipHandler }: FetchChange
117123
entityTypeStatistics.added = added.length
118124
entityTypeStatistics.removed = removed.length
119125

126+
logger.startScope()
120127
return Promise.resolve(context)
121128
},
122129
}

‎src/engine/logger/create-scoped-logger.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { ILogger, LogLevel } from './types'
2+
3+
export function createScopedLogger(logger: ILogger, scope: string) {
4+
const scopeStr = `[${scope}]`
5+
return {
6+
...logger,
7+
log: (level: LogLevel, message?: string | Error | undefined) => logger.log(level, `${scopeStr}\t${message}`),
8+
startScope: () => logger.log(LogLevel.INFO, `${scopeStr}\tSTART`),
9+
stopScope: () => logger.log(LogLevel.INFO, `${scopeStr}\tDONE`),
10+
}
11+
}

0 commit comments

Comments
 (0)