From 6916492083bcf3352093807e449f2942568f6228 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 10 Jan 2024 10:41:09 +0100 Subject: [PATCH 1/4] Add a chapter on the long filter for magento plugin --- content/connectivity/magento-2/_index.en.md | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/content/connectivity/magento-2/_index.en.md b/content/connectivity/magento-2/_index.en.md index 076bbed..732bc0a 100644 --- a/content/connectivity/magento-2/_index.en.md +++ b/content/connectivity/magento-2/_index.en.md @@ -137,6 +137,34 @@ In this example we will search for customers that were updated after 1985 (`@dat # ... ``` +#### With long filter +Filters are passed on the url. +But the most popular web browsers will not work with URLs over 2000 characters, and would return a 414 (Request-URI Too Long). +You can use the method withLongFilter to avoid this limitation and batch your request in multiple requests. + +In this example we will search for specifics orders with a lot of data, we have 214 increment_id, and we use a withLongFilter with parameters '@order_filter' (like the withFilter method), but we have 2 additional parameters who defines an offset and a lenght. +The offset can help you to start the request at the chosen index, by default we have 0. +The lenght help you to define a batch lenght, by default we have 200. +Here we have defined an offset to 0 and a lenght to 150, it means we are starting the request to the first element and make multiple request with 150 items max. +```yaml +# ... + order_filter_group: + class: Kiboko\Component\Flow\Magento2\FilterGroup + calls: + - withLongFilter: [ '@order_filter' ] + order_filter: + class: Kiboko\Component\Flow\Magento2\FilterGroup + calls: + - withLongFilter: ['@order_increment_id', 0, 150] + order_increment_id: + class: Kiboko\Component\Flow\Magento2\Filter + arguments: + - 'increment_id' + - 'in' + - '000000526,4000000026,00000918,000001754,6000000123,4000000150,6000000185,000003798,6000000211,6000000230,000004664,000004950,6000000265,6000000293,000008038,000008842,6000000475,6000000494,6000000500,000012074,000012247,000012453,000012589,2000000159,000014903,6000000760,6000000776,000016545,4000000530,000016553,4000000531,000016567,000016593,000016623,000016631,000016636,000016642,000016646,000016649,000016662,000016684,000016687,000016690,000016700,000016706,000016731,000016743,000016751,000016755,000016770,000016771,000016775,000016782,000016805,000016820,000016826,000016832,000016836,000016837,000016839,000016841,000016842,000016843,000016845,000016849,000016851,000016852,000016858,000016859,000016862,000016863,000018513,6000000947,4000000604,000021935,000021938,000021944,000022086,000022515,000022582,000022613,000022682,000022998,000023200,000023201,000023202,000023203,6000001188,000023204,000023205,000023206,6000001189,000023207,000023208,000023209,000023210,000023211,000023212,000023213,000023216,000023218,000023219,000023220,000023221,6000001190,000023229,000023230,6000001191,000023232,000023236,6000001192,000023238,000023243,000023244,000023245,000023249,000023250,3000000086,23253,000023257,000023258,6000001194,6000001195,000023269,000023274,6000001196,000023287,000023292,000023294,000023297,000023299,000023300,000023312,000023319,000023320,000023322,000023324,000023325,000023326,000023327,000023328,000023331,6000001202,000023337,000023338,000023339,5000000362,000023340,000023341,000023342,000023343,000023345,000023346,000023347,000023349,000023350,000023352,000023355,000023360,000023376,000023396,000023451,000023465,000023466,000023469,000023470,000023472,5000000363,000023478,000023479,000023480,000023482,000023483,000023484,000023486,000023488,000023489,000023492,000023494,4000000654,000023498,000022925,000023506,000023507,000023510,6000001217,000023513,000023515,000023520,000023521,000023524,000023528,000023532,000023536,6000001316,000025518,000026204,6000001390,000026947,000026948,4000000734,000027673,000027772,000027936,000027990,6000001487,000028108,000028118,000028129,000028130,000028131,000028132,000028133,5000000445' +# ... +``` + ### Building a lookup There is a lookup class for Categories, and one for product Attributes. From 8d9be307edd24b34800b0f34d53618f850f1f4ac Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 10 Jan 2024 17:09:23 +0100 Subject: [PATCH 2/4] Improve description, add some fancy things --- content/connectivity/magento-2/_index.en.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/content/connectivity/magento-2/_index.en.md b/content/connectivity/magento-2/_index.en.md index 732bc0a..e6e527d 100644 --- a/content/connectivity/magento-2/_index.en.md +++ b/content/connectivity/magento-2/_index.en.md @@ -138,14 +138,17 @@ In this example we will search for customers that were updated after 1985 (`@dat ``` #### With long filter -Filters are passed on the url. +Filters are passed to the url. But the most popular web browsers will not work with URLs over 2000 characters, and would return a 414 (Request-URI Too Long). -You can use the method withLongFilter to avoid this limitation and batch your request in multiple requests. +You can use the method `withLongFilter` to avoid this limitation and batch your request in multiple smaller requests. -In this example we will search for specifics orders with a lot of data, we have 214 increment_id, and we use a withLongFilter with parameters '@order_filter' (like the withFilter method), but we have 2 additional parameters who defines an offset and a lenght. -The offset can help you to start the request at the chosen index, by default we have 0. -The lenght help you to define a batch lenght, by default we have 200. -Here we have defined an offset to 0 and a lenght to 150, it means we are starting the request to the first element and make multiple request with 150 items max. +In this example we will search for specific orders with a lot of elements in the request's filter. +We have 214 increment_id, and we use a `withLongFilter` with parameters: + - `@order_increment_id` references our order's filter. + - `offset`, starts the request at the chosen index, by default we have 0. + - `length`, defines a batch length, by default we have 200. + +Here we have set an offset to 0 and a length to 150, it means we are starting the request from the first element and make multiple requests with 150 items max. ```yaml # ... order_filter_group: @@ -161,7 +164,7 @@ Here we have defined an offset to 0 and a lenght to 150, it means we are startin arguments: - 'increment_id' - 'in' - - '000000526,4000000026,00000918,000001754,6000000123,4000000150,6000000185,000003798,6000000211,6000000230,000004664,000004950,6000000265,6000000293,000008038,000008842,6000000475,6000000494,6000000500,000012074,000012247,000012453,000012589,2000000159,000014903,6000000760,6000000776,000016545,4000000530,000016553,4000000531,000016567,000016593,000016623,000016631,000016636,000016642,000016646,000016649,000016662,000016684,000016687,000016690,000016700,000016706,000016731,000016743,000016751,000016755,000016770,000016771,000016775,000016782,000016805,000016820,000016826,000016832,000016836,000016837,000016839,000016841,000016842,000016843,000016845,000016849,000016851,000016852,000016858,000016859,000016862,000016863,000018513,6000000947,4000000604,000021935,000021938,000021944,000022086,000022515,000022582,000022613,000022682,000022998,000023200,000023201,000023202,000023203,6000001188,000023204,000023205,000023206,6000001189,000023207,000023208,000023209,000023210,000023211,000023212,000023213,000023216,000023218,000023219,000023220,000023221,6000001190,000023229,000023230,6000001191,000023232,000023236,6000001192,000023238,000023243,000023244,000023245,000023249,000023250,3000000086,23253,000023257,000023258,6000001194,6000001195,000023269,000023274,6000001196,000023287,000023292,000023294,000023297,000023299,000023300,000023312,000023319,000023320,000023322,000023324,000023325,000023326,000023327,000023328,000023331,6000001202,000023337,000023338,000023339,5000000362,000023340,000023341,000023342,000023343,000023345,000023346,000023347,000023349,000023350,000023352,000023355,000023360,000023376,000023396,000023451,000023465,000023466,000023469,000023470,000023472,5000000363,000023478,000023479,000023480,000023482,000023483,000023484,000023486,000023488,000023489,000023492,000023494,4000000654,000023498,000022925,000023506,000023507,000023510,6000001217,000023513,000023515,000023520,000023521,000023524,000023528,000023532,000023536,6000001316,000025518,000026204,6000001390,000026947,000026948,4000000734,000027673,000027772,000027936,000027990,6000001487,000028108,000028118,000028129,000028130,000028131,000028132,000028133,5000000445' + - '000000526,4000000026,00000918,000001754,6000000123,4000000150,6000000185,000003798,6000000211,[..],5000000445' # ... ``` From c3f4f79b626f869126dd7896ccfe2174f7ef3baa Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 24 Jan 2024 12:13:09 +0100 Subject: [PATCH 3/4] Edit the chapter on magento to include the refacto of filters, fix the section on the long filter too --- content/connectivity/magento-2/_index.en.md | 55 +++++++++++---------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/content/connectivity/magento-2/_index.en.md b/content/connectivity/magento-2/_index.en.md index 6990ea3..dc132b6 100644 --- a/content/connectivity/magento-2/_index.en.md +++ b/content/connectivity/magento-2/_index.en.md @@ -35,12 +35,12 @@ The package includes the following extractor classes: `CustomerExtractor`, `Invo Extractor classes take 4 arguments: -| name | description | type | default value | -|---------------|--------------------------------------------------------------------------------------------------|--------------------------|---------------| -| logger | the service that will log exceptions | \Psr\Log\LoggerInterface | | -| client | client to choose depending on the Magento version. Available clients are: V2_1, V2_2, V2_3, V2_4 | Client | | -| page size | (Optional) maximum amount of entities to retrieve in a single payload | int | 100 | -| filter groups | (Optional) groups of filters to use when searching for entities | array | [] | +| name | description | type | default value | +|------------------|------------------------------------------------------------------------------------------------------|--------------------------|---------------| +| logger | the service that will log exceptions | \Psr\Log\LoggerInterface | | +| client | client to choose depending on the Magento version. Available clients are: V2_1, V2_2, V2_3, V2_4 | Client | | +| query parameters | query parameters sent to the api which contains groups of filters to use when searching for entities | array | [] | +| page size | (Optional) maximum amount of entities to retrieve in a single payload | int | 100 | ```yaml custom: @@ -52,8 +52,8 @@ custom: arguments: - '@Monolog\Logger' # Logger - '@Kiboko\Magento\V2_1\Client' # Client + - [] # QueryParameters, contains filters - 500 # Page size - - [] # Filter groups Kiboko\Magento\V2_1\Client: factory: @@ -89,7 +89,7 @@ custom: - 300 # Log level. 300 for Warning, 200 for Info... ``` -#### With filters +#### With ScalarFilters Filters and filter groups can be specified. Filters in a group are chained with `OR`. Groups are chained with `AND`. @@ -101,18 +101,22 @@ In this example we will search for customers that were updated after 1985 (`@dat arguments: - '@Monolog\Logger' - '@Kiboko\Magento\V2_1\Client' + - '@query_parameters' - 500 - - [ '@date_filter_group', '@id_filter_group' ] - # updated_at >= 1985-10-26 11:25:00 AND (entity_id = 17 OR entity_id = 46) # ... + query_parameters: + class: Kiboko\Component\Flow\Magento2\QueryParameters + calls: + - withGroups: [ '@id_filter_group', '@date_filter_group' ] + # updated_at >= 1985-10-26 11:25:00 AND (entity_id = 17 OR entity_id = 46) date_filter_group: class: Kiboko\Component\Flow\Magento2\FilterGroup calls: - withFilter: [ '@last_execution' ] last_execution: - class: Kiboko\Component\Flow\Magento2\Filter + class: Kiboko\Component\Flow\Magento2\Filter\ScalarFilter arguments: - 'updated_at' - 'gteq' @@ -123,13 +127,13 @@ In this example we will search for customers that were updated after 1985 (`@dat calls: - withFilter: [ '@id_to_check', '@other_id' ] id_to_check: - class: Kiboko\Component\Flow\Magento2\Filter + class: Kiboko\Component\Flow\Magento2\Filter\ScalarFilter arguments: - 'entity_id' - 'eq' - '17' other_id: - class: Kiboko\Component\Flow\Magento2\Filter + class: Kiboko\Component\Flow\Magento2\Filter\ScalarFilter arguments: - 'entity_id' - 'eq' @@ -137,34 +141,35 @@ In this example we will search for customers that were updated after 1985 (`@dat # ... ``` -#### With long filter +#### With ArrayFilter Filters are passed to the url. But the most popular web browsers will not work with URLs over 2000 characters, and would return a 414 (Request-URI Too Long). -You can use the method `withLongFilter` to avoid this limitation and batch your request in multiple smaller requests. +You can use the class `ArrayFilter` to avoid this limitation and batch your request in multiple smaller requests. In this example we will search for specific orders with a lot of elements in the request's filter. -We have 214 increment_id, and we use a `withLongFilter` with parameters: - - `@order_increment_id` references our order's filter. - - `offset`, starts the request at the chosen index, by default we have 0. - - `length`, defines a batch length, by default we have 200. +We have 214 increment_id, and we use the `ArrayFilter` with parameters: + - `increment_id`: name of the field to filter. + - `in` operator to the filter, you need the 'in' operator to use the ArrayFilter. + - `000000526,4000000026,00000918,000001754,6000000123,4000000150,6000000185,000003798,6000000211,[..],5000000445` defines the target values. + - `150` defines the lenght of your smaller request (by default set to 200). -Here we have set an offset to 0 and a length to 150, it means we are starting the request from the first element and make multiple requests with 150 items max. ```yaml # ... - order_filter_group: - class: Kiboko\Component\Flow\Magento2\FilterGroup + query_parameters: + class: Kiboko\Component\Flow\Magento2\QueryParameters calls: - - withLongFilter: [ '@order_filter' ] + - withGroup: [ '@order_filter' ] order_filter: class: Kiboko\Component\Flow\Magento2\FilterGroup calls: - - withLongFilter: ['@order_increment_id', 0, 150] + - withFilter: ['@order_increment_id'] order_increment_id: - class: Kiboko\Component\Flow\Magento2\Filter + class: Kiboko\Component\Flow\Magento2\Filter\ArrayFilter arguments: - 'increment_id' - 'in' - '000000526,4000000026,00000918,000001754,6000000123,4000000150,6000000185,000003798,6000000211,[..],5000000445' + - 150 # ... ``` From 935b3946e5f78269144028f4be5771cb251a9169 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Fri, 9 Feb 2024 12:30:31 +0100 Subject: [PATCH 4/4] Edit the chapter on magento to include the refacto of filters, fix the section on the long filter too --- content/connectivity/magento-2/_index.en.md | 40 ++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/content/connectivity/magento-2/_index.en.md b/content/connectivity/magento-2/_index.en.md index dc132b6..403e21c 100644 --- a/content/connectivity/magento-2/_index.en.md +++ b/content/connectivity/magento-2/_index.en.md @@ -25,6 +25,10 @@ This package includes classes to extract data from Magento, using a [custom conn ```shell composer require php-etl/magento2-flow:'*' ``` +if you are still on satellite 0.6, use this : +```shell +composer require php-etl/magento2-flow:'v0.2' +``` This package includes classes and code that you will be able to use in your custom connector. @@ -35,12 +39,12 @@ The package includes the following extractor classes: `CustomerExtractor`, `Invo Extractor classes take 4 arguments: -| name | description | type | default value | -|------------------|------------------------------------------------------------------------------------------------------|--------------------------|---------------| -| logger | the service that will log exceptions | \Psr\Log\LoggerInterface | | -| client | client to choose depending on the Magento version. Available clients are: V2_1, V2_2, V2_3, V2_4 | Client | | -| query parameters | query parameters sent to the api which contains groups of filters to use when searching for entities | array | [] | -| page size | (Optional) maximum amount of entities to retrieve in a single payload | int | 100 | +| name | description | type | default value | +|------------------|-------------------------------------------------------------------------------------------------------------|--------------------------|---------------| +| logger | the service that will log exceptions | \Psr\Log\LoggerInterface | | +| client | client parameter to communicate with magento2 api. Compatibility magento2 api versions : 2.1, 2.2, 2.3, 2.4 | Client | | +| query parameters | query parameters sent to the api which contains groups of filters to use when searching for entities | array | [] | +| page size | (Optional) maximum amount of entities to retrieve in a single payload | int | 100 | ```yaml custom: @@ -51,13 +55,13 @@ custom: public: true arguments: - '@Monolog\Logger' # Logger - - '@Kiboko\Magento\V2_1\Client' # Client + - '@Kiboko\Magento\Client' # Client - [] # QueryParameters, contains filters - 500 # Page size - Kiboko\Magento\V2_1\Client: + Kiboko\Magento\Client: factory: - class: 'Kiboko\Magento\V2_1\Client' # Client + class: 'Kiboko\Magento\Client' # Client method: 'create' arguments: - '@Http\Client\Common\PluginClient' @@ -100,7 +104,7 @@ In this example we will search for customers that were updated after 1985 (`@dat public: true arguments: - '@Monolog\Logger' - - '@Kiboko\Magento\V2_1\Client' + - '@Kiboko\Magento\Client' - '@query_parameters' - 500 @@ -187,17 +191,15 @@ custom: public: true arguments: - '@Monolog\Logger' - - '@Kiboko\Magento\V2_3\Client' # Client to use depending on the Magento version. - # Available clients are: - # V2_1, V2_2, V2_3, V2_4 + - '@Kiboko\Magento\Client' - '@Symfony\Component\Cache\Psr16Cache' - 'category.%s' - '@Acme\Custom\LookupMapper' # Your custom mapper class - 'category_name' # Index of the category ID, in your line. - Kiboko\Magento\V2_3\Client: + Kiboko\Magento\Client: factory: - class: 'Kiboko\Magento\V2_3\Client' # Client + class: 'Kiboko\Magento\Client' # Client method: 'create' arguments: - '@Http\Client\Common\PluginClient' @@ -246,18 +248,16 @@ custom: public: true arguments: - '@Monolog\Logger' - - '@Kiboko\Magento\V2_3\Client' # Client to use depending on the Magento version. - # Available clients are: - # V2_1, V2_2, V2_3, V2_4 + - '@Kiboko\Magento\Client' - '@Symfony\Component\Cache\Psr16Cache' - 'collection.%s' # Cache key - '@Acme\Custom\LookupMapper' # Your custom mapper class - 'Collection' # Index of the attribute ID, in your line. - 'qv_collection' # Attribute code - Kiboko\Magento\V2_3\Client: + Kiboko\Magento\Client: factory: - class: 'Kiboko\Magento\V2_3\Client' # Client + class: 'Kiboko\Magento\Client' # Client method: 'create' arguments: - '@Http\Client\Common\PluginClient'