diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..116ac5b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,81 @@ +# Copyright (c) 2019, Xiaomi, Inc. All rights reserved. +# This source code is licensed under the Apache License Version 2.0, which +# can be found in the LICENSE file in the root directory of this source tree. +# +### Maven template +target/ +logs/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# Avoid ignoring Maven wrapper jar file (.jar files are usually ignored) +!/.mvn/wrapper/maven-wrapper.jar +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### macOS template +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +**/node_modules/ +**/node/ +**/package-lock.json +**/junit-report +/shell/job_execution* \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..afaf360d --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git "a/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" new file mode 100644 index 00000000..1b1cc5d5 --- /dev/null +++ "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" @@ -0,0 +1,30 @@ + +# Deployment process + +1. Download the full code + +1. The operating environment requires jdk 8 or above, maven3 + + 1. Configuration database + + 1. Execute in order under the project root directory + + ```shell + cd thain-fe + npm install + npm run build + cd .. + mvn -U clean package + ``` + + 1. Once the package is complete, you can run it. + + ```shell + java -jar thain-server/target/thain-server-1.0.0.jar + ``` + + 1. Open localhost:9900 to see the effect diff --git "a/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" new file mode 100644 index 00000000..b4a87dc9 --- /dev/null +++ "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" @@ -0,0 +1,18 @@ + +# Database configuration + +modify ```thain-server/src/main/resources/application.properties``` + +The following three items are set for the database. + +```properties +spring.datasource.url= +spring.datasource.username= +spring.datasource.password= +``` + +Execute sql build table ```/sql``` \ No newline at end of file diff --git "a/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" new file mode 100644 index 00000000..aa3f3991 --- /dev/null +++ "b/docs/en/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" @@ -0,0 +1,140 @@ + +# User Management + +## DB user management + +1. Configure the database connection and set the authentication method to DB + + After properly configuring the database connection through [Database Configuration](./2.数据库配置.md) + + modify thain-web/src/main/resources/application.properties + + ```properties + #User authentication via database + thain.login.source=DBAuthentication + ``` + +1. Add user + + The current version does not provide background management yet. Adding users can use this method.`com.xiaomi.thain.server.dao.UserDao.insertUser(ThainUser user)` + +## Ldap User Management + +1. Configuring Ldap and starting the service[Reference openLdap](http://www.openldap.org/doc) + + The LDIF is defined as follows (the basic user admin: admin is defined) + + ```ldif + dn: {your base DN} + objectClass: dcObject + objectClass: organization + objectClass: top + dc: {your DC} + o: {your organization} + + dn: ou=people,{your base DN} + objectClass: organizationalUnit + objectClass: top + ou: people + + dn: uid=admin,ou=people,{your base DN} + objectClass: inetOrgPerson + objectClass: organizationalPerson + objectClass: person + objectClass: top + cn: admin + sn: admin + uid: admin + userPassword:: e0NSWVBUfVo2VGdhNGVkQkFjekU= + ``` + +1. Configure the Ldap connection and set the authentication method to Ldap + + thain-server/src/main/resources/application.properties中添加 + + ```properties + spring.ldap.urls=ldap:{ldap-url} + spring.ldap.username={ldap-username} + spring.ldap.password={ldap-password} + spring.ldap.base={ldap-baseDn} + ``` + And modify + ```properties + #User authentication via Ldap + thain.login.source=LdapAuthentication + ``` + +1. Add user + + Adding users can use this method`com.xiaomi.thain.server.dao.LdapUserDao.save(LdapUser user)` + +## Third party authorized login + +1. Apply the client-id and client-secret of the third-party application that needs to log in and add two basic configurations: + + spring.security.oauth2.client.registration.{clientName}.client-id=${clientId} + spring.security.oauth2.client.registration.{clientName}.client-secret=${clientSecret} + + More specific configuration can refer to the spring-security document + +1. To manually register ClientRegistration, you can configure it as follows + + ```java + @Configuration + public class OAuth2LoginConfig { + //添加客户端并注册bean + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + return new InMemoryClientRegistrationRepository(this.googleClientRegistration()); + } + + //添加google客户端的配置 + private ClientRegistration googleClientRegistration() { + return ClientRegistration.withRegistrationId("google") + .clientId("google-client-id") + .clientSecret("google-client-secret") + //... + .clientName("Google") + .build(); + } + } + ``` + + And modify + + ```java + @EnableWebSecurity + @Slf4j + public class WebSecurityConfig extends WebSecurityConfigurerAdapter { + + @Autowired + private final ClientRegistrationRepository clientRegistrationRepository; + + //... + /** + * Third-party login configuration + * + * @param http HttpSecurity + */ + private void thirdConfig(HttpSecurity http) throws Exception { + http.oauth2Login() + .clientRegistrationRepository(clientRegistrationRepository); + //... + } + } + ``` + +1. Modify the login page of the front-end project thain-fe, add the corresponding third-party login link, and the system provides google third-party login by default. + + ```html + + ``` +1. If you do not need a third-party login, delete the configuration + +## More + +Project user permission verification uses spring security framework, more content reference [spring-security](https://docs.spring.io/spring-security/site/docs/5.2.0.BUILD-SNAPSHOT/reference/htmlsingle) diff --git "a/docs/en/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" "b/docs/en/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" new file mode 100644 index 00000000..3aeb5eb7 --- /dev/null +++ "b/docs/en/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" @@ -0,0 +1,50 @@ + +# Create a task + +A Thain Flow is composed of multiple jobs. + +There are two ways to create a task, one is through page creation, and the other is created through the http interface. + +## Page creation + +Open the editing page of Thain. + +## httpinterface creation + +Reference java sdk + +## flowParameter description + +### Required parameters + +1. id: If you modify the required id of the task, you do not need to fill in the new task. +1. name: flow name +1. createUser: the user who created the task, if it is the page creation will get the current login user + +### Optional parameters + +1. cron: trigger flow according to cron timing, do not fill will not automatically trigger +1. modifyCallbackUrl: Pause scheduling and continue scheduling will call back this address, post form, pause {"status":1}, continue {"status":2} +1. pauseContinuousFailure: fill in the number, after pauseuseContinuousFailure times, pause the schedule +1. emailContinuousFailure: mailbox, after the pauseContinuousFailure time, send a notification email +1. callbackUrl: The task status changes the url of the callback, the post form starts executing `{"code":1}`, the execution fails `{"code":3, "message": the reason for the failure}`, the run succeeds `{"code ": 2}` +1. "sla related": + - slaKill: You can only fill in the truth temporarily. + - slaDuration: flow runs for the longest number of seconds, the timeout will be killed + - slaEmail: Mailbox for task timeout notification + +## job parameter description + +### Required parameters + +- name: job name, job name in a flow cannot be repeated +- component: component name + +### Optional parameters +- condition: If this job depends on other jobs, you need to fill in the job name here, multiple separated by && +- properties: component parameters, what needs to be filled out refer to specific components +- callbackUrl: The job status changes the url of the callback, the post form starts executing `{"code":1}`, the execution fails `{"code":3, "message": the reason for the failure}`, the run succeeds `{"code" : 2}` \ No newline at end of file diff --git "a/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" "b/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" new file mode 100644 index 00000000..85c8dc16 --- /dev/null +++ "b/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" @@ -0,0 +1,13 @@ + +# Existing components + + +|Component Name|Required Parameters|Optional Parameters|Production Results|Others| +|:------|:------|:-----|:------|:--| +Std::http|1. url: url 2. method: GET or POST|1. contentType: Content-Type
2. referenceData: Process data reference, separated by commas, such as: ` aName: a.name, bAge:b.age`, in front of the colon is the key sent by http, followed by value
3. forwardData: forward data, this will be sent to the url intact, key is forwardData |result : The result of the http request || +Std::mail|1. title: mail header
2. contentHtml: mail content
3. recipient: recipient (multiple separated by commas) | | | can be in the message header and content Use ${} to quote the results of other jobs, such as: `${a.name}`| +Std::shell|1. shellBase64: shell script file base64|1.environmentVariable: shell variable (xxx=yyy form, multiple separated by newline)| |Recommended to use this component when creating the page, if it is a third-party system recommendation Implemented in http with your own system| \ No newline at end of file diff --git "a/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" "b/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" new file mode 100644 index 00000000..98da5e8d --- /dev/null +++ "b/docs/en/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" @@ -0,0 +1,70 @@ + +# Custom component + +1. Can refer to + - com.xiaomi.thain.component.std.MailComponent + - com.xiaomi.thain.component.std.HttpComponent + + +1. Use annotations: com.xiaomi.thain.component.annotation.ThainComponent + + - group: The group to which the component belongs, such as std, xdata, etc., used to distinguish components + - name: the component name, such as: http + - defineJson + - component-defined json + - This json is used for field validation, front-end component display, etc. + +1. json definition + - Format: + + ```ts + /** + * https://app.quicktype.io/#l=schema This site generates json + * + * Component front-end schema + * + */ + Export type ComponentDefineJson = Array<{ + /** + * Key passed to the backend + */ + Property: string; + /** + * Required or not, false or not written is not required + */ + Required?: boolean; + /** + * Enter the identifier in front of the box, do not write the default property + */ + Label?: string; + Input: + | { + Id: 'line' | 'textarea' | 'sql' | 'shell' | 'richText'; + } + | { + Id: 'select'; + Options: Array<{ + /** + * the value of the attribute + */ + Id: string; + /** + * The candidate in the drop-down box is not written with id instead + */ + Name?: string; + }>; + }; + }>; + ``` + +1. java class design + - Define the property in json in the java class, and the runtime will automatically inject according to the information filled in by the user. + - You can define a property of type com.xiaomi.thain.component.tools.ComponentTools + - ComponentTools has many features in this interface + +1. ComponentTools interface + - com.xiaomi.thain.component.tools.ComponentTools See comments \ No newline at end of file diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100755 index 00000000..2b70d51f Binary files /dev/null and b/docs/images/logo.png differ diff --git "a/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" new file mode 100644 index 00000000..d565369b --- /dev/null +++ "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/1.\351\203\250\347\275\262\346\265\201\347\250\213.md" @@ -0,0 +1,30 @@ + +# 部署流程 + +1. 下载完整代码 + +1. 运行环境需要jdk 8 以上, maven3 + + 1. 配置数据库 + + 1. 在项目根目录下依次执行 + + ```shell + cd thain-fe + npm install + npm run build + cd .. + mvn -U clean package + ``` + + 1. 打包完成后,即可运行 + + ```shell + java -jar thain-server/target/thain-server-1.0.0.jar + ``` + + 1. 打开 localhost:9900 就可以看到效果了 diff --git "a/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" new file mode 100644 index 00000000..da448910 --- /dev/null +++ "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/2.\346\225\260\346\215\256\345\272\223\351\205\215\347\275\256.md" @@ -0,0 +1,21 @@ + +# 数据库配置 + +修改 thain-server/src/main/resources/application.properties + +下面三项为数据库设置 + +```properties +spring.datasource.url= +spring.datasource.username= +spring.datasource.password= +``` + +数据库初始化 (1 建表并初始化数据, 2 初始化数据, 其他 不做初始化) +datasource.initialization.level=1 + +建表sql ```/thain-core/src/main/resources/sql``` \ No newline at end of file diff --git "a/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" new file mode 100644 index 00000000..635d540b --- /dev/null +++ "b/docs/zh/1.\351\203\250\347\275\262\345\220\257\345\212\250/3.\347\224\250\346\210\267\347\256\241\347\220\206.md" @@ -0,0 +1,142 @@ + +# 用户管理 + +## DB用户管理 + +1. 创建数据库(推荐Mysql) + +1. 配置数据库连接并设置验证方式为DB + + 通过[数据库配置](./2.数据库配置.md)正确配置数据库连接之后 + + 修改 thain-web/src/main/resources/application.properties中 + + ```properties + #通过数据库进行用户验证 + thain.login.source=DBAuthentication + ``` + +1. 添加用户 + + 当前版本暂未提供后台管理,添加用户可以通过此方法`com.xiaomi.thain.server.dao.UserDao.insertUser(ThainUser user)` + +## Ldap用户管理 + +1. 配置Ldap并启动服务[参考openLdap](http://www.openldap.org/doc) + + LDIF定义如下(定义了基本的用户admin:admin) + + ```ldif + dn: {your base DN} + objectClass: dcObject + objectClass: organization + objectClass: top + dc: {your DC} + o: {your organization} + + dn: ou=people,{your base DN} + objectClass: organizationalUnit + objectClass: top + ou: people + + dn: uid=admin,ou=people,{your base DN} + objectClass: inetOrgPerson + objectClass: organizationalPerson + objectClass: person + objectClass: top + cn: admin + sn: admin + uid: admin + userPassword:: e0NSWVBUfVo2VGdhNGVkQkFjekU= + ``` + +1. 配置Ldap连接并设置验证方式为Ldap + + thain-server/src/main/resources/application.properties中添加 + + ```properties + spring.ldap.urls=ldap:{ldap-url} + spring.ldap.username={ldap-username} + spring.ldap.password={ldap-password} + spring.ldap.base={ldap-baseDn} + ``` + 并修改 + ```properties + #通过Ldap进行用户验证 + thain.login.source=LdapAuthentication + ``` + +1. 添加用户 + + 添加用户可以通过此方法`com.xiaomi.thain.server.dao.LdapUserDao.save(LdapUser user)` + +## 第三方授权登录 + +1. 申请需要登录的第三方应用的client-id和client-secret并添加两条基本配置: + + spring.security.oauth2.client.registration.{clientName}.client-id=${clientId} + spring.security.oauth2.client.registration.{clientName}.client-secret=${clientSecret} + + 更具体配置可以参考spring-security文档 + +1. 如需手动注册ClientRegistration,可以按照如下配置 + + ```java + @Configuration + public class OAuth2LoginConfig { + //添加客户端并注册bean + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + return new InMemoryClientRegistrationRepository(this.googleClientRegistration()); + } + + //添加google客户端的配置 + private ClientRegistration googleClientRegistration() { + return ClientRegistration.withRegistrationId("google") + .clientId("google-client-id") + .clientSecret("google-client-secret") + //... + .clientName("Google") + .build(); + } + } + ``` + + 并修改 + + ```java + @EnableWebSecurity + @Slf4j + public class WebSecurityConfig extends WebSecurityConfigurerAdapter { + + @Autowired + private final ClientRegistrationRepository clientRegistrationRepository; + + //... + /** + * 第三方登录配置 + * + * @param http HttpSecurity + */ + private void thirdConfig(HttpSecurity http) throws Exception { + http.oauth2Login() + .clientRegistrationRepository(clientRegistrationRepository); + //... + } + } + ``` + +1. 修改前端项目thain-fe的登录页面,添加相应第三方登录链接,系统默认提供了google的第三方登录 + + ```html + + ``` +1. 如果不需要第三方登录,则删除配置 + +## 更多 + +项目用户权限验证采用spring security框架,更多内容参考[spring-security](https://docs.spring.io/spring-security/site/docs/5.2.0.BUILD-SNAPSHOT/reference/htmlsingle) diff --git "a/docs/zh/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" "b/docs/zh/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" new file mode 100644 index 00000000..e8956e4a --- /dev/null +++ "b/docs/zh/2.\344\275\277\347\224\250\346\225\231\347\250\213/\345\210\233\345\273\272\344\273\273\345\212\241.md" @@ -0,0 +1,51 @@ + +# 创建任务 + +一个Thain Flow 是由多个Job组成的. + +创建任务有两种方法, 一种是通过页面创建, 另一种是通过http接口创建. + +## 页面创建 + +打开Thain的编辑页面, 多拽即可 + +## http接口创建 + +参考java sdk + +## flow参数说明 + +### 必填参数 + +1. id: 如果修改任务必填id, 创建新任务不需要填写 +1. name: flow名称 +1. createUser: 创建任务的用户, 如果是页面创建会获取当前登录用户 + +### 选填参数 + +1. cron: 根据cron定时触发flow, 不填写不会自动触发 +1. modifyCallbackUrl: 暂停调度 和 继续调度 会回调这个地址, post form, 暂停{"status":1}, 继续{"status":2} +1. pauseContinuousFailure: 填数字, 失败pauseContinuousFailure次后, 暂停调度 +1. emailContinuousFailure: 邮箱, 失败pauseContinuousFailure次后 发送通知邮件 +1. callbackUrl: 任务状态改变回调的url, post form 开始执行 `{"code":1}`, 执行失败 `{"code":3, "message":失败原因}`, 运行成功`{"code": 2}` +1. "sla相关": + - slaKill: 暂时只能填true + - slaDuration: flow运行最长秒数, 超时将会被kill + - slaEmail: 任务超时通知的邮箱 + +## job参数说明 + +### 必填参数 + +- name: job名称, 一个flow中job name 不能重复 +- component: 组件名称 + +### 选填参数 + +- condition: 如果这个job依赖于其他job, 需要在这填写job name, 多个用&&分隔 +- properties: 组件参数, 需要填写的内容参考具体的组件 +- callbackUrl: Job状态改变回调的url, post form 开始执行 `{"code":1}`, 执行失败 `{"code":3, "message":失败原因}`, 运行成功`{"code": 2}` diff --git "a/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" "b/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" new file mode 100644 index 00000000..6b2d7d8e --- /dev/null +++ "b/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/1.\345\267\262\346\234\211\347\273\204\344\273\266.md" @@ -0,0 +1,13 @@ + +# 已有组件 + + +|组件名称|必填参数|选填参数|产生结果|其他| +|:------|:------|:-----|:------|:--| +std::http|1. url: url
2. method: GET 或 POST|1. contentType: Content-Type
2. referenceData: 流程数据引用, 多个用逗号分开, 如: `aName:a.name,bAge:b.age`, 冒号前面的是http发过去的key, 后面是value
3. forwardData: 转发数据, 这个会原封不动的发给url, key是forwardData|result : http请求的返回结果|| +std::mail|1. title: 邮件标题
2. contentHtml:邮件内容
3. recipient:收件人(多个用逗号隔开)| | |可以在邮件标题和内容中用${}引用别的job产生的结果, 如: `${a.name}`| +std::shell|1. shellBase64: shell 脚本文件 base64|1.environmentVariable: shell 变量(xxx=yyy 形式,多个用换行隔开)| |推荐在页面创建时候使用该组件, 如果是第三方系统建议用http在自己的系统实现| diff --git "a/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" "b/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" new file mode 100644 index 00000000..72449a88 --- /dev/null +++ "b/docs/zh/3.\347\273\204\344\273\266\350\257\264\346\230\216/2.\350\207\252\345\256\232\344\271\211\347\273\204\344\273\266.md" @@ -0,0 +1,68 @@ + +# 自定义组件 + +1. 可以参考 + - com.xiaomi.thain.component.std.MailComponent + - com.xiaomi.thain.component.std.HttpComponent + + +1. 使用注解: com.xiaomi.thain.component.annotation.ThainComponent + - group: 组件所属的组,如:std, xdata等,用于区分组件 + - name: 组件名称, 如:http + - defineJson + - 组件定义的json + - 这个json用来字段校验,前端组件展示等 + +1. json 定义 + - 格式: + ```ts + /** + * https://app.quicktype.io/#l=schema 这个网站生成json + * + * 组件前端 schema + * + */ + export type ComponentDefineJson = Array<{ + /** + * 传到后端的key + */ + property: string; + /** + * 是否必填,false 或不写 则为不是必填 + */ + required?: boolean; + /** + * 输入框前面的标识,不写默认用property + */ + label?: string; + input: + | { + id: 'line' | 'textarea' | 'sql' | 'shell' | 'richText'; + } + | { + id: 'select'; + options: Array<{ + /** + * 属性的值 + */ + id: string; + /** + * 下拉框中的候选项,不写用id代替 + */ + name?: string; + }>; + }; + }>; + ``` + +1. java Class 设计 + - 把json中的property 定义在java class 中, 运行时会根据用户填写的信息自动注入 + - 可以定义一个 com.xiaomi.thain.component.tools.ComponentTools 类型的属性 + - ComponentTools 这个接口中有很多的功能 + +1. ComponentTools 接口 + - com.xiaomi.thain.component.tools.ComponentTools 见注释 diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..26200323 --- /dev/null +++ b/pom.xml @@ -0,0 +1,24 @@ + + + + + 4.0.0 + com.xiaomi.thain + thain + pom + 1.0.0 + thain + + + thain-common + thain-component + thain-core + thain-server + thain-sdk + + diff --git a/readme.md b/readme.md new file mode 100644 index 00000000..0d05fbb4 --- /dev/null +++ b/readme.md @@ -0,0 +1,46 @@ + +# Thain + +![Thain Logo](./docs/images/logo.png) + +## Other language versions + +[简体中文](./readme_zh.md) + +## Introduction + +Thain is a distributed flow schedule platform, it was invented at XiaoMi Technology and the project has been proven easily-learning, developed-rapidly, stable in the XiaoMi internal departments. +Thain has completed backend management system which supports depending, re-run, roll-back and monitoring of the flows. + +## Documentation Contents + +1. Product Introduction +1. Deployment + - [deployment process](./docs/en/1.部署启动/1.部署流程.md) + - [database configuration](./docs/en/1.部署启动/2.数据库配置.md) + - [user management](./docs/en/1.部署启动/3.用户管理.md) +1. Use Tutorial + - [Create Flow](./docs/en/2.使用教程/创建任务.md) +1. Component + - [Existing component](./docs/en/3.组件说明/1.已有组件.md) + - [Custom component](./docs/en/3.组件说明/2.自定义组件.md) + +## Maintainers + +@liangyongrui + +## Help Us Do Better + +Now Thain continues to be developed by xiaomi internal developers. If you have any questions, you can ask in the issues or send email to: + +- zuojianguang@xiaomi.com +- liangyongrui@xiaomi.com +- miaoyu3@xiaomi.com + +## License + +[Apache License Version 2.0](LICENSE) © Xiaomi \ No newline at end of file diff --git a/readme_zh.md b/readme_zh.md new file mode 100644 index 00000000..69c203ac --- /dev/null +++ b/readme_zh.md @@ -0,0 +1,47 @@ + +# Thain + +![Thain Logo](./docs/images/logo.png) + +## Other language versions + +[English](./readme.md) + +## 产品简介 + +Thain是小米自研的新一代分布式任务调度平台,提供定时、任务编排、分布式跑等功能。 +Thain提供了任务调度与执行的一整套解决方案,在小米集团内部使用并久经考验,具有易学习、易上手、开发高效稳定的特点。 +有完善的后台管理界面,支持任务的依赖,重跑,回溯,任务执行情况查看。 + +## 文档目录 + +1. 产品简介 +1. 部署启动 + - [部署流程](./docs/zh/1.部署启动/1.部署流程.md) + - [数据库配置](./docs/zh/1.部署启动/2.数据库配置.md) + - [用户管理](./docs/zh/1.部署启动/3.用户管理.md) +1. 使用教程 + - [创建任务](./docs/zh/2.使用教程/创建任务.md) +1. 组件说明 + - [已有组件](./docs/zh/3.组件说明/1.已有组件.md) + - [自定义组件](./docs/zh/3.组件说明/2.自定义组件.md) + +## Maintainers + +@liangyongrui + +## 帮助我们做的更好 + +有任何疑问都可以在issues中提出,或发送邮件到: + +- zuojianguang@xiaomi.com +- liangyongrui@xiaomi.com +- miaoyu3@xiaomi.com + +## License + +[Apache License Version 2.0](LICENSE) © Xiaomi \ No newline at end of file diff --git a/thain-common/pom.xml b/thain-common/pom.xml new file mode 100644 index 00000000..0508b304 --- /dev/null +++ b/thain-common/pom.xml @@ -0,0 +1,170 @@ + + + + + + thain + com.xiaomi.thain + 1.0.0 + + 4.0.0 + + thain-common + + thain-common + + + UTF-8 + 1.8 + 1.8 + + + + + org.json + json + 20180130 + + + org.everit.json + org.everit.json.schema + 1.5.1 + + + json + org.json + + + guava + com.google.guava + + + commons-collections + commons-collections + + + commons-logging + commons-logging + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.10.0 + compile + + + log4j-core + org.apache.logging.log4j + + + + + org.apache.logging.log4j + log4j-core + 2.10.0 + compile + + + org.slf4j + jul-to-slf4j + 1.7.25 + compile + + + slf4j-api + org.slf4j + + + + + com.google.guava + guava + 28.0-jre + + + + + org.projectlombok + lombok + 1.18.8 + + + + org.junit.jupiter + junit-jupiter-api + 5.5.0-RC1 + test + + + org.apache.commons + commons-collections4 + 4.3 + + + org.apache.httpcomponents + httpcore + 4.4.11 + + + org.apache.commons + commons-lang3 + 3.9 + + + org.apache.httpcomponents + httpclient + 4.5.8 + + + httpcore + org.apache.httpcomponents + + + + + com.fasterxml.jackson.core + jackson-annotations + 2.9.5 + + + com.alibaba + fastjson + 1.2.58 + + + com.fasterxml.jackson.core + jackson-core + 2.9.8 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.8 + + + jackson-core + com.fasterxml.jackson.core + + + jackson-annotations + com.fasterxml.jackson.core + + + + + + javax.mail + javax.mail-api + 1.6.2 + + + + + diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowExecutionStatus.java b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowExecutionStatus.java new file mode 100644 index 00000000..0ac3260c --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowExecutionStatus.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ + +package com.xiaomi.thain.common.constant; + +/** + * flow的最后一次运行状态 + * + * @author liangyongrui + */ +public enum FlowExecutionStatus { + + /** + * 1 正在运行 + */ + RUNNING(1), + /** + * 2 运行成功 + */ + SUCCESS(2), + /** + * 3 运行异常 + */ + ERROR(3), + + /** + * 4 手动kill + */ + KILLED(4); + + public final int code; + + FlowExecutionStatus(int code) { + this.code = code; + } + + public static FlowExecutionStatus getInstance(int status) { + switch (status) { + case 1: + return RUNNING; + case 2: + return SUCCESS; + case 3: + return ERROR; + case 4: + return KILLED; + default: + } + return RUNNING; + } +} diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowLastRunStatus.java b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowLastRunStatus.java new file mode 100644 index 00000000..20deeefd --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowLastRunStatus.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ + +package com.xiaomi.thain.common.constant; + +import lombok.val; + +/** + * flow的最后一次运行状态 + * + * @author liangyongrui + */ +public enum FlowLastRunStatus { + /** + * 1 未运行 + */ + NEVER(1), + /** + * 2 运行成功 + */ + SUCCESS(2), + /** + * 3 运行异常 + */ + ERROR(3), + /** + * 4 正在运行 + */ + RUNNING(4), + /** + * 5 手动杀死 + */ + KILLED(5), + /** + * 6 暂停运行 + */ + PAUSE(6); + + public final int code; + + FlowLastRunStatus(int code) { + this.code = code; + } + + public static FlowLastRunStatus getInstance(int code) { + for (val t : FlowLastRunStatus.values()) { + if (t.code == code) { + return t; + } + } + return SUCCESS; + } +} diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowSchedulingStatus.java b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowSchedulingStatus.java new file mode 100644 index 00000000..0b9ffc66 --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/constant/FlowSchedulingStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ +package com.xiaomi.thain.common.constant; + +/** + * flow的调度状态 + * + * @author liangyongrui + */ +public enum FlowSchedulingStatus { + /** + * 1 调度中 + */ + SCHEDULING(1), + /** + * 2 暂停调度 + */ + PAUSE(2), + + /** + * 3 未设置调度(只运行一次的任务) + */ + NOT_SET(3); + + public final int code; + + FlowSchedulingStatus(int code) { + this.code = code; + } + +} diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/constant/HttpCallbackStatus.java b/thain-common/src/main/java/com/xiaomi/thain/common/constant/HttpCallbackStatus.java new file mode 100644 index 00000000..e6a3f558 --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/constant/HttpCallbackStatus.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ +package com.xiaomi.thain.common.constant; + +/** + * Date 19-5-21 上午10:23 + * http 回调状态 + * + * @author liangyongrui@xiaomi.com + */ +public enum HttpCallbackStatus { + /** + * 1 开始运行 + */ + START(1), + /** + * 2 运行成功 + */ + SUCCESS(2), + /** + * 3 运行异常 + */ + ERROR(3), + /** + * 4 手动杀死 + */ + KILLED(4), + /** + * 5 暂停运行 + */ + PAUSE(5); + + public final int code; + + HttpCallbackStatus(int code) { + this.code = code; + } +} diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/constant/JobExecutionStatus.java b/thain-common/src/main/java/com/xiaomi/thain/common/constant/JobExecutionStatus.java new file mode 100644 index 00000000..a28ab239 --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/constant/JobExecutionStatus.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ +package com.xiaomi.thain.common.constant; + +/** + * JobExecution的执行状态 + * + * @author liangyongrui + */ +public enum JobExecutionStatus { + /** + * 1 未运行 + */ + NEVER(1), + /** + * 2 正在运行 + */ + RUNNING(2), + /** + * 3 执行成功 + */ + SUCCESS(3), + /** + * 4 执行异常 + */ + ERROR(4); + + public final int code; + + JobExecutionStatus(int code) { + this.code = code; + } + +} diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/entity/ApiResult.java b/thain-common/src/main/java/com/xiaomi/thain/common/entity/ApiResult.java new file mode 100644 index 00000000..be5d9f18 --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/entity/ApiResult.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ + +package com.xiaomi.thain.common.entity; + +import com.google.common.collect.ImmutableMap; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NonNull; +import lombok.ToString; + +import javax.annotation.Nullable; +import java.util.List; + +/** + * @author liangyongrui + */ +@Builder +@ToString +@AllArgsConstructor +public class ApiResult { + public final int status; + @NonNull + public final String message; + @Nullable + public final Object data; + + public static ApiResult success() { + return success(""); + } + + public static ApiResult success(@NonNull List data, long count, int page, int pageSize) { + return new ApiResult(200, "success", ImmutableMap.of( + "data", data, + "count", count, + "page", page, + "pageSize", pageSize + )); + } + + public static ApiResult success(@Nullable Object data) { + return new ApiResult(200, "success", data); + } + + public static ApiResult fail(@Nullable String message) { + return new ApiResult(400, message == null ? "unknown error" : message, ""); + } +} + diff --git a/thain-common/src/main/java/com/xiaomi/thain/common/entity/ComponentItemDefine.java b/thain-common/src/main/java/com/xiaomi/thain/common/entity/ComponentItemDefine.java new file mode 100644 index 00000000..5ec582e9 --- /dev/null +++ b/thain-common/src/main/java/com/xiaomi/thain/common/entity/ComponentItemDefine.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Xiaomi, Inc. All rights reserved. + * This source code is licensed under the Apache License Version 2.0, which + * can be found in the LICENSE file in the root directory of this source tree. + */ +package com.xiaomi.thain.common.entity; + +import lombok.AllArgsConstructor; +import lombok.NonNull; + +import java.util.List; + +/** + * Date 19-5-27 下午5:28 + * + * @author liangyongrui@xiaomi.com + */ +@AllArgsConstructor +public class ComponentItemDefine { + + @AllArgsConstructor + public static class Option { + @NonNull + public final String id; + @NonNull + public final String name; + } + + @AllArgsConstructor + public static class Input { + @NonNull + public final String id; + @NonNull + public final List