Skip to content

Commit d5b292e

Browse files
hyponethyponet
authored
Merge pull request #46 from basenana/feature/workflowdef
workflow db model def
2 parents f6122d2 + 3e3b83a commit d5b292e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1667
-379
lines changed

‎.github/configs/nanafs-fuse.conf

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"api": {
3+
"enable": true,
4+
"host": "127.0.0.1",
5+
"port": 7081,
6+
"pprof": false
7+
},
8+
"fuse": {
9+
"enable": true,
10+
"root_path": "/tmp/mnt/nanafs",
11+
"display_name": "nanafs"
12+
},
13+
"webdav": {},
14+
"meta": {
15+
"type": "sqlite",
16+
"path": "/var/lib/nanafs/sqlite.db"
17+
},
18+
"storages": [
19+
{
20+
"id": "local-0",
21+
"type": "local",
22+
"local_dir": "/var/lib/nanafs/local"
23+
}
24+
],
25+
"cache_dir": "/var/lib/nanafs/local",
26+
"cache_size": 10
27+
}

‎.github/workflows/pjdfstest.yml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: pjdfstest
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- 'main'
8+
paths-ignore:
9+
- 'docs/**'
10+
- '**.md'
11+
12+
jobs:
13+
pjdfstest:
14+
strategy:
15+
fail-fast: true
16+
runs-on: ubuntu-latest
17+
18+
steps:
19+
- name: Checkout code
20+
uses: actions/checkout@v2
21+
22+
- name: Set up Go 1.18
23+
uses: actions/setup-go@v3
24+
with:
25+
go-version: 1.18
26+
27+
- name: init nanafs dep
28+
run: |
29+
sudo apt-get install -y libfuse3-dev fuse3 libssl-dev
30+
sudo mkdir -p /var/lib/nanafs
31+
sudo mkdir -p /etc/nanafs
32+
mkdir -p /tmp/mnt/nanafs
33+
sudo cp .github/configs/nanafs-fuse.conf /etc/nanafs/nanafs.conf
34+
35+
- name: Build nanafs binary
36+
run: sudo make buildbin
37+
38+
- name: Run nanafs fuse daemon
39+
run: sudo nanafs serve --config /etc/nanafs/nanafs.conf > /tmp/nanafs.log 2>&1 &
40+
41+
- name: Make pjdfstest
42+
run: |
43+
cp -r e2e/pjdfstest /tmp
44+
cd /tmp/pjdfstest && sudo autoreconf -ifs && sudo ./configure && sudo make pjdfstest
45+
46+
- name: Run pjdfstest
47+
run: |
48+
df
49+
sudo chmod 777 /tmp/mnt/nanafs && cd /tmp/mnt/nanafs
50+
sudo prove -rv /tmp/pjdfstest
51+
52+
- name: Print nanafs log
53+
if: failure()
54+
run: cat /tmp/nanafs.log
55+
56+
# - name: Setup upterm session
57+
# if: failure()
58+
# timeout-minutes: 60
59+
# uses: lhotari/action-upterm@v1

‎.github/workflows/unittest.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: unittest
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- 'main'
8+
paths-ignore:
9+
- 'docs/**'
10+
- '**.md'
11+
12+
jobs:
13+
unittest:
14+
strategy:
15+
fail-fast: true
16+
runs-on: ubuntu-latest
17+
18+
steps:
19+
- name: Checkout code
20+
uses: actions/checkout@v2
21+
22+
- name: Set up Go 1.18
23+
uses: actions/setup-go@v3
24+
with:
25+
go-version: 1.18
26+
27+
- name: Run unit tests
28+
run: make test

‎Makefile

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
1-
.PHONY: all build buildinci clean run check cover lint docker help
1+
.PHONY: all build buildbin clean run check cover lint docker help
22
BIN_DIR=bin
33
BASE_PATH=$(shell pwd)
4+
$(shell git fetch --tags)
5+
GIT_COMMIT := $(shell git rev-parse HEAD)
6+
LATEST_TAG := $(shell git describe --tags --abbrev=0)
7+
48
all: check build
59
build:
610
docker run --rm -v $(BASE_PATH):/go/src/github.com/basenana/nanafs \
11+
-e GIT_COMMIT=$(GIT_COMMIT) \
12+
-e GIT_TAG=$(LATEST_TAG) \
713
-v $(BASE_PATH)/bin:/bin/nanafs \
814
-w /go/src/github.com/basenana/nanafs \
915
golang:1.18 sh ./hack/multibuild.sh ./cmd /bin/nanafs
10-
buildinci:
11-
CGO_ENABLED=0 GOOS=$1 GOARCH=$2 go build -ldflags="-s -w" -o /usr/bin/nanafs ./cmd
16+
buildbin:
17+
CGO_ENABLED=0 GOOS=$1 GOARCH=$2 go build \
18+
-ldflags="-s -w -X github.com/basenana/nanafs/config.gitCommit=${GIT_COMMIT} -X github.com/basenana/nanafs/config.gitTag=${GIT_TAG}" \
19+
-o /usr/bin/nanafs ./cmd
1220
clean:
1321
@go clean
1422
test:

‎README.md

Lines changed: 67 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# NanaFS
22

3+
![unittest workflow](https://github.com/basenana/nanafs/actions/workflows/unittest.yml/badge.svg)
4+
![pages-build-deployment](https://github.com/basenana/nanafs/actions/workflows/pages/pages-build-deployment/badge.svg)
5+
6+
<p align="right">[ English | <a href="https://github.com/basenana/nanafs/blob/main/README_zh.md">简体中文</a> ]</p>
7+
38
NanaFS is a workflow engine that simplifies data management
49
by allowing users to manage structured and unstructured data in one place,
510
rather than across multiple sources. It's like a filing cabinet
@@ -10,104 +15,67 @@ NanaFS is also customizable through plugin support,
1015
meaning users can tailor the workflow engine to their specific needs.
1116
This makes NanaFS a versatile and valuable tool for personal, academic, and professional use.
1217

18+
## Key Features
19+
20+
### Cloud-Based Storage
21+
22+
NanaFS utilizes cloud-based storage as main backend storage,
23+
supporting not only object storage but also file-hosting service.
24+
With the cloud-based storage feature, NanaFS can have unlimited storage capacity at an affordable price,
25+
and users can access their data saved in the cloud anywhere and anytime.
26+
27+
The following storage options are already supported or planned to be supported:
28+
29+
- **Object Storage**: AWS S3, AlibabaCloud OSS, Cloudflare R2
30+
- **Cloud Drive**: Google Drive, OneDrive, AliyunDrive, BaiduWangpan
31+
- **Other Storage Protocols**: WebDAV
32+
33+
### POSIX Compatibility
34+
35+
NanaFS offers a file system interface that complies with the POSIX standard through FUSE.
36+
This makes it easy to mount NanaFS onto the directory tree and manage NanaFS data using tools such as Finder on Linux
37+
and MacOS operating systems.
38+
39+
Additionally, NanaFS has passed the majority of pjdfstest's compatibility tests, ensuring compatibility with Linux/Unix
40+
systems.
41+
This means that existing commands and tools can be used to efficiently process data in NanaFS. For specific needs,
42+
custom scripts or programs can also be written to process files in NanaFS.
43+
44+
### File-Centric Workflow
45+
46+
The actual value of data storage lies in its use. To facilitate this,
47+
NanaFS provides a file-centric workflow engine equipped with rule-based automatic file processing capabilities.
48+
49+
With the workflow engine, tasks such as batch file renaming and creating semantic indexing based on file content become
50+
very simple.
51+
This eases data manipulation, makes data no longer "cold," and helps uncover more inherent value in the data.
52+
53+
### Plugin Support
54+
55+
NanaFS supports multiple types of plugins to extend its capabilities. Currently, NanaFS primarily supports three types
56+
of plugins:
57+
58+
- **Source Plugin**: synchronizes data periodically from a source address and integrates it into NanaFS. This includes
59+
aggregating RSS information and filing emails according to the SMTP protocol.
60+
- **Mirror Plugin**: maps external storage systems into NanaFS, allowing NanaFS to manage data from multiple storage
61+
systems through a unified interface.
62+
- **Process Plugin**: provides file processing capabilities and enhances the functionality of workflows by extending
63+
Process Plugins.
64+
65+
### Data Security
66+
67+
Data security is a prerequisite for data storage and usage.
68+
NanaFS provides end-to-end encryption from storage to transmission, ensuring that your cloud data cannot be accessed
69+
even if it is stolen by hackers.
70+
Similarly, cloud service providers cannot access or modify your data, ensuring that your data is not leaked or misused.
71+
1372
## Usage
1473

15-
### Requirements
16-
17-
1. NanaFS requires the FUSE library for POSIX-FS use cases, but it is optional if you are using the API or WebDAV
18-
exclusively.
19-
2. In order to build the NanaFS binary, Docker is needed. So if you plan to build it yourself, you will need to have
20-
Docker installed first.
21-
22-
Fortunately, installing FUSE or Docker is a straightforward process.
23-
24-
For Ubuntu, FUSE can be installed using `apt`:
25-
26-
```bash
27-
sudo apt-get install -y libfuse3-dev fuse3 libssl-dev
28-
```
29-
30-
For macOS, same operation like linux, `brew` will make things done:
31-
32-
```bash
33-
brew install --cask osxfuse
34-
```
35-
36-
### Deploy using the binary file
37-
38-
The current and historical versions of the binary files can be downloaded on
39-
the [release page](https://github.com/basenana/nanafs/releases).
40-
41-
### Build your own version
42-
43-
If your own version needs to be built, just `make` it:
44-
45-
```bash
46-
make build
47-
```
48-
49-
### Run
50-
51-
Before running a trigger in NanaFS, ensure the configuration is correct.
52-
53-
We provide a tool to quickly edit and generate configuration files.
54-
Generate a default local configuration file using the provided tool,
55-
but keep in mind that the generated configuration may not be optimal.
56-
57-
```bash
58-
nanafs config init
59-
```
60-
61-
The generated configuration file can be found in the `~/.nana` directory:
62-
63-
```json
64-
{
65-
"api": {
66-
"enable": true,
67-
"host": "127.0.0.1",
68-
"port": 7081,
69-
"pprof": false
70-
},
71-
"fuse": {
72-
"enable": false,
73-
"root_path": "/your/path/to/mount",
74-
"display_name": "nanafs"
75-
},
76-
"webdav": {
77-
"enable": false,
78-
"host": "127.0.0.1",
79-
"port": 7082,
80-
"overwrite_users": [
81-
{
82-
"uid": 0,
83-
"gid": 0,
84-
"username": "admin",
85-
"password": "changeme"
86-
}
87-
]
88-
},
89-
"meta": {
90-
"type": "sqlite",
91-
"path": "/your/data/path/sqlite.db"
92-
},
93-
"storages": [
94-
{
95-
"id": "local-0",
96-
"type": "local",
97-
"local_dir": "/your/data/path/local"
98-
}
99-
],
100-
"cache_dir": "/your/data/path/cache",
101-
"cache_size": 10
102-
}
103-
```
104-
105-
Run nanafs:
106-
107-
```bash
108-
nanafs serve
109-
```
110-
111-
## Architecture
112-
113-
TODO
74+
NanaFS's usage guidelines, including its parameters and examples of tools and commands, are documented
75+
in [Instructions For Usage](https://github.com/basenana/nanafs/blob/main/docs/usage.md).
76+
77+
## Feedback
78+
79+
If you encounter any problems while using NanaFS, whether it's related to usage, bugs, or you have suggestions for new
80+
features,
81+
please feel free to create an issue.

‎README_zh.md

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# NanaFS
2+
3+
![unittest workflow](https://github.com/basenana/nanafs/actions/workflows/unittest.yml/badge.svg)
4+
![pages-build-deployment](https://github.com/basenana/nanafs/actions/workflows/pages/pages-build-deployment/badge.svg)
5+
6+
<p align="right">[ <a href="https://github.com/basenana/nanafs/blob/main/README.md">English</a> | 简体中文 ]</p>
7+
8+
NanaFS 是一款文件系统风格的工作流引擎，也是一款尝试将结构化和非结构化数据的统一管理的数据仓库。
9+
10+
在个人工作、学习和生活中，大量的数据和材料存储在不同的信息孤岛之中，例如 Office 文件、电子邮件、RSS、工作笔记、待办记事等。
11+
孤立的数据没有价值，无论是一个刚起步的项目还是酝酿一篇论文，数据是为具体场景服务的。 我曾花费很多时间编写胶水脚本，使它们可以被一起管理。
12+
为什么不构建一个内建工作流，并提供具体场景数据入口的文件系统呢，于是 NanaFS 应运而生。
13+
14+
NanaFS 采用一个统一的文件模型，将来自多种数据源的数据汇总至同一位置，并基于统一的存储提供通用的工作流能力。
15+
16+
## 核心特性
17+
18+
### 基于云端存储
19+
20+
NanaFS 采用云端存储技术作为其主要后端存储方式，不仅仅支持对象存储，也支持网络硬盘。因此，凭借云端存储的能力，NanaFS
21+
不但可以价格可控的拥有近乎无限的存储容量，也使得用户可以随时随地访问其保存在云端的数据。
22+
23+
已经支持或者计划支持的存储包括：
24+
25+
- **对象存储类**：AWS S3、阿里云OSS、Cloudflare R2
26+
- **云盘类**：Google Drive、OneDrive、阿里云盘、百度网盘
27+
- **其他存储协议**：WebDAV
28+
29+
### POSIX 兼容
30+
31+
NanaFS 通过 FUSE 提供了符合 POSIX 标准的文件系统接口。这使得在 Linux 和 MacOS 操作系统中，可以轻松地将 NanaFS
32+
挂载到目录树上，并通过访达等工具管理 NanaFS 中的数据。
33+
34+
同时，NanaFS 通过了绝大多数的 pjdfstest 兼容性测试，保证了与 Linux/Unix 系统的兼容，而这也意味着可以利用现有的命令和工具对
35+
NanaFS 中的数据进行处理。
36+
对于特定需求，也可以通过编写处理文件的脚本或程序，实现自定义处理 NanaFS 中的文件。
37+
38+
### 面向文件的工作流
39+
40+
数据存储的实际价值在于数据的使用。NanaFS 为此提供了面向文件的工作流引擎，并配备了基于规则的文件自动处理能力。
41+
42+
基于工作流的能力，无论是文件的批量重命名，还是创建基于文件内容语义的索引都变得十分简单。轻松地操控数据，让数据不再冰冷，发掘数据中蕴含的更多价值。
43+
44+
### 支持插件
45+
46+
NanaFS 通过支持多种类型的插件，以实现对功能的拓展。目前 NanaFS 主要支持三种类型的插件：
47+
48+
- **Source Plugin**：定期的从源地址同步数据并收纳到 NanaFS 中，比如聚合 RSS 信息，根据 SMTP 协议归档电子邮件；
49+
- **Mirror Plugin**：将外部存储系统映射到 NanaFS 中，使得 NanaFS 可以统一入口的管理多个存储系统的数据；
50+
- **Process Plugin**：提供文件处理能力，通过拓展 Process Plugin，增强 Workflow 的功能。
51+
52+
### 数据安全
53+
54+
数据安全是进行数据存储和使用的前提条件。NanaFS
55+
提供从存储到传输的全链路加密，即使您的云端数据被窃取，黑客也无法获取您的数据。同样，云服务提供商也无法访问和修改您的数据，保证了您的数据不会被泄露和滥用。
56+
57+
## 使用
58+
59+
NanaFS 的使用方式参数文档 [Instructions For Usage](https://github.com/basenana/nanafs/blob/main/docs/usage_zh.md)。
60+
61+
## 反馈
62+
63+
如果您在使用NanaFS时遇到任何问题，无论是相关于使用方式、Bugs，还是您有新功能的建议，请随时创建 issue。

‎cmd/apps/config/cmd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func init() {
3434

3535
var RunCmd = &cobra.Command{
3636
Use: "config",
37-
Short: "nanafs config management",
37+
Short: "NanaFS config management",
3838
Run: func(cmd *cobra.Command, args []string) {
3939
configPath := localConfigFilePath(WorkSpace)
4040
fmt.Printf("Workspace Config: %s\n\n", configPath)

0 commit comments

Comments
 (0)