add readme and .env support

Author: githubesson

.env

@@ -0,0 +1,3 @@
+# SQL Parser Configuration
+BATCH_SIZE=100000
+WORKER_COUNT=1 

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 SQL Parser Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

README.md

@@ -0,0 +1,115 @@
+# SQL Parser
+
+A high-performance SQL INSERT statement parser that processes large SQL files and outputs the data in various formats.
+
+## Features
+
+- Processes large SQL files with INSERT statements
+- Memory-efficient batch processing
+- Parallel processing with configurable worker count
+- Multiple output formats:
+  - JSON
+  - CSV
+  - Text
+- Buffered I/O for optimal performance
+- Configurable batch size and worker count via environment variables
+
+## Installation
+
+```bash
+git clone https://github.com/githubesson/sqlparser
+cd sqlparser
+go build cmd/sqlparser/main.go
+```
+
+## Usage
+
+```bash
+sqlparser [-format=txt|csv|json] [-output=filename] [-workers=N] <sqlfile>
+```
+
+### Arguments
+
+- `-format`: Output format (default: txt)
+  - `txt`: Human-readable text format
+  - `csv`: CSV format with headers
+  - `json`: JSON format with table structure
+- `-output`: Output file path (default: stdout)
+- `-workers`: Number of worker threads (default: 1)
+- `<sqlfile>`: Input SQL file containing INSERT statements
+
+### Environment Variables
+
+The application can be configured using environment variables in a `.env` file:
+
+```env
+BATCH_SIZE=100000  # Number of rows to process in each batch
+WORKER_COUNT=1     # Default number of worker threads
+```
+
+### Examples
+
+1. Process SQL file and output as JSON:
+```bash
+sqlparser -format=json -output=output.json input.sql
+```
+
+2. Process SQL file with 4 workers and output as CSV:
+```bash
+sqlparser -format=csv -workers=4 -output=output.csv input.sql
+```
+
+3. Process SQL file and print to console in text format:
+```bash
+sqlparser input.sql
+```
+
+## Performance Optimization
+
+The parser is optimized for performance through:
+- Batch processing to manage memory usage
+- Parallel processing with configurable worker count
+- Buffered I/O operations
+- Memory pooling for row data
+- Efficient string handling
+
+## Output Formats
+
+### Text Format
+```
+Table: users
+Row 1:
+  id: 1
+  name: John Doe
+  email: [email protected]
+```
+
+### CSV Format
+```
+Table:,users
+Row,id,name,email
+1,1,John Doe,[email protected]
+```
+
+### JSON Format
+```json
+[
+  {
+    "table_name": "users",
+    "rows": [
+      {
+        "row_number": 1,
+        "data": {
+          "id": "1",
+          "name": "John Doe",
+          "email": "[email protected]"
+        }
+      }
+    ]
+  }
+]
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details. 

cmd/sqlparser/main.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime"
+	"strconv"
 
 	"sqlparser/pkg/models"
 	"sqlparser/pkg/parser"
@@ -14,15 +14,15 @@ import (
 func main() {
 	format := flag.String("format", "txt", "Output format (txt, csv, json)")
 	output := flag.String("output", "", "Output file (if not specified, prints to stdout)")
-	workers := flag.Int("workers", runtime.NumCPU(), "Number of worker threads (default: number of CPU cores)")
+	workers := flag.Int("workers", getWorkerCount(), "Number of worker threads")
 	flag.Parse()
 
 	args := flag.Args()
 	if len(args) < 1 {
 		fmt.Printf("Usage: sqlparser [-format=txt|csv|json] [-output=filename] [-workers=N] <sqlfile>\n")
 		fmt.Printf("  -format: Output format (default: txt)\n")
 		fmt.Printf("  -output: Output file (default: stdout)\n")
-		fmt.Printf("  -workers: Number of worker threads (default: %d)\n", runtime.NumCPU())
+		fmt.Printf("  -workers: Number of worker threads (default: %d)\n", getWorkerCount())
 		os.Exit(1)
 	}
 
@@ -59,3 +59,12 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func getWorkerCount() int {
+	if val := os.Getenv("WORKER_COUNT"); val != "" {
+		if count, err := strconv.Atoi(val); err == nil && count > 0 {
+			return count
+		}
+	}
+	return 1
+}

pkg/models/types.go

@@ -1,9 +1,23 @@
 package models
 
-const (
-	BatchSize = 100000
+import (
+	"os"
+	"strconv"
+)
+
+var (
+	BatchSize = getBatchSize()
 )
 
+func getBatchSize() int {
+	if val := os.Getenv("BATCH_SIZE"); val != "" {
+		if size, err := strconv.Atoi(val); err == nil && size > 0 {
+			return size
+		}
+	}
+	return 100000 // default batch size
+}
+
 type OutputFormat string
 
 const (