Memoization bug, redundant transformation, semicolon bug

autoquote.py

@@ -3,7 +3,149 @@
 with open("RUN.scm") as file:
     allcode = file.read()
 basecode, usercode = allcode.split(";__METTACODE__:")
-usercode_nocomments = "\n".join([line.split(";")[0] for line in usercode.split("\n")])
+
+# Helper function to remove comments while preserving strings
+def remove_comments_preserve_strings(line):
+    """Remove comments from a line while preserving content inside string literals."""
+    result = []
+    in_string = False
+    escaped = False
+
+    for i, char in enumerate(line):
+        if escaped:
+            result.append(char)
+            escaped = False
+            continue
+
+        if char == '\\' and in_string:
+            result.append(char)
+            escaped = True
+            continue
+
+        if char == '"':
+            result.append(char)
+            in_string = not in_string
+            continue
+
+        if char == ';' and not in_string:
+            # Found a comment outside of a string
+            break
+
+        result.append(char)
+
+    return ''.join(result)
+
+# Helper function to extract tokens while respecting string boundaries
+def extract_tokens_outside_strings(text):
+    """Extract tokens from text, excluding content inside string literals."""
+    tokens = []
+    current_token = []
+    in_string = False
+    escaped = False
+
+    for char in text:
+        if escaped:
+            escaped = False
+            continue
+
+        if char == '\\' and in_string:
+            escaped = True
+            continue
+
+        if char == '"':
+            if current_token and not in_string:
+                tokens.append(''.join(current_token))
+                current_token = []
+            in_string = not in_string
+            continue
+
+        if in_string:
+            # Skip content inside strings
+            continue
+
+        if char in ' ()\n\t':
+            if current_token:
+                tokens.append(''.join(current_token))
+                current_token = []
+        else:
+            current_token.append(char)
+
+    if current_token:
+        tokens.append(''.join(current_token))
+
+    return tokens
+
+# Helper function to quote symbols while preserving string content
+def quote_symbols_outside_strings(text, symbols_to_quote):
+    """Quote symbols in text, but only outside of string literals."""
+    result = []
+    current_token = []
+    in_string = False
+    escaped = False
+
+    i = 0
+    while i < len(text):
+        char = text[i]
+
+        if escaped:
+            result.append(char)
+            escaped = False
+            i += 1
+            continue
+
+        if char == '\\' and in_string:
+            result.append(char)
+            escaped = True
+            i += 1
+            continue
+
+        if char == '"':
+            # Flush current token if any
+            if current_token:
+                token = ''.join(current_token)
+                if token in symbols_to_quote and not in_string:
+                    result.append("'" + token)
+                else:
+                    result.append(token)
+                current_token = []
+            result.append(char)
+            in_string = not in_string
+            i += 1
+            continue
+
+        if in_string:
+            # Inside string, just copy characters
+            result.append(char)
+            i += 1
+            continue
+
+        if char in ' ()\n\t':
+            # Delimiter found, flush current token
+            if current_token:
+                token = ''.join(current_token)
+                if token in symbols_to_quote:
+                    result.append("'" + token)
+                else:
+                    result.append(token)
+                current_token = []
+            result.append(char)
+            i += 1
+        else:
+            # Build up current token
+            current_token.append(char)
+            i += 1
+
+    # Flush any remaining token
+    if current_token:
+        token = ''.join(current_token)
+        if token in symbols_to_quote and not in_string:
+            result.append("'" + token)
+        else:
+            result.append(token)
+
+    return ''.join(result)
+
+usercode_nocomments = "\n".join([remove_comments_preserve_strings(line) for line in usercode.split("\n")])
 
 #2. Starting from a list of builtin functions we automatically add the defined MeTTa and Scheme functions&macros:
 functions = set(["-", "+", "*", "/",        #arithmetic functions
@@ -34,22 +176,17 @@
         if name:
             functions.add(name)
 
-#3. We now check for yet unquoted symbols:
-identified_symbols = set([x for x in usercode_nocomments.replace("(", " ").replace(")"," ").replace("\n","").split(" ") \
+#3. We now check for yet unquoted symbols (using proper string-aware tokenization):
+tokens = extract_tokens_outside_strings(usercode_nocomments)
+identified_symbols = set([x for x in tokens
                           if x != "" and x[0] != "$" #not a variable
                                      and not x in functions #not a function or macro
                                      and not x.replace("-","").replace(".","").isnumeric() #not a number
                                      and x != "#f" and x != "#t" #not a boolean
                                      and x[0] != '"']) #not a string
 
-#4. We quote the identified symbols in the user code:
-newcode = usercode
-for x in identified_symbols:
-    possible_prefix = [" ", "(", "\n"]
-    possible_postfix = [")", " ", "\n"]
-    for prefix in possible_prefix:
-        for postfix in possible_postfix:
-            newcode=newcode.replace(prefix + x + postfix, prefix + "'" + x + postfix)
+#4. We quote the identified symbols in the user code (string-aware):
+newcode = quote_symbols_outside_strings(usercode, identified_symbols)
 
 #5. Additionally, we remove quotes in type definitions, which is easier than keeping track of all defined types:
 newcodefinal = ""
@@ -61,4 +198,3 @@
 #6. Now, override the file with the one which has the properly quoted symbols
 with open("RUN.scm", "w") as file:
     file.write(basecode + "\n;__METTACODE__:" + newcodefinal.replace("''","'"))
-

mettamorph.scm

@@ -363,11 +363,11 @@
 
 ;Memoize the results of function f
 (define (memoized f)
-  (lambda (n)
-    (let ((key (cons f n)))
+  (lambda args
+    (let ((key (cons f args)))
       (if (hash-table-exists? cache key)
           (hash-table-ref cache key)
-          (let ((result (f n)))
+          (let ((result (apply f args)))
             (hash-table-set! cache key result)
             result)))))
 

run_scheme.sh

@@ -1,6 +1,6 @@
 (cd ./temp && cd ..) || mkdir temp
 cp $1 ./temp/CODE.metta > /dev/null 2>&1 #copy for repl only
-awk '{gsub(/\(if /, "(If ") gsub("==", "equal?") gsub(/\%void\%/, "Empty") gsub("new-space", "new-space ()") gsub(/\(\, /, "(MatchChain ") gsub(/self \(= \(/, "self (\047=def (\047") gsub(/self \(: /, "self (\047:def \047") gsub(/True/, "#t") gsub(/False/, "#f") gsub(/\|\-/, "DERIVE") gsub(/\(\)/, "\047()") gsub(/\(match /, "(Match ") gsub(/\(let /, "(Let ") gsub(/\(let\* /, "(Let* ") gsub(/\(match /, "(Match ") gsub(/\(case /, "(Case "); if($0 ~ /^[(]/ && !($0 ~ /^\(define/) && !($0 ~ /^[(][:|=]/)) { gsub(/^[(]/, "!(add-atom \047\\&self (", $0); $0 = $0 ")" } gsub(/\!\(/, "(! ", $0) gsub(/\(: /, "(Typedef ")}1' $1 > ./temp/PROGRAM.scm 2> /dev/null
+awk '{gsub(/\(if /, "(If ") gsub("==", "equal?") gsub(/\%void\%/, "Empty") gsub("new-space", "new-space ()") gsub(/\(\, /, "(MatchChain ") gsub(/self \(= \(/, "self (\047=def (\047") gsub(/self \(: /, "self (\047:def \047") gsub(/True/, "#t") gsub(/False/, "#f") gsub(/\|\-/, "DERIVE") gsub(/\(\)/, "\047()") gsub(/\(match /, "(Match ") gsub(/\(let /, "(Let ") gsub(/\(let\* /, "(Let* ") gsub(/\(case /, "(Case "); if($0 ~ /^[(]/ && !($0 ~ /^\(define/) && !($0 ~ /^[(][:|=]/)) { gsub(/^[(]/, "!(add-atom \047\\&self (", $0); $0 = $0 ")" } gsub(/\!\(/, "(! ", $0) gsub(/\(: /, "(Typedef ")}1' $1 > ./temp/PROGRAM.scm 2> /dev/null
 cat mettamorph.scm > RUN.scm
 echo ";__METTACODE__:" >> RUN.scm
 cat ./temp/PROGRAM.scm >> RUN.scm