ast_generator/gazprea_ast_grammar.py

@@ -0,0 +1,53 @@
+from constants import Grammar
+
+GAZPREA_TOP_LEVEL: Grammar = {
+    # Top level elements
+    '<start>': ['<topBlock>'],
+    '<topBlock>': ['<XML_OPEN_TAG>block<XML_CLOSE_TAG><routine_list><main_routine><routine_list><XML_OPEN_SLASH>block<XML_CLOSE_TAG>'],
+    # TODO constants
+
+    # Routines
+    '<routine>': ['<function>', '<procedure>'],  # TODO forward_declaration
+    '<function>': [
+        '<XML_OPEN_TAG>function name="_NAME_" return_type="_TYPE_"  args="_ARGS_"<XML_CLOSE_TAG><return_block><XML_OPEN_SLASH>function<XML_CLOSE_TAG>'],
+    '<procedure>': [
+        '<XML_OPEN_TAG>procedure name="_NAME_" return_type="_TYPE_"  args="_ARGS_"<XML_CLOSE_TAG><block><XML_OPEN_SLASH>procedure<XML_CLOSE_TAG>'],
+    '<main_routine>': [
+        '<XML_OPEN_TAG>procedure name="main" return_type="int"  args="()"<XML_CLOSE_TAG><return_block><XML_OPEN_SLASH>procedure<XML_CLOSE_TAG>'],
+    '<routine_list>': ['<routine><routine_list><routine>', '<routine>'],
+
+    # Blocks
+    '<block>': ['<XML_OPEN_TAG>block<XML_CLOSE_TAG><statement_list><XML_OPEN_SLASH>block<XML_CLOSE_TAG>'],
+    '<return_block>': ['<XML_OPEN_TAG>block<XML_CLOSE_TAG><statement_list><return><XML_OPEN_SLASH>block<XML_CLOSE_TAG>'],
+    '<statement>': [
+        '<declaration>',
+        '<stream>',
+        # '<call>',
+        # '<return>',  # TODO if/else, loop
+    ],
+    '<statement_list>': ['<statement><statement_list><statement>', '<statement>'],
+
+    # Things that belong on their own lines
+    '<declaration>': ['<XML_OPEN_TAG>declaration<XML_CLOSE_TAG><variable><rhs><XML_OPEN_SLASH>declaration<XML_CLOSE_TAG>'],
+    '<stream>': ['<out_stream>'], #, '<in_stream>'],
+    '<return>': ['<XML_OPEN_TAG>return<XML_CLOSE_TAG><has_value><XML_OPEN_SLASH>return<XML_CLOSE_TAG>'],
+
+    '<out_stream>': ['<XML_OPEN_TAG>stream type="std_output"<XML_CLOSE_TAG><has_value><XML_OPEN_SLASH>stream<XML_CLOSE_TAG>'],
+    # '<in_stream>': ['<XML_OPEN_TAG>stream type="std_input"<XML_CLOSE_TAG><has_value><XML_OPEN_SLASH>stream<XML_CLOSE_TAG>'],
+
+    # Things that are part of lines
+    '<has_value>': ['<variable>', '<literal>', '<operator>'],
+    '<lhs>': ['<XML_OPEN_TAG>lhs<XML_CLOSE_TAG><has_value><XML_OPEN_SLASH>lhs<XML_CLOSE_TAG>'],
+    '<rhs>': ['<XML_OPEN_TAG>rhs<XML_CLOSE_TAG><has_value><XML_OPEN_SLASH>rhs<XML_CLOSE_TAG>'],
+
+    # Things that have values
+    '<operator>': ['<XML_OPEN_TAG>operator<XML_CLOSE_TAG><lhs><rhs><XML_OPEN_SLASH>operator<XML_CLOSE_TAG>'],
+    '<variable>': ['<XML_OPEN_TAG>variable mut="_MODIFIER_" type="_TYPE_" name="_NAME_"<XML_SLASH_TAG>'],
+    '<literal>': ['<XML_OPEN_TAG>literal type="_TYPE_" value="_VALUE_"<XML_SLASH_TAG>'],
+
+    # Helper rules
+    '<XML_OPEN_TAG>': ['<'],
+    '<XML_CLOSE_TAG>': ['>'],
+    '<XML_SLASH_TAG>': ['/>'],
+    '<XML_OPEN_SLASH>': ['</'],
+}

ast_generator/test/config.yaml

@@ -5,21 +5,19 @@ generation-options:
     max-conditionals-loops: 5                    # maximum number of loops/conditionals per routine
     max-number-of-routines: 5                    # maximum number of routines (main will always be generated)
     generate-dead-code: True                     # generate dead code
-    max-loop-iterations: 100                     # maximum number of iterations in a loop
-    max-globals: 5                               # maximum number of global variables
 properties:
     max-range-length: 5                          # maximum length of ranges, vectors and tuples, (AxA matrices can exist)
     use-english-words: True                      # use english words instead of random names (this may limit the maximum number of names)
     id-length:                                   # length of identifiers
         min: 1
-        max: 5
+        max: 10
     function-name-length:                        # length of function names
         min: 1
         max: 10
     number-of-arguments:                         # number of arguments to a routine
         min: 1
         max: 10
-    generate-max-int: False                       # if False, generate integers between [-1000, 1000] else
+    generate-max-int: True                       # if False, generate integers between [-1000, 1000] else
 expression-weights:                              # weights for expressions
                                                  #  the higher a weight, the more likely (0, 10000), 0 to exclude, 10000 for only that
     brackets: 10

ast_generator/test/test_ast_generator.py

@@ -60,7 +60,7 @@ class TestGeneration(unittest.TestCase):
     def test_generate_assignment(self):
         self.ast_gen.ast = ET.Element("block")
         self.ast_gen.current_ast_element = self.ast_gen.ast
-        self.ast_gen.generate_declaration(mut='var')
+        self.ast_gen.generate_declaration()
         self.ast_gen.generate_assignment()
 
         self.assertIsNotNone(self.ast_gen.ast.find("assignment"))
@@ -130,21 +130,13 @@ class TestGeneration(unittest.TestCase):
         self.assertIsNotNone(self.ast_gen.current_ast_element.find("conditional"))
         conditional = self.ast_gen.ast.find("conditional")
 
-        print(ET.tostring(conditional, 'utf-8').decode('utf-8'))
+        # print(ET.tostring(conditional, 'utf-8').decode('utf-8'))
 
-        self.has_child(conditional)
+        self.assertIsNotNone(conditional.find("operator") or conditional.find("unary_operator") or conditional.find("literal"))
 
         block = conditional.findall("block")
         self.assertEqual(2, len(block))
 
-    def has_child(self, conditional):
-        opts = ['operator', 'unary_operator', 'literal', 'brackets']
-        res = []
-        for i in opts:
-            res.append(conditional.find(i))
-        res_list = list(filter(lambda x: x is not None, res))
-        self.assertGreater(len(res_list), 0)
-
     def test_generate_loop(self):
         self.ast_gen.ast = ET.Element("block")
         self.ast_gen.current_ast_element = self.ast_gen.ast
@@ -155,7 +147,7 @@ class TestGeneration(unittest.TestCase):
 
         # print(ET.tostring(loop, 'utf-8').decode('utf-8'))
 
-        self.has_child(loop)
+        self.assertIsNotNone(loop.find("operator") or loop.find("unary_operator") or loop.find("literal"))
 
         block = loop.findall("block")
         self.assertEqual(1, len(block))
@@ -219,7 +211,7 @@ class TestGeneration(unittest.TestCase):
 
         self.assertIsNotNone(self.ast_gen.ast)
 
-        print(ET.tostring(self.ast_gen.ast, 'utf-8').decode('utf-8'))
+        # print(ET.tostring(self.ast_gen.ast, 'utf-8').decode('utf-8'))
 
         procedures = self.ast_gen.ast.findall("procedure")
         self.assertLess(0, len(procedures))
@@ -292,20 +284,14 @@ class TestGeneration(unittest.TestCase):
         else:
             lhs = operator.find("lhs")
             rhs = operator.find("rhs")
-            if lhs is None:
-                if rhs.find("operator") is not None:
-                    res = self.is_no_op(rhs.find("operator"))
-                elif rhs.find("unary") is not None:
-                    res = self.is_no_op(rhs.find("unary"))
-            else:
-                if lhs.find("operator") is not None:
-                    res = self.is_no_op(lhs.find("operator"))
-                elif lhs.find("unary") is not None:
-                    res = self.is_no_op(lhs.find("unary"))
-                elif rhs.find("operator") is not None:
-                    res = self.is_no_op(rhs.find("operator"))
-                elif rhs.find("unary") is not None:
-                    res = self.is_no_op(rhs.find("unary"))
+            if lhs.find("operator") is not None:
+                res = self.is_no_op(lhs.find("operator"))
+            elif lhs.find("unary") is not None:
+                res = self.is_no_op(lhs.find("unary"))
+            elif rhs.find("operator") is not None:
+                res = self.is_no_op(rhs.find("operator"))
+            elif rhs.find("unary") is not None:
+                res = self.is_no_op(lhs.find("unary"))
 
         return res
 

ast_generator/utils.py

@@ -1,4 +1,3 @@
-import random
 from xml.etree import ElementTree as ET
 
 from constants import GAZ_VAR_TAG, GAZ_ARG_TAG
@@ -98,84 +97,3 @@ def build_xml_element(*keys, name):
     for key in list(keys)[0]:  # TODO refactor
         elem.set(key[0], key[1])
     return elem
-
-
-def get_numberlines(settings_section: str, subsettings: list[str], excluded_values, settings):
-    assert len(subsettings) == len(excluded_values)
-
-    number_line = 0
-    cutoffs = []
-    cutoff = 0
-    options = {}
-    option = 0
-
-    valid_settings = []
-
-    for key, value in settings[settings_section].items():
-        if key in subsettings and key not in excluded_values:  # this check needs to be done recursively
-            if isinstance(value, int):
-                t = {
-                    key: value
-                }
-                valid_settings.append(t)
-            elif isinstance(value, dict):
-                valid_settings.append(value)
-            else:
-                raise TypeError("invalid setting type. Found " + str(value) + " instead of expected int or dict")
-
-    for v in range(len(valid_settings)):
-        for i in excluded_values:
-            for j in i:
-                if j in valid_settings[v]:
-                    valid_settings[v].pop(j)
-
-    for v in valid_settings:
-        if isinstance(v, dict):
-            for key, value in v.items():
-                number_line += value
-                cutoffs.append(cutoff + value)
-                cutoff += value
-                options[option] = key
-                option += 1
-        elif isinstance(v, int):
-            number_line += v
-            cutoffs.append(cutoff + v)
-            cutoff += v
-            options[option] = v
-            option += 1
-        else:
-            raise TypeError("invalid setting type. Found " + str(v) + " instead of expected int")
-
-    return options, cutoffs, number_line
-
-
-def filter_options(exclude, include, options, opts):
-    if include is not None and exclude is not None:
-        raise ValueError("Cannot specify both include and exclude")
-    elif include is not None and include in opts:
-        for i in range(len(opts)):
-            if opts[i] in include:
-                continue
-            else:
-                options.pop(opts.index(opts[i]))
-    elif exclude is not None and exclude in opts:
-        options.pop(opts.index(exclude))
-    elif include is None and exclude is None:
-        pass
-    else:
-        raise ValueError("Invalid include/exclude options " + str(include) + " " + str(exclude))
-
-
-def _choose_option(cutoffs, number_line, options):
-    op = ""
-    a = random.randint(0, number_line - 1)
-    i = 0
-    for i in range(len(cutoffs) - 1):
-        if i == 0:
-            if a < cutoffs[i]:
-                op = options[i]
-                break
-        if cutoffs[i] <= a < cutoffs[i + 1]:
-            op = options[i]
-            break
-    return op

config.yaml

@@ -6,10 +6,9 @@ generation-options:
     max-number-of-routines: 5                    # maximum number of routines (main will always be generated)
     generate-dead-code: True                     # generate dead code
     max-loop-iterations: 100                     # maximum number of iterations in a loop
-    max-globals: 5                               # maximum number of global variables
 properties:
     max-range-length: 5                          # maximum length of ranges, vectors and tuples, (AxA matrices can exist)
-    use-english-words: True                      # use english words instead of random names (if we run out, we switch to random)
+    use-english-words: True                      # use english words instead of random names (this may limit the maximum number of names)
     id-length:                                   # length of identifiers
         min: 1
         max: 5
@@ -68,12 +67,12 @@ statement-weights:                               # set to 0 for any statements y
     in-stream: 5
 
 type-weights:
-    atomic-types:
-        int: 50  # TODO change these to the gaz types
-        float: 50
-        bool: 50
-        char: 50
-        void: 0  # TODO add support for void
+    value-types:
+        integer: 50
+        real: 50
+        boolean: 50
+        character: 50
+        void: 10
     composite-types:
         vector: 20
         tuple: 5

constants.py

@@ -47,12 +47,6 @@ GAZ_STRING_KEY = "string"
 GAZ_CHAR_KEY = "char"
 GAZ_BRACKET_TAG = "brackets"
 GAZ_BREAK_TAG = "break"
-GAZ_ATOMIC_TYPE_KEY = "atomic"
-GAZ_COMPOSITE_TYPE_KEY = "composite"
-GAZ_COMPOSITE_KEY = "composite"
-GAZ_VECTOR_KEY = "vector"
-GAZ_TUPLE_KEY = "tuple"
-GAZ_MATRIX_KEY = "matrix"
 
 
 class NoneTagException(Exception):

gazprea_fuzzer.py

@@ -1,6 +1,5 @@
 import os
 import random
-import sys
 import warnings
 from contextlib import redirect_stdout
 
@@ -31,11 +30,10 @@ class Fuzzer():
         os.mkdir("fuzzer")
         os.mkdir("fuzzer/input")
         os.mkdir("fuzzer/debug")
-        os.mkdir("fuzzer/debug/ast_err")
+        os.mkdir("fuzzer/debug/ast")
         os.mkdir("fuzzer/instream")
         os.mkdir("fuzzer/outputs")
         os.mkdir("fuzzer/ground_truth")
-        os.system("cp tester_config.json fuzzer/tester_config.json")
         for i in range(self.batch):
             try:
                 self.fuzzer.fuzz()
@@ -44,7 +42,7 @@ class Fuzzer():
                 warnings.warn("None Tag Exception encountered, writing stack trace and xml to debug/ast/{}.xml\n"
                               "Look for a top-level <argument> tag or send it to Ayrton and I'll see what I can see"
                               "".format(r))
-                with open("fuzzer/debug/ast_err/{}.xml".format(r), 'w') as f:
+                with open("fuzzer/debug/ast/{}.xml".format(r), 'w') as f:
                     f.write(xml.dom.minidom.parseString(ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml())
                 continue
             dom = xml.dom.minidom.parseString(ET.tostring(self.fuzzer.ast).decode('utf-8'))
@@ -60,16 +58,9 @@ class Fuzzer():
                         except (OverflowError, ZeroDivisionError, ValueError):
                             os.system("rm -f fuzzer/ground_truth/{}_{}.py".format(self.file_name, i))
                             continue
-                        except KeyboardInterrupt:
-                            r = random.randint(0, 1000000)
-                            warnings.warn("Execution halted, result written to debug/ast/{}.xml\n"
-                                          "".format(r))
-                            with open("fuzzer/debug/ast_err/{}.xml".format(r), 'w') as f:
-                                f.write(xml.dom.minidom.parseString(ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml())
-                            sys.exit(1)
             with open("fuzzer/input/{}_{}.in".format(self.file_name, i), 'w') as f:
                 f.write(self.fuzzer.source)
-            with open("fuzzer/debug/{}_{}.xml".format(self.file_name, i), 'w') as f:
+            with open("fuzzer/debug/{}_{}.out".format(self.file_name, i), 'w') as f:
                 f.write(pretty)
                     # y.write(self.fuzzer.out)
             # with open("fuzzer/instream/{}.in".format(i), 'w') as f:

tester_config.json

@@ -1,36 +0,0 @@
-{
-  "inDir": "<inDir>",
-  "outDir": "<outDir>",
-  "inStrDir": "<inStrDir>",
-  "testedExecutablePaths": {
-    "<team id>": "<path_to_gazc_exe>"
-  },
-  "runtimes": {
-    "<team id>": "<path_to_libgazrt.so>"
-  },
-  "toolchains": {
-    "gazprea": [
-      {
-        "stepName": "gazc",
-        "executablePath": "$EXE",
-        "arguments": [
-          "$INPUT",
-          "$OUTPUT"
-          ],
-        "output": "gazc.ll",
-        "allowError": true
-      },
-      {
-        "stepName": "lli",
-        "executablePath": "/cshome/cmput415/415-resources/llvm-project/build/bin/lli",
-        "arguments": [
-          "$INPUT"
-        ],
-        "output": "-",
-        "usesRuntime": true,
-        "usesInStr": true,
-        "allowError": true
-      }
-    ]
-  }
-}