From f911c05e3d1436cadaf9b13a9f9c889c23712b99 Mon Sep 17 00:00:00 2001 From: ayrton Date: Sat, 18 Nov 2023 12:21:52 -0700 Subject: [PATCH] Refactoring of the ast_parser class Took 48 minutes --- ast_generator/ast_generator.py | 2 +- ast_generator/gazprea_ast_grammar.py | 2 +- ast_parser/ast_parser.py | 165 ----------------- ast_parser/gaz_unparser.py | 166 ++++++++++++++++++ ast_parser/general_unparser.py | 82 +++++++++ ast_parser/python_unparser.py | 0 .../{test_parse_code.py => test_unparse.py} | 35 ++-- ast_generator/constants.py => constants.py | 2 + gazprea_fuzzer.py | 2 +- 9 files changed, 264 insertions(+), 192 deletions(-) delete mode 100644 ast_parser/ast_parser.py create mode 100644 ast_parser/gaz_unparser.py create mode 100644 ast_parser/general_unparser.py create mode 100644 ast_parser/python_unparser.py rename ast_parser/test/{test_parse_code.py => test_unparse.py} (83%) rename ast_generator/constants.py => constants.py (96%) diff --git a/ast_generator/ast_generator.py b/ast_generator/ast_generator.py index 64e146a..80f5e97 100644 --- a/ast_generator/ast_generator.py +++ b/ast_generator/ast_generator.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from english_words import get_english_words_set -from ast_generator.constants import * +from constants import * class AstGenerator: diff --git a/ast_generator/gazprea_ast_grammar.py b/ast_generator/gazprea_ast_grammar.py index d714e70..a35b590 100644 --- a/ast_generator/gazprea_ast_grammar.py +++ b/ast_generator/gazprea_ast_grammar.py @@ -1,4 +1,4 @@ -from ast_generator.constants import Grammar +from constants import Grammar GAZPREA_TOP_LEVEL: Grammar = { # Top level elements diff --git a/ast_parser/ast_parser.py b/ast_parser/ast_parser.py deleted file mode 100644 index 9b700f0..0000000 --- a/ast_parser/ast_parser.py +++ /dev/null @@ -1,165 +0,0 @@ -import os -import shutil -import xml.etree.ElementTree as ET - - -def to_gazprea_type(ty: str): - if ty == "int": - return "integer" - elif ty == "bool": - return "boolean" - elif ty == "string": - return "string" - elif ty == 'void': - return 'void' - else: - raise Exception("Unknown type: " + ty) - - -class AstParser: - def __init__(self, input: str or ET.Element, from_xml: bool = False): - if from_xml: - self.xml = input - self.input = None - else: - self.input = input - self.xml = None - - self.indentation = 0 - - - def parse(self): - if os.path.isdir("/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp"): - os.system("rm -rf /home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp") - os.mkdir("/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp") - else: - os.mkdir("/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp") - with open("/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp/input.in", "x") as f: - f.write(self.input) - os.system("/home/stormblessed/.local/bin/gazc " - "/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp/input.in " - "/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp/output.out " - "/home/stormblessed/Code/gazprea_fuzzer_v0.2/ast_parser/test/.tmp/xml.xml") - self.xml = ET.parse(".tmp/xml.xml") - - def unparse(self): - """ - @brief unparses the xml into valid gazprea code - - :return: a string of valid gazprea code - """ - self.input = "" - for node in self.xml: - self._unparse_node(node) - - def _unparse_node(self, node): - if node.tag not in ["variable", "rhs", "lhs", "literal", "operator"]: - self.input += " " * self.indentation - - if node.tag == "block": - self._block_unparse(node) - elif node.tag == "declaration": - self._declaration_unparse(node) - elif node.tag == "return": - self._return_unparse(node) - elif node.tag == "operator": - self._operator_unparse(node) - elif node.tag == "stream": - self._stream_unparse(node) - elif node.tag == "literal": - self._literal_unparse(node) - elif node.tag == "procedure" or node.tag == "function": - self._routine_unparse(node) - elif node.tag == "variable": - self._variable_unparse(node) - elif node.tag == "rhs" or node.tag == "lhs": - self._xhs_unparse(node) - elif node.tag == "literal": - self._literal_unparse(node) - else: - raise Exception("Unknown tag: " + node.tag) - - def _block_unparse(self, node): - self.input += "{\n" - self.indentation += 4 - for child in node: - self._unparse_node(child) - self.indentation -= 4 - self.input += "}\n\n" - - def _declaration_unparse(self, node): - variable = node.find("variable") - rhs = node.find("rhs") - self._variable_unparse(variable, True) - self.input += "=" - self._unparse_node(rhs) - self.input += ";\n" - - def _variable_unparse(self, node, is_declaration = False): - if is_declaration: - mut = node.get("mut") - type = to_gazprea_type(node.get("type")) - name = node.get("name") - - self.input += "{} {} {} ".format(mut, type, name) - else: - self.input += " {} ".format(node.get("name")) - - def _stream_unparse(self, node): - for child in node: - self._unparse_node(child) - - self.input += "-> {};\n".format(node.get("type")) - - def _literal_unparse(self, node): - self.input += " {} ".format(node.get("value")) - - def _xhs_unparse(self, node): - for child in node: - self._unparse_node(child) - - def _operator_unparse(self, node): - self._xhs_unparse(node.find("lhs")) - self.input += "{}".format(node.get("op")) - self._xhs_unparse(node.find("rhs")) - - def _return_unparse(self, node): - self.input += "return" - for child in node: - self._unparse_node(child) - self.input += ";\n" - - def _routine_unparse(self, node): - return_type = "" - if node.get("return_type") != "": - return_type = "returns " + to_gazprea_type(node.get("return_type")) - - self.input += "{} {}{} {} ".format( - node.tag, - node.get("name"), - node.get("args"), - return_type, - ) - - for child in node: - self._unparse_node(child) - - # cls.input += "}\n\n" #blocks are already there - - -if __name__ == '__main__': - input = """ - function art() returns integer { - return 3; - } - procedure main() returns integer { - integer b = art(); - integer a = 1; - a * 42 -> std_output; - return 0; - } - """ - - parser = AstParser(input) - parser.parse() - diff --git a/ast_parser/gaz_unparser.py b/ast_parser/gaz_unparser.py new file mode 100644 index 0000000..546b416 --- /dev/null +++ b/ast_parser/gaz_unparser.py @@ -0,0 +1,166 @@ +import os +import shutil +import xml.etree.ElementTree as ET + +from ast_parser.general_unparser import GeneralUnparser +from constants import * + + +def to_gazprea_type(ty: str): #TODO implement the compound types + if ty == GAZ_INT_KEY: + return "integer" + elif ty == GAZ_BOOL_KEY: + return "boolean" + elif ty == GAZ_FLOAT_KEY: + return "real" + elif ty == GAZ_CHAR_KEY: + return "character" + elif ty == GAZ_STRING_KEY: + return "string[*]" + else: + raise Exception("Unknown type: " + ty) + + +class AstParser(GeneralUnparser): + def __init__(self, input: str or ET.Element, from_xml: bool = False): + if from_xml: + self.xml = input + self.input = None + else: + self.input = input + self.xml = None + + self.indentation = 0 + + def unparse(self): + """ + @brief unparses the xml into valid gazprea code + + :return: a string of valid gazprea code + """ + self.input = "" + for node in self.xml: + self.unparse_node(node) + + def unparse_node(self, node): + if node.tag not in [GAZ_VAR_TAG, GAZ_RHS_TAG, GAZ_LHS_TAG, GAZ_LIT_TAG, GAZ_OPERATOR_TAG]: + self.input += " " * self.indentation + + if node.tag == GAZ_BLOCK_TAG: + self.unparse_block(node) + elif node.tag == GAZ_DECLARATION_TAG: + self.unparse_declaration(node) + elif node.tag == GAZ_RETURN_TAG: + self.unparse_return(node) + elif node.tag == GAZ_OPERATOR_TAG: + self.unparse_operator(node) + elif node.tag == GAZ_STREAM_TAG: + self.unparse_stream(node) + elif node.tag == GAZ_LIT_TAG: + self.unparse_literal(node) + elif node.tag == GAZ_PROCEDURE_TAG or node.tag == GAZ_FUNCTION_TAG: + self.unparse_routine(node) + elif node.tag == GAZ_VAR_TAG: + self.unparse_variable(node) + elif node.tag == GAZ_RHS_TAG or node.tag == GAZ_LHS_TAG: + self._xhs_unparse(node) + elif node.tag == GAZ_LIT_TAG: + self.unparse_literal(node) + else: + raise Exception("Unknown tag: " + node.tag) + + def unparse_block(self, node): + self.input += "{\n" + self.indentation += 4 + for child in node: + self.unparse_node(child) + self.indentation -= 4 + self.input += "}\n\n" + + def unparse_declaration(self, node): + variable = node.find(GAZ_VAR_TAG) + rhs = node.find(GAZ_RHS_TAG) + self.unparse_variable(variable, True) + self.input += " = " + self.unparse_node(rhs) + self.input += ";\n" + + def unparse_variable(self, node, is_declaration = False): + if is_declaration: + mut = node.get(GAZ_QUALIFIER_KEY) + type = to_gazprea_type(node.get(GAZ_TY_KEY)) + name = node.get(GAZ_NAME_KEY) + + self.input += "{} {} {}".format(mut, type, name) + else: + self.input += "{}".format(node.get(GAZ_NAME_KEY)) + + def unparse_stream(self, node): + if node.tag == GAZ_OUT_STREAM: + self.unparse_outstream(node) + elif node.tag == GAZ_IN_STREAM: + self.unparse_instream(node) + + def unparse_instream(self, node): + for child in node: + self.unparse_node(child) + + self.input += "<- {};\n".format(GAZ_IN_STREAM) + + def unparse_outstream(self, node): + for child in node: + self.unparse_node(child) + + self.input += "-> {};\n".format(GAZ_OUT_STREAM) + + def unparse_literal(self, node): + self.input += "{}".format(node.get(GAZ_VAL_KEY)) + + def _xhs_unparse(self, node): + for child in node: + self.unparse_node(child) + + def unparse_operator(self, node): + self._xhs_unparse(node.find(GAZ_LHS_TAG)) + self.input += " {} ".format(node.get("op")) + self._xhs_unparse(node.find(GAZ_RHS_TAG)) + + def unparse_return(self, node): + self.input += "return " + for child in node: + self.unparse_node(child) + self.input += ";\n" + + def unparse_routine(self, node): + return_type = "" + if node.get(GAZ_RETURN_KEY) not in [None, GAZ_VOID_TYPE, ""]: + return_type = "returns " + to_gazprea_type(node.get(GAZ_RETURN_KEY)) + + args = self.unparse_argument(node.findall(GAZ_ARG_TAG)) + + self.input += "{} {}{} {} ".format( + node.tag, + node.get(GAZ_NAME_KEY), + args, + return_type, + ) + + for child in node: + self.unparse_node(child) + + # cls.input += "}\n\n" #blocks are already there + + def unparse_argument(self, nodes): + if len(nodes) == 0: + return "()" + args = "(" + for i in range(len(nodes)): + args += self._unparse_arg(nodes[i]) + if i < len(nodes) - 1: + args += ", " + + args += ")" + return args + + def _unparse_arg(self, node): + return "{} {}".format(to_gazprea_type(node.get(GAZ_TY_KEY)), node.get(GAZ_NAME_KEY)) diff --git a/ast_parser/general_unparser.py b/ast_parser/general_unparser.py new file mode 100644 index 0000000..c57740f --- /dev/null +++ b/ast_parser/general_unparser.py @@ -0,0 +1,82 @@ +import xml.etree.ElementTree as ET +from abc import ABC, abstractmethod + + +class GeneralUnparser(ABC): + @abstractmethod + def unparse(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_node(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_top_block(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_block(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_routine(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_argument(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_statement(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_expression(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_declaration(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_operator(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_unary(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_stream(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_assignment(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_conditional(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_loop(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_routine_call(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_return(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_literal(self, element_in: ET.Element): + pass + + @abstractmethod + def unparse_variable(self, element_in: ET.Element): + pass + + diff --git a/ast_parser/python_unparser.py b/ast_parser/python_unparser.py new file mode 100644 index 0000000..e69de29 diff --git a/ast_parser/test/test_parse_code.py b/ast_parser/test/test_unparse.py similarity index 83% rename from ast_parser/test/test_parse_code.py rename to ast_parser/test/test_unparse.py index 0b90547..6ae7798 100644 --- a/ast_parser/test/test_parse_code.py +++ b/ast_parser/test/test_unparse.py @@ -1,36 +1,23 @@ import unittest -from ast_parser.ast_parser import AstParser +from ast_parser.gaz_unparser import AstParser class TestParseCode(unittest.TestCase): - def test_parse_code(self): - input = """ - procedure main() returns integer { - integer a = 1; - a * 42 -> std_output; - return 0; - } - """ - - parser = AstParser(input) - parser.parse() - self.assertIsNotNone(parser.xml) - self.assertEqual(parser.xml.getroot().tag, "block") def test_unparse_variable_regular(self): input = '' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) - self.assertEqual(" a ", parser.input) + self.assertEqual("a", parser.input) def test_unparse_variable_declaration(self): input = '' parser = AstParser(input, True) parser.input = "" - parser._variable_unparse(parser.xml, True) + parser.unparse_variable(parser.xml, True) self.assertIsNotNone(parser.input) self.assertEqual("var integer a ", parser.input) @@ -39,7 +26,7 @@ class TestParseCode(unittest.TestCase): input = '' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual(" 1 ", parser.input) @@ -47,7 +34,7 @@ class TestParseCode(unittest.TestCase): input = '' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual("var integer a = 1 ;\n", parser.input) @@ -55,7 +42,7 @@ class TestParseCode(unittest.TestCase): input = ' ' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual(" a * 42 -> std_output;\n", parser.input) @@ -63,7 +50,7 @@ class TestParseCode(unittest.TestCase): input = ' ' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual("{\n var integer a = 1 ;\n a * 42 -> std_output;\n return 0 ;\n}\n\n", parser.input) @@ -71,7 +58,7 @@ class TestParseCode(unittest.TestCase): input = ' ' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual(" a * 42 ", parser.input) @@ -79,7 +66,7 @@ class TestParseCode(unittest.TestCase): input = ' ' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) self.assertEqual("return 0 ;\n", parser.input) @@ -87,7 +74,7 @@ class TestParseCode(unittest.TestCase): input = '' parser = AstParser(input, True) parser.input = "" - parser._unparse_node(parser.xml) + parser.unparse_node(parser.xml) self.assertIsNotNone(parser.input) i = ' ' * parser.indentation self.assertEqual("procedure main() returns integer {\n var integer a = 1 ;\n a * 42 -> std_output;\n return 0 ;\n}\n\n", parser.input) diff --git a/ast_generator/constants.py b/constants.py similarity index 96% rename from ast_generator/constants.py rename to constants.py index 72dd7e7..0937ba3 100644 --- a/ast_generator/constants.py +++ b/constants.py @@ -43,3 +43,5 @@ GAZ_LOOP_TAG = "loop" GAZ_TRUE_BLOCK_TAG = "true" GAZ_FALSE_BLOCK_TAG = "false" GAZ_ARG_TAG = "argument" +GAZ_STRING_KEY = "string" +GAZ_CHAR_KEY = "char" diff --git a/gazprea_fuzzer.py b/gazprea_fuzzer.py index fd51e5c..7898d61 100644 --- a/gazprea_fuzzer.py +++ b/gazprea_fuzzer.py @@ -7,7 +7,7 @@ from isla.solver import ISLaSolver import ast_generator.ast_generator from ast_parser.ast_solver import AstSolver -from ast_parser.ast_parser import AstParser +from ast_parser.gaz_unparser import AstParser from ast_generator import gazprea_ast_grammar import xml.etree.ElementTree as ET import xml.dom.minidom