From 9f915bd80ee7f8b373c616e433da96784ec36131 Mon Sep 17 00:00:00 2001 From: ayrton Date: Sat, 25 Nov 2023 11:40:46 -0700 Subject: [PATCH] Fixed conditionals rarely evaluating Added the setting in the yaml for truthiness, and added a timeout error to the code generation. Need to fix the generation error that prevents a large amount of code from being generated... Took 1 hour 47 minutes --- ast_generator/ast_generator.py | 56 ++++++++++++- ast_generator/test/config.yaml | 4 +- ast_generator/test/test_ast_generator.py | 70 ++++++++++++++-- ast_generator/test/xml/conditional.xml | 8 ++ ast_generator/test/xml/conditional_2.xml | 8 ++ ast_generator/test/xml/conditional_3.xml | 15 ++++ ast_generator/tiny_py_unparse.py | 101 +++++++++++++++++++++++ ast_generator/utils.py | 3 +- config.yaml | 6 +- gazprea_fuzzer.py | 45 +++++++++- 10 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 ast_generator/test/xml/conditional.xml create mode 100644 ast_generator/test/xml/conditional_2.xml create mode 100644 ast_generator/test/xml/conditional_3.xml create mode 100644 ast_generator/tiny_py_unparse.py diff --git a/ast_generator/ast_generator.py b/ast_generator/ast_generator.py index 19b8e7d..b8b801e 100644 --- a/ast_generator/ast_generator.py +++ b/ast_generator/ast_generator.py @@ -3,6 +3,7 @@ import warnings from english_words import get_english_words_set +from ast_generator.tiny_py_unparse import TinyPyUnparser from ast_generator.utils import * from ast_generator.utils import filter_options, _choose_option from constants import * @@ -436,13 +437,66 @@ class AstGenerator: self.make_scoped_element(GAZ_LOOP_TAG, []) self.current_control_flow_nesting_depth += 1 - self.generate_expression(GAZ_BOOL_KEY) # the loop entry condition #TODO force true + truthiness = self.get_truthiness() + + while True: + self.generate_expression(GAZ_BOOL_KEY) # the loop entry condition #TODO force true, if false, generate a simple block and return, no need to nest code we won't use + + iterator = self.current_ast_element.iter() + next(iterator) + conditional = next(iterator) + if not self.truth_check(conditional, truthiness): + self.current_ast_element.remove(conditional) + else: + break + self.generate_block(block_type=GAZ_LOOP_TAG, loop_var=init_var) # append a variable increment and prepend a break statement if var is > max loop iterations self.current_control_flow_nesting_depth -= 1 self.exit_scoped_element(parent) + def get_truthiness(self): + return random.random() < self.settings['misc-weights']['truthiness'] + + def truth_check(self, element: ET.Element, truthiness: bool): + # evaluate the element + uncompiled = self.unparse_conditional_statement(element) + substr1 = uncompiled.find("**") + substr2 = uncompiled.find("**", substr1 + 1) + + if -1 not in [substr1, substr2]: + return False + + # print(uncompiled) + try: + res = eval(uncompiled) + except (OverflowError, ZeroDivisionError): + res = False + + # check it against truthiness and return the comparison + return res == truthiness + + def unparse_conditional_statement(self, element: ET.Element): + py_unparse = TinyPyUnparser(element, True) + unparsed = "" + if element.tag == GAZ_LIT_TAG: + py_unparse.unparse_literal(element) + unparsed = py_unparse.source + elif element.tag == GAZ_OPERATOR_TAG: + py_unparse.unparse_operator(element) + unparsed = py_unparse.source + elif element.tag == GAZ_VAR_TAG: + xml = self.current_scope.resolve(element.get("name")) + py_unparse.unparse_variable(xml) + unparsed = py_unparse.source + elif element.tag == GAZ_BRACKET_TAG: + py_unparse.unparse_brackets(element) + unparsed = py_unparse.source + else: + raise ValueError("Unknown element type for conditional argument: " + element.tag) + return unparsed + def generate_zero_declaration(self): """ @brief generate a declaration int a = 0 for some a diff --git a/ast_generator/test/config.yaml b/ast_generator/test/config.yaml index 5ad7b7a..8334169 100644 --- a/ast_generator/test/config.yaml +++ b/ast_generator/test/config.yaml @@ -90,9 +90,7 @@ misc-weights: type-qualifier-weights: const: 10 var: 60 - conditional-eval: - true: 50 - false: 50 + truthiness: 0.9 # Probability of conditionals being true block-termination-probability: 0.2 # probability for a block to terminate diff --git a/ast_generator/test/test_ast_generator.py b/ast_generator/test/test_ast_generator.py index 98fc68d..246b784 100644 --- a/ast_generator/test/test_ast_generator.py +++ b/ast_generator/test/test_ast_generator.py @@ -245,12 +245,13 @@ class TestGeneration(unittest.TestCase): self.assertFalse(self.is_no_op(operator)) def test_create_global(self): - self.ast_gen.ast = ET.Element("block") + element = ET.Element("block") + self.ast_gen.ast = element self.ast_gen.current_ast_element = self.ast_gen.ast self.ast_gen.generate_main() - global_block = self.ast_gen.current_ast_element - global_scope = self.ast_gen.current_scope + global_block = element + global_scope = self.ast_gen.current_scope.get_top_scope() self.assertIsNotNone(self.ast_gen.ast.find("procedure")) self.ast_gen.current_ast_element = self.ast_gen.ast.find("procedure") @@ -310,8 +311,10 @@ class TestGeneration(unittest.TestCase): return res def test_print_all_current_scope_vars(self): - self.ast_gen.ast = ET.Element("block") - self.ast_gen.current_ast_element = self.ast_gen.ast + element = ET.Element("block") + self.ast_gen.ast = element + self.ast_gen.current_ast_element = element + self.ast_gen.current_scope = Scope(None) self.ast_gen.generate_declaration(mut='var') self.ast_gen.generate_declaration(mut='var') self.ast_gen.generate_declaration(mut='var') @@ -319,12 +322,67 @@ class TestGeneration(unittest.TestCase): self.ast_gen.print_all_current_scope_vars() - print(ET.tostring(self.ast_gen.ast)) + + # print(ET.tostring(self.ast_gen.ast)) streams = self.ast_gen.ast.findall("stream") self.assertEqual(4, len(streams)) + def test_conditional_test_1(self): + element = ET.fromstring('') + + self.assertIsNotNone(element) + + res = self.ast_gen.truth_check(element, True) + self.assertFalse(res) + + def test_conditional_test_2(self): + element = ET.fromstring('') + self.assertIsNotNone(element) + + res = self.ast_gen.truth_check(element, True) + self.assertTrue(res) + + def test_conditional_test_3(self): + with open("xml/conditional.xml", 'r') as f: + element = ET.fromstring(f.read()) + + self.assertIsNotNone(element) + + res = self.ast_gen.truth_check(element, True) + self.assertTrue(res) + + def test_conditional_test_4(self): + with open("xml/conditional_2.xml", 'r') as f: + element = ET.fromstring(f.read()) + + self.assertIsNotNone(element) + + res = self.ast_gen.truth_check(element, True) + self.assertFalse(res) + + def test_conditional_test_5(self): + with open("xml/conditional_3.xml", 'r') as f: + element = ET.fromstring(f.read()) + + self.assertIsNotNone(element) + + res = self.ast_gen.truth_check(element, True) + self.assertTrue(res) + + # def test_conditional_test_variable(self): + # element = ET.fromstring('') + # + # var = Variable("harold", "int", "var") + # var.decl_xml = ET.fromstring('') + # self.ast_gen.current_scope.append("harold", element) + # self.assertIsNotNone(element) + # + # res = self.ast_gen.truth_check(element, True) + # self.assertTrue(res) + + def write_ast(self): dom = xml.dom.minidom.parseString(ET.tostring(self.ast_gen.ast).decode('utf-8')) pretty: str = dom.toprettyxml() diff --git a/ast_generator/test/xml/conditional.xml b/ast_generator/test/xml/conditional.xml new file mode 100644 index 0000000..1e90dff --- /dev/null +++ b/ast_generator/test/xml/conditional.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/ast_generator/test/xml/conditional_2.xml b/ast_generator/test/xml/conditional_2.xml new file mode 100644 index 0000000..8610df4 --- /dev/null +++ b/ast_generator/test/xml/conditional_2.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/ast_generator/test/xml/conditional_3.xml b/ast_generator/test/xml/conditional_3.xml new file mode 100644 index 0000000..8e8f9cc --- /dev/null +++ b/ast_generator/test/xml/conditional_3.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ast_generator/tiny_py_unparse.py b/ast_generator/tiny_py_unparse.py new file mode 100644 index 0000000..6468e84 --- /dev/null +++ b/ast_generator/tiny_py_unparse.py @@ -0,0 +1,101 @@ +from ast_parser.general_unparser import GeneralUnparser + + +import warnings +from xml.etree import ElementTree as ET + +from ast_parser.gaz_unparser import GazUnparser +from ast_parser.general_unparser import GeneralUnparser +from constants import * + + +def to_python_type(ty): + if ty == GAZ_INT_KEY: + return "int" + elif ty == GAZ_BOOL_KEY: + return "bool" + elif ty == GAZ_FLOAT_KEY: + return "float" + elif ty == GAZ_CHAR_KEY: + return "str" + elif ty == GAZ_STRING_KEY: + return "str" + else: + raise Exception("Unknown type: " + ty) + + +def to_python_op(param, ty): + if param == "negation" or param == "subtraction": + return "-" + elif param == "addition" or param == "noop": + return "+" + elif param == "multiplication": + return "*" + elif param == "division" and ty != GAZ_INT_KEY: + return "/" + elif param == "division" and ty == GAZ_INT_KEY: + return "//" + elif param == "modulo": + return "%" + elif param == "power": + return "**" + elif param == "equality": + return "==" + elif param == "inequality": + return "!=" + elif param == "less-than": + return "<" + elif param == "less-than-or-equal": + return "<=" + elif param == "greater-than": + return ">" + elif param == "greater-than-or-equal": + return ">=" + elif param == "xor": + return "!=" + else: + warnings.warn("Warning, unknown operator: " + param) + return param + + +class TinyPyUnparser(GeneralUnparser): + def __init__(self, ast: ET.Element, debug=False): + super().__init__(ast, debug, + endline='\n', + outstream_begin_delimiter="gprint(", + outstream_end_delimiter=", end='')", + function_return_type_indicator_predicate="->", + loop_start_delimiter="while ", + loop_end_delimiter=":", + conditional_case_delimiter="elif ", + conditional_start_delimiter="if ", + conditional_else_delimiter="else:", + conditional_end_delimiter=":", + block_start_delimiter="", + block_end_delimiter="", # TODO can this contain the pass? + strip_conditionals=True) + + self.source = '' + + def format_variable(self, mut, ty, name, declaration: bool = False): + if declaration: + return "{}: {}".format(name, ty) + else: + return "{}".format(name) + + def translate_value(self, val): + return str(val) + + def translate_op(self, param, ty=None): + return to_python_op(param, ty) + + def translate_type(self, ty): + return to_python_type(ty) + + def unparse_block(self, node): + raise TypeError("Cannot unparse blocks for this type of evaluation") + + def setup(self): + pass + + diff --git a/ast_generator/utils.py b/ast_generator/utils.py index a4d723f..292306e 100644 --- a/ast_generator/utils.py +++ b/ast_generator/utils.py @@ -5,12 +5,13 @@ from constants import GAZ_VAR_TAG, GAZ_ARG_TAG class Variable: - def __init__(self, name: str, type: str, qualifier: str, value: any = None): + def __init__(self, name: str, type: str, qualifier: str, value: any = None): # decl_xml: ET.Element = None, self.name = name self.type = type self.value = value self.qualifier = qualifier self.xml = self._build_xml() + # self.decl_xml = decl_xml def _build_xml(self): args = [ diff --git a/config.yaml b/config.yaml index 5ad7b7a..5010022 100644 --- a/config.yaml +++ b/config.yaml @@ -4,7 +4,7 @@ generation-options: max-nesting-depth: 5 # maximum nesting depth for statements max-conditionals-loops: 5 # maximum number of loops/conditionals per routine max-number-of-routines: 5 # maximum number of routines (main will always be generated) - generate-dead-code: True # generate dead code + generate-dead-code: False # generate dead code max-loop-iterations: 100 # maximum number of iterations in a loop max-globals: 5 # maximum number of global variables properties: @@ -90,9 +90,7 @@ misc-weights: type-qualifier-weights: const: 10 var: 60 - conditional-eval: - true: 50 - false: 50 + truthiness: 0.9 # Probability of conditionals being true block-termination-probability: 0.2 # probability for a block to terminate diff --git a/gazprea_fuzzer.py b/gazprea_fuzzer.py index e7d2722..c7088f6 100644 --- a/gazprea_fuzzer.py +++ b/gazprea_fuzzer.py @@ -14,6 +14,8 @@ import xml.etree.ElementTree as ET from constants import NoneTagException +import signal as sig + def gprint(expr, end=''): if type(expr) is bool: @@ -37,6 +39,9 @@ class Fuzzer(): self.fuzzer = fz.GazpreaFuzzer(config) def fuzz(self): + sig.signal(sig.SIGALRM, handler) + sig.alarm(30) + base_dir = os.getcwd() fuzzer_root = base_dir + '/fuzzer' @@ -71,7 +76,10 @@ class Fuzzer(): "Look for a top-level tag or send it to Ayrton and I'll see what I can see" "".format(r)) with open(f"{fuzzer_asterr}/{r}.xml", 'w') as f: - f.write(xml.dom.minidom.parseString(ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml()) + try: + f.write(xml.dom.minidom.parseString(ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml()) + except AttributeError: + print("Failed to write to the file", file=sys.stderr) if i - 1 >= min_i: i -= 1 else: @@ -90,7 +98,7 @@ class Fuzzer(): exec(read, globals()) except (OverflowError, ZeroDivisionError, ValueError, TypeError, SyntaxError): os.system("rm -f {}/{}_{}.py".format(fuzzer_ground_truth, self.file_name, i)) - os.system("rm -f {}/{}_{}.py".format(fuzzer_outputs, self.file_name, i)) + os.system("rm -f {}/{}_{}.out".format(fuzzer_outputs, self.file_name, i)) warnings.warn("Runtime error encountered, retrying") if i - 1 >= min_i: i -= 1 @@ -105,6 +113,31 @@ class Fuzzer(): f.write(xml.dom.minidom.parseString( ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml()) sys.exit(1) + except TimeoutError: + r = random.randint(0, 1000000) + warnings.warn("Execution timed out, result written to debug/ast/{}.xml\n" + "".format(r)) + with open("{}/{}.xml".format(fuzzer_asterr, r), 'w') as f: + f.write(xml.dom.minidom.parseString( + ET.tostring(self.fuzzer.ast).decode('utf-8')).toprettyxml()) + if i - 1 >= min_i: + i -= 1 + else: + i = min_i + continue + + with open("{}/{}_{}.out".format(fuzzer_outputs, self.file_name, i), 'r') as y: + out = y.read() + if out == "": + os.system("rm -f {}/{}_{}.py".format(fuzzer_ground_truth, self.file_name, i)) + os.system("rm -f {}/{}_{}.out".format(fuzzer_outputs, self.file_name, i)) + if i - 1 >= min_i: + i -= 1 + else: + i = min_i + continue + + with open("{}/{}_{}.in".format(fuzzer_input, self.file_name, i), 'w') as f: f.write(self.fuzzer.source) with open("{}/{}_{}.xml".format(fuzzer_debug, self.file_name, i), 'w') as f: @@ -114,11 +147,19 @@ class Fuzzer(): # f.write(self.fuzzer.source) # with open("fuzzer/outputs/{}.out".format(i), 'w') as f: # f.write(self.fuzzer.out) + print("test {}/{} generated".format(i, self.batch)) i += 1 min_i = i +def handler(signum, frame): + print("Execution Timeout") + raise TimeoutError + + if __name__ == '__main__': + + parser = argparse.ArgumentParser( description='Procedurally generate a test case for Gazprea' )