From 3ceae2b41814bf826352ac52041fa3200b0d22b0 Mon Sep 17 00:00:00 2001 From: Jonatan Waern Date: Thu, 31 Mar 2022 13:32:40 +0200 Subject: [PATCH 1/3] Add UDI backend UDI (unrolled device info) is an extensive description of the parameters AND semantics of registers within a device --- Makefile | 1 + py/dml/dmlc.py | 27 +++++- py/dml/udi_backend.py | 207 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 py/dml/udi_backend.py diff --git a/Makefile b/Makefile index a4079e7eb..80d929902 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ PYFILES := dml/__init__.py \ dml/ast.py \ dml/c_backend.py \ dml/g_backend.py \ + dml/udi_backend.py \ dml/codegen.py \ dml/crep.py \ dml/ctree.py \ diff --git a/py/dml/dmlc.py b/py/dml/dmlc.py index ff742fb5d..cb3e020c3 100644 --- a/py/dml/dmlc.py +++ b/py/dml/dmlc.py @@ -16,6 +16,7 @@ import dml.c_backend import dml.info_backend import dml.g_backend +import dml.udi_backend import dml.globals import dml.dmlparse from .logging import * @@ -347,6 +348,21 @@ def set_debuggable(option, opt, value, parser): callback = set_debuggable, help = 'generate artifacts and C code that allow for easier debugging') + #
-g
+ #
Generate extensive information about banks and registers within a + # device, suitable for static, off-line, device validation.
+ optpar.add_option( + '-u', "--udi", dest = 'udi_enabled', action = 'store_true', + help = 'generate static unrolled device info') + #
-g
+ #
Specify a template that is of interest to the unrolled device info. + # Implies --udi.
+ optpar.add_option( + "--udi-template", dest = 'udi_extra_templates', action = 'append', + metavar = 'TEMPLATE', + default = [], + help = 'specify tracked templates for unrolled device info') + #
--warn=tag
#
Enable selected warnings. The tags can be found using # the -T option.
@@ -642,17 +658,26 @@ def show_illegal_attributes(option, opt, value, parser): dml.info_backend.generate(dev, outputbase + '.xml') logtime("info") - if output_c: + + if output_c or options.output_udi: dml.c_backend.generate(dev, headers, footers, outputbase, [inputfilename] + list(imported.keys()), options.full_module) logtime("c") structure.check_unused_and_warn(dev) + + if output_c: if dml.globals.debuggable: dml.g_backend.generate(expr_util.param_str(dev, 'classname'), dev, dml_version, outputbase + '.g') logtime("g") + if options.udi_enabled or options.udi_extra_templates: + dml.udi_backend.generate(expr_util.param_str(dev, 'classname'), + options.udi_extra_templates, + dev, dml_version, outputbase + '.udi') + logtime("udi") + if not logging.failure: # report WREF for broken unused parameters. But just ignore it # if there's already a hard error somewhere, because if the diff --git a/py/dml/udi_backend.py b/py/dml/udi_backend.py new file mode 100644 index 000000000..38557e03c --- /dev/null +++ b/py/dml/udi_backend.py @@ -0,0 +1,207 @@ +# © 2021-2022 Intel Corporation +# SPDX-License-Identifier: MPL-2.0 + +# Generates Unrolled Device Info + +__all__ = ('generate',) + +import itertools +import json +from . import ctree, crep, expr_util, types +from . import logging +from .expr import mkLit +from .logging import dollar +from .codegen import eval_initializer +from .symtab import global_scope +import dml.globals + +# Names of traits to track should be listed here +# Note: user can specify additional templates to search for +# through commandline arguments +interesting_traits = { + "read_only", + "write_only", + "ignore_write", + "ignore_read", + "read_zero", + "write_1_clears", + "clear_on_read", + "write_1_only", + "write_0_only", + "read_constant", + "constant", + "no_reset", + "reserved", + "unimpl", + "read_unimpl", + "write_unimpl", + "silent_unimpl", + "unmapped", + "sticky", + "function_mapped_bank", + "poreset", + "hreset", + "sreset", +} + +def array_info(obj): + return list(zip(obj._arraylens, obj._idxvars)) + +def enc(expr): + if expr.constant: + val = expr.value + if isinstance(val, list): + return [enc(e) for e in val] + elif isinstance(val, bytes): + return val.decode("utf-8") + else: + return val + else: + return str(expr) + +def en_parameter(node, indices = []): + try: + expr = node.get_expr(tuple(indices)) + except logging.DMLError: + import os, sys, traceback + if os.getenv('DMLC_DEBUG'): + sys.stderr.write("error encoding parameter: %s" % node) + traceback.print_exc() + return None + else: + return {node.name : enc(expr)} + +def evaluate_init_value(node, indices): + if node.astinit: + return enc(eval_initializer( + node.site, node._type, node.astinit, + ctree.Location(node.parent, tuple(indices)), + global_scope, True).as_expr(node._type)) + else: + return None + +def en_data(node, indices = []): + return {node.name : { "init_value" : evaluate_init_value(node, indices) }} + +def en_var(node, indices = []): + return {node.name : { "type" : str(node._type), + "init_value" : evaluate_init_value(node, indices) }} + +def en_compobj(node, indices = []): + def do_enc(node, proper_indices): + content = {} + for trait in node.traits.ancestors: + if trait.name in interesting_traits: + content.setdefault("templates", []).append(trait.name) + en_subobjs(content, node, proper_indices) + return content + + new_indices = [[ctree.mkIntegerConstant(node.site, i, False) + for i in range(arrsize)] for arrsize in node._arraylens] + if new_indices: + new_indices = itertools.product(*new_indices) + expanded = {} + for extra_indices in new_indices: + expanded["{}{}".format( + node.name, + "".join("[{}]".format(enc(ind)) for + ind in extra_indices))] = do_enc( + node, indices + list(extra_indices)) + return expanded + else: + content = {} + en_subobjs(content, node, indices) + return {node.name : do_enc(node, indices)} + +obj_encoder_map = { + 'bank' : ("banks", en_compobj), + 'attribute' : ("attributes", en_compobj), + 'parameter' : ("parameters", en_parameter), + 'register' : ("registers", en_compobj), + 'field' : ("fields", en_compobj), + 'data' : ("data", en_data), + 'session' : ("data", en_var), + 'saved' : ("data", en_var) +} + +def en_obj(output, obj, indices = []): + if obj.objtype in obj_encoder_map: + (collection, encoder) = obj_encoder_map[obj.objtype] + subobjdict = encoder(obj, indices) + output.setdefault(collection, {}).update(subobjdict) + # Group is a special case, merge subobj names into the group name + elif obj.objtype == 'group': + for s in subobjs(obj): + subdict = encoder(s, indices) + for sub in subobjdict: + output.setdefault( + collection, + {})["{}.{}".format(obj.name, sub)] = subdict[sub] + +def subobjs(node): + for s in node.get_components(): + # Skip these + if s.name.startswith("_"): + continue + + # skip auto and non-interested parameters + if s.objtype == 'parameter': + if s.name in ('this', 'name', 'qname', 'parent', 'index', + 'indexvar', 'shown_desc', 'objtype', + 'dev', 'bank', + 'documentation', 'shown_documentation', + 'limitations', 'shown_limitations', + 'dml_1_4'): + continue + if (node.objtype == 'device' and + s.name in ('obj', 'logobj', 'simics_api_version', + 'banks', 'simics_bool_is_int')): + continue + if (node.objtype == 'bank' and + s.name in ('mapped_registers', 'unmapped_registers', + 'numbered_registers')): + continue + if (node.objtype == 'register' and + s.name in ('notinregister', 'fields', '_regname')): + continue + if (node.objtype == 'field' and + s.name in ('notinfield', 'reg',)): + continue + # skip implicit field + if s.objtype == 'field' and not s.name: + continue + yield s + +def en_subobjs(output, obj, indices = []): + for s in subobjs(obj): + en_obj(output, s, indices) + +# layout: +# {classname: +# DML: , +# banks: { +# : { +# stuff, +# registers: { +# : { +# stuff, +# fields: { +# : {stuff} +# }, +# }, +# }, +# }, +# } +# } +def generate(classname, extra_templates, device, dml_version, out_file): + interesting_traits.update(set(extra_templates)) + # Setting things up like this, allows easy combination and sorting of + # devices in later tools + complete_json = { + classname : { + "DML" : dml_version, + } + } + complete_json[classname].update(en_compobj(device)[device.name]) + with open(out_file, "w") as outfile: + json.dump(complete_json, outfile) From 31f03f04dbe8d870fffb147eca02c9a3f39819ac Mon Sep 17 00:00:00 2001 From: Jonatan Waern Date: Tue, 5 Apr 2022 11:56:33 +0200 Subject: [PATCH 2/3] Fix behaviour of group arrays --- py/dml/udi_backend.py | 48 ++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/py/dml/udi_backend.py b/py/dml/udi_backend.py index 38557e03c..7b51cc7d9 100644 --- a/py/dml/udi_backend.py +++ b/py/dml/udi_backend.py @@ -102,11 +102,10 @@ def do_enc(node, proper_indices): new_indices = itertools.product(*new_indices) expanded = {} for extra_indices in new_indices: - expanded["{}{}".format( - node.name, - "".join("[{}]".format(enc(ind)) for - ind in extra_indices))] = do_enc( - node, indices + list(extra_indices)) + full_name = "{}{}".format(node.name, + "".join("[{}]".format(enc(ind)) for + ind in extra_indices)) + expanded[full_name] = do_enc(node, indices + list(extra_indices)) return expanded else: content = {} @@ -131,16 +130,36 @@ def en_obj(output, obj, indices = []): output.setdefault(collection, {}).update(subobjdict) # Group is a special case, merge subobj names into the group name elif obj.objtype == 'group': - for s in subobjs(obj): - subdict = encoder(s, indices) - for sub in subobjdict: - output.setdefault( - collection, - {})["{}.{}".format(obj.name, sub)] = subdict[sub] + new_indices = [[ctree.mkIntegerConstant(obj.site, i, False) + for i in range(arrsize)] for arrsize in obj._arraylens] + if new_indices: + new_indices = itertools.product(*new_indices) + for extra_indices in new_indices: + full_name = "{}{}".format(obj.name, + "".join("[{}]".format(enc(ind)) for + ind in extra_indices)) + for s in subobjs(obj): + (collection, encoder) = obj_encoder_map[s.objtype] + subdict = encoder(s, indices + list(extra_indices)) + for sub in subdict: + output.setdefault( + collection, + {})["{}.{}".format(full_name, sub)] = subdict[sub] + else: + for s in subobjs(obj): + (collection, encoder) = obj_encoder_map[s.objtype] + subdict = encoder(s, indices) + for sub in subobjdict: + output.setdefault( + collection, + {})["{}.{}".format(obj.name, sub)] = subdict[sub] def subobjs(node): for s in node.get_components(): - # Skip these + # skip implicit field + if s.objtype == 'field' and not s.name: + continue + # skip "internal" objects if s.name.startswith("_"): continue @@ -151,7 +170,7 @@ def subobjs(node): 'dev', 'bank', 'documentation', 'shown_documentation', 'limitations', 'shown_limitations', - 'dml_1_4'): + 'dml_1_4', "dml_1_2"): continue if (node.objtype == 'device' and s.name in ('obj', 'logobj', 'simics_api_version', @@ -167,9 +186,6 @@ def subobjs(node): if (node.objtype == 'field' and s.name in ('notinfield', 'reg',)): continue - # skip implicit field - if s.objtype == 'field' and not s.name: - continue yield s def en_subobjs(output, obj, indices = []): From 0b437a3a4dbe915ecb26c5c1575df16d33620b26 Mon Sep 17 00:00:00 2001 From: Jonatan Waern Date: Mon, 11 Apr 2022 09:03:52 +0200 Subject: [PATCH 3/3] Move towards a dump-everything strategy We are still not dumping every single parameter, some of them are clearly uninteresting --- py/dml/dmlc.py | 16 ++-------- py/dml/udi_backend.py | 69 +++++-------------------------------------- 2 files changed, 10 insertions(+), 75 deletions(-) diff --git a/py/dml/dmlc.py b/py/dml/dmlc.py index cb3e020c3..85015b02e 100644 --- a/py/dml/dmlc.py +++ b/py/dml/dmlc.py @@ -354,14 +354,6 @@ def set_debuggable(option, opt, value, parser): optpar.add_option( '-u', "--udi", dest = 'udi_enabled', action = 'store_true', help = 'generate static unrolled device info') - #
-g
- #
Specify a template that is of interest to the unrolled device info. - # Implies --udi.
- optpar.add_option( - "--udi-template", dest = 'udi_extra_templates', action = 'append', - metavar = 'TEMPLATE', - default = [], - help = 'specify tracked templates for unrolled device info') #
--warn=tag
#
Enable selected warnings. The tags can be found using @@ -658,23 +650,19 @@ def show_illegal_attributes(option, opt, value, parser): dml.info_backend.generate(dev, outputbase + '.xml') logtime("info") - - if output_c or options.output_udi: + if output_c: dml.c_backend.generate(dev, headers, footers, outputbase, [inputfilename] + list(imported.keys()), options.full_module) logtime("c") structure.check_unused_and_warn(dev) - - if output_c: if dml.globals.debuggable: dml.g_backend.generate(expr_util.param_str(dev, 'classname'), dev, dml_version, outputbase + '.g') logtime("g") - if options.udi_enabled or options.udi_extra_templates: + if options.udi_enabled: dml.udi_backend.generate(expr_util.param_str(dev, 'classname'), - options.udi_extra_templates, dev, dml_version, outputbase + '.udi') logtime("udi") diff --git a/py/dml/udi_backend.py b/py/dml/udi_backend.py index 7b51cc7d9..31d24bd7f 100644 --- a/py/dml/udi_backend.py +++ b/py/dml/udi_backend.py @@ -15,35 +15,6 @@ from .symtab import global_scope import dml.globals -# Names of traits to track should be listed here -# Note: user can specify additional templates to search for -# through commandline arguments -interesting_traits = { - "read_only", - "write_only", - "ignore_write", - "ignore_read", - "read_zero", - "write_1_clears", - "clear_on_read", - "write_1_only", - "write_0_only", - "read_constant", - "constant", - "no_reset", - "reserved", - "unimpl", - "read_unimpl", - "write_unimpl", - "silent_unimpl", - "unmapped", - "sticky", - "function_mapped_bank", - "poreset", - "hreset", - "sreset", -} - def array_info(obj): return list(zip(obj._arraylens, obj._idxvars)) @@ -91,8 +62,7 @@ def en_compobj(node, indices = []): def do_enc(node, proper_indices): content = {} for trait in node.traits.ancestors: - if trait.name in interesting_traits: - content.setdefault("templates", []).append(trait.name) + content.setdefault("templates", []).append(trait.name) en_subobjs(content, node, proper_indices) return content @@ -113,12 +83,15 @@ def do_enc(node, proper_indices): return {node.name : do_enc(node, indices)} obj_encoder_map = { - 'bank' : ("banks", en_compobj), 'attribute' : ("attributes", en_compobj), + 'bank' : ("banks", en_compobj), + 'connect' : ("connects", en_compobj), + 'data' : ("data", en_data), + 'field' : ("fields", en_compobj), + 'group' : ("groups", en_compobj), + 'implement' : ("implements", en_compobj), 'parameter' : ("parameters", en_parameter), 'register' : ("registers", en_compobj), - 'field' : ("fields", en_compobj), - 'data' : ("data", en_data), 'session' : ("data", en_var), 'saved' : ("data", en_var) } @@ -128,31 +101,6 @@ def en_obj(output, obj, indices = []): (collection, encoder) = obj_encoder_map[obj.objtype] subobjdict = encoder(obj, indices) output.setdefault(collection, {}).update(subobjdict) - # Group is a special case, merge subobj names into the group name - elif obj.objtype == 'group': - new_indices = [[ctree.mkIntegerConstant(obj.site, i, False) - for i in range(arrsize)] for arrsize in obj._arraylens] - if new_indices: - new_indices = itertools.product(*new_indices) - for extra_indices in new_indices: - full_name = "{}{}".format(obj.name, - "".join("[{}]".format(enc(ind)) for - ind in extra_indices)) - for s in subobjs(obj): - (collection, encoder) = obj_encoder_map[s.objtype] - subdict = encoder(s, indices + list(extra_indices)) - for sub in subdict: - output.setdefault( - collection, - {})["{}.{}".format(full_name, sub)] = subdict[sub] - else: - for s in subobjs(obj): - (collection, encoder) = obj_encoder_map[s.objtype] - subdict = encoder(s, indices) - for sub in subobjdict: - output.setdefault( - collection, - {})["{}.{}".format(obj.name, sub)] = subdict[sub] def subobjs(node): for s in node.get_components(): @@ -209,8 +157,7 @@ def en_subobjs(output, obj, indices = []): # }, # } # } -def generate(classname, extra_templates, device, dml_version, out_file): - interesting_traits.update(set(extra_templates)) +def generate(classname, device, dml_version, out_file): # Setting things up like this, allows easy combination and sorting of # devices in later tools complete_json = {