1from collections import namedtuple
2import enum
3import os.path
4import re
5
6from c_common import fsutil
7from c_common.clsutil import classonly
8import c_common.misc as _misc
9import c_common.strutil as _strutil
10import c_common.tables as _tables
11from .parser._regexes import SIMPLE_TYPE, _STORAGE
12
13
14FIXED_TYPE = _misc.Labeled('FIXED_TYPE')
15
16STORAGE = frozenset(_STORAGE)
17
18
19#############################
20# kinds
21
22@enum.unique
23class KIND(enum.Enum):
24
25    # XXX Use these in the raw parser code.
26    TYPEDEF = 'typedef'
27    STRUCT = 'struct'
28    UNION = 'union'
29    ENUM = 'enum'
30    FUNCTION = 'function'
31    VARIABLE = 'variable'
32    STATEMENT = 'statement'
33
34    @classonly
35    def _from_raw(cls, raw):
36        if raw is None:
37            return None
38        elif isinstance(raw, cls):
39            return raw
40        elif type(raw) is str:
41            # We could use cls[raw] for the upper-case form,
42            # but there's no need to go to the trouble.
43            return cls(raw.lower())
44        else:
45            raise NotImplementedError(raw)
46
47    @classonly
48    def by_priority(cls, group=None):
49        if group is None:
50            return cls._ALL_BY_PRIORITY.copy()
51        elif group == 'type':
52            return cls._TYPE_DECLS_BY_PRIORITY.copy()
53        elif group == 'decl':
54            return cls._ALL_DECLS_BY_PRIORITY.copy()
55        elif isinstance(group, str):
56            raise NotImplementedError(group)
57        else:
58            # XXX Treat group as a set of kinds & return in priority order?
59            raise NotImplementedError(group)
60
61    @classonly
62    def is_type_decl(cls, kind):
63        if kind in cls.TYPES:
64            return True
65        if not isinstance(kind, cls):
66            raise TypeError(f'expected KIND, got {kind!r}')
67        return False
68
69    @classonly
70    def is_decl(cls, kind):
71        if kind in cls.DECLS:
72            return True
73        if not isinstance(kind, cls):
74            raise TypeError(f'expected KIND, got {kind!r}')
75        return False
76
77    @classonly
78    def get_group(cls, kind, *, groups=None):
79        if not isinstance(kind, cls):
80            raise TypeError(f'expected KIND, got {kind!r}')
81        if groups is None:
82            groups = ['type']
83        elif not groups:
84            groups = ()
85        elif isinstance(groups, str):
86            group = groups
87            if group not in cls._GROUPS:
88                raise ValueError(f'unsupported group {group!r}')
89            groups = [group]
90        else:
91            unsupported = [g for g in groups if g not in cls._GROUPS]
92            if unsupported:
93                raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}')
94        for group in groups:
95            if kind in cls._GROUPS[group]:
96                return group
97        else:
98            return kind.value
99
100    @classonly
101    def resolve_group(cls, group):
102        if isinstance(group, cls):
103            return {group}
104        elif isinstance(group, str):
105            try:
106                return cls._GROUPS[group].copy()
107            except KeyError:
108                raise ValueError(f'unsupported group {group!r}')
109        else:
110            resolved = set()
111            for gr in group:
112                resolve.update(cls.resolve_group(gr))
113            return resolved
114            #return {*cls.resolve_group(g) for g in group}
115
116
117KIND._TYPE_DECLS_BY_PRIORITY = [
118    # These are in preferred order.
119    KIND.TYPEDEF,
120    KIND.STRUCT,
121    KIND.UNION,
122    KIND.ENUM,
123]
124KIND._ALL_DECLS_BY_PRIORITY = [
125    # These are in preferred order.
126    *KIND._TYPE_DECLS_BY_PRIORITY,
127    KIND.FUNCTION,
128    KIND.VARIABLE,
129]
130KIND._ALL_BY_PRIORITY = [
131    # These are in preferred order.
132    *KIND._ALL_DECLS_BY_PRIORITY,
133    KIND.STATEMENT,
134]
135
136KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY)
137KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY)
138KIND._GROUPS = {
139    'type': KIND.TYPES,
140    'decl': KIND.DECLS,
141}
142KIND._GROUPS.update((k.value, {k}) for k in KIND)
143
144
145def get_kind_group(item):
146    return KIND.get_group(item.kind)
147
148
149#############################
150# low-level
151
152def _fix_filename(filename, relroot, *,
153                  formatted=True,
154                  **kwargs):
155    if formatted:
156        fix = fsutil.format_filename
157    else:
158        fix = fsutil.fix_filename
159    return fix(filename, relroot=relroot, **kwargs)
160
161
162class FileInfo(namedtuple('FileInfo', 'filename lno')):
163    @classmethod
164    def from_raw(cls, raw):
165        if isinstance(raw, cls):
166            return raw
167        elif isinstance(raw, tuple):
168            return cls(*raw)
169        elif not raw:
170            return None
171        elif isinstance(raw, str):
172            return cls(raw, -1)
173        else:
174            raise TypeError(f'unsupported "raw": {raw:!r}')
175
176    def __str__(self):
177        return self.filename
178
179    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
180        filename = _fix_filename(self.filename, relroot, **kwargs)
181        if filename == self.filename:
182            return self
183        return self._replace(filename=filename)
184
185
186class SourceLine(namedtuple('Line', 'file kind data conditions')):
187    KINDS = (
188        #'directive',  # data is ...
189        'source',  # "data" is the line
190        #'comment',  # "data" is the text, including comment markers
191    )
192
193    @property
194    def filename(self):
195        return self.file.filename
196
197    @property
198    def lno(self):
199        return self.file.lno
200
201
202class DeclID(namedtuple('DeclID', 'filename funcname name')):
203    """The globally-unique identifier for a declaration."""
204
205    @classmethod
206    def from_row(cls, row, **markers):
207        row = _tables.fix_row(row, **markers)
208        return cls(*row)
209
210    # We have to provde _make() becaose we implemented __new__().
211
212    @classmethod
213    def _make(cls, iterable):
214        try:
215            return cls(*iterable)
216        except Exception:
217            super()._make(iterable)
218            raise  # re-raise
219
220    def __new__(cls, filename, funcname, name):
221        self = super().__new__(
222            cls,
223            filename=str(filename) if filename else None,
224            funcname=str(funcname) if funcname else None,
225            name=str(name) if name else None,
226        )
227        self._compare = tuple(v or '' for v in self)
228        return self
229
230    def __hash__(self):
231        return super().__hash__()
232
233    def __eq__(self, other):
234        try:
235            other = tuple(v or '' for v in other)
236        except TypeError:
237            return NotImplemented
238        return self._compare == other
239
240    def __gt__(self, other):
241        try:
242            other = tuple(v or '' for v in other)
243        except TypeError:
244            return NotImplemented
245        return self._compare > other
246
247    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
248        filename = _fix_filename(self.filename, relroot, **kwargs)
249        if filename == self.filename:
250            return self
251        return self._replace(filename=filename)
252
253
254class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')):
255
256    @classmethod
257    def from_raw(cls, raw):
258        if isinstance(raw, cls):
259            return raw
260        elif isinstance(raw, tuple):
261            return cls(*raw)
262        else:
263            raise TypeError(f'unsupported "raw": {raw:!r}')
264
265    @classmethod
266    def from_row(cls, row, columns=None):
267        if not columns:
268            colnames = 'filename funcname name kind data'.split()
269        else:
270            colnames = list(columns)
271            for i, column in enumerate(colnames):
272                if column == 'file':
273                    colnames[i] = 'filename'
274                elif column == 'funcname':
275                    colnames[i] = 'parent'
276        if len(row) != len(set(colnames)):
277            raise NotImplementedError(columns, row)
278        kwargs = {}
279        for column, value in zip(colnames, row):
280            if column == 'filename':
281                kwargs['file'] = FileInfo.from_raw(value)
282            elif column == 'kind':
283                kwargs['kind'] = KIND(value)
284            elif column in cls._fields:
285                kwargs[column] = value
286            else:
287                raise NotImplementedError(column)
288        return cls(**kwargs)
289
290    @property
291    def id(self):
292        try:
293            return self._id
294        except AttributeError:
295            if self.kind is KIND.STATEMENT:
296                self._id = None
297            else:
298                self._id = DeclID(str(self.file), self.funcname, self.name)
299            return self._id
300
301    @property
302    def filename(self):
303        if not self.file:
304            return None
305        return self.file.filename
306
307    @property
308    def lno(self):
309        if not self.file:
310            return -1
311        return self.file.lno
312
313    @property
314    def funcname(self):
315        if not self.parent:
316            return None
317        if type(self.parent) is str:
318            return self.parent
319        else:
320            return self.parent.name
321
322    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
323        fixed = self.file.fix_filename(relroot, **kwargs)
324        if fixed == self.file:
325            return self
326        return self._replace(file=fixed)
327
328    def as_row(self, columns=None):
329        if not columns:
330            columns = self._fields
331        row = []
332        for column in columns:
333            if column == 'file':
334                value = self.filename
335            elif column == 'kind':
336                value = self.kind.value
337            elif column == 'data':
338                value = self._render_data()
339            else:
340                value = getattr(self, column)
341            row.append(value)
342        return row
343
344    def _render_data(self):
345        if not self.data:
346            return None
347        elif isinstance(self.data, str):
348            return self.data
349        else:
350            # XXX
351            raise NotImplementedError
352
353
354def _get_vartype(data):
355    try:
356        vartype = dict(data['vartype'])
357    except KeyError:
358        vartype = dict(data)
359        storage = data.get('storage')
360    else:
361        storage = data.get('storage') or vartype.get('storage')
362    del vartype['storage']
363    return storage, vartype
364
365
366def get_parsed_vartype(decl):
367    kind = getattr(decl, 'kind', None)
368    if isinstance(decl, ParsedItem):
369        storage, vartype = _get_vartype(decl.data)
370        typequal = vartype['typequal']
371        typespec = vartype['typespec']
372        abstract = vartype['abstract']
373    elif isinstance(decl, dict):
374        kind = decl.get('kind')
375        storage, vartype = _get_vartype(decl)
376        typequal = vartype['typequal']
377        typespec = vartype['typespec']
378        abstract = vartype['abstract']
379    elif isinstance(decl, VarType):
380        storage = None
381        typequal, typespec, abstract = decl
382    elif isinstance(decl, TypeDef):
383        storage = None
384        typequal, typespec, abstract = decl.vartype
385    elif isinstance(decl, Variable):
386        storage = decl.storage
387        typequal, typespec, abstract = decl.vartype
388    elif isinstance(decl, Function):
389        storage = decl.storage
390        typequal, typespec, abstract = decl.signature.returntype
391    elif isinstance(decl, str):
392        vartype, storage = VarType.from_str(decl)
393        typequal, typespec, abstract = vartype
394    else:
395        raise NotImplementedError(decl)
396    return kind, storage, typequal, typespec, abstract
397
398
399def get_default_storage(decl):
400    if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION):
401        return None
402    return 'extern' if decl.parent is None else 'auto'
403
404
405def get_effective_storage(decl, *, default=None):
406    # Note that "static" limits access to just that C module
407    # and "extern" (the default for module-level) allows access
408    # outside the C module.
409    if default is None:
410        default = get_default_storage(decl)
411        if default is None:
412            return None
413    try:
414        storage = decl.storage
415    except AttributeError:
416        storage, _ = _get_vartype(decl.data)
417    return storage or default
418
419
420#############################
421# high-level
422
423class HighlevelParsedItem:
424
425    kind = None
426
427    FIELDS = ('file', 'parent', 'name', 'data')
428
429    @classmethod
430    def from_parsed(cls, parsed):
431        if parsed.kind is not cls.kind:
432            raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})')
433        data, extra = cls._resolve_data(parsed.data)
434        self = cls(
435            cls._resolve_file(parsed),
436            parsed.name,
437            data,
438            cls._resolve_parent(parsed) if parsed.parent else None,
439            **extra or {}
440        )
441        self._parsed = parsed
442        return self
443
444    @classmethod
445    def _resolve_file(cls, parsed):
446        fileinfo = FileInfo.from_raw(parsed.file)
447        if not fileinfo:
448            raise NotImplementedError(parsed)
449        return fileinfo
450
451    @classmethod
452    def _resolve_data(cls, data):
453        return data, None
454
455    @classmethod
456    def _raw_data(cls, data, extra):
457        if isinstance(data, str):
458            return data
459        else:
460            raise NotImplementedError(data)
461
462    @classmethod
463    def _data_as_row(cls, data, extra, colnames):
464        row = {}
465        for colname in colnames:
466            if colname in row:
467                continue
468            rendered = cls._render_data_row_item(colname, data, extra)
469            if rendered is iter(rendered):
470                rendered, = rendered
471            row[colname] = rendered
472        return row
473
474    @classmethod
475    def _render_data_row_item(cls, colname, data, extra):
476        if colname == 'data':
477            return str(data)
478        else:
479            return None
480
481    @classmethod
482    def _render_data_row(cls, fmt, data, extra, colnames):
483        if fmt != 'row':
484            raise NotImplementedError
485        datarow = cls._data_as_row(data, extra, colnames)
486        unresolved = [c for c, v in datarow.items() if v is None]
487        if unresolved:
488            raise NotImplementedError(unresolved)
489        for colname, value in datarow.items():
490            if type(value) != str:
491                if colname == 'kind':
492                    datarow[colname] = value.value
493                else:
494                    datarow[colname] = str(value)
495        return datarow
496
497    @classmethod
498    def _render_data(cls, fmt, data, extra):
499        row = cls._render_data_row(fmt, data, extra, ['data'])
500        yield ' '.join(row.values())
501
502    @classmethod
503    def _resolve_parent(cls, parsed, *, _kind=None):
504        fileinfo = FileInfo(parsed.file.filename, -1)
505        if isinstance(parsed.parent, str):
506            if parsed.parent.isidentifier():
507                name = parsed.parent
508            else:
509                # XXX It could be something like "<kind> <name>".
510                raise NotImplementedError(repr(parsed.parent))
511            parent = ParsedItem(fileinfo, _kind, None, name, None)
512        elif type(parsed.parent) is tuple:
513            # XXX It could be something like (kind, name).
514            raise NotImplementedError(repr(parsed.parent))
515        else:
516            return parsed.parent
517        Parent = KIND_CLASSES.get(_kind, Declaration)
518        return Parent.from_parsed(parent)
519
520    @classmethod
521    def _parse_columns(cls, columns):
522        colnames = {}  # {requested -> actual}
523        columns = list(columns or cls.FIELDS)
524        datacolumns = []
525        for i, colname in enumerate(columns):
526            if colname == 'file':
527                columns[i] = 'filename'
528                colnames['file'] = 'filename'
529            elif colname == 'lno':
530                columns[i] = 'line'
531                colnames['lno'] = 'line'
532            elif colname in ('filename', 'line'):
533                colnames[colname] = colname
534            elif colname == 'data':
535                datacolumns.append(colname)
536                colnames[colname] = None
537            elif colname in cls.FIELDS or colname == 'kind':
538                colnames[colname] = colname
539            else:
540                datacolumns.append(colname)
541                colnames[colname] = None
542        return columns, datacolumns, colnames
543
544    def __init__(self, file, name, data, parent=None, *,
545                 _extra=None,
546                 _shortkey=None,
547                 _key=None,
548                 ):
549        self.file = file
550        self.parent = parent or None
551        self.name = name
552        self.data = data
553        self._extra = _extra or {}
554        self._shortkey = _shortkey
555        self._key = _key
556
557    def __repr__(self):
558        args = [f'{n}={getattr(self, n)!r}'
559                for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]]
560        return f'{type(self).__name__}({", ".join(args)})'
561
562    def __str__(self):
563        try:
564            return self._str
565        except AttributeError:
566            self._str = next(self.render())
567            return self._str
568
569    def __getattr__(self, name):
570        try:
571            return self._extra[name]
572        except KeyError:
573            raise AttributeError(name)
574
575    def __hash__(self):
576        return hash(self._key)
577
578    def __eq__(self, other):
579        if isinstance(other, HighlevelParsedItem):
580            return self._key == other._key
581        elif type(other) is tuple:
582            return self._key == other
583        else:
584            return NotImplemented
585
586    def __gt__(self, other):
587        if isinstance(other, HighlevelParsedItem):
588            return self._key > other._key
589        elif type(other) is tuple:
590            return self._key > other
591        else:
592            return NotImplemented
593
594    @property
595    def id(self):
596        return self.parsed.id
597
598    @property
599    def shortkey(self):
600        return self._shortkey
601
602    @property
603    def key(self):
604        return self._key
605
606    @property
607    def filename(self):
608        if not self.file:
609            return None
610        return self.file.filename
611
612    @property
613    def parsed(self):
614        try:
615            return self._parsed
616        except AttributeError:
617            parent = self.parent
618            if parent is not None and not isinstance(parent, str):
619                parent = parent.name
620            self._parsed = ParsedItem(
621                self.file,
622                self.kind,
623                parent,
624                self.name,
625                self._raw_data(),
626            )
627            return self._parsed
628
629    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
630        if self.file:
631            self.file = self.file.fix_filename(relroot, **kwargs)
632        return self
633
634    def as_rowdata(self, columns=None):
635        columns, datacolumns, colnames = self._parse_columns(columns)
636        return self._as_row(colnames, datacolumns, self._data_as_row)
637
638    def render_rowdata(self, columns=None):
639        columns, datacolumns, colnames = self._parse_columns(columns)
640        def data_as_row(data, ext, cols):
641            return self._render_data_row('row', data, ext, cols)
642        rowdata = self._as_row(colnames, datacolumns, data_as_row)
643        for column, value in rowdata.items():
644            colname = colnames.get(column)
645            if not colname:
646                continue
647            if column == 'kind':
648                value = value.value
649            else:
650                if column == 'parent':
651                    if self.parent:
652                        value = f'({self.parent.kind.value} {self.parent.name})'
653                if not value:
654                    value = '-'
655                elif type(value) is VarType:
656                    value = repr(str(value))
657                else:
658                    value = str(value)
659            rowdata[column] = value
660        return rowdata
661
662    def _as_row(self, colnames, datacolumns, data_as_row):
663        try:
664            data = data_as_row(self.data, self._extra, datacolumns)
665        except NotImplementedError:
666            data = None
667        row = data or {}
668        for column, colname in colnames.items():
669            if colname == 'filename':
670                value = self.file.filename if self.file else None
671            elif colname == 'line':
672                value = self.file.lno if self.file else None
673            elif colname is None:
674                value = getattr(self, column, None)
675            else:
676                value = getattr(self, colname, None)
677            row.setdefault(column, value)
678        return row
679
680    def render(self, fmt='line'):
681        fmt = fmt or 'line'
682        try:
683            render = _FORMATS[fmt]
684        except KeyError:
685            raise TypeError(f'unsupported fmt {fmt!r}')
686        try:
687            data = self._render_data(fmt, self.data, self._extra)
688        except NotImplementedError:
689            data = '-'
690        yield from render(self, data)
691
692
693### formats ###
694
695def _fmt_line(parsed, data=None):
696    parts = [
697        f'<{parsed.kind.value}>',
698    ]
699    parent = ''
700    if parsed.parent:
701        parent = parsed.parent
702        if not isinstance(parent, str):
703            if parent.kind is KIND.FUNCTION:
704                parent = f'{parent.name}()'
705            else:
706                parent = parent.name
707        name = f'<{parent}>.{parsed.name}'
708    else:
709        name = parsed.name
710    if data is None:
711        data = parsed.data
712    elif data is iter(data):
713        data, = data
714    parts.extend([
715        name,
716        f'<{data}>' if data else '-',
717        f'({str(parsed.file or "<unknown file>")})',
718    ])
719    yield '\t'.join(parts)
720
721
722def _fmt_full(parsed, data=None):
723    if parsed.kind is KIND.VARIABLE and parsed.parent:
724        prefix = 'local '
725        suffix = f' ({parsed.parent.name})'
726    else:
727        # XXX Show other prefixes (e.g. global, public)
728        prefix = suffix = ''
729    yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}'
730    for column, info in parsed.render_rowdata().items():
731        if column == 'kind':
732            continue
733        if column == 'name':
734            continue
735        if column == 'parent' and parsed.kind is not KIND.VARIABLE:
736            continue
737        if column == 'data':
738            if parsed.kind in (KIND.STRUCT, KIND.UNION):
739                column = 'members'
740            elif parsed.kind is KIND.ENUM:
741                column = 'enumerators'
742            elif parsed.kind is KIND.STATEMENT:
743                column = 'text'
744                data, = data
745            else:
746                column = 'signature'
747                data, = data
748            if not data:
749#                yield f'\t{column}:\t-'
750                continue
751            elif isinstance(data, str):
752                yield f'\t{column}:\t{data!r}'
753            else:
754                yield f'\t{column}:'
755                for line in data:
756                    yield f'\t\t- {line}'
757        else:
758            yield f'\t{column}:\t{info}'
759
760
761_FORMATS = {
762    'raw': (lambda v, _d: [repr(v)]),
763    'brief': _fmt_line,
764    'line': _fmt_line,
765    'full': _fmt_full,
766}
767
768
769### declarations ##
770
771class Declaration(HighlevelParsedItem):
772
773    @classmethod
774    def from_row(cls, row, **markers):
775        fixed = tuple(_tables.fix_row(row, **markers))
776        if cls is Declaration:
777            _, _, _, kind, _ = fixed
778            sub = KIND_CLASSES.get(KIND(kind))
779            if not sub or not issubclass(sub, Declaration):
780                raise TypeError(f'unsupported kind, got {row!r}')
781        else:
782            sub = cls
783        return sub._from_row(fixed)
784
785    @classmethod
786    def _from_row(cls, row):
787        filename, funcname, name, kind, data = row
788        kind = KIND._from_raw(kind)
789        if kind is not cls.kind:
790            raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}')
791        fileinfo = FileInfo.from_raw(filename)
792        if isinstance(data, str):
793            data, extra = cls._parse_data(data, fmt='row')
794        if extra:
795            return cls(fileinfo, name, data, funcname, _extra=extra)
796        else:
797            return cls(fileinfo, name, data, funcname)
798
799    @classmethod
800    def _resolve_parent(cls, parsed, *, _kind=None):
801        if _kind is None:
802            raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})')
803        return super()._resolve_parent(parsed, _kind=_kind)
804
805    @classmethod
806    def _render_data(cls, fmt, data, extra):
807        if not data:
808            # XXX There should be some!  Forward?
809            yield '???'
810        else:
811            yield from cls._format_data(fmt, data, extra)
812
813    @classmethod
814    def _render_data_row_item(cls, colname, data, extra):
815        if colname == 'data':
816            return cls._format_data('row', data, extra)
817        else:
818            return None
819
820    @classmethod
821    def _format_data(cls, fmt, data, extra):
822        raise NotImplementedError(fmt)
823
824    @classmethod
825    def _parse_data(cls, datastr, fmt=None):
826        """This is the reverse of _render_data."""
827        if not datastr or datastr is _tables.UNKNOWN or datastr == '???':
828            return None, None
829        elif datastr is _tables.EMPTY or datastr == '-':
830            # All the kinds have *something* even it is unknown.
831            raise TypeError('all declarations have data of some sort, got none')
832        else:
833            return cls._unformat_data(datastr, fmt)
834
835    @classmethod
836    def _unformat_data(cls, datastr, fmt=None):
837        raise NotImplementedError(fmt)
838
839
840class VarType(namedtuple('VarType', 'typequal typespec abstract')):
841
842    @classmethod
843    def from_str(cls, text):
844        orig = text
845        storage, sep, text = text.strip().partition(' ')
846        if not sep:
847            text = storage
848            storage = None
849        elif storage not in ('auto', 'register', 'static', 'extern'):
850            text = orig
851            storage = None
852        return cls._from_str(text), storage
853
854    @classmethod
855    def _from_str(cls, text):
856        orig = text
857        if text.startswith(('const ', 'volatile ')):
858            typequal, _, text = text.partition(' ')
859        else:
860            typequal = None
861
862        # Extract a series of identifiers/keywords.
863        m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text)
864        if not m:
865            raise ValueError(f'invalid vartype text {orig!r}')
866        typespec, abstract = m.groups()
867
868        return cls(typequal, typespec, abstract or None)
869
870    def __str__(self):
871        parts = []
872        if self.qualifier:
873            parts.append(self.qualifier)
874        parts.append(self.spec + (self.abstract or ''))
875        return ' '.join(parts)
876
877    @property
878    def qualifier(self):
879        return self.typequal
880
881    @property
882    def spec(self):
883        return self.typespec
884
885
886class Variable(Declaration):
887    kind = KIND.VARIABLE
888
889    @classmethod
890    def _resolve_parent(cls, parsed):
891        return super()._resolve_parent(parsed, _kind=KIND.FUNCTION)
892
893    @classmethod
894    def _resolve_data(cls, data):
895        if not data:
896            return None, None
897        storage, vartype = _get_vartype(data)
898        return VarType(**vartype), {'storage': storage}
899
900    @classmethod
901    def _raw_data(self, data, extra):
902        vartype = data._asdict()
903        return {
904            'storage': extra['storage'],
905            'vartype': vartype,
906        }
907
908    @classmethod
909    def _format_data(cls, fmt, data, extra):
910        storage = extra.get('storage')
911        text = f'{storage} {data}' if storage else str(data)
912        if fmt in ('line', 'brief'):
913            yield text
914        #elif fmt == 'full':
915        elif fmt == 'row':
916            yield text
917        else:
918            raise NotImplementedError(fmt)
919
920    @classmethod
921    def _unformat_data(cls, datastr, fmt=None):
922        if fmt in ('line', 'brief'):
923            vartype, storage = VarType.from_str(datastr)
924            return vartype, {'storage': storage}
925        #elif fmt == 'full':
926        elif fmt == 'row':
927            vartype, storage = VarType.from_str(datastr)
928            return vartype, {'storage': storage}
929        else:
930            raise NotImplementedError(fmt)
931
932    def __init__(self, file, name, data, parent=None, storage=None):
933        super().__init__(file, name, data, parent,
934                         _extra={'storage': storage or None},
935                         _shortkey=f'({parent.name}).{name}' if parent else name,
936                         _key=(str(file),
937                               # Tilde comes after all other ascii characters.
938                               f'~{parent or ""}~',
939                               name,
940                               ),
941                         )
942        if storage:
943            if storage not in STORAGE:
944                # The parser must need an update.
945                raise NotImplementedError(storage)
946            # Otherwise we trust the compiler to have validated it.
947
948    @property
949    def vartype(self):
950        return self.data
951
952
953class Signature(namedtuple('Signature', 'params returntype inline isforward')):
954
955    @classmethod
956    def from_str(cls, text):
957        orig = text
958        storage, sep, text = text.strip().partition(' ')
959        if not sep:
960            text = storage
961            storage = None
962        elif storage not in ('auto', 'register', 'static', 'extern'):
963            text = orig
964            storage = None
965        return cls._from_str(text), storage
966
967    @classmethod
968    def _from_str(cls, text):
969        orig = text
970        inline, sep, text = text.partition('|')
971        if not sep:
972            text = inline
973            inline = None
974
975        isforward = False
976        if text.endswith(';'):
977            text = text[:-1]
978            isforward = True
979        elif text.endswith('{}'):
980            text = text[:-2]
981
982        index = text.rindex('(')
983        if index < 0:
984            raise ValueError(f'bad signature text {orig!r}')
985        params = text[index:]
986        while params.count('(') <= params.count(')'):
987            index = text.rindex('(', 0, index)
988            if index < 0:
989                raise ValueError(f'bad signature text {orig!r}')
990            params = text[index:]
991        text = text[:index]
992
993        returntype = VarType._from_str(text.rstrip())
994
995        return cls(params, returntype, inline, isforward)
996
997    def __str__(self):
998        parts = []
999        if self.inline:
1000            parts.extend([
1001                self.inline,
1002                '|',
1003            ])
1004        parts.extend([
1005            str(self.returntype),
1006            self.params,
1007            ';' if self.isforward else '{}',
1008        ])
1009        return ' '.join(parts)
1010
1011    @property
1012    def returns(self):
1013        return self.returntype
1014
1015
1016class Function(Declaration):
1017    kind = KIND.FUNCTION
1018
1019    @classmethod
1020    def _resolve_data(cls, data):
1021        if not data:
1022            return None, None
1023        kwargs = dict(data)
1024        returntype = dict(data['returntype'])
1025        del returntype['storage']
1026        kwargs['returntype'] = VarType(**returntype)
1027        storage = kwargs.pop('storage')
1028        return Signature(**kwargs), {'storage': storage}
1029
1030    @classmethod
1031    def _raw_data(self, data):
1032        # XXX finish!
1033        return data
1034
1035    @classmethod
1036    def _format_data(cls, fmt, data, extra):
1037        storage = extra.get('storage')
1038        text = f'{storage} {data}' if storage else str(data)
1039        if fmt in ('line', 'brief'):
1040            yield text
1041        #elif fmt == 'full':
1042        elif fmt == 'row':
1043            yield text
1044        else:
1045            raise NotImplementedError(fmt)
1046
1047    @classmethod
1048    def _unformat_data(cls, datastr, fmt=None):
1049        if fmt in ('line', 'brief'):
1050            sig, storage = Signature.from_str(sig)
1051            return sig, {'storage': storage}
1052        #elif fmt == 'full':
1053        elif fmt == 'row':
1054            sig, storage = Signature.from_str(sig)
1055            return sig, {'storage': storage}
1056        else:
1057            raise NotImplementedError(fmt)
1058
1059    def __init__(self, file, name, data, parent=None, storage=None):
1060        super().__init__(file, name, data, parent, _extra={'storage': storage})
1061        self._shortkey = f'~{name}~ {self.data}'
1062        self._key = (
1063            str(file),
1064            self._shortkey,
1065        )
1066
1067    @property
1068    def signature(self):
1069        return self.data
1070
1071
1072class TypeDeclaration(Declaration):
1073
1074    def __init__(self, file, name, data, parent=None, *, _shortkey=None):
1075        if not _shortkey:
1076            _shortkey = f'{self.kind.value} {name}'
1077        super().__init__(file, name, data, parent,
1078                         _shortkey=_shortkey,
1079                         _key=(
1080                             str(file),
1081                             _shortkey,
1082                             ),
1083                         )
1084
1085
1086class POTSType(TypeDeclaration):
1087
1088    def __init__(self, name):
1089        _file = _data = _parent = None
1090        super().__init__(_file, name, _data, _parent, _shortkey=name)
1091
1092
1093class FuncPtr(TypeDeclaration):
1094
1095    def __init__(self, vartype):
1096        _file = _name = _parent = None
1097        data = vartype
1098        self.vartype = vartype
1099        super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>')
1100
1101
1102class TypeDef(TypeDeclaration):
1103    kind = KIND.TYPEDEF
1104
1105    @classmethod
1106    def _resolve_data(cls, data):
1107        if not data:
1108            raise NotImplementedError(data)
1109        vartype = dict(data)
1110        del vartype['storage']
1111        return VarType(**vartype), None
1112
1113    @classmethod
1114    def _raw_data(self, data):
1115        # XXX finish!
1116        return data
1117
1118    @classmethod
1119    def _format_data(cls, fmt, data, extra):
1120        text = str(data)
1121        if fmt in ('line', 'brief'):
1122            yield text
1123        elif fmt == 'full':
1124            yield text
1125        elif fmt == 'row':
1126            yield text
1127        else:
1128            raise NotImplementedError(fmt)
1129
1130    @classmethod
1131    def _unformat_data(cls, datastr, fmt=None):
1132        if fmt in ('line', 'brief'):
1133            vartype, _ = VarType.from_str(datastr)
1134            return vartype, None
1135        #elif fmt == 'full':
1136        elif fmt == 'row':
1137            vartype, _ = VarType.from_str(datastr)
1138            return vartype, None
1139        else:
1140            raise NotImplementedError(fmt)
1141
1142    def __init__(self, file, name, data, parent=None):
1143        super().__init__(file, name, data, parent, _shortkey=name)
1144
1145    @property
1146    def vartype(self):
1147        return self.data
1148
1149
1150class Member(namedtuple('Member', 'name vartype size')):
1151
1152    @classmethod
1153    def from_data(cls, raw, index):
1154        name = raw.name if raw.name else index
1155        vartype = size = None
1156        if type(raw.data) is int:
1157            size = raw.data
1158        elif isinstance(raw.data, str):
1159            size = int(raw.data)
1160        elif raw.data:
1161            vartype = dict(raw.data)
1162            del vartype['storage']
1163            if 'size' in vartype:
1164                size = vartype.pop('size')
1165                if isinstance(size, str) and size.isdigit():
1166                    size = int(size)
1167            vartype = VarType(**vartype)
1168        return cls(name, vartype, size)
1169
1170    @classmethod
1171    def from_str(cls, text):
1172        name, _, vartype = text.partition(': ')
1173        if name.startswith('#'):
1174            name = int(name[1:])
1175        if vartype.isdigit():
1176            size = int(vartype)
1177            vartype = None
1178        else:
1179            vartype, _ = VarType.from_str(vartype)
1180            size = None
1181        return cls(name, vartype, size)
1182
1183    def __str__(self):
1184        name = self.name if isinstance(self.name, str) else f'#{self.name}'
1185        return f'{name}: {self.vartype or self.size}'
1186
1187
1188class _StructUnion(TypeDeclaration):
1189
1190    @classmethod
1191    def _resolve_data(cls, data):
1192        if not data:
1193            # XXX There should be some!  Forward?
1194            return None, None
1195        return [Member.from_data(v, i) for i, v in enumerate(data)], None
1196
1197    @classmethod
1198    def _raw_data(self, data):
1199        # XXX finish!
1200        return data
1201
1202    @classmethod
1203    def _format_data(cls, fmt, data, extra):
1204        if fmt in ('line', 'brief'):
1205            members = ', '.join(f'<{m}>' for m in data)
1206            yield f'[{members}]'
1207        elif fmt == 'full':
1208            for member in data:
1209                yield f'{member}'
1210        elif fmt == 'row':
1211            members = ', '.join(f'<{m}>' for m in data)
1212            yield f'[{members}]'
1213        else:
1214            raise NotImplementedError(fmt)
1215
1216    @classmethod
1217    def _unformat_data(cls, datastr, fmt=None):
1218        if fmt in ('line', 'brief'):
1219            members = [Member.from_str(m[1:-1])
1220                       for m in datastr[1:-1].split(', ')]
1221            return members, None
1222        #elif fmt == 'full':
1223        elif fmt == 'row':
1224            members = [Member.from_str(m.rstrip('>').lstrip('<'))
1225                       for m in datastr[1:-1].split('>, <')]
1226            return members, None
1227        else:
1228            raise NotImplementedError(fmt)
1229
1230    def __init__(self, file, name, data, parent=None):
1231        super().__init__(file, name, data, parent)
1232
1233    @property
1234    def members(self):
1235        return self.data
1236
1237
1238class Struct(_StructUnion):
1239    kind = KIND.STRUCT
1240
1241
1242class Union(_StructUnion):
1243    kind = KIND.UNION
1244
1245
1246class Enum(TypeDeclaration):
1247    kind = KIND.ENUM
1248
1249    @classmethod
1250    def _resolve_data(cls, data):
1251        if not data:
1252            # XXX There should be some!  Forward?
1253            return None, None
1254        enumerators = [e if isinstance(e, str) else e.name
1255                       for e in data]
1256        return enumerators, None
1257
1258    @classmethod
1259    def _raw_data(self, data):
1260        # XXX finish!
1261        return data
1262
1263    @classmethod
1264    def _format_data(cls, fmt, data, extra):
1265        if fmt in ('line', 'brief'):
1266            yield repr(data)
1267        elif fmt == 'full':
1268            for enumerator in data:
1269                yield f'{enumerator}'
1270        elif fmt == 'row':
1271            # XXX This won't work with CSV...
1272            yield ','.join(data)
1273        else:
1274            raise NotImplementedError(fmt)
1275
1276    @classmethod
1277    def _unformat_data(cls, datastr, fmt=None):
1278        if fmt in ('line', 'brief'):
1279            return _strutil.unrepr(datastr), None
1280        #elif fmt == 'full':
1281        elif fmt == 'row':
1282            return datastr.split(','), None
1283        else:
1284            raise NotImplementedError(fmt)
1285
1286    def __init__(self, file, name, data, parent=None):
1287        super().__init__(file, name, data, parent)
1288
1289    @property
1290    def enumerators(self):
1291        return self.data
1292
1293
1294### statements ###
1295
1296class Statement(HighlevelParsedItem):
1297    kind = KIND.STATEMENT
1298
1299    @classmethod
1300    def _resolve_data(cls, data):
1301        # XXX finish!
1302        return data, None
1303
1304    @classmethod
1305    def _raw_data(self, data):
1306        # XXX finish!
1307        return data
1308
1309    @classmethod
1310    def _render_data(cls, fmt, data, extra):
1311        # XXX Handle other formats?
1312        return repr(data)
1313
1314    @classmethod
1315    def _parse_data(self, datastr, fmt=None):
1316        # XXX Handle other formats?
1317        return _strutil.unrepr(datastr), None
1318
1319    def __init__(self, file, name, data, parent=None):
1320        super().__init__(file, name, data, parent,
1321                         _shortkey=data or '',
1322                         _key=(
1323                             str(file),
1324                             file.lno,
1325                             # XXX Only one stmt per line?
1326                             ),
1327                         )
1328
1329    @property
1330    def text(self):
1331        return self.data
1332
1333
1334###
1335
1336KIND_CLASSES = {cls.kind: cls for cls in [
1337    Variable,
1338    Function,
1339    TypeDef,
1340    Struct,
1341    Union,
1342    Enum,
1343    Statement,
1344]}
1345
1346
1347def resolve_parsed(parsed):
1348    if isinstance(parsed, HighlevelParsedItem):
1349        return parsed
1350    try:
1351        cls = KIND_CLASSES[parsed.kind]
1352    except KeyError:
1353        raise ValueError(f'unsupported kind in {parsed!r}')
1354    return cls.from_parsed(parsed)
1355
1356
1357def set_flag(item, name, value):
1358    try:
1359        setattr(item, name, value)
1360    except AttributeError:
1361        object.__setattr__(item, name, value)
1362
1363
1364#############################
1365# composite
1366
1367class Declarations:
1368
1369    @classmethod
1370    def from_decls(cls, decls):
1371        return cls(decls)
1372
1373    @classmethod
1374    def from_parsed(cls, items):
1375        decls = (resolve_parsed(item)
1376                 for item in items
1377                 if item.kind is not KIND.STATEMENT)
1378        return cls.from_decls(decls)
1379
1380    @classmethod
1381    def _resolve_key(cls, raw):
1382        if isinstance(raw, str):
1383            raw = [raw]
1384        elif isinstance(raw, Declaration):
1385            raw = (
1386                raw.filename if cls._is_public(raw) else None,
1387                # `raw.parent` is always None for types and functions.
1388                raw.parent if raw.kind is KIND.VARIABLE else None,
1389                raw.name,
1390            )
1391
1392        extra = None
1393        if len(raw) == 1:
1394            name, = raw
1395            if name:
1396                name = str(name)
1397                if name.endswith(('.c', '.h')):
1398                    # This is only legit as a query.
1399                    key = (name, None, None)
1400                else:
1401                    key = (None, None, name)
1402            else:
1403                key = (None, None, None)
1404        elif len(raw) == 2:
1405            parent, name = raw
1406            name = str(name)
1407            if isinstance(parent, Declaration):
1408                key = (None, parent.name, name)
1409            elif not parent:
1410                key = (None, None, name)
1411            else:
1412                parent = str(parent)
1413                if parent.endswith(('.c', '.h')):
1414                    key = (parent, None, name)
1415                else:
1416                    key = (None, parent, name)
1417        else:
1418            key, extra = raw[:3], raw[3:]
1419            filename, funcname, name = key
1420            filename = str(filename) if filename else None
1421            if isinstance(funcname, Declaration):
1422                funcname = funcname.name
1423            else:
1424                funcname = str(funcname) if funcname else None
1425            name = str(name) if name else None
1426            key = (filename, funcname, name)
1427        return key, extra
1428
1429    @classmethod
1430    def _is_public(cls, decl):
1431        # For .c files don't we need info from .h files to make this decision?
1432        # XXX Check for "extern".
1433        # For now we treat all decls a "private" (have filename set).
1434        return False
1435
1436    def __init__(self, decls):
1437        # (file, func, name) -> decl
1438        # "public":
1439        #   * (None, None, name)
1440        # "private", "global":
1441        #   * (file, None, name)
1442        # "private", "local":
1443        #   * (file, func, name)
1444        if hasattr(decls, 'items'):
1445            self._decls = decls
1446        else:
1447            self._decls = {}
1448            self._extend(decls)
1449
1450        # XXX always validate?
1451
1452    def validate(self):
1453        for key, decl in self._decls.items():
1454            if type(key) is not tuple or len(key) != 3:
1455                raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})')
1456            filename, funcname, name = key
1457            if not name:
1458                raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})')
1459            elif type(name) is not str:
1460                raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})')
1461            # XXX Check filename type?
1462            # XXX Check funcname type?
1463
1464            if decl.kind is KIND.STATEMENT:
1465                raise ValueError(f'expected a declaration, got {decl!r}')
1466
1467    def __repr__(self):
1468        return f'{type(self).__name__}({list(self)})'
1469
1470    def __len__(self):
1471        return len(self._decls)
1472
1473    def __iter__(self):
1474        yield from self._decls
1475
1476    def __getitem__(self, key):
1477        # XXX Be more exact for the 3-tuple case?
1478        if type(key) not in (str, tuple):
1479            raise KeyError(f'unsupported key {key!r}')
1480        resolved, extra = self._resolve_key(key)
1481        if extra:
1482            raise KeyError(f'key must have at most 3 parts, got {key!r}')
1483        if not resolved[2]:
1484            raise ValueError(f'expected name in key, got {key!r}')
1485        try:
1486            return self._decls[resolved]
1487        except KeyError:
1488            if type(key) is tuple and len(key) == 3:
1489                filename, funcname, name = key
1490            else:
1491                filename, funcname, name = resolved
1492            if filename and not filename.endswith(('.c', '.h')):
1493                raise KeyError(f'invalid filename in key {key!r}')
1494            elif funcname and funcname.endswith(('.c', '.h')):
1495                raise KeyError(f'invalid funcname in key {key!r}')
1496            elif name and name.endswith(('.c', '.h')):
1497                raise KeyError(f'invalid name in key {key!r}')
1498            else:
1499                raise  # re-raise
1500
1501    @property
1502    def types(self):
1503        return self._find(kind=KIND.TYPES)
1504
1505    @property
1506    def functions(self):
1507        return self._find(None, None, None, KIND.FUNCTION)
1508
1509    @property
1510    def variables(self):
1511        return self._find(None, None, None, KIND.VARIABLE)
1512
1513    def iter_all(self):
1514        yield from self._decls.values()
1515
1516    def get(self, key, default=None):
1517        try:
1518            return self[key]
1519        except KeyError:
1520            return default
1521
1522    #def add_decl(self, decl, key=None):
1523    #    decl = _resolve_parsed(decl)
1524    #    self._add_decl(decl, key)
1525
1526    def find(self, *key, **explicit):
1527        if not key:
1528            if not explicit:
1529                return iter(self)
1530            return self._find(**explicit)
1531
1532        resolved, extra = self._resolve_key(key)
1533        filename, funcname, name = resolved
1534        if not extra:
1535            kind = None
1536        elif len(extra) == 1:
1537            kind, = extra
1538        else:
1539            raise KeyError(f'key must have at most 4 parts, got {key!r}')
1540
1541        implicit= {}
1542        if filename:
1543            implicit['filename'] = filename
1544        if funcname:
1545            implicit['funcname'] = funcname
1546        if name:
1547            implicit['name'] = name
1548        if kind:
1549            implicit['kind'] = kind
1550        return self._find(**implicit, **explicit)
1551
1552    def _find(self, filename=None, funcname=None, name=None, kind=None):
1553        for decl in self._decls.values():
1554            if filename and decl.filename != filename:
1555                continue
1556            if funcname:
1557                if decl.kind is not KIND.VARIABLE:
1558                    continue
1559                if decl.parent.name != funcname:
1560                    continue
1561            if name and decl.name != name:
1562                continue
1563            if kind:
1564                kinds = KIND.resolve_group(kind)
1565                if decl.kind not in kinds:
1566                    continue
1567            yield decl
1568
1569    def _add_decl(self, decl, key=None):
1570        if key:
1571            if type(key) not in (str, tuple):
1572                raise NotImplementedError((key, decl))
1573            # Any partial key will be turned into a full key, but that
1574            # same partial key will still match a key lookup.
1575            resolved, _ = self._resolve_key(key)
1576            if not resolved[2]:
1577                raise ValueError(f'expected name in key, got {key!r}')
1578            key = resolved
1579            # XXX Also add with the decl-derived key if not the same?
1580        else:
1581            key, _ = self._resolve_key(decl)
1582        self._decls[key] = decl
1583
1584    def _extend(self, decls):
1585        decls = iter(decls)
1586        # Check only the first item.
1587        for decl in decls:
1588            if isinstance(decl, Declaration):
1589                self._add_decl(decl)
1590                # Add the rest without checking.
1591                for decl in decls:
1592                    self._add_decl(decl)
1593            elif isinstance(decl, HighlevelParsedItem):
1594                raise NotImplementedError(decl)
1595            else:
1596                try:
1597                    key, decl = decl
1598                except ValueError:
1599                    raise NotImplementedError(decl)
1600                if not isinstance(decl, Declaration):
1601                    raise NotImplementedError(decl)
1602                self._add_decl(decl, key)
1603                # Add the rest without checking.
1604                for key, decl in decls:
1605                    self._add_decl(decl, key)
1606            # The iterator will be exhausted at this point.
1607