xref: /aosp_15_r20/external/fonttools/Tests/feaLib/lexer_test.py (revision e1fe3e4ad2793916b15cccdc4a7da52a7e1dd0e9) 
1from fontTools.feaLib.error import FeatureLibError, IncludedFeaNotFound
2from fontTools.feaLib.lexer import IncludingLexer, Lexer
3from fontTools.misc.textTools import tobytes
4from io import StringIO
5import os
6import shutil
7import tempfile
8import unittest
9
10
11def lex(s):
12    return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")]
13
14
15class LexerTest(unittest.TestCase):
16    def __init__(self, methodName):
17        unittest.TestCase.__init__(self, methodName)
18        # Python 3 renamed assertRaisesRegexp to assertRaisesRegex,
19        # and fires deprecation warnings if a program uses the old name.
20        if not hasattr(self, "assertRaisesRegex"):
21            self.assertRaisesRegex = self.assertRaisesRegexp
22
23    def test_empty(self):
24        self.assertEqual(lex(""), [])
25        self.assertEqual(lex(" \t "), [])
26
27    def test_name(self):
28        self.assertEqual(lex("a17"), [(Lexer.NAME, "a17")])
29        self.assertEqual(lex(".notdef"), [(Lexer.NAME, ".notdef")])
30        self.assertEqual(lex("two.oldstyle"), [(Lexer.NAME, "two.oldstyle")])
31        self.assertEqual(lex("_"), [(Lexer.NAME, "_")])
32        self.assertEqual(lex("\\table"), [(Lexer.NAME, "\\table")])
33        self.assertEqual(lex("a+*:^~!"), [(Lexer.NAME, "a+*:^~!")])
34        self.assertEqual(lex("with-dash"), [(Lexer.NAME, "with-dash")])
35
36    def test_cid(self):
37        self.assertEqual(lex("\\0 \\987"), [(Lexer.CID, 0), (Lexer.CID, 987)])
38
39    def test_glyphclass(self):
40        self.assertEqual(lex("@Vowel.sc"), [(Lexer.GLYPHCLASS, "Vowel.sc")])
41        self.assertEqual(lex("@Vowel-sc"), [(Lexer.GLYPHCLASS, "Vowel-sc")])
42        self.assertRaisesRegex(FeatureLibError, "Expected glyph class", lex, "@(a)")
43        self.assertRaisesRegex(FeatureLibError, "Expected glyph class", lex, "@ A")
44        self.assertEqual(lex("@" + ("A" * 600)), [(Lexer.GLYPHCLASS, "A" * 600)])
45        self.assertRaisesRegex(
46            FeatureLibError, "Glyph class names must consist of", lex, "@Ab:c"
47        )
48
49    def test_include(self):
50        self.assertEqual(
51            lex("include (~/foo/bar baz.fea);"),
52            [
53                (Lexer.NAME, "include"),
54                (Lexer.FILENAME, "~/foo/bar baz.fea"),
55                (Lexer.SYMBOL, ";"),
56            ],
57        )
58        self.assertEqual(
59            lex("include # Comment\n    (foo) \n;"),
60            [
61                (Lexer.NAME, "include"),
62                (Lexer.COMMENT, "# Comment"),
63                (Lexer.FILENAME, "foo"),
64                (Lexer.SYMBOL, ";"),
65            ],
66        )
67        self.assertRaises(FeatureLibError, lex, "include blah")
68        self.assertRaises(FeatureLibError, lex, "include (blah")
69
70    def test_number(self):
71        self.assertEqual(lex("123 -456"), [(Lexer.NUMBER, 123), (Lexer.NUMBER, -456)])
72        self.assertEqual(lex("0xCAFED00D"), [(Lexer.HEXADECIMAL, 0xCAFED00D)])
73        self.assertEqual(lex("0xcafed00d"), [(Lexer.HEXADECIMAL, 0xCAFED00D)])
74        self.assertEqual(lex("010"), [(Lexer.OCTAL, 0o10)])
75
76    def test_float(self):
77        self.assertEqual(lex("1.23 -4.5"), [(Lexer.FLOAT, 1.23), (Lexer.FLOAT, -4.5)])
78
79    def test_symbol(self):
80        self.assertEqual(lex("a'"), [(Lexer.NAME, "a"), (Lexer.SYMBOL, "'")])
81        self.assertEqual(lex("-A-B"), [(Lexer.SYMBOL, "-"), (Lexer.NAME, "A-B")])
82        self.assertEqual(
83            lex("foo - -2"),
84            [(Lexer.NAME, "foo"), (Lexer.SYMBOL, "-"), (Lexer.NUMBER, -2)],
85        )
86
87    def test_comment(self):
88        self.assertEqual(
89            lex("# Comment\n#"), [(Lexer.COMMENT, "# Comment"), (Lexer.COMMENT, "#")]
90        )
91
92    def test_string(self):
93        self.assertEqual(
94            lex('"foo" "bar"'), [(Lexer.STRING, "foo"), (Lexer.STRING, "bar")]
95        )
96        self.assertEqual(
97            lex('"foo \nbar\r baz \r\nqux\n\n "'), [(Lexer.STRING, "foo bar baz qux ")]
98        )
99        # The lexer should preserve escape sequences because they have
100        # different interpretations depending on context. For better
101        # or for worse, that is how the OpenType Feature File Syntax
102        # has been specified; see section 9.e (name table) for examples.
103        self.assertEqual(
104            lex(r'"M\00fcller-Lanc\00e9"'),  # 'nameid 9'
105            [(Lexer.STRING, r"M\00fcller-Lanc\00e9")],
106        )
107        self.assertEqual(
108            lex(r'"M\9fller-Lanc\8e"'),  # 'nameid 9 1'
109            [(Lexer.STRING, r"M\9fller-Lanc\8e")],
110        )
111        self.assertRaises(FeatureLibError, lex, '"foo\n bar')
112
113    def test_bad_character(self):
114        self.assertRaises(FeatureLibError, lambda: lex("123 \u0001"))
115
116    def test_newline(self):
117        def lines(s):
118            return [loc.line for (_, _, loc) in Lexer(s, "test.fea")]
119
120        self.assertEqual(lines("FOO\n\nBAR\nBAZ"), [1, 3, 4])  # Unix
121        self.assertEqual(lines("FOO\r\rBAR\rBAZ"), [1, 3, 4])  # Macintosh
122        self.assertEqual(lines("FOO\r\n\r\n BAR\r\nBAZ"), [1, 3, 4])  # Windows
123        self.assertEqual(lines("FOO\n\rBAR\r\nBAZ"), [1, 3, 4])  # mixed
124
125    def test_location(self):
126        def locs(s):
127            return [str(loc) for (_, _, loc) in Lexer(s, "test.fea")]
128
129        self.assertEqual(
130            locs("a b # Comment\n12 @x"),
131            [
132                "test.fea:1:1",
133                "test.fea:1:3",
134                "test.fea:1:5",
135                "test.fea:2:1",
136                "test.fea:2:4",
137            ],
138        )
139
140    def test_scan_over_(self):
141        lexer = Lexer("abbacabba12", "test.fea")
142        self.assertEqual(lexer.pos_, 0)
143        lexer.scan_over_("xyz")
144        self.assertEqual(lexer.pos_, 0)
145        lexer.scan_over_("abc")
146        self.assertEqual(lexer.pos_, 9)
147        lexer.scan_over_("abc")
148        self.assertEqual(lexer.pos_, 9)
149        lexer.scan_over_("0123456789")
150        self.assertEqual(lexer.pos_, 11)
151
152    def test_scan_until_(self):
153        lexer = Lexer("foo'bar", "test.fea")
154        self.assertEqual(lexer.pos_, 0)
155        lexer.scan_until_("'")
156        self.assertEqual(lexer.pos_, 3)
157        lexer.scan_until_("'")
158        self.assertEqual(lexer.pos_, 3)
159
160
161class IncludingLexerTest(unittest.TestCase):
162    @staticmethod
163    def getpath(filename):
164        path, _ = os.path.split(__file__)
165        return os.path.join(path, "data", filename)
166
167    def test_include(self):
168        lexer = IncludingLexer(self.getpath("include/include4.fea"))
169        result = [
170            "%s %s:%d" % (token, os.path.split(loc.file)[1], loc.line)
171            for _, token, loc in lexer
172        ]
173        self.assertEqual(
174            result,
175            [
176                "I4a include4.fea:1",
177                "I3a include3.fea:1",
178                "I2a include2.fea:1",
179                "I1a include1.fea:1",
180                "I0 include0.fea:1",
181                "I1b include1.fea:3",
182                "; include2.fea:2",
183                "I2b include2.fea:3",
184                "; include3.fea:2",
185                "I3b include3.fea:3",
186                "; include4.fea:2",
187                "I4b include4.fea:3",
188            ],
189        )
190
191    def test_include_limit(self):
192        lexer = IncludingLexer(self.getpath("include/include6.fea"))
193        self.assertRaises(FeatureLibError, lambda: list(lexer))
194
195    def test_include_self(self):
196        lexer = IncludingLexer(self.getpath("include/includeself.fea"))
197        self.assertRaises(FeatureLibError, lambda: list(lexer))
198
199    def test_include_missing_file(self):
200        lexer = IncludingLexer(self.getpath("include/includemissingfile.fea"))
201        self.assertRaisesRegex(
202            IncludedFeaNotFound,
203            "includemissingfile.fea:1:8: The following feature file "
204            "should be included but cannot be found: "
205            "missingfile.fea",
206            lambda: list(lexer),
207        )
208
209    def test_featurefilepath_None(self):
210        lexer = IncludingLexer(StringIO("# foobar"))
211        self.assertIsNone(lexer.featurefilepath)
212        files = set(loc.file for _, _, loc in lexer)
213        self.assertIn("<features>", files)
214
215    def test_include_absolute_path(self):
216        with tempfile.NamedTemporaryFile(delete=False) as included:
217            included.write(
218                tobytes(
219                    """
220                feature kern {
221                    pos A B -40;
222                } kern;
223                """,
224                    encoding="utf-8",
225                )
226            )
227        including = StringIO("include(%s);" % included.name)
228        try:
229            lexer = IncludingLexer(including)
230            files = set(loc.file for _, _, loc in lexer)
231            self.assertIn(included.name, files)
232        finally:
233            os.remove(included.name)
234
235    def test_include_relative_to_cwd(self):
236        # save current working directory, to be restored later
237        cwd = os.getcwd()
238        tmpdir = tempfile.mkdtemp()
239        try:
240            # create new feature file in a temporary directory
241            with open(
242                os.path.join(tmpdir, "included.fea"), "w", encoding="utf-8"
243            ) as included:
244                included.write(
245                    """
246                    feature kern {
247                        pos A B -40;
248                    } kern;
249                    """
250                )
251            # change current folder to the temporary dir
252            os.chdir(tmpdir)
253            # instantiate a new lexer that includes the above file
254            # using a relative path; the IncludingLexer does not
255            # itself have a path, because it was initialized from
256            # an in-memory stream, so it will use the current working
257            # directory to resolve relative include statements
258            lexer = IncludingLexer(StringIO("include(included.fea);"))
259            files = set(os.path.realpath(loc.file) for _, _, loc in lexer)
260            expected = os.path.realpath(included.name)
261            self.assertIn(expected, files)
262        finally:
263            # remove temporary folder and restore previous working directory
264            os.chdir(cwd)
265            shutil.rmtree(tmpdir)
266
267
268if __name__ == "__main__":
269    import sys
270
271    sys.exit(unittest.main())
272