Sources/TreeSitterScanners/python/scanner.c
437 lines · 15470 bytes
1#include "tree_sitter/array.h"
2#include "tree_sitter/parser.h"
3
4#include <assert.h>
5#include <stdint.h>
6#include <stdio.h>
7#include <string.h>
8
9enum TokenType {
10 NEWLINE,
11 INDENT,
12 DEDENT,
13 STRING_START,
14 STRING_CONTENT,
15 ESCAPE_INTERPOLATION,
16 STRING_END,
17 COMMENT,
18 CLOSE_PAREN,
19 CLOSE_BRACKET,
20 CLOSE_BRACE,
21 EXCEPT,
22};
23
24typedef enum {
25 SingleQuote = 1 << 0,
26 DoubleQuote = 1 << 1,
27 BackQuote = 1 << 2,
28 Raw = 1 << 3,
29 Format = 1 << 4,
30 Triple = 1 << 5,
31 Bytes = 1 << 6,
32} Flags;
33
34typedef struct {
35 char flags;
36} Delimiter;
37
38static inline Delimiter new_delimiter() { return (Delimiter){0}; }
39
40static inline bool is_format(Delimiter *delimiter) { return delimiter->flags & Format; }
41
42static inline bool is_raw(Delimiter *delimiter) { return delimiter->flags & Raw; }
43
44static inline bool is_triple(Delimiter *delimiter) { return delimiter->flags & Triple; }
45
46static inline bool is_bytes(Delimiter *delimiter) { return delimiter->flags & Bytes; }
47
48static inline int32_t end_character(Delimiter *delimiter) {
49 if (delimiter->flags & SingleQuote) {
50 return '\'';
51 }
52 if (delimiter->flags & DoubleQuote) {
53 return '"';
54 }
55 if (delimiter->flags & BackQuote) {
56 return '`';
57 }
58 return 0;
59}
60
61static inline void set_format(Delimiter *delimiter) { delimiter->flags |= Format; }
62
63static inline void set_raw(Delimiter *delimiter) { delimiter->flags |= Raw; }
64
65static inline void set_triple(Delimiter *delimiter) { delimiter->flags |= Triple; }
66
67static inline void set_bytes(Delimiter *delimiter) { delimiter->flags |= Bytes; }
68
69static inline void set_end_character(Delimiter *delimiter, int32_t character) {
70 switch (character) {
71 case '\'':
72 delimiter->flags |= SingleQuote;
73 break;
74 case '"':
75 delimiter->flags |= DoubleQuote;
76 break;
77 case '`':
78 delimiter->flags |= BackQuote;
79 break;
80 default:
81 assert(false);
82 }
83}
84
85typedef struct {
86 Array(uint16_t) indents;
87 Array(Delimiter) delimiters;
88 bool inside_interpolated_string;
89} Scanner;
90
91static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
92
93static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
94
95bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
96 Scanner *scanner = (Scanner *)payload;
97
98 bool error_recovery_mode = valid_symbols[STRING_CONTENT] && valid_symbols[INDENT];
99 bool within_brackets = valid_symbols[CLOSE_BRACE] || valid_symbols[CLOSE_PAREN] || valid_symbols[CLOSE_BRACKET];
100
101 bool advanced_once = false;
102 if (valid_symbols[ESCAPE_INTERPOLATION] && scanner->delimiters.size > 0 &&
103 (lexer->lookahead == '{' || lexer->lookahead == '}') && !error_recovery_mode) {
104 Delimiter *delimiter = array_back(&scanner->delimiters);
105 if (is_format(delimiter)) {
106 lexer->mark_end(lexer);
107 bool is_left_brace = lexer->lookahead == '{';
108 advance(lexer);
109 advanced_once = true;
110 if ((lexer->lookahead == '{' && is_left_brace) || (lexer->lookahead == '}' && !is_left_brace)) {
111 advance(lexer);
112 lexer->mark_end(lexer);
113 lexer->result_symbol = ESCAPE_INTERPOLATION;
114 return true;
115 }
116 return false;
117 }
118 }
119
120 if (valid_symbols[STRING_CONTENT] && scanner->delimiters.size > 0 && !error_recovery_mode) {
121 Delimiter *delimiter = array_back(&scanner->delimiters);
122 int32_t end_char = end_character(delimiter);
123 bool has_content = advanced_once;
124 while (lexer->lookahead) {
125 if ((advanced_once || lexer->lookahead == '{' || lexer->lookahead == '}') && is_format(delimiter)) {
126 lexer->mark_end(lexer);
127 lexer->result_symbol = STRING_CONTENT;
128 return has_content;
129 }
130 if (lexer->lookahead == '\\') {
131 if (is_raw(delimiter)) {
132 // Step over the backslash.
133 advance(lexer);
134 // Step over any escaped quotes.
135 if (lexer->lookahead == end_character(delimiter) || lexer->lookahead == '\\') {
136 advance(lexer);
137 }
138 // Step over newlines
139 if (lexer->lookahead == '\r') {
140 advance(lexer);
141 if (lexer->lookahead == '\n') {
142 advance(lexer);
143 }
144 } else if (lexer->lookahead == '\n') {
145 advance(lexer);
146 }
147 continue;
148 }
149 if (is_bytes(delimiter)) {
150 lexer->mark_end(lexer);
151 advance(lexer);
152 if (lexer->lookahead == 'N' || lexer->lookahead == 'u' || lexer->lookahead == 'U') {
153 // In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are
154 // not escape sequences
155 // https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
156 advance(lexer);
157 } else {
158 lexer->result_symbol = STRING_CONTENT;
159 return has_content;
160 }
161 } else {
162 lexer->mark_end(lexer);
163 lexer->result_symbol = STRING_CONTENT;
164 return has_content;
165 }
166 } else if (lexer->lookahead == end_char) {
167 if (is_triple(delimiter)) {
168 lexer->mark_end(lexer);
169 advance(lexer);
170 if (lexer->lookahead == end_char) {
171 advance(lexer);
172 if (lexer->lookahead == end_char) {
173 if (has_content) {
174 lexer->result_symbol = STRING_CONTENT;
175 } else {
176 advance(lexer);
177 lexer->mark_end(lexer);
178 array_pop(&scanner->delimiters);
179 lexer->result_symbol = STRING_END;
180 scanner->inside_interpolated_string = false;
181 }
182 return true;
183 }
184 lexer->mark_end(lexer);
185 lexer->result_symbol = STRING_CONTENT;
186 return true;
187 }
188 lexer->mark_end(lexer);
189 lexer->result_symbol = STRING_CONTENT;
190 return true;
191 }
192 if (has_content) {
193 lexer->result_symbol = STRING_CONTENT;
194 } else {
195 advance(lexer);
196 array_pop(&scanner->delimiters);
197 lexer->result_symbol = STRING_END;
198 scanner->inside_interpolated_string = false;
199 }
200 lexer->mark_end(lexer);
201 return true;
202
203 } else if (lexer->lookahead == '\n' && has_content && !is_triple(delimiter)) {
204 return false;
205 }
206 advance(lexer);
207 has_content = true;
208 }
209 }
210
211 lexer->mark_end(lexer);
212
213 bool found_end_of_line = false;
214 uint16_t indent_length = 0;
215 int32_t first_comment_indent_length = -1;
216 for (;;) {
217 if (lexer->lookahead == '\n') {
218 found_end_of_line = true;
219 indent_length = 0;
220 skip(lexer);
221 } else if (lexer->lookahead == ' ') {
222 indent_length++;
223 skip(lexer);
224 } else if (lexer->lookahead == '\r' || lexer->lookahead == '\f') {
225 indent_length = 0;
226 skip(lexer);
227 } else if (lexer->lookahead == '\t') {
228 indent_length += 8;
229 skip(lexer);
230 } else if (lexer->lookahead == '#' && (valid_symbols[INDENT] || valid_symbols[DEDENT] ||
231 valid_symbols[NEWLINE] || valid_symbols[EXCEPT])) {
232 // If we haven't found an EOL yet,
233 // then this is a comment after an expression:
234 // foo = bar # comment
235 // Just return, since we don't want to generate an indent/dedent
236 // token.
237 if (!found_end_of_line) {
238 return false;
239 }
240 if (first_comment_indent_length == -1) {
241 first_comment_indent_length = (int32_t)indent_length;
242 }
243 while (lexer->lookahead && lexer->lookahead != '\n') {
244 skip(lexer);
245 }
246 skip(lexer);
247 indent_length = 0;
248 } else if (lexer->lookahead == '\\') {
249 skip(lexer);
250 if (lexer->lookahead == '\r') {
251 skip(lexer);
252 }
253 if (lexer->lookahead == '\n' || lexer->eof(lexer)) {
254 skip(lexer);
255 } else {
256 return false;
257 }
258 } else if (lexer->eof(lexer)) {
259 indent_length = 0;
260 found_end_of_line = true;
261 break;
262 } else {
263 break;
264 }
265 }
266
267 if (found_end_of_line) {
268 if (scanner->indents.size > 0) {
269 uint16_t current_indent_length = *array_back(&scanner->indents);
270
271 if (valid_symbols[INDENT] && indent_length > current_indent_length) {
272 array_push(&scanner->indents, indent_length);
273 lexer->result_symbol = INDENT;
274 return true;
275 }
276
277 bool next_tok_is_string_start =
278 lexer->lookahead == '\"' || lexer->lookahead == '\'' || lexer->lookahead == '`';
279
280 if ((valid_symbols[DEDENT] ||
281 (!valid_symbols[NEWLINE] && !(valid_symbols[STRING_START] && next_tok_is_string_start) &&
282 !within_brackets)) &&
283 indent_length < current_indent_length && !scanner->inside_interpolated_string &&
284
285 // Wait to create a dedent token until we've consumed any
286 // comments
287 // whose indentation matches the current block.
288 first_comment_indent_length < (int32_t)current_indent_length) {
289 array_pop(&scanner->indents);
290 lexer->result_symbol = DEDENT;
291 return true;
292 }
293 }
294
295 if (valid_symbols[NEWLINE] && !error_recovery_mode) {
296 lexer->result_symbol = NEWLINE;
297 return true;
298 }
299 }
300
301 if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
302 Delimiter delimiter = new_delimiter();
303
304 bool has_flags = false;
305 while (lexer->lookahead) {
306 if (lexer->lookahead == 'f' || lexer->lookahead == 'F' || lexer->lookahead == 't' ||
307 lexer->lookahead == 'T') {
308 set_format(&delimiter);
309 } else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
310 set_raw(&delimiter);
311 } else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
312 set_bytes(&delimiter);
313 } else if (lexer->lookahead != 'u' && lexer->lookahead != 'U') {
314 break;
315 }
316 has_flags = true;
317 advance(lexer);
318 }
319
320 if (lexer->lookahead == '`') {
321 set_end_character(&delimiter, '`');
322 advance(lexer);
323 lexer->mark_end(lexer);
324 } else if (lexer->lookahead == '\'') {
325 set_end_character(&delimiter, '\'');
326 advance(lexer);
327 lexer->mark_end(lexer);
328 if (lexer->lookahead == '\'') {
329 advance(lexer);
330 if (lexer->lookahead == '\'') {
331 advance(lexer);
332 lexer->mark_end(lexer);
333 set_triple(&delimiter);
334 }
335 }
336 } else if (lexer->lookahead == '"') {
337 set_end_character(&delimiter, '"');
338 advance(lexer);
339 lexer->mark_end(lexer);
340 if (lexer->lookahead == '"') {
341 advance(lexer);
342 if (lexer->lookahead == '"') {
343 advance(lexer);
344 lexer->mark_end(lexer);
345 set_triple(&delimiter);
346 }
347 }
348 }
349
350 if (end_character(&delimiter)) {
351 array_push(&scanner->delimiters, delimiter);
352 lexer->result_symbol = STRING_START;
353 scanner->inside_interpolated_string = is_format(&delimiter);
354 return true;
355 }
356 if (has_flags) {
357 return false;
358 }
359 }
360
361 return false;
362}
363
364unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buffer) {
365 Scanner *scanner = (Scanner *)payload;
366
367 size_t size = 0;
368
369 buffer[size++] = (char)scanner->inside_interpolated_string;
370
371 size_t delimiter_count = scanner->delimiters.size;
372 if (delimiter_count > UINT8_MAX) {
373 delimiter_count = UINT8_MAX;
374 }
375 buffer[size++] = (char)delimiter_count;
376
377 if (delimiter_count > 0) {
378 memcpy(&buffer[size], scanner->delimiters.contents, delimiter_count);
379 }
380 size += delimiter_count;
381
382 uint32_t iter = 1;
383 for (; iter < scanner->indents.size && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) {
384 uint16_t indent_value = *array_get(&scanner->indents, iter);
385 buffer[size++] = (char)(indent_value & 0xFF);
386 buffer[size++] = (char)((indent_value >> 8) & 0xFF);
387 }
388
389 return size;
390}
391
392void tree_sitter_python_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
393 Scanner *scanner = (Scanner *)payload;
394
395 array_delete(&scanner->delimiters);
396 array_delete(&scanner->indents);
397 array_push(&scanner->indents, 0);
398
399 if (length > 0) {
400 size_t size = 0;
401
402 scanner->inside_interpolated_string = (bool)buffer[size++];
403
404 size_t delimiter_count = (uint8_t)buffer[size++];
405 if (delimiter_count > 0) {
406 array_reserve(&scanner->delimiters, delimiter_count);
407 scanner->delimiters.size = delimiter_count;
408 memcpy(scanner->delimiters.contents, &buffer[size], delimiter_count);
409 size += delimiter_count;
410 }
411
412 for (; size + 1 < length; size += 2) {
413 uint16_t indent_value = (unsigned char)buffer[size] | ((unsigned char)buffer[size + 1] << 8);
414 array_push(&scanner->indents, indent_value);
415 }
416 }
417}
418
419void *tree_sitter_python_external_scanner_create() {
420#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
421 _Static_assert(sizeof(Delimiter) == sizeof(char), "");
422#else
423 assert(sizeof(Delimiter) == sizeof(char));
424#endif
425 Scanner *scanner = calloc(1, sizeof(Scanner));
426 array_init(&scanner->indents);
427 array_init(&scanner->delimiters);
428 tree_sitter_python_external_scanner_deserialize(scanner, NULL, 0);
429 return scanner;
430}
431
432void tree_sitter_python_external_scanner_destroy(void *payload) {
433 Scanner *scanner = (Scanner *)payload;
434 array_delete(&scanner->indents);
435 array_delete(&scanner->delimiters);
436 free(scanner);
437}