Coverage for fpdf2_textindex/parser.py: 92.02%
278 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-06-01 14:22 +0000
1"""Text Index Parser."""
3from __future__ import annotations
5import itertools
6import logging
7import re
8from typing import Final, TYPE_CHECKING
10from fpdf2_textindex import constants as const
11from fpdf2_textindex.alias import AliasRegistry
12from fpdf2_textindex.constants import LOGGER
13from fpdf2_textindex.errors import FPDF2TextindexError
14from fpdf2_textindex.interface import Alias
15from fpdf2_textindex.interface import CrossReferenceType
16from fpdf2_textindex.interface import LabelPath
17from fpdf2_textindex.interface import TextIndexEntry
18from fpdf2_textindex.md_emphasis import MDEmphasis
19from fpdf2_textindex.utils import insert_at_match
20from fpdf2_textindex.utils import remove_match_from_str
21from fpdf2_textindex.utils import remove_quotes
23if TYPE_CHECKING:
24 from collections.abc import Iterable, Iterator
26 from fpdf2_textindex.interface import LabelPathT
29class TextIndexParser:
30 """Text Index Parser.
32 Parses text(s), finds text index directives, creates the corresponding
33 entries and replaces the directives by corresponding markdown links.
34 """
36 _LEADING_BRACKET_SPAN: Final[str] = (
37 r"(?<!\\)\[(?P<leading_bracket_span>[^\]<>]+)(?<!\\)\]"
38 )
39 _LEADING_NON_WHITESPACE_SPAN: Final[str] = (
40 r"(?P<leading_non_whitespace_span>[^\s\[\]\{\}<>]+?)"
41 rf"{MDEmphasis.MARKER_PATTERN.format(name='md_center'):s}"
42 )
43 _PARAMS: Final[str] = r"\{\^(?P<params>[^\}<\n]*)\}"
44 _DIRECTIVE_PATTERN: re.Pattern[str] = re.compile(
45 rf"{MDEmphasis.MARKER_PATTERN.format(name='md_start'):s}"
46 rf"(?:{_LEADING_NON_WHITESPACE_SPAN:s}|{_LEADING_BRACKET_SPAN:s})?"
47 rf"(?<!>){_PARAMS:s}"
48 rf"{MDEmphasis.MARKER_PATTERN.format(name='md_end'):s}"
49 )
51 _CROSS_REF_IN_PARAMS_PATTERN: re.Pattern[str] = re.compile(r"\|(.+)$")
52 _LABEL_PATH_IN_PARAMS_PATTERN: re.Pattern[str] = re.compile(
53 rf"^((?:[^\|\[~]|{MDEmphasis.STRIKETHROUGH.marker:s})+)"
54 )
55 _SEARCH_WILDCARD_PATTERN: re.Pattern[str] = re.compile(r"\*\^(\-?)")
56 _SORT_KEY_IN_PARAMS_PATTERN: re.Pattern[str] = re.compile(
57 r"\s*\~(['\"]?)(.+)\1$"
58 )
59 _SUFFIX_IN_PARAMS_PATTERN: re.Pattern[str] = re.compile(
60 r"\s*\[(?P<suffix>(?:[^\]\"]+|\"[^\"]+\")+)(?<!\\)\]\s*"
61 )
63 def __init__(
64 self,
65 *,
66 strict: bool = True,
67 ) -> None:
68 """Initializes the parser.
70 Args:
71 strict: If `True` and an entry will have a normal reference
72 (locator) and a SEE-cross reference, a `FPDF2TextindexError`
73 will be raised. Else, it will just be a warning and the
74 SEE-cross reference will be automatically converted to SEE ALSO.
75 Defaults to `True`.
76 """
77 self._alias_reg = AliasRegistry()
78 self._enabled = True
79 self._directive_id = -1
80 self._root = TextIndexEntry(label="root")
81 self._strict = bool(strict)
83 def __iter__(self) -> Iterator[TextIndexEntry]:
84 yield from itertools.islice(iter(self._root), 1, None)
86 def __len__(self) -> int:
87 return sum(1 for _ in self)
89 def __repr__(self) -> str:
90 return f"{type(self).__name__:s}({len(self):d} entries)"
92 @property
93 def aliases(self) -> list[Alias]:
94 """The parsed aliases."""
95 return list(self._alias_reg.values())
97 @property
98 def entries(self) -> list[TextIndexEntry]:
99 """The parsed entries."""
100 return list(iter(self))
102 @property
103 def last_directive_id(self) -> int:
104 """Last directive id."""
105 return self._directive_id
107 @property
108 def last_index_id(self) -> str:
109 """Last index id."""
110 return f"{const.INDEX_ID_PREFIX:s}{self._directive_id:d}"
112 def entry_at_label_path(
113 self,
114 label_path: LabelPathT,
115 *,
116 create: bool = False,
117 ) -> tuple[TextIndexEntry | None, bool]:
118 """Returns an entry by its label path.
120 If `create=True` and the entry does not exist, it will be created.
122 Args:
123 label_path: The label path.
124 create: Whether to create the entry if it does not exist already.
125 Defaults to `False`.
127 Returns:
128 The found :py:class:`fpdf2_textindex.TextIndexEntry` or `None` and
129 whether the entry has existed before.
130 """
131 created = False
132 node = self._root
133 for label in LabelPath(label_path):
134 child = node.get_child(label)
135 if child is None:
136 if not create: 136 ↛ 137line 136 didn't jump to line 137 because the condition on line 136 was never true
137 LOGGER.warning("Failed to find %r", label)
138 return None, False
139 LOGGER.debug(
140 "Making new entry %r (%s)",
141 label,
142 f"within {node.label!r:s}" if node.parent else "at root",
143 )
144 child = TextIndexEntry(label=label, parent=node)
145 created = True
146 node = child
147 return node, not created
149 def parse_text(self, text: str) -> str:
150 """Parses a text, finds text index directives, creates the corresponding
151 entries and replaces the directives by corresponding markdown links.
153 Args:
154 text: The text to parse.
156 Returns:
157 The parsed text.
159 Raises:
160 FPDF2TextindexError:
161 If a directive cannot be parsed.
162 If the label cannot be identified correctly.
163 If `strict=True` and and adding a SEE-cross reference to an
164 entry with a former "normal" reference (locator) or viceversa.
165 """
166 LOGGER.info(
167 "Parsing text %r",
168 text if len(text) < 45 else text[:20] + "..." + text[-20:],
169 )
171 former_len = len(self)
172 offset = 0 # Account for replacements
174 for directive in self._DIRECTIVE_PATTERN.finditer(text):
175 # Parse and encapsulate each entry, either as object or range-end
176 LOGGER.debug("Directive found: %r", directive.group(0))
177 params = directive.group("params").strip()
179 params, toggling, status_toggled = self._parse_toggling_directive(
180 params
181 )
182 if toggling and (self._enabled or status_toggled):
183 # This was a toggling mark, and we are either now enabled or we
184 # were when we encountered it, remove the mark.
185 text = remove_match_from_str(text, directive, offset=offset)
186 offset += -len(directive.group(0))
187 continue
188 if not toggling and not self._enabled:
189 LOGGER.debug(
190 "Disabled, ignoring directive: %r", directive.group(0)
191 )
192 continue
194 self._directive_id += 1
195 label, content = self._parse_label(directive)
197 params, closing, locator_emphasis = self._parse_final_marker(params)
198 params, label_path, label, unreferenced_alias = (
199 self._parse_label_path(
200 params, label, content, directive.group(0)
201 )
202 )
203 # Found unreferenced alias
204 if unreferenced_alias:
205 LOGGER.log(
206 logging.INFO if label else logging.WARNING,
207 "\tUnreferenced alias %s; skipping rest of directive: %r",
208 "created" if label else "definition without a label",
209 directive.group(0),
210 )
211 # Replace directive in text
212 text = insert_at_match(text, directive, content, offset=offset)
213 offset += len(content) - len(directive.group(0))
214 self._directive_id -= 1
215 continue
217 LOGGER.debug("\tLabel path: %s", label_path)
218 LOGGER.debug("\tLabel: %r", label)
219 if not label:
220 LOGGER.warning(
221 "No entry label specified in directive, ignoring: %r",
222 directive.group(0),
223 )
224 self._directive_id -= 1
225 continue
227 params, suffix = self._parse_suffix(params)
228 params, sort_key = self._parse_sort_key(params, content)
229 params, create_ref, cref_type_label_path = self._parse_cross_ref(
230 params, label_path, label, content
231 )
232 if params.strip(): 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true
233 msg = f"Unparsed directive content: {params!r:s}"
234 LOGGER.error(msg)
235 raise FPDF2TextindexError(msg)
237 # Insert into entries tree
238 replace_directive = self._update_index(
239 label_path,
240 label,
241 create_ref,
242 cref_type_label_path,
243 closing,
244 directive.group(0),
245 locator_emphasis,
246 sort_key,
247 suffix,
248 )
249 if not replace_directive: 249 ↛ 250line 249 didn't jump to line 250 because the condition on line 249 was never true
250 self._directive_id -= 1
251 continue
253 # Replace directive in text with suitable link
254 link = self._create_link(content)
255 text = insert_at_match(text, directive, link, offset=offset)
256 offset += len(link) - len(directive.group(0))
258 LOGGER.info("Parsed text: %d entries created", len(self) - former_len)
259 LOGGER.debug(
260 "Created text: %r",
261 text if len(text) < 60 else text[:30] + "..." + text[-30:],
262 )
263 return text
265 def _create_link(self, content: str) -> str:
266 unstyled_label, label_emphasis = MDEmphasis.parse(content)
267 return label_emphasis.format(
268 f"[{unstyled_label:s}](#{self.last_index_id:s})"
269 )
271 def _parse_cross_ref(
272 self,
273 params: str,
274 label_path: Iterable[str],
275 label: str,
276 content: str,
277 ) -> tuple[str, bool, list[tuple[CrossReferenceType, LabelPath]]]:
278 create_ref = True
279 cref_type_label_path: list[tuple[CrossReferenceType, LabelPath]] = []
280 params = params.strip()
281 cross_match = self._CROSS_REF_IN_PARAMS_PATTERN.match(params)
282 if cross_match is None:
283 return params, create_ref, cref_type_label_path
285 refs_string = cross_match.group(1).strip()
287 # Process aliases before splitting path
288 refs_string = self._alias_reg.replace_aliases(refs_string)
290 # Handle wildcards in cross references
291 refs_string = self._parse_wildcards(refs_string, content)
293 refs = refs_string.split(const.REFS_DELIMITER)
294 for ref in refs:
295 ref = ref.strip()
297 inbound = ref.startswith(const.INBOUND_MARKER)
298 if inbound:
299 ref = ref[len(const.INBOUND_MARKER) :]
301 ref_type = (
302 CrossReferenceType.ALSO
303 if ref.startswith(const.ALSO_MARKER)
304 else CrossReferenceType.SEE
305 )
306 if ref_type == CrossReferenceType.ALSO:
307 ref = ref[len(const.ALSO_MARKER) :]
308 elif not inbound:
309 # Do not create a (page-) reference for this mark's entry if
310 # there is a SEE-cross reference.
311 create_ref = False
313 # Split reference label path
314 ref_label_path = LabelPath.split_str(ref)
316 # Cross reference in different entry, referencing this mark's entry
317 if inbound:
318 source_entry, _ = self.entry_at_label_path(
319 ref_label_path, create=True
320 )
321 if TYPE_CHECKING:
322 assert isinstance(source_entry, TextIndexEntry)
323 LOGGER.debug(
324 "\tCreating inbound %s cross reference from entry %s (%s)",
325 ref_type.upper(),
326 ref_label_path[-1],
327 f"Path: {source_entry.joined_label_path!r:s}"
328 if len(ref_label_path) > 1
329 else "at root",
330 )
331 source_entry.add_cross_reference(
332 self._directive_id,
333 ref_type,
334 LabelPath((*label_path, label)),
335 strict=self._strict,
336 )
338 # Cross reference within this mark's entry
339 else:
340 cref_type_label_path.append((ref_type, ref_label_path))
342 params = remove_match_from_str(params, cross_match)
343 if len(cref_type_label_path) > 0:
344 LOGGER.debug("\tCross references: %r", cref_type_label_path)
345 return params, create_ref, cref_type_label_path
347 def _parse_final_marker(self, params: str) -> tuple[str, bool, bool]:
348 params = params.strip()
349 closing = params.endswith(const.CLOSING_MARKER)
350 locator_emphasis = params.endswith(const.EMPHASIS_MARKER)
351 if closing:
352 params = params[: -len(const.CLOSING_MARKER)]
353 LOGGER.debug("\tClosing mark: %r", const.CLOSING_MARKER)
354 elif locator_emphasis:
355 params = params[: -len(const.EMPHASIS_MARKER)]
356 LOGGER.debug("\tLocator Emphasis: %r", const.EMPHASIS_MARKER)
357 return params, closing, locator_emphasis
359 def _parse_label(
360 self,
361 directive: re.Match[str],
362 ) -> tuple[str | None, str]:
363 label = None
364 # Leading bracketed span "[x]{^}"
365 if directive.group("leading_bracket_span"):
366 label = directive.group("leading_bracket_span")
367 if ( 367 ↛ 392line 367 didn't jump to line 392 because the condition on line 367 was always true
368 directive.group("md_start") is not None
369 and directive.group("md_start")
370 == directive.group("md_end")[::-1]
371 ):
372 label = (
373 directive.group("md_start")
374 + label
375 + directive.group("md_end")
376 )
377 # Leading implicit non-whitespace span "X{^}"
378 elif directive.group("leading_non_whitespace_span"):
379 label = directive.group("leading_non_whitespace_span")
380 for end in ("md_center", "md_end"):
381 if (
382 directive.group("md_start") is not None
383 and directive.group("md_start")
384 == directive.group(end)[::-1]
385 ):
386 label = (
387 directive.group("md_start")
388 + label
389 + directive.group(end)
390 )
392 content = label or ""
393 LOGGER.debug("\tContent: %r", content)
394 return label, content
396 def _parse_label_path(
397 self,
398 params: str,
399 label: str | None,
400 content: str,
401 directive_str: str,
402 ) -> tuple[str, LabelPath, str | None, bool]:
403 label_path_match = self._LABEL_PATH_IN_PARAMS_PATTERN.match(params)
404 if not label_path_match:
405 return params, LabelPath(), label, False
407 label_path_str = label_path_match.group(0).strip()
409 # Process aliases before splitting path.
410 label_path_str = self._alias_reg.replace_aliases(label_path_str)
412 # Handle wildcards in label path
413 label_path_str = self._parse_wildcards(label_path_str, label)
415 # Having already replaced alias references, check for alias
416 # definition at end of label path
417 label_path_str, alias_name, alias_start = self._alias_reg.strip_alias(
418 label_path_str
419 )
421 # Split label path
422 label_path = LabelPath.split_str(label_path_str)
424 # Last item is now the label
425 if label_path[-1] not in {"", const.PATH_DELIMITER}:
426 label = label_path[-1]
427 label_path = label_path[:-1]
428 assert isinstance(label, str)
430 # Remove empty last label
431 if label_path and label_path[-1] == "":
432 label_path = label_path[:-1]
434 # Assert label
435 if label is None: 435 ↛ 436line 435 didn't jump to line 436 because the condition on line 435 was never true
436 msg = "cannot identify label: %r"
437 LOGGER.error(msg, directive_str)
438 raise FPDF2TextindexError(msg % directive_str)
440 # Trim label path from params.
441 params = remove_match_from_str(params, label_path_match)
443 # Check for alias definition.
444 label_path, label, unreferenced_alias = (
445 self._alias_reg.define_or_replace_from_label_path(
446 label_path,
447 label,
448 content,
449 alias_name,
450 alias_start,
451 directive_str,
452 )
453 )
455 return params, label_path, label, unreferenced_alias
457 def _parse_sort_key(
458 self,
459 params: str,
460 content: str,
461 ) -> tuple[str, str | None]:
462 params = params.strip()
463 match = self._SORT_KEY_IN_PARAMS_PATTERN.search(params)
464 if match is not None:
465 sort_key = match.group(2)
466 # Handle wildcards in sort key
467 sort_key = self._parse_wildcards(
468 sort_key,
469 content,
470 force_label_only=True,
471 )
472 params = remove_match_from_str(params, match)
473 LOGGER.debug("\tSort key: %r", sort_key)
474 return params, sort_key
475 return params, None
477 def _parse_suffix(self, params: str) -> tuple[str, str | None]:
478 params = params.strip()
479 match = self._SUFFIX_IN_PARAMS_PATTERN.search(params)
480 if match is not None:
481 suffix = match.group("suffix")
482 suffix = remove_quotes(suffix)
483 params = remove_match_from_str(params, match)
484 LOGGER.debug("\tSuffix: %r", suffix)
485 return params, suffix
486 return params, None
488 def _parse_toggling_directive(self, params: str) -> tuple[str, bool, bool]:
489 params = params.strip()
490 toggling = params in {const.DISABLE_MARKER, const.ENABLE_MARKER}
491 if not toggling:
492 return params, toggling, False
494 status_toggled = False
495 if params == const.ENABLE_MARKER and not self._enabled:
496 self._enabled = True
497 status_toggled = True
498 LOGGER.info("============ Processing enabled. ============")
499 elif params == const.DISABLE_MARKER and self._enabled: 499 ↛ 503line 499 didn't jump to line 503 because the condition on line 499 was always true
500 self._enabled = False
501 status_toggled = True
502 LOGGER.info("============ Processing disabled. ============")
503 return "", toggling, status_toggled
505 def _parse_wildcards(
506 self,
507 directive_str: str,
508 label: str | None,
509 *,
510 force_label_only: bool = False,
511 ) -> str:
512 if not label:
513 return directive_str
515 found_wildcards = list(
516 self._SEARCH_WILDCARD_PATTERN.finditer(directive_str)
517 )
518 found_item = (
519 self._prefix_search(label) if len(found_wildcards) > 0 else None
520 )
521 if isinstance(found_item, TextIndexEntry):
522 replace_label = f'"{found_item.label:s}"'
523 replace_path = found_item.joined_label_path
524 for found_wildcard in reversed(found_wildcards):
525 label_only = (found_wildcard.group(1) != "") or force_label_only
526 replacement = replace_label if label_only else replace_path
527 directive_str = (
528 directive_str[: found_wildcard.start()]
529 + replacement
530 + directive_str[found_wildcard.end() :]
531 )
532 LOGGER.debug(
533 "\tFound %sprefix match for %r: %r",
534 "(label-only) " if label_only else "",
535 label,
536 replacement,
537 )
538 else:
539 for found_wildcard in reversed(found_wildcards): 539 ↛ 540line 539 didn't jump to line 540 because the loop on line 539 never started
540 directive_str = (
541 directive_str[: found_wildcard.start()]
542 + "*" # Fallback on basic wildcard functionality.
543 + directive_str[found_wildcard.end() :]
544 )
545 unstyled_label = MDEmphasis.parse(label)[0]
546 directive_str = directive_str.replace("**", unstyled_label.lower())
547 directive_str = directive_str.replace("*", unstyled_label)
548 return directive_str
550 def _prefix_search(self, text: str) -> TextIndexEntry | None:
551 for entry in self: 551 ↛ 554line 551 didn't jump to line 554 because the loop on line 551 didn't complete
552 if entry.label.startswith(text):
553 return entry
554 return None
556 def _update_index(
557 self,
558 label_path: Iterable[str],
559 label: str,
560 create_ref: bool,
561 cref_type_label_path: list[tuple[CrossReferenceType, LabelPath]],
562 closing: bool,
563 directive: str,
564 locator_emphasis: bool,
565 sort_key: str | None,
566 suffix: str | None,
567 ) -> bool:
568 entry, existed = self.entry_at_label_path(
569 LabelPath((*label_path, label)),
570 create=not closing,
571 )
572 if not entry and closing: 572 ↛ 573line 572 didn't jump to line 573 because the condition on line 572 was never true
573 LOGGER.warning(
574 "Attempted to close a non-existent entry %r; ignoring: %r",
575 LabelPath((*label_path, label)).join(),
576 directive,
577 )
578 return False
580 # Entry exists and we are closing its range,
581 if entry and closing:
582 if entry.references: 582 ↛ 601line 582 didn't jump to line 601 because the condition on line 582 was always true
583 # If it already has a closing ID, update it, but warn
584 if entry.references[-1].end_id is not None: 584 ↛ 585line 584 didn't jump to line 585 because the condition on line 584 was never true
585 LOGGER.warning(
586 "Altering existing end-location of reference %r: %r",
587 entry.joined_label_path,
588 directive,
589 )
590 entry.update_latest_reference_end(
591 self._directive_id,
592 end_suffix=suffix,
593 )
594 LOGGER.debug(
595 "\tSet end-location for reference to %r",
596 entry.joined_label_path,
597 )
598 else:
599 # Entry exists, but has no references, so we can't set a
600 # closing id.
601 LOGGER.warning(
602 "Attempted to close non-existent reference for "
603 "existing entry %r; ignoring: %r",
604 entry.joined_label_path,
605 directive,
606 )
607 return True
609 # We now have the correct entry, whether it existed before or not
610 if TYPE_CHECKING:
611 assert isinstance(entry, TextIndexEntry)
612 if create_ref:
613 entry.add_reference(
614 self._directive_id,
615 locator_emphasis=locator_emphasis,
616 start_suffix=suffix,
617 strict=self._strict,
618 )
619 elif suffix or locator_emphasis:
620 LOGGER.warning(
621 "Ignoring suffix/locator emphasis in cross reference: %r",
622 directive,
623 )
625 if sort_key:
626 if entry.sort_key and entry.sort_key != sort_key: 626 ↛ 627line 626 didn't jump to line 627 because the condition on line 626 was never true
627 LOGGER.warning(
628 "Altering existing sort-key for reference %r: "
629 "before: %r, now: %r, directive: %r",
630 entry.joined_label_path,
631 entry.sort_key,
632 sort_key,
633 directive,
634 )
635 entry.sort_key = sort_key
637 if len(cref_type_label_path) > 0:
638 if existed:
639 LOGGER.debug(
640 "\tAdding cross references to existing entry %r",
641 entry.joined_label_path,
642 )
643 for ref_type, ref_label_path in cref_type_label_path:
644 entry.add_cross_reference(
645 self._directive_id,
646 ref_type,
647 ref_label_path,
648 strict=self._strict,
649 )
651 return True