Coverage for src/debputy/installations.py: 65%
499 statements
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
1import collections
2import dataclasses
3import os.path
4import re
5from enum import IntEnum
6from typing import (
7 List,
8 Dict,
9 FrozenSet,
10 Callable,
11 Union,
12 Iterator,
13 Tuple,
14 Set,
15 Sequence,
16 Optional,
17 Iterable,
18 TYPE_CHECKING,
19 cast,
20 Any,
21 Mapping,
22)
24from debputy.exceptions import DebputyRuntimeError
25from debputy.filesystem_scan import FSPath
26from debputy.manifest_conditions import (
27 ConditionContext,
28 ManifestCondition,
29 _BUILD_DOCS_BDO,
30)
31from debputy.manifest_parser.base_types import (
32 FileSystemMatchRule,
33 FileSystemExactMatchRule,
34 DebputyDispatchableType,
35)
36from debputy.packages import BinaryPackage
37from debputy.path_matcher import MatchRule, ExactFileSystemPath, MATCH_ANYTHING
38from debputy.substitution import Substitution
39from debputy.util import _error, _warn
41if TYPE_CHECKING:
42 from debputy.packager_provided_files import PackagerProvidedFile
43 from debputy.plugin.api import VirtualPath
44 from debputy.plugin.api.impl_types import PluginProvidedDiscardRule
47_MAN_TH_LINE = re.compile(r'^[.]TH\s+\S+\s+"?(\d+[^"\s]*)"?')
48_MAN_DT_LINE = re.compile(r"^[.]Dt\s+\S+\s+(\d+\S*)")
49_MAN_SECTION_BASENAME = re.compile(r"[.]([1-9]\w*)(?:[.]gz)?$")
50_MAN_REAL_SECTION = re.compile(r"^(\d+)")
51_MAN_INST_BASENAME = re.compile(r"[.][^.]+$")
52MAN_GUESS_LANG_FROM_PATH = re.compile(
53 r"(?:^|/)man/(?:([a-z][a-z](?:_[A-Z][A-Z])?)(?:\.[^/]+)?)?/man[1-9]/"
54)
55MAN_GUESS_FROM_BASENAME = re.compile(r"[.]([a-z][a-z](?:_[A-Z][A-Z])?)[.](?:[1-9]|man)")
58class InstallRuleError(DebputyRuntimeError):
59 pass
62class PathAlreadyInstalledOrDiscardedError(InstallRuleError):
63 @property
64 def path(self) -> str:
65 return cast("str", self.args[0])
67 @property
68 def into(self) -> FrozenSet[BinaryPackage]:
69 return cast("FrozenSet[BinaryPackage]", self.args[1])
71 @property
72 def definition_source(self) -> str:
73 return cast("str", self.args[2])
76class ExactPathMatchTwiceError(InstallRuleError):
77 @property
78 def path(self) -> str:
79 return cast("str", self.args[1])
81 @property
82 def into(self) -> BinaryPackage:
83 return cast("BinaryPackage", self.args[2])
85 @property
86 def definition_source(self) -> str:
87 return cast("str", self.args[3])
90class NoMatchForInstallPatternError(InstallRuleError):
91 @property
92 def pattern(self) -> str:
93 return cast("str", self.args[1])
95 @property
96 def search_dirs(self) -> Sequence["SearchDir"]:
97 return cast("Sequence[SearchDir]", self.args[2])
99 @property
100 def definition_source(self) -> str:
101 return cast("str", self.args[3])
104@dataclasses.dataclass(slots=True, frozen=True)
105class SearchDir:
106 search_dir: "VirtualPath"
107 applies_to: FrozenSet[BinaryPackage]
110@dataclasses.dataclass(slots=True, frozen=True)
111class BinaryPackageInstallRuleContext:
112 binary_package: BinaryPackage
113 fs_root: FSPath
114 doc_main_package: BinaryPackage
116 def replace(self, **changes: Any) -> "BinaryPackageInstallRuleContext":
117 return dataclasses.replace(self, **changes)
120@dataclasses.dataclass(slots=True, frozen=True)
121class InstallSearchDirContext:
122 search_dirs: Sequence[SearchDir]
123 check_for_uninstalled_dirs: Sequence["VirtualPath"]
124 # TODO: Support search dirs per-package
125 debian_pkg_dirs: Mapping[str, "VirtualPath"] = dataclasses.field(
126 default_factory=dict
127 )
130@dataclasses.dataclass(slots=True)
131class InstallRuleContext:
132 # TODO: Search dirs should be per-package
133 search_dirs: Sequence[SearchDir]
134 binary_package_contexts: Dict[str, BinaryPackageInstallRuleContext] = (
135 dataclasses.field(default_factory=dict)
136 )
138 def __getitem__(self, item: str) -> BinaryPackageInstallRuleContext:
139 return self.binary_package_contexts[item]
141 def __setitem__(self, key: str, value: BinaryPackageInstallRuleContext) -> None:
142 self.binary_package_contexts[key] = value
144 def replace(self, **changes: Any) -> "InstallRuleContext":
145 return dataclasses.replace(self, **changes)
148@dataclasses.dataclass(slots=True, frozen=True)
149class PathMatch:
150 path: "VirtualPath"
151 search_dir: "VirtualPath"
152 is_exact_match: bool
153 into: FrozenSet[BinaryPackage]
156class DiscardState(IntEnum):
157 UNCHECKED = 0
158 NOT_DISCARDED = 1
159 DISCARDED_BY_PLUGIN_PROVIDED_RULE = 2
160 DISCARDED_BY_MANIFEST_RULE = 3
163def _determine_manpage_section(
164 match_rule: PathMatch,
165 provided_section: Optional[int],
166 definition_source: str,
167) -> Optional[str]:
168 section = str(provided_section) if provided_section is not None else None
169 if section is None:
170 detected_section = None
171 with open(match_rule.path.fs_path, "r") as fd:
172 for line in fd:
173 if not line.startswith((".TH", ".Dt")):
174 continue
176 m = _MAN_DT_LINE.match(line)
177 if not m:
178 m = _MAN_TH_LINE.match(line)
179 if not m:
180 continue
181 detected_section = m.group(1)
182 if "." in detected_section:
183 _warn(
184 f"Ignoring detected section {detected_section} in {match_rule.path.fs_path}"
185 f" (detected via {definition_source}): It looks too much like a version"
186 )
187 detected_section = None
188 break
189 if detected_section is None:
190 m = _MAN_SECTION_BASENAME.search(os.path.basename(match_rule.path.path))
191 if m:
192 detected_section = m.group(1)
193 section = detected_section
195 return section
198def _determine_manpage_real_section(
199 match_rule: PathMatch,
200 section: Optional[str],
201 definition_source: str,
202) -> int:
203 real_section = None
204 if section is not None:
205 m = _MAN_REAL_SECTION.match(section)
206 if m:
207 real_section = int(m.group(1))
208 if real_section is None or real_section < 0 or real_section > 9:
209 if real_section is not None:
210 _warn(
211 f"Computed section for {match_rule.path.fs_path} was {real_section} (section: {section}),"
212 f" which is not a valid section (must be between 1 and 9 incl.)"
213 )
214 _error(
215 f"Could not determine the section for {match_rule.path.fs_path} automatically. The man page"
216 f" was detected via {definition_source}. Consider using `section: <number>` to"
217 " explicitly declare the section. Keep in mind that it applies to all man pages for that"
218 " rule and you may have to split the rule into two for this reason."
219 )
220 return real_section
223def _determine_manpage_language(
224 match_rule: PathMatch,
225 provided_language: Optional[str],
226) -> Optional[str]:
227 if provided_language is not None:
228 if provided_language not in ("derive-from-basename", "derive-from-path"):
229 return provided_language if provided_language != "C" else None
230 if provided_language == "derive-from-basename":
231 m = MAN_GUESS_FROM_BASENAME.search(match_rule.path.name)
232 if m is None:
233 return None
234 return m.group(1)
235 # Fall-through for derive-from-path case
236 m = MAN_GUESS_LANG_FROM_PATH.search(match_rule.path.path)
237 if m is None:
238 return None
239 return m.group(1)
242def _dest_path_for_manpage(
243 provided_section: Optional[int],
244 provided_language: Optional[str],
245 definition_source: str,
246) -> Callable[["PathMatch"], str]:
247 def _manpage_dest_path(match_rule: PathMatch) -> str:
248 inst_basename = _MAN_INST_BASENAME.sub("", match_rule.path.name)
249 section = _determine_manpage_section(
250 match_rule, provided_section, definition_source
251 )
252 real_section = _determine_manpage_real_section(
253 match_rule, section, definition_source
254 )
255 assert section is not None
256 language = _determine_manpage_language(match_rule, provided_language)
257 if language is None:
258 maybe_language = ""
259 else:
260 maybe_language = f"{language}/"
261 lang_suffix = f".{language}"
262 if inst_basename.endswith(lang_suffix):
263 inst_basename = inst_basename[: -len(lang_suffix)]
265 return (
266 f"usr/share/man/{maybe_language}man{real_section}/{inst_basename}.{section}"
267 )
269 return _manpage_dest_path
272class SourcePathMatcher:
273 def __init__(self, auto_discard_rules: List["PluginProvidedDiscardRule"]) -> None:
274 self._already_matched: Dict[
275 str,
276 Tuple[FrozenSet[BinaryPackage], str],
277 ] = {}
278 self._exact_match_request: Set[Tuple[str, str]] = set()
279 self._discarded: Dict[str, DiscardState] = {}
280 self._auto_discard_rules = auto_discard_rules
281 self.used_auto_discard_rules: Dict[str, Set[str]] = collections.defaultdict(set)
283 def is_reserved(self, path: "VirtualPath") -> bool:
284 fs_path = path.fs_path
285 if fs_path in self._already_matched:
286 return True
287 result = self._discarded.get(fs_path, DiscardState.UNCHECKED)
288 if result == DiscardState.UNCHECKED: 288 ↛ 290line 288 didn't jump to line 290, because the condition on line 288 was never false
289 result = self._check_plugin_provided_exclude_state_for(path)
290 if result == DiscardState.NOT_DISCARDED:
291 return False
293 return True
295 def exclude(self, path: str) -> None:
296 self._discarded[path] = DiscardState.DISCARDED_BY_MANIFEST_RULE
298 def _run_plugin_provided_discard_rules_on(self, path: "VirtualPath") -> bool:
299 for dr in self._auto_discard_rules:
300 verdict = dr.should_discard(path)
301 if verdict:
302 self.used_auto_discard_rules[dr.name].add(path.fs_path)
303 return True
304 return False
306 def _check_plugin_provided_exclude_state_for(
307 self,
308 path: "VirtualPath",
309 ) -> DiscardState:
310 cache_misses = []
311 current_path = path
312 while True:
313 fs_path = current_path.fs_path
314 exclude_state = self._discarded.get(fs_path, DiscardState.UNCHECKED)
315 if exclude_state != DiscardState.UNCHECKED:
316 verdict = exclude_state
317 break
318 cache_misses.append(fs_path)
319 if self._run_plugin_provided_discard_rules_on(current_path):
320 verdict = DiscardState.DISCARDED_BY_PLUGIN_PROVIDED_RULE
321 break
322 # We cannot trust a "NOT_DISCARDED" until we check its parent (the directory could
323 # be excluded without the files in it triggering the rule).
324 parent_dir = current_path.parent_dir
325 if not parent_dir:
326 verdict = DiscardState.NOT_DISCARDED
327 break
328 current_path = parent_dir
329 if cache_misses: 329 ↛ 332line 329 didn't jump to line 332, because the condition on line 329 was never false
330 for p in cache_misses:
331 self._discarded[p] = verdict
332 return verdict
334 def may_match(
335 self,
336 match: PathMatch,
337 *,
338 is_exact_match: bool = False,
339 ) -> Tuple[FrozenSet[BinaryPackage], bool]:
340 m = self._already_matched.get(match.path.fs_path)
341 if m: 341 ↛ 342line 341 didn't jump to line 342, because the condition on line 341 was never true
342 return m[0], False
343 current_path = match.path.fs_path
344 discard_state = self._discarded.get(current_path, DiscardState.UNCHECKED)
346 if discard_state == DiscardState.UNCHECKED:
347 discard_state = self._check_plugin_provided_exclude_state_for(match.path)
349 assert discard_state is not None and discard_state != DiscardState.UNCHECKED
351 is_discarded = discard_state != DiscardState.NOT_DISCARDED
352 if (
353 is_exact_match
354 and discard_state == DiscardState.DISCARDED_BY_PLUGIN_PROVIDED_RULE
355 ):
356 is_discarded = False
357 return frozenset(), is_discarded
359 def reserve(
360 self,
361 path: "VirtualPath",
362 reserved_by: FrozenSet[BinaryPackage],
363 definition_source: str,
364 *,
365 is_exact_match: bool = False,
366 ) -> None:
367 fs_path = path.fs_path
368 self._already_matched[fs_path] = reserved_by, definition_source
369 if not is_exact_match: 369 ↛ 371line 369 didn't jump to line 371, because the condition on line 369 was never false
370 return
371 for pkg in reserved_by:
372 m_key = (pkg.name, fs_path)
373 self._exact_match_request.add(m_key)
374 try:
375 del self._discarded[fs_path]
376 except KeyError:
377 pass
378 for discarded_paths in self.used_auto_discard_rules.values():
379 discarded_paths.discard(fs_path)
381 def detect_missing(self, search_dir: "VirtualPath") -> Iterator["VirtualPath"]:
382 stack = list(search_dir.iterdir)
383 while stack:
384 m = stack.pop()
385 if m.is_dir:
386 s_len = len(stack)
387 stack.extend(m.iterdir)
389 if s_len == len(stack) and not self.is_reserved(m):
390 # "Explicitly" empty dir
391 yield m
392 elif not self.is_reserved(m):
393 yield m
395 def find_and_reserve_all_matches(
396 self,
397 match_rule: MatchRule,
398 search_dirs: Sequence[SearchDir],
399 dir_only_match: bool,
400 match_filter: Optional[Callable[["VirtualPath"], bool]],
401 reserved_by: FrozenSet[BinaryPackage],
402 definition_source: str,
403 ) -> Tuple[List[PathMatch], Tuple[int, ...]]:
404 matched = []
405 already_installed_paths = 0
406 already_excluded_paths = 0
407 glob_expand = False if isinstance(match_rule, ExactFileSystemPath) else True
409 for match in _resolve_path(
410 match_rule,
411 search_dirs,
412 dir_only_match,
413 match_filter,
414 reserved_by,
415 ):
416 installed_into, excluded = self.may_match(
417 match, is_exact_match=not glob_expand
418 )
419 if installed_into: 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true
420 if glob_expand:
421 already_installed_paths += 1
422 continue
423 packages = ", ".join(p.name for p in installed_into)
424 raise PathAlreadyInstalledOrDiscardedError(
425 f'The "{match.path.fs_path}" has been reserved by and installed into {packages}.'
426 f" The definition that triggered this issue is {definition_source}.",
427 match,
428 installed_into,
429 definition_source,
430 )
431 if excluded:
432 if glob_expand: 432 ↛ 435line 432 didn't jump to line 435, because the condition on line 432 was never false
433 already_excluded_paths += 1
434 continue
435 raise PathAlreadyInstalledOrDiscardedError(
436 f'The "{match.path.fs_path}" has been excluded. If you want this path installed, move it'
437 f" above the exclusion rule that excluded it. The definition that triggered this"
438 f" issue is {definition_source}.",
439 match,
440 installed_into,
441 definition_source,
442 )
443 if not glob_expand:
444 for pkg in match.into:
445 m_key = (pkg.name, match.path.fs_path)
446 if m_key in self._exact_match_request: 446 ↛ 447line 446 didn't jump to line 447, because the condition on line 446 was never true
447 raise ExactPathMatchTwiceError(
448 f'The path "{match.path.fs_path}" (via exact match) has already been installed'
449 f" into {pkg.name}. The second installation triggered by {definition_source}",
450 match.path,
451 pkg,
452 definition_source,
453 )
454 self._exact_match_request.add(m_key)
456 if reserved_by: 456 ↛ 462line 456 didn't jump to line 462, because the condition on line 456 was never false
457 self._already_matched[match.path.fs_path] = (
458 match.into,
459 definition_source,
460 )
461 else:
462 self.exclude(match.path.fs_path)
463 matched.append(match)
464 exclude_counts = already_installed_paths, already_excluded_paths
465 return matched, exclude_counts
468def _resolve_path(
469 match_rule: MatchRule,
470 search_dirs: Iterable["SearchDir"],
471 dir_only_match: bool,
472 match_filter: Optional[Callable[["VirtualPath"], bool]],
473 into: FrozenSet[BinaryPackage],
474) -> Iterator[PathMatch]:
475 missing_matches = set(into)
476 for sdir in search_dirs:
477 matched = False
478 if into and missing_matches.isdisjoint(sdir.applies_to): 478 ↛ 480line 478 didn't jump to line 480, because the condition on line 478 was never true
479 # All the packages, where this search dir applies, already got a match
480 continue
481 applicable = sdir.applies_to & missing_matches
482 for matched_path in match_rule.finditer(
483 sdir.search_dir,
484 ignore_paths=match_filter,
485 ):
486 if dir_only_match and not matched_path.is_dir: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true
487 continue
488 if matched_path.parent_dir is None:
489 if match_rule is MATCH_ANYTHING: 489 ↛ 491line 489 didn't jump to line 491, because the condition on line 489 was never false
490 continue
491 _error(
492 f"The pattern {match_rule.describe_match_short()} matched the root dir."
493 )
494 yield PathMatch(matched_path, sdir.search_dir, False, applicable)
495 matched = True
496 # continue; we want to match everything we can from this search directory.
498 if matched:
499 missing_matches -= applicable
500 if into and not missing_matches:
501 # For install rules, we can stop as soon as all packages had a match
502 # For discard rules, all search directories must be visited. Otherwise,
503 # you would have to repeat the discard rule once per search dir to be
504 # sure something is fully discarded
505 break
508def _resolve_dest_paths(
509 match: PathMatch,
510 dest_paths: Sequence[Tuple[str, bool]],
511 install_context: "InstallRuleContext",
512) -> Sequence[Tuple[str, "FSPath"]]:
513 dest_and_roots = []
514 for dest_path, dest_path_is_format in dest_paths:
515 if dest_path_is_format:
516 for pkg in match.into:
517 parent_dir = match.path.parent_dir
518 pkg_install_context = install_context[pkg.name]
519 fs_root = pkg_install_context.fs_root
520 dpath = dest_path.format(
521 basename=match.path.name,
522 dirname=parent_dir.path if parent_dir is not None else "",
523 package_name=pkg.name,
524 doc_main_package_name=pkg_install_context.doc_main_package.name,
525 )
526 if dpath.endswith("/"): 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true
527 raise ValueError(
528 f'Provided destination (when resolved for {pkg.name}) for "{match.path.path}" ended'
529 f' with "/" ("{dest_path}"), which it must not!'
530 )
531 dest_and_roots.append((dpath, fs_root))
532 else:
533 if dest_path.endswith("/"): 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true
534 raise ValueError(
535 f'Provided destination for "{match.path.path}" ended with "/" ("{dest_path}"),'
536 " which it must not!"
537 )
538 dest_and_roots.extend(
539 (dest_path, install_context[pkg.name].fs_root) for pkg in match.into
540 )
541 return dest_and_roots
544def _resolve_matches(
545 matches: List[PathMatch],
546 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]],
547 install_context: "InstallRuleContext",
548) -> Iterator[Tuple[PathMatch, Sequence[Tuple[str, "FSPath"]]]]:
549 if callable(dest_paths): 549 ↛ 550line 549 didn't jump to line 550, because the condition on line 549 was never true
550 compute_dest_path = dest_paths
551 for match in matches:
552 dpath = compute_dest_path(match)
553 if dpath.endswith("/"):
554 raise ValueError(
555 f'Provided destination for "{match.path.path}" ended with "/" ("{dpath}"), which it must not!'
556 )
557 dest_and_roots = [
558 (dpath, install_context[pkg.name].fs_root) for pkg in match.into
559 ]
560 yield match, dest_and_roots
561 else:
562 for match in matches:
563 dest_and_roots = _resolve_dest_paths(
564 match,
565 dest_paths,
566 install_context,
567 )
568 yield match, dest_and_roots
571class InstallRule(DebputyDispatchableType):
572 __slots__ = (
573 "_already_matched",
574 "_exact_match_request",
575 "_condition",
576 "_match_filter",
577 "_definition_source",
578 )
580 def __init__(
581 self,
582 condition: Optional[ManifestCondition],
583 definition_source: str,
584 *,
585 match_filter: Optional[Callable[["VirtualPath"], bool]] = None,
586 ) -> None:
587 self._condition = condition
588 self._definition_source = definition_source
589 self._match_filter = match_filter
591 def _check_single_match(
592 self, source: FileSystemMatchRule, matches: List[PathMatch]
593 ) -> None:
594 seen_pkgs = set()
595 problem_pkgs = frozenset()
596 for m in matches:
597 problem_pkgs = seen_pkgs & m.into
598 if problem_pkgs: 598 ↛ 599line 598 didn't jump to line 599, because the condition on line 598 was never true
599 break
600 seen_pkgs.update(problem_pkgs)
601 if problem_pkgs: 601 ↛ 602line 601 didn't jump to line 602, because the condition on line 601 was never true
602 pkg_names = ", ".join(sorted(p.name for p in problem_pkgs))
603 _error(
604 f'The pattern "{source.raw_match_rule}" matched multiple entries for the packages: {pkg_names}.'
605 "However, it should matched exactly one item. Please tighten the pattern defined"
606 f" in {self._definition_source}"
607 )
609 def _match_pattern(
610 self,
611 path_matcher: SourcePathMatcher,
612 fs_match_rule: FileSystemMatchRule,
613 condition_context: ConditionContext,
614 search_dirs: Sequence[SearchDir],
615 into: FrozenSet[BinaryPackage],
616 ) -> List[PathMatch]:
617 (matched, exclude_counts) = path_matcher.find_and_reserve_all_matches(
618 fs_match_rule.match_rule,
619 search_dirs,
620 fs_match_rule.raw_match_rule.endswith("/"),
621 self._match_filter,
622 into,
623 self._definition_source,
624 )
626 already_installed_paths, already_excluded_paths = exclude_counts
628 if into: 628 ↛ 633line 628 didn't jump to line 633, because the condition on line 628 was never false
629 allow_empty_match = all(not p.should_be_acted_on for p in into) 629 ↛ exitline 629 didn't finish the generator expression on line 629
630 else:
631 # discard rules must match provided at least one search dir exist. If none of them
632 # exist, then we assume the discard rule is for a package that will not be built
633 allow_empty_match = any(s.search_dir.is_dir for s in search_dirs)
634 if self._condition is not None and not self._condition.evaluate( 634 ↛ 637line 634 didn't jump to line 637, because the condition on line 634 was never true
635 condition_context
636 ):
637 allow_empty_match = True
639 if not matched and not allow_empty_match:
640 search_dir_text = ", ".join(x.search_dir.fs_path for x in search_dirs)
641 if already_excluded_paths and already_installed_paths: 641 ↛ 642line 641 didn't jump to line 642, because the condition on line 641 was never true
642 total_paths = already_excluded_paths + already_installed_paths
643 msg = (
644 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring"
645 f" {total_paths} path(s) already been matched previously either by install or"
646 f" exclude rules. If you wanted to install some of these paths into multiple"
647 f" packages, please tweak the definition that installed them to install them"
648 f' into multiple packages (usually change "into: foo" to "into: [foo, bar]".'
649 f" If you wanted to install these paths and exclude rules are getting in your"
650 f" way, then please move this install rule before the exclusion rule that causes"
651 f" issue or, in case of built-in excludes, list the paths explicitly (without"
652 f" using patterns). Source for this issue is {self._definition_source}. Match rule:"
653 f" {fs_match_rule.match_rule.describe_match_exact()}"
654 )
655 elif already_excluded_paths: 655 ↛ 656line 655 didn't jump to line 656
656 msg = (
657 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring"
658 f" {already_excluded_paths} path(s) that have been excluded."
659 " If you wanted to install some of these paths, please move the install rule"
660 " before the exclusion rule or, in case of built-in excludes, list the paths explicitly"
661 f" (without using patterns). Source for this issue is {self._definition_source}. Match rule:"
662 f" {fs_match_rule.match_rule.describe_match_exact()}"
663 )
664 elif already_installed_paths: 664 ↛ 665line 664 didn't jump to line 665
665 msg = (
666 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring"
667 f" {already_installed_paths} path(s) already been matched previously."
668 " If you wanted to install some of these paths into multiple packages,"
669 f" please tweak the definition that installed them to install them into"
670 f' multiple packages (usually change "into: foo" to "into: [foo, bar]".'
671 f" Source for this issue is {self._definition_source}. Match rule:"
672 f" {fs_match_rule.match_rule.describe_match_exact()}"
673 )
674 else:
675 # TODO: Try harder to find the match and point out possible typos
676 msg = (
677 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} (definition:"
678 f" {self._definition_source}). Match rule: {fs_match_rule.match_rule.describe_match_exact()}"
679 )
680 raise NoMatchForInstallPatternError(
681 msg,
682 fs_match_rule,
683 search_dirs,
684 self._definition_source,
685 )
686 return matched
688 def _install_matches(
689 self,
690 path_matcher: SourcePathMatcher,
691 matches: List[PathMatch],
692 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]],
693 install_context: "InstallRuleContext",
694 into: FrozenSet[BinaryPackage],
695 condition_context: ConditionContext,
696 ) -> None:
697 if ( 697 ↛ exit, 697 ↛ 7032 missed branches: 1) line 697 didn't jump to the function exit, 2) line 697 didn't jump to line 703, because the condition on line 697 was never true
698 self._condition is not None
699 and not self._condition.evaluate(condition_context)
700 ) or not any(p.should_be_acted_on for p in into):
701 # Rule is disabled; skip all its actions - also allow empty matches
702 # for this particular case.
703 return
705 if not matches: 705 ↛ 706line 705 didn't jump to line 706, because the condition on line 705 was never true
706 raise ValueError("matches must not be empty")
708 for match, dest_paths_and_roots in _resolve_matches(
709 matches,
710 dest_paths,
711 install_context,
712 ):
713 install_recursively_into_dirs = []
714 for dest, fs_root in dest_paths_and_roots:
715 dir_part, basename = os.path.split(dest)
716 # We do not associate these with the FS path. First off,
717 # it is complicated to do in most cases (indeed, debhelper
718 # does not preserve these directories either) and secondly,
719 # it is "only" mtime and mode - mostly irrelevant as the
720 # directory is 99.9% likely to be 0755 (we are talking
721 # directories like "/usr", "/usr/share").
722 dir_path = fs_root.mkdirs(dir_part)
723 existing_path = dir_path.get(basename)
725 if match.path.is_dir:
726 if existing_path is not None and not existing_path.is_dir: 726 ↛ 727line 726 didn't jump to line 727, because the condition on line 726 was never true
727 existing_path.unlink()
728 existing_path = None
729 current_dir = existing_path
731 if current_dir is None: 731 ↛ 735line 731 didn't jump to line 735, because the condition on line 731 was never false
732 current_dir = dir_path.mkdir(
733 basename, reference_path=match.path
734 )
735 install_recursively_into_dirs.append(current_dir)
736 else:
737 if existing_path is not None and existing_path.is_dir: 737 ↛ 738line 737 didn't jump to line 738, because the condition on line 737 was never true
738 _error(
739 f"Cannot install {match.path} ({match.path.fs_path}) as {dest}. That path already exist"
740 f" and is a directory. This error was triggered via {self._definition_source}."
741 )
743 if match.path.is_symlink:
744 dir_path.add_symlink(
745 basename, match.path.readlink(), reference_path=match.path
746 )
747 else:
748 dir_path.insert_file_from_fs_path(
749 basename,
750 match.path.fs_path,
751 follow_symlinks=False,
752 use_fs_path_mode=True,
753 reference_path=match.path,
754 )
755 if install_recursively_into_dirs:
756 self._install_dir_recursively(
757 path_matcher, install_recursively_into_dirs, match, into
758 )
760 def _install_dir_recursively(
761 self,
762 path_matcher: SourcePathMatcher,
763 parent_dirs: Sequence[FSPath],
764 match: PathMatch,
765 into: FrozenSet[BinaryPackage],
766 ) -> None:
767 stack = [
768 (parent_dirs, e)
769 for e in match.path.iterdir
770 if not path_matcher.is_reserved(e)
771 ]
773 while stack:
774 current_dirs, dir_entry = stack.pop()
775 path_matcher.reserve(
776 dir_entry,
777 into,
778 self._definition_source,
779 is_exact_match=False,
780 )
781 if dir_entry.is_dir: 781 ↛ 782line 781 didn't jump to line 782, because the condition on line 781 was never true
782 new_dirs = [
783 d.mkdir(dir_entry.name, reference_path=dir_entry)
784 for d in current_dirs
785 ]
786 stack.extend(
787 (new_dirs, de)
788 for de in dir_entry.iterdir
789 if not path_matcher.is_reserved(de)
790 )
791 elif dir_entry.is_symlink:
792 for current_dir in current_dirs:
793 current_dir.add_symlink(
794 dir_entry.name,
795 dir_entry.readlink(),
796 reference_path=dir_entry,
797 )
798 elif dir_entry.is_file: 798 ↛ 808line 798 didn't jump to line 808, because the condition on line 798 was never false
799 for current_dir in current_dirs:
800 current_dir.insert_file_from_fs_path(
801 dir_entry.name,
802 dir_entry.fs_path,
803 use_fs_path_mode=True,
804 follow_symlinks=False,
805 reference_path=dir_entry,
806 )
807 else:
808 _error(
809 f"Unsupported file type: {dir_entry.fs_path} - neither a file, directory or symlink"
810 )
812 def perform_install(
813 self,
814 path_matcher: SourcePathMatcher,
815 install_context: InstallRuleContext,
816 condition_context: ConditionContext,
817 ) -> None:
818 raise NotImplementedError
820 @classmethod
821 def install_as(
822 cls,
823 source: FileSystemMatchRule,
824 dest_path: str,
825 into: FrozenSet[BinaryPackage],
826 definition_source: str,
827 condition: Optional[ManifestCondition],
828 ) -> "InstallRule":
829 return GenericInstallationRule(
830 [source],
831 [(dest_path, False)],
832 into,
833 condition,
834 definition_source,
835 require_single_match=True,
836 )
838 @classmethod
839 def install_dest(
840 cls,
841 sources: Sequence[FileSystemMatchRule],
842 dest_dir: Optional[str],
843 into: FrozenSet[BinaryPackage],
844 definition_source: str,
845 condition: Optional[ManifestCondition],
846 ) -> "InstallRule":
847 if dest_dir is None:
848 dest_dir = "{dirname}/{basename}"
849 else:
850 dest_dir = os.path.join(dest_dir, "{basename}")
851 return GenericInstallationRule(
852 sources,
853 [(dest_dir, True)],
854 into,
855 condition,
856 definition_source,
857 )
859 @classmethod
860 def install_multi_as(
861 cls,
862 source: FileSystemMatchRule,
863 dest_paths: Sequence[str],
864 into: FrozenSet[BinaryPackage],
865 definition_source: str,
866 condition: Optional[ManifestCondition],
867 ) -> "InstallRule":
868 if len(dest_paths) < 2: 868 ↛ 869line 868 didn't jump to line 869, because the condition on line 868 was never true
869 raise ValueError(
870 "Please use `install_as` when there is less than 2 dest path"
871 )
872 dps = tuple((dp, False) for dp in dest_paths)
873 return GenericInstallationRule(
874 [source],
875 dps,
876 into,
877 condition,
878 definition_source,
879 require_single_match=True,
880 )
882 @classmethod
883 def install_multi_dest(
884 cls,
885 sources: Sequence[FileSystemMatchRule],
886 dest_dirs: Sequence[str],
887 into: FrozenSet[BinaryPackage],
888 definition_source: str,
889 condition: Optional[ManifestCondition],
890 ) -> "InstallRule":
891 if len(dest_dirs) < 2: 891 ↛ 892line 891 didn't jump to line 892, because the condition on line 891 was never true
892 raise ValueError(
893 "Please use `install_dest` when there is less than 2 dest dir"
894 )
895 dest_paths = tuple((os.path.join(dp, "{basename}"), True) for dp in dest_dirs)
896 return GenericInstallationRule(
897 sources,
898 dest_paths,
899 into,
900 condition,
901 definition_source,
902 )
904 @classmethod
905 def install_doc(
906 cls,
907 sources: Sequence[FileSystemMatchRule],
908 dest_dir: Optional[str],
909 into: FrozenSet[BinaryPackage],
910 definition_source: str,
911 condition: Optional[ManifestCondition],
912 ) -> "InstallRule":
913 cond: ManifestCondition = _BUILD_DOCS_BDO
914 if condition is not None:
915 cond = ManifestCondition.all_of([cond, condition])
916 dest_path_is_format = False
917 if dest_dir is None:
918 dest_dir = "usr/share/doc/{doc_main_package_name}/{basename}"
919 dest_path_is_format = True
921 return GenericInstallationRule(
922 sources,
923 [(dest_dir, dest_path_is_format)],
924 into,
925 cond,
926 definition_source,
927 )
929 @classmethod
930 def install_doc_as(
931 cls,
932 source: FileSystemMatchRule,
933 dest_path: str,
934 into: FrozenSet[BinaryPackage],
935 definition_source: str,
936 condition: Optional[ManifestCondition],
937 ) -> "InstallRule":
938 cond: ManifestCondition = _BUILD_DOCS_BDO
939 if condition is not None:
940 cond = ManifestCondition.all_of([cond, condition])
942 return GenericInstallationRule(
943 [source],
944 [(dest_path, False)],
945 into,
946 cond,
947 definition_source,
948 require_single_match=True,
949 )
951 @classmethod
952 def install_examples(
953 cls,
954 sources: Sequence[FileSystemMatchRule],
955 into: FrozenSet[BinaryPackage],
956 definition_source: str,
957 condition: Optional[ManifestCondition],
958 ) -> "InstallRule":
959 cond: ManifestCondition = _BUILD_DOCS_BDO
960 if condition is not None: 960 ↛ 961line 960 didn't jump to line 961, because the condition on line 960 was never true
961 cond = ManifestCondition.all_of([cond, condition])
962 return GenericInstallationRule(
963 sources,
964 [("usr/share/doc/{doc_main_package_name}/examples/{basename}", True)],
965 into,
966 cond,
967 definition_source,
968 )
970 @classmethod
971 def install_man(
972 cls,
973 sources: Sequence[FileSystemMatchRule],
974 into: FrozenSet[BinaryPackage],
975 section: Optional[int],
976 language: Optional[str],
977 definition_source: str,
978 condition: Optional[ManifestCondition],
979 ) -> "InstallRule":
980 cond: ManifestCondition = _BUILD_DOCS_BDO
981 if condition is not None: 981 ↛ 982line 981 didn't jump to line 982, because the condition on line 981 was never true
982 cond = ManifestCondition.all_of([cond, condition])
984 dest_path_computer = _dest_path_for_manpage(
985 section, language, definition_source
986 )
988 return GenericInstallationRule( 988 ↛ exitline 988 didn't jump to the function exit
989 sources,
990 dest_path_computer,
991 into,
992 cond,
993 definition_source,
994 match_filter=lambda m: not m.is_file,
995 )
997 @classmethod
998 def discard_paths(
999 cls,
1000 paths: Sequence[FileSystemMatchRule],
1001 definition_source: str,
1002 condition: Optional[ManifestCondition],
1003 *,
1004 limit_to: Optional[Sequence[FileSystemExactMatchRule]] = None,
1005 ) -> "InstallRule":
1006 return DiscardRule(
1007 paths,
1008 condition,
1009 tuple(limit_to) if limit_to is not None else tuple(),
1010 definition_source,
1011 )
1014class PPFInstallRule(InstallRule):
1015 __slots__ = (
1016 "_ppfs",
1017 "_substitution",
1018 "_into",
1019 )
1021 def __init__(
1022 self,
1023 into: BinaryPackage,
1024 substitution: Substitution,
1025 ppfs: Sequence["PackagerProvidedFile"],
1026 ) -> None:
1027 super().__init__(
1028 None,
1029 "<built-in; PPF install rule>",
1030 )
1031 self._substitution = substitution
1032 self._ppfs = ppfs
1033 self._into = into
1035 def perform_install(
1036 self,
1037 path_matcher: SourcePathMatcher,
1038 install_context: InstallRuleContext,
1039 condition_context: ConditionContext,
1040 ) -> None:
1041 binary_install_context = install_context[self._into.name]
1042 fs_root = binary_install_context.fs_root
1043 for ppf in self._ppfs:
1044 source_path = ppf.path.fs_path
1045 dest_dir, name = ppf.compute_dest()
1046 dir_path = fs_root.mkdirs(dest_dir)
1048 dir_path.insert_file_from_fs_path(
1049 name,
1050 source_path,
1051 follow_symlinks=True,
1052 use_fs_path_mode=False,
1053 mode=ppf.definition.default_mode,
1054 )
1057class GenericInstallationRule(InstallRule):
1058 __slots__ = (
1059 "_sources",
1060 "_into",
1061 "_dest_paths",
1062 "_require_single_match",
1063 )
1065 def __init__(
1066 self,
1067 sources: Sequence[FileSystemMatchRule],
1068 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]],
1069 into: FrozenSet[BinaryPackage],
1070 condition: Optional[ManifestCondition],
1071 definition_source: str,
1072 *,
1073 require_single_match: bool = False,
1074 match_filter: Optional[Callable[["VirtualPath"], bool]] = None,
1075 ) -> None:
1076 super().__init__(
1077 condition,
1078 definition_source,
1079 match_filter=match_filter,
1080 )
1081 self._sources = sources
1082 self._into = into
1083 self._dest_paths = dest_paths
1084 self._require_single_match = require_single_match
1085 if self._require_single_match and len(sources) != 1: 1085 ↛ 1086line 1085 didn't jump to line 1086, because the condition on line 1085 was never true
1086 raise ValueError("require_single_match implies sources must have len 1")
1088 def perform_install(
1089 self,
1090 path_matcher: SourcePathMatcher,
1091 install_context: InstallRuleContext,
1092 condition_context: ConditionContext,
1093 ) -> None:
1094 for source in self._sources:
1095 matches = self._match_pattern(
1096 path_matcher,
1097 source,
1098 condition_context,
1099 install_context.search_dirs,
1100 self._into,
1101 )
1102 if self._require_single_match and len(matches) > 1:
1103 self._check_single_match(source, matches)
1104 self._install_matches(
1105 path_matcher,
1106 matches,
1107 self._dest_paths,
1108 install_context,
1109 self._into,
1110 condition_context,
1111 )
1114class DiscardRule(InstallRule):
1115 __slots__ = ("_fs_match_rules", "_limit_to")
1117 def __init__(
1118 self,
1119 fs_match_rules: Sequence[FileSystemMatchRule],
1120 condition: Optional[ManifestCondition],
1121 limit_to: Sequence[FileSystemExactMatchRule],
1122 definition_source: str,
1123 ) -> None:
1124 super().__init__(condition, definition_source)
1125 self._fs_match_rules = fs_match_rules
1126 self._limit_to = limit_to
1128 def perform_install(
1129 self,
1130 path_matcher: SourcePathMatcher,
1131 install_context: InstallRuleContext,
1132 condition_context: ConditionContext,
1133 ) -> None:
1134 into = frozenset()
1135 limit_to = self._limit_to
1136 if limit_to:
1137 matches = {x.match_rule.path for x in limit_to}
1138 search_dirs = tuple(
1139 s
1140 for s in install_context.search_dirs
1141 if s.search_dir.fs_path in matches
1142 )
1143 if len(limit_to) != len(search_dirs):
1144 matches.difference(s.search_dir.fs_path for s in search_dirs)
1145 paths = ":".join(matches)
1146 _error(
1147 f"The discard rule defined at {self._definition_source} mentions the following"
1148 f" search directories that were not known to debputy: {paths}."
1149 " Either the search dir is missing somewhere else or it should be removed from"
1150 " the discard rule."
1151 )
1152 else:
1153 search_dirs = install_context.search_dirs
1155 for fs_match_rule in self._fs_match_rules:
1156 self._match_pattern(
1157 path_matcher,
1158 fs_match_rule,
1159 condition_context,
1160 search_dirs,
1161 into,
1162 )