Coverage for src/debputy/installations.py: 65%

499 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2024-04-07 12:14 +0200

1import collections 

2import dataclasses 

3import os.path 

4import re 

5from enum import IntEnum 

6from typing import ( 

7 List, 

8 Dict, 

9 FrozenSet, 

10 Callable, 

11 Union, 

12 Iterator, 

13 Tuple, 

14 Set, 

15 Sequence, 

16 Optional, 

17 Iterable, 

18 TYPE_CHECKING, 

19 cast, 

20 Any, 

21 Mapping, 

22) 

23 

24from debputy.exceptions import DebputyRuntimeError 

25from debputy.filesystem_scan import FSPath 

26from debputy.manifest_conditions import ( 

27 ConditionContext, 

28 ManifestCondition, 

29 _BUILD_DOCS_BDO, 

30) 

31from debputy.manifest_parser.base_types import ( 

32 FileSystemMatchRule, 

33 FileSystemExactMatchRule, 

34 DebputyDispatchableType, 

35) 

36from debputy.packages import BinaryPackage 

37from debputy.path_matcher import MatchRule, ExactFileSystemPath, MATCH_ANYTHING 

38from debputy.substitution import Substitution 

39from debputy.util import _error, _warn 

40 

41if TYPE_CHECKING: 

42 from debputy.packager_provided_files import PackagerProvidedFile 

43 from debputy.plugin.api import VirtualPath 

44 from debputy.plugin.api.impl_types import PluginProvidedDiscardRule 

45 

46 

47_MAN_TH_LINE = re.compile(r'^[.]TH\s+\S+\s+"?(\d+[^"\s]*)"?') 

48_MAN_DT_LINE = re.compile(r"^[.]Dt\s+\S+\s+(\d+\S*)") 

49_MAN_SECTION_BASENAME = re.compile(r"[.]([1-9]\w*)(?:[.]gz)?$") 

50_MAN_REAL_SECTION = re.compile(r"^(\d+)") 

51_MAN_INST_BASENAME = re.compile(r"[.][^.]+$") 

52MAN_GUESS_LANG_FROM_PATH = re.compile( 

53 r"(?:^|/)man/(?:([a-z][a-z](?:_[A-Z][A-Z])?)(?:\.[^/]+)?)?/man[1-9]/" 

54) 

55MAN_GUESS_FROM_BASENAME = re.compile(r"[.]([a-z][a-z](?:_[A-Z][A-Z])?)[.](?:[1-9]|man)") 

56 

57 

58class InstallRuleError(DebputyRuntimeError): 

59 pass 

60 

61 

62class PathAlreadyInstalledOrDiscardedError(InstallRuleError): 

63 @property 

64 def path(self) -> str: 

65 return cast("str", self.args[0]) 

66 

67 @property 

68 def into(self) -> FrozenSet[BinaryPackage]: 

69 return cast("FrozenSet[BinaryPackage]", self.args[1]) 

70 

71 @property 

72 def definition_source(self) -> str: 

73 return cast("str", self.args[2]) 

74 

75 

76class ExactPathMatchTwiceError(InstallRuleError): 

77 @property 

78 def path(self) -> str: 

79 return cast("str", self.args[1]) 

80 

81 @property 

82 def into(self) -> BinaryPackage: 

83 return cast("BinaryPackage", self.args[2]) 

84 

85 @property 

86 def definition_source(self) -> str: 

87 return cast("str", self.args[3]) 

88 

89 

90class NoMatchForInstallPatternError(InstallRuleError): 

91 @property 

92 def pattern(self) -> str: 

93 return cast("str", self.args[1]) 

94 

95 @property 

96 def search_dirs(self) -> Sequence["SearchDir"]: 

97 return cast("Sequence[SearchDir]", self.args[2]) 

98 

99 @property 

100 def definition_source(self) -> str: 

101 return cast("str", self.args[3]) 

102 

103 

104@dataclasses.dataclass(slots=True, frozen=True) 

105class SearchDir: 

106 search_dir: "VirtualPath" 

107 applies_to: FrozenSet[BinaryPackage] 

108 

109 

110@dataclasses.dataclass(slots=True, frozen=True) 

111class BinaryPackageInstallRuleContext: 

112 binary_package: BinaryPackage 

113 fs_root: FSPath 

114 doc_main_package: BinaryPackage 

115 

116 def replace(self, **changes: Any) -> "BinaryPackageInstallRuleContext": 

117 return dataclasses.replace(self, **changes) 

118 

119 

120@dataclasses.dataclass(slots=True, frozen=True) 

121class InstallSearchDirContext: 

122 search_dirs: Sequence[SearchDir] 

123 check_for_uninstalled_dirs: Sequence["VirtualPath"] 

124 # TODO: Support search dirs per-package 

125 debian_pkg_dirs: Mapping[str, "VirtualPath"] = dataclasses.field( 

126 default_factory=dict 

127 ) 

128 

129 

130@dataclasses.dataclass(slots=True) 

131class InstallRuleContext: 

132 # TODO: Search dirs should be per-package 

133 search_dirs: Sequence[SearchDir] 

134 binary_package_contexts: Dict[str, BinaryPackageInstallRuleContext] = ( 

135 dataclasses.field(default_factory=dict) 

136 ) 

137 

138 def __getitem__(self, item: str) -> BinaryPackageInstallRuleContext: 

139 return self.binary_package_contexts[item] 

140 

141 def __setitem__(self, key: str, value: BinaryPackageInstallRuleContext) -> None: 

142 self.binary_package_contexts[key] = value 

143 

144 def replace(self, **changes: Any) -> "InstallRuleContext": 

145 return dataclasses.replace(self, **changes) 

146 

147 

148@dataclasses.dataclass(slots=True, frozen=True) 

149class PathMatch: 

150 path: "VirtualPath" 

151 search_dir: "VirtualPath" 

152 is_exact_match: bool 

153 into: FrozenSet[BinaryPackage] 

154 

155 

156class DiscardState(IntEnum): 

157 UNCHECKED = 0 

158 NOT_DISCARDED = 1 

159 DISCARDED_BY_PLUGIN_PROVIDED_RULE = 2 

160 DISCARDED_BY_MANIFEST_RULE = 3 

161 

162 

163def _determine_manpage_section( 

164 match_rule: PathMatch, 

165 provided_section: Optional[int], 

166 definition_source: str, 

167) -> Optional[str]: 

168 section = str(provided_section) if provided_section is not None else None 

169 if section is None: 

170 detected_section = None 

171 with open(match_rule.path.fs_path, "r") as fd: 

172 for line in fd: 

173 if not line.startswith((".TH", ".Dt")): 

174 continue 

175 

176 m = _MAN_DT_LINE.match(line) 

177 if not m: 

178 m = _MAN_TH_LINE.match(line) 

179 if not m: 

180 continue 

181 detected_section = m.group(1) 

182 if "." in detected_section: 

183 _warn( 

184 f"Ignoring detected section {detected_section} in {match_rule.path.fs_path}" 

185 f" (detected via {definition_source}): It looks too much like a version" 

186 ) 

187 detected_section = None 

188 break 

189 if detected_section is None: 

190 m = _MAN_SECTION_BASENAME.search(os.path.basename(match_rule.path.path)) 

191 if m: 

192 detected_section = m.group(1) 

193 section = detected_section 

194 

195 return section 

196 

197 

198def _determine_manpage_real_section( 

199 match_rule: PathMatch, 

200 section: Optional[str], 

201 definition_source: str, 

202) -> int: 

203 real_section = None 

204 if section is not None: 

205 m = _MAN_REAL_SECTION.match(section) 

206 if m: 

207 real_section = int(m.group(1)) 

208 if real_section is None or real_section < 0 or real_section > 9: 

209 if real_section is not None: 

210 _warn( 

211 f"Computed section for {match_rule.path.fs_path} was {real_section} (section: {section})," 

212 f" which is not a valid section (must be between 1 and 9 incl.)" 

213 ) 

214 _error( 

215 f"Could not determine the section for {match_rule.path.fs_path} automatically. The man page" 

216 f" was detected via {definition_source}. Consider using `section: <number>` to" 

217 " explicitly declare the section. Keep in mind that it applies to all man pages for that" 

218 " rule and you may have to split the rule into two for this reason." 

219 ) 

220 return real_section 

221 

222 

223def _determine_manpage_language( 

224 match_rule: PathMatch, 

225 provided_language: Optional[str], 

226) -> Optional[str]: 

227 if provided_language is not None: 

228 if provided_language not in ("derive-from-basename", "derive-from-path"): 

229 return provided_language if provided_language != "C" else None 

230 if provided_language == "derive-from-basename": 

231 m = MAN_GUESS_FROM_BASENAME.search(match_rule.path.name) 

232 if m is None: 

233 return None 

234 return m.group(1) 

235 # Fall-through for derive-from-path case 

236 m = MAN_GUESS_LANG_FROM_PATH.search(match_rule.path.path) 

237 if m is None: 

238 return None 

239 return m.group(1) 

240 

241 

242def _dest_path_for_manpage( 

243 provided_section: Optional[int], 

244 provided_language: Optional[str], 

245 definition_source: str, 

246) -> Callable[["PathMatch"], str]: 

247 def _manpage_dest_path(match_rule: PathMatch) -> str: 

248 inst_basename = _MAN_INST_BASENAME.sub("", match_rule.path.name) 

249 section = _determine_manpage_section( 

250 match_rule, provided_section, definition_source 

251 ) 

252 real_section = _determine_manpage_real_section( 

253 match_rule, section, definition_source 

254 ) 

255 assert section is not None 

256 language = _determine_manpage_language(match_rule, provided_language) 

257 if language is None: 

258 maybe_language = "" 

259 else: 

260 maybe_language = f"{language}/" 

261 lang_suffix = f".{language}" 

262 if inst_basename.endswith(lang_suffix): 

263 inst_basename = inst_basename[: -len(lang_suffix)] 

264 

265 return ( 

266 f"usr/share/man/{maybe_language}man{real_section}/{inst_basename}.{section}" 

267 ) 

268 

269 return _manpage_dest_path 

270 

271 

272class SourcePathMatcher: 

273 def __init__(self, auto_discard_rules: List["PluginProvidedDiscardRule"]) -> None: 

274 self._already_matched: Dict[ 

275 str, 

276 Tuple[FrozenSet[BinaryPackage], str], 

277 ] = {} 

278 self._exact_match_request: Set[Tuple[str, str]] = set() 

279 self._discarded: Dict[str, DiscardState] = {} 

280 self._auto_discard_rules = auto_discard_rules 

281 self.used_auto_discard_rules: Dict[str, Set[str]] = collections.defaultdict(set) 

282 

283 def is_reserved(self, path: "VirtualPath") -> bool: 

284 fs_path = path.fs_path 

285 if fs_path in self._already_matched: 

286 return True 

287 result = self._discarded.get(fs_path, DiscardState.UNCHECKED) 

288 if result == DiscardState.UNCHECKED: 288 ↛ 290line 288 didn't jump to line 290, because the condition on line 288 was never false

289 result = self._check_plugin_provided_exclude_state_for(path) 

290 if result == DiscardState.NOT_DISCARDED: 

291 return False 

292 

293 return True 

294 

295 def exclude(self, path: str) -> None: 

296 self._discarded[path] = DiscardState.DISCARDED_BY_MANIFEST_RULE 

297 

298 def _run_plugin_provided_discard_rules_on(self, path: "VirtualPath") -> bool: 

299 for dr in self._auto_discard_rules: 

300 verdict = dr.should_discard(path) 

301 if verdict: 

302 self.used_auto_discard_rules[dr.name].add(path.fs_path) 

303 return True 

304 return False 

305 

306 def _check_plugin_provided_exclude_state_for( 

307 self, 

308 path: "VirtualPath", 

309 ) -> DiscardState: 

310 cache_misses = [] 

311 current_path = path 

312 while True: 

313 fs_path = current_path.fs_path 

314 exclude_state = self._discarded.get(fs_path, DiscardState.UNCHECKED) 

315 if exclude_state != DiscardState.UNCHECKED: 

316 verdict = exclude_state 

317 break 

318 cache_misses.append(fs_path) 

319 if self._run_plugin_provided_discard_rules_on(current_path): 

320 verdict = DiscardState.DISCARDED_BY_PLUGIN_PROVIDED_RULE 

321 break 

322 # We cannot trust a "NOT_DISCARDED" until we check its parent (the directory could 

323 # be excluded without the files in it triggering the rule). 

324 parent_dir = current_path.parent_dir 

325 if not parent_dir: 

326 verdict = DiscardState.NOT_DISCARDED 

327 break 

328 current_path = parent_dir 

329 if cache_misses: 329 ↛ 332line 329 didn't jump to line 332, because the condition on line 329 was never false

330 for p in cache_misses: 

331 self._discarded[p] = verdict 

332 return verdict 

333 

334 def may_match( 

335 self, 

336 match: PathMatch, 

337 *, 

338 is_exact_match: bool = False, 

339 ) -> Tuple[FrozenSet[BinaryPackage], bool]: 

340 m = self._already_matched.get(match.path.fs_path) 

341 if m: 341 ↛ 342line 341 didn't jump to line 342, because the condition on line 341 was never true

342 return m[0], False 

343 current_path = match.path.fs_path 

344 discard_state = self._discarded.get(current_path, DiscardState.UNCHECKED) 

345 

346 if discard_state == DiscardState.UNCHECKED: 

347 discard_state = self._check_plugin_provided_exclude_state_for(match.path) 

348 

349 assert discard_state is not None and discard_state != DiscardState.UNCHECKED 

350 

351 is_discarded = discard_state != DiscardState.NOT_DISCARDED 

352 if ( 

353 is_exact_match 

354 and discard_state == DiscardState.DISCARDED_BY_PLUGIN_PROVIDED_RULE 

355 ): 

356 is_discarded = False 

357 return frozenset(), is_discarded 

358 

359 def reserve( 

360 self, 

361 path: "VirtualPath", 

362 reserved_by: FrozenSet[BinaryPackage], 

363 definition_source: str, 

364 *, 

365 is_exact_match: bool = False, 

366 ) -> None: 

367 fs_path = path.fs_path 

368 self._already_matched[fs_path] = reserved_by, definition_source 

369 if not is_exact_match: 369 ↛ 371line 369 didn't jump to line 371, because the condition on line 369 was never false

370 return 

371 for pkg in reserved_by: 

372 m_key = (pkg.name, fs_path) 

373 self._exact_match_request.add(m_key) 

374 try: 

375 del self._discarded[fs_path] 

376 except KeyError: 

377 pass 

378 for discarded_paths in self.used_auto_discard_rules.values(): 

379 discarded_paths.discard(fs_path) 

380 

381 def detect_missing(self, search_dir: "VirtualPath") -> Iterator["VirtualPath"]: 

382 stack = list(search_dir.iterdir) 

383 while stack: 

384 m = stack.pop() 

385 if m.is_dir: 

386 s_len = len(stack) 

387 stack.extend(m.iterdir) 

388 

389 if s_len == len(stack) and not self.is_reserved(m): 

390 # "Explicitly" empty dir 

391 yield m 

392 elif not self.is_reserved(m): 

393 yield m 

394 

395 def find_and_reserve_all_matches( 

396 self, 

397 match_rule: MatchRule, 

398 search_dirs: Sequence[SearchDir], 

399 dir_only_match: bool, 

400 match_filter: Optional[Callable[["VirtualPath"], bool]], 

401 reserved_by: FrozenSet[BinaryPackage], 

402 definition_source: str, 

403 ) -> Tuple[List[PathMatch], Tuple[int, ...]]: 

404 matched = [] 

405 already_installed_paths = 0 

406 already_excluded_paths = 0 

407 glob_expand = False if isinstance(match_rule, ExactFileSystemPath) else True 

408 

409 for match in _resolve_path( 

410 match_rule, 

411 search_dirs, 

412 dir_only_match, 

413 match_filter, 

414 reserved_by, 

415 ): 

416 installed_into, excluded = self.may_match( 

417 match, is_exact_match=not glob_expand 

418 ) 

419 if installed_into: 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true

420 if glob_expand: 

421 already_installed_paths += 1 

422 continue 

423 packages = ", ".join(p.name for p in installed_into) 

424 raise PathAlreadyInstalledOrDiscardedError( 

425 f'The "{match.path.fs_path}" has been reserved by and installed into {packages}.' 

426 f" The definition that triggered this issue is {definition_source}.", 

427 match, 

428 installed_into, 

429 definition_source, 

430 ) 

431 if excluded: 

432 if glob_expand: 432 ↛ 435line 432 didn't jump to line 435, because the condition on line 432 was never false

433 already_excluded_paths += 1 

434 continue 

435 raise PathAlreadyInstalledOrDiscardedError( 

436 f'The "{match.path.fs_path}" has been excluded. If you want this path installed, move it' 

437 f" above the exclusion rule that excluded it. The definition that triggered this" 

438 f" issue is {definition_source}.", 

439 match, 

440 installed_into, 

441 definition_source, 

442 ) 

443 if not glob_expand: 

444 for pkg in match.into: 

445 m_key = (pkg.name, match.path.fs_path) 

446 if m_key in self._exact_match_request: 446 ↛ 447line 446 didn't jump to line 447, because the condition on line 446 was never true

447 raise ExactPathMatchTwiceError( 

448 f'The path "{match.path.fs_path}" (via exact match) has already been installed' 

449 f" into {pkg.name}. The second installation triggered by {definition_source}", 

450 match.path, 

451 pkg, 

452 definition_source, 

453 ) 

454 self._exact_match_request.add(m_key) 

455 

456 if reserved_by: 456 ↛ 462line 456 didn't jump to line 462, because the condition on line 456 was never false

457 self._already_matched[match.path.fs_path] = ( 

458 match.into, 

459 definition_source, 

460 ) 

461 else: 

462 self.exclude(match.path.fs_path) 

463 matched.append(match) 

464 exclude_counts = already_installed_paths, already_excluded_paths 

465 return matched, exclude_counts 

466 

467 

468def _resolve_path( 

469 match_rule: MatchRule, 

470 search_dirs: Iterable["SearchDir"], 

471 dir_only_match: bool, 

472 match_filter: Optional[Callable[["VirtualPath"], bool]], 

473 into: FrozenSet[BinaryPackage], 

474) -> Iterator[PathMatch]: 

475 missing_matches = set(into) 

476 for sdir in search_dirs: 

477 matched = False 

478 if into and missing_matches.isdisjoint(sdir.applies_to): 478 ↛ 480line 478 didn't jump to line 480, because the condition on line 478 was never true

479 # All the packages, where this search dir applies, already got a match 

480 continue 

481 applicable = sdir.applies_to & missing_matches 

482 for matched_path in match_rule.finditer( 

483 sdir.search_dir, 

484 ignore_paths=match_filter, 

485 ): 

486 if dir_only_match and not matched_path.is_dir: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true

487 continue 

488 if matched_path.parent_dir is None: 

489 if match_rule is MATCH_ANYTHING: 489 ↛ 491line 489 didn't jump to line 491, because the condition on line 489 was never false

490 continue 

491 _error( 

492 f"The pattern {match_rule.describe_match_short()} matched the root dir." 

493 ) 

494 yield PathMatch(matched_path, sdir.search_dir, False, applicable) 

495 matched = True 

496 # continue; we want to match everything we can from this search directory. 

497 

498 if matched: 

499 missing_matches -= applicable 

500 if into and not missing_matches: 

501 # For install rules, we can stop as soon as all packages had a match 

502 # For discard rules, all search directories must be visited. Otherwise, 

503 # you would have to repeat the discard rule once per search dir to be 

504 # sure something is fully discarded 

505 break 

506 

507 

508def _resolve_dest_paths( 

509 match: PathMatch, 

510 dest_paths: Sequence[Tuple[str, bool]], 

511 install_context: "InstallRuleContext", 

512) -> Sequence[Tuple[str, "FSPath"]]: 

513 dest_and_roots = [] 

514 for dest_path, dest_path_is_format in dest_paths: 

515 if dest_path_is_format: 

516 for pkg in match.into: 

517 parent_dir = match.path.parent_dir 

518 pkg_install_context = install_context[pkg.name] 

519 fs_root = pkg_install_context.fs_root 

520 dpath = dest_path.format( 

521 basename=match.path.name, 

522 dirname=parent_dir.path if parent_dir is not None else "", 

523 package_name=pkg.name, 

524 doc_main_package_name=pkg_install_context.doc_main_package.name, 

525 ) 

526 if dpath.endswith("/"): 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true

527 raise ValueError( 

528 f'Provided destination (when resolved for {pkg.name}) for "{match.path.path}" ended' 

529 f' with "/" ("{dest_path}"), which it must not!' 

530 ) 

531 dest_and_roots.append((dpath, fs_root)) 

532 else: 

533 if dest_path.endswith("/"): 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true

534 raise ValueError( 

535 f'Provided destination for "{match.path.path}" ended with "/" ("{dest_path}"),' 

536 " which it must not!" 

537 ) 

538 dest_and_roots.extend( 

539 (dest_path, install_context[pkg.name].fs_root) for pkg in match.into 

540 ) 

541 return dest_and_roots 

542 

543 

544def _resolve_matches( 

545 matches: List[PathMatch], 

546 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]], 

547 install_context: "InstallRuleContext", 

548) -> Iterator[Tuple[PathMatch, Sequence[Tuple[str, "FSPath"]]]]: 

549 if callable(dest_paths): 549 ↛ 550line 549 didn't jump to line 550, because the condition on line 549 was never true

550 compute_dest_path = dest_paths 

551 for match in matches: 

552 dpath = compute_dest_path(match) 

553 if dpath.endswith("/"): 

554 raise ValueError( 

555 f'Provided destination for "{match.path.path}" ended with "/" ("{dpath}"), which it must not!' 

556 ) 

557 dest_and_roots = [ 

558 (dpath, install_context[pkg.name].fs_root) for pkg in match.into 

559 ] 

560 yield match, dest_and_roots 

561 else: 

562 for match in matches: 

563 dest_and_roots = _resolve_dest_paths( 

564 match, 

565 dest_paths, 

566 install_context, 

567 ) 

568 yield match, dest_and_roots 

569 

570 

571class InstallRule(DebputyDispatchableType): 

572 __slots__ = ( 

573 "_already_matched", 

574 "_exact_match_request", 

575 "_condition", 

576 "_match_filter", 

577 "_definition_source", 

578 ) 

579 

580 def __init__( 

581 self, 

582 condition: Optional[ManifestCondition], 

583 definition_source: str, 

584 *, 

585 match_filter: Optional[Callable[["VirtualPath"], bool]] = None, 

586 ) -> None: 

587 self._condition = condition 

588 self._definition_source = definition_source 

589 self._match_filter = match_filter 

590 

591 def _check_single_match( 

592 self, source: FileSystemMatchRule, matches: List[PathMatch] 

593 ) -> None: 

594 seen_pkgs = set() 

595 problem_pkgs = frozenset() 

596 for m in matches: 

597 problem_pkgs = seen_pkgs & m.into 

598 if problem_pkgs: 598 ↛ 599line 598 didn't jump to line 599, because the condition on line 598 was never true

599 break 

600 seen_pkgs.update(problem_pkgs) 

601 if problem_pkgs: 601 ↛ 602line 601 didn't jump to line 602, because the condition on line 601 was never true

602 pkg_names = ", ".join(sorted(p.name for p in problem_pkgs)) 

603 _error( 

604 f'The pattern "{source.raw_match_rule}" matched multiple entries for the packages: {pkg_names}.' 

605 "However, it should matched exactly one item. Please tighten the pattern defined" 

606 f" in {self._definition_source}" 

607 ) 

608 

609 def _match_pattern( 

610 self, 

611 path_matcher: SourcePathMatcher, 

612 fs_match_rule: FileSystemMatchRule, 

613 condition_context: ConditionContext, 

614 search_dirs: Sequence[SearchDir], 

615 into: FrozenSet[BinaryPackage], 

616 ) -> List[PathMatch]: 

617 (matched, exclude_counts) = path_matcher.find_and_reserve_all_matches( 

618 fs_match_rule.match_rule, 

619 search_dirs, 

620 fs_match_rule.raw_match_rule.endswith("/"), 

621 self._match_filter, 

622 into, 

623 self._definition_source, 

624 ) 

625 

626 already_installed_paths, already_excluded_paths = exclude_counts 

627 

628 if into: 628 ↛ 633line 628 didn't jump to line 633, because the condition on line 628 was never false

629 allow_empty_match = all(not p.should_be_acted_on for p in into) 629 ↛ exitline 629 didn't finish the generator expression on line 629

630 else: 

631 # discard rules must match provided at least one search dir exist. If none of them 

632 # exist, then we assume the discard rule is for a package that will not be built 

633 allow_empty_match = any(s.search_dir.is_dir for s in search_dirs) 

634 if self._condition is not None and not self._condition.evaluate( 634 ↛ 637line 634 didn't jump to line 637, because the condition on line 634 was never true

635 condition_context 

636 ): 

637 allow_empty_match = True 

638 

639 if not matched and not allow_empty_match: 

640 search_dir_text = ", ".join(x.search_dir.fs_path for x in search_dirs) 

641 if already_excluded_paths and already_installed_paths: 641 ↛ 642line 641 didn't jump to line 642, because the condition on line 641 was never true

642 total_paths = already_excluded_paths + already_installed_paths 

643 msg = ( 

644 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring" 

645 f" {total_paths} path(s) already been matched previously either by install or" 

646 f" exclude rules. If you wanted to install some of these paths into multiple" 

647 f" packages, please tweak the definition that installed them to install them" 

648 f' into multiple packages (usually change "into: foo" to "into: [foo, bar]".' 

649 f" If you wanted to install these paths and exclude rules are getting in your" 

650 f" way, then please move this install rule before the exclusion rule that causes" 

651 f" issue or, in case of built-in excludes, list the paths explicitly (without" 

652 f" using patterns). Source for this issue is {self._definition_source}. Match rule:" 

653 f" {fs_match_rule.match_rule.describe_match_exact()}" 

654 ) 

655 elif already_excluded_paths: 655 ↛ 656line 655 didn't jump to line 656

656 msg = ( 

657 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring" 

658 f" {already_excluded_paths} path(s) that have been excluded." 

659 " If you wanted to install some of these paths, please move the install rule" 

660 " before the exclusion rule or, in case of built-in excludes, list the paths explicitly" 

661 f" (without using patterns). Source for this issue is {self._definition_source}. Match rule:" 

662 f" {fs_match_rule.match_rule.describe_match_exact()}" 

663 ) 

664 elif already_installed_paths: 664 ↛ 665line 664 didn't jump to line 665

665 msg = ( 

666 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} after ignoring" 

667 f" {already_installed_paths} path(s) already been matched previously." 

668 " If you wanted to install some of these paths into multiple packages," 

669 f" please tweak the definition that installed them to install them into" 

670 f' multiple packages (usually change "into: foo" to "into: [foo, bar]".' 

671 f" Source for this issue is {self._definition_source}. Match rule:" 

672 f" {fs_match_rule.match_rule.describe_match_exact()}" 

673 ) 

674 else: 

675 # TODO: Try harder to find the match and point out possible typos 

676 msg = ( 

677 f"There were no matches for {fs_match_rule.raw_match_rule} in {search_dir_text} (definition:" 

678 f" {self._definition_source}). Match rule: {fs_match_rule.match_rule.describe_match_exact()}" 

679 ) 

680 raise NoMatchForInstallPatternError( 

681 msg, 

682 fs_match_rule, 

683 search_dirs, 

684 self._definition_source, 

685 ) 

686 return matched 

687 

688 def _install_matches( 

689 self, 

690 path_matcher: SourcePathMatcher, 

691 matches: List[PathMatch], 

692 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]], 

693 install_context: "InstallRuleContext", 

694 into: FrozenSet[BinaryPackage], 

695 condition_context: ConditionContext, 

696 ) -> None: 

697 if ( 697 ↛ exit,   697 ↛ 7032 missed branches: 1) line 697 didn't jump to the function exit, 2) line 697 didn't jump to line 703, because the condition on line 697 was never true

698 self._condition is not None 

699 and not self._condition.evaluate(condition_context) 

700 ) or not any(p.should_be_acted_on for p in into): 

701 # Rule is disabled; skip all its actions - also allow empty matches 

702 # for this particular case. 

703 return 

704 

705 if not matches: 705 ↛ 706line 705 didn't jump to line 706, because the condition on line 705 was never true

706 raise ValueError("matches must not be empty") 

707 

708 for match, dest_paths_and_roots in _resolve_matches( 

709 matches, 

710 dest_paths, 

711 install_context, 

712 ): 

713 install_recursively_into_dirs = [] 

714 for dest, fs_root in dest_paths_and_roots: 

715 dir_part, basename = os.path.split(dest) 

716 # We do not associate these with the FS path. First off, 

717 # it is complicated to do in most cases (indeed, debhelper 

718 # does not preserve these directories either) and secondly, 

719 # it is "only" mtime and mode - mostly irrelevant as the 

720 # directory is 99.9% likely to be 0755 (we are talking 

721 # directories like "/usr", "/usr/share"). 

722 dir_path = fs_root.mkdirs(dir_part) 

723 existing_path = dir_path.get(basename) 

724 

725 if match.path.is_dir: 

726 if existing_path is not None and not existing_path.is_dir: 726 ↛ 727line 726 didn't jump to line 727, because the condition on line 726 was never true

727 existing_path.unlink() 

728 existing_path = None 

729 current_dir = existing_path 

730 

731 if current_dir is None: 731 ↛ 735line 731 didn't jump to line 735, because the condition on line 731 was never false

732 current_dir = dir_path.mkdir( 

733 basename, reference_path=match.path 

734 ) 

735 install_recursively_into_dirs.append(current_dir) 

736 else: 

737 if existing_path is not None and existing_path.is_dir: 737 ↛ 738line 737 didn't jump to line 738, because the condition on line 737 was never true

738 _error( 

739 f"Cannot install {match.path} ({match.path.fs_path}) as {dest}. That path already exist" 

740 f" and is a directory. This error was triggered via {self._definition_source}." 

741 ) 

742 

743 if match.path.is_symlink: 

744 dir_path.add_symlink( 

745 basename, match.path.readlink(), reference_path=match.path 

746 ) 

747 else: 

748 dir_path.insert_file_from_fs_path( 

749 basename, 

750 match.path.fs_path, 

751 follow_symlinks=False, 

752 use_fs_path_mode=True, 

753 reference_path=match.path, 

754 ) 

755 if install_recursively_into_dirs: 

756 self._install_dir_recursively( 

757 path_matcher, install_recursively_into_dirs, match, into 

758 ) 

759 

760 def _install_dir_recursively( 

761 self, 

762 path_matcher: SourcePathMatcher, 

763 parent_dirs: Sequence[FSPath], 

764 match: PathMatch, 

765 into: FrozenSet[BinaryPackage], 

766 ) -> None: 

767 stack = [ 

768 (parent_dirs, e) 

769 for e in match.path.iterdir 

770 if not path_matcher.is_reserved(e) 

771 ] 

772 

773 while stack: 

774 current_dirs, dir_entry = stack.pop() 

775 path_matcher.reserve( 

776 dir_entry, 

777 into, 

778 self._definition_source, 

779 is_exact_match=False, 

780 ) 

781 if dir_entry.is_dir: 781 ↛ 782line 781 didn't jump to line 782, because the condition on line 781 was never true

782 new_dirs = [ 

783 d.mkdir(dir_entry.name, reference_path=dir_entry) 

784 for d in current_dirs 

785 ] 

786 stack.extend( 

787 (new_dirs, de) 

788 for de in dir_entry.iterdir 

789 if not path_matcher.is_reserved(de) 

790 ) 

791 elif dir_entry.is_symlink: 

792 for current_dir in current_dirs: 

793 current_dir.add_symlink( 

794 dir_entry.name, 

795 dir_entry.readlink(), 

796 reference_path=dir_entry, 

797 ) 

798 elif dir_entry.is_file: 798 ↛ 808line 798 didn't jump to line 808, because the condition on line 798 was never false

799 for current_dir in current_dirs: 

800 current_dir.insert_file_from_fs_path( 

801 dir_entry.name, 

802 dir_entry.fs_path, 

803 use_fs_path_mode=True, 

804 follow_symlinks=False, 

805 reference_path=dir_entry, 

806 ) 

807 else: 

808 _error( 

809 f"Unsupported file type: {dir_entry.fs_path} - neither a file, directory or symlink" 

810 ) 

811 

812 def perform_install( 

813 self, 

814 path_matcher: SourcePathMatcher, 

815 install_context: InstallRuleContext, 

816 condition_context: ConditionContext, 

817 ) -> None: 

818 raise NotImplementedError 

819 

820 @classmethod 

821 def install_as( 

822 cls, 

823 source: FileSystemMatchRule, 

824 dest_path: str, 

825 into: FrozenSet[BinaryPackage], 

826 definition_source: str, 

827 condition: Optional[ManifestCondition], 

828 ) -> "InstallRule": 

829 return GenericInstallationRule( 

830 [source], 

831 [(dest_path, False)], 

832 into, 

833 condition, 

834 definition_source, 

835 require_single_match=True, 

836 ) 

837 

838 @classmethod 

839 def install_dest( 

840 cls, 

841 sources: Sequence[FileSystemMatchRule], 

842 dest_dir: Optional[str], 

843 into: FrozenSet[BinaryPackage], 

844 definition_source: str, 

845 condition: Optional[ManifestCondition], 

846 ) -> "InstallRule": 

847 if dest_dir is None: 

848 dest_dir = "{dirname}/{basename}" 

849 else: 

850 dest_dir = os.path.join(dest_dir, "{basename}") 

851 return GenericInstallationRule( 

852 sources, 

853 [(dest_dir, True)], 

854 into, 

855 condition, 

856 definition_source, 

857 ) 

858 

859 @classmethod 

860 def install_multi_as( 

861 cls, 

862 source: FileSystemMatchRule, 

863 dest_paths: Sequence[str], 

864 into: FrozenSet[BinaryPackage], 

865 definition_source: str, 

866 condition: Optional[ManifestCondition], 

867 ) -> "InstallRule": 

868 if len(dest_paths) < 2: 868 ↛ 869line 868 didn't jump to line 869, because the condition on line 868 was never true

869 raise ValueError( 

870 "Please use `install_as` when there is less than 2 dest path" 

871 ) 

872 dps = tuple((dp, False) for dp in dest_paths) 

873 return GenericInstallationRule( 

874 [source], 

875 dps, 

876 into, 

877 condition, 

878 definition_source, 

879 require_single_match=True, 

880 ) 

881 

882 @classmethod 

883 def install_multi_dest( 

884 cls, 

885 sources: Sequence[FileSystemMatchRule], 

886 dest_dirs: Sequence[str], 

887 into: FrozenSet[BinaryPackage], 

888 definition_source: str, 

889 condition: Optional[ManifestCondition], 

890 ) -> "InstallRule": 

891 if len(dest_dirs) < 2: 891 ↛ 892line 891 didn't jump to line 892, because the condition on line 891 was never true

892 raise ValueError( 

893 "Please use `install_dest` when there is less than 2 dest dir" 

894 ) 

895 dest_paths = tuple((os.path.join(dp, "{basename}"), True) for dp in dest_dirs) 

896 return GenericInstallationRule( 

897 sources, 

898 dest_paths, 

899 into, 

900 condition, 

901 definition_source, 

902 ) 

903 

904 @classmethod 

905 def install_doc( 

906 cls, 

907 sources: Sequence[FileSystemMatchRule], 

908 dest_dir: Optional[str], 

909 into: FrozenSet[BinaryPackage], 

910 definition_source: str, 

911 condition: Optional[ManifestCondition], 

912 ) -> "InstallRule": 

913 cond: ManifestCondition = _BUILD_DOCS_BDO 

914 if condition is not None: 

915 cond = ManifestCondition.all_of([cond, condition]) 

916 dest_path_is_format = False 

917 if dest_dir is None: 

918 dest_dir = "usr/share/doc/{doc_main_package_name}/{basename}" 

919 dest_path_is_format = True 

920 

921 return GenericInstallationRule( 

922 sources, 

923 [(dest_dir, dest_path_is_format)], 

924 into, 

925 cond, 

926 definition_source, 

927 ) 

928 

929 @classmethod 

930 def install_doc_as( 

931 cls, 

932 source: FileSystemMatchRule, 

933 dest_path: str, 

934 into: FrozenSet[BinaryPackage], 

935 definition_source: str, 

936 condition: Optional[ManifestCondition], 

937 ) -> "InstallRule": 

938 cond: ManifestCondition = _BUILD_DOCS_BDO 

939 if condition is not None: 

940 cond = ManifestCondition.all_of([cond, condition]) 

941 

942 return GenericInstallationRule( 

943 [source], 

944 [(dest_path, False)], 

945 into, 

946 cond, 

947 definition_source, 

948 require_single_match=True, 

949 ) 

950 

951 @classmethod 

952 def install_examples( 

953 cls, 

954 sources: Sequence[FileSystemMatchRule], 

955 into: FrozenSet[BinaryPackage], 

956 definition_source: str, 

957 condition: Optional[ManifestCondition], 

958 ) -> "InstallRule": 

959 cond: ManifestCondition = _BUILD_DOCS_BDO 

960 if condition is not None: 960 ↛ 961line 960 didn't jump to line 961, because the condition on line 960 was never true

961 cond = ManifestCondition.all_of([cond, condition]) 

962 return GenericInstallationRule( 

963 sources, 

964 [("usr/share/doc/{doc_main_package_name}/examples/{basename}", True)], 

965 into, 

966 cond, 

967 definition_source, 

968 ) 

969 

970 @classmethod 

971 def install_man( 

972 cls, 

973 sources: Sequence[FileSystemMatchRule], 

974 into: FrozenSet[BinaryPackage], 

975 section: Optional[int], 

976 language: Optional[str], 

977 definition_source: str, 

978 condition: Optional[ManifestCondition], 

979 ) -> "InstallRule": 

980 cond: ManifestCondition = _BUILD_DOCS_BDO 

981 if condition is not None: 981 ↛ 982line 981 didn't jump to line 982, because the condition on line 981 was never true

982 cond = ManifestCondition.all_of([cond, condition]) 

983 

984 dest_path_computer = _dest_path_for_manpage( 

985 section, language, definition_source 

986 ) 

987 

988 return GenericInstallationRule( 988 ↛ exitline 988 didn't jump to the function exit

989 sources, 

990 dest_path_computer, 

991 into, 

992 cond, 

993 definition_source, 

994 match_filter=lambda m: not m.is_file, 

995 ) 

996 

997 @classmethod 

998 def discard_paths( 

999 cls, 

1000 paths: Sequence[FileSystemMatchRule], 

1001 definition_source: str, 

1002 condition: Optional[ManifestCondition], 

1003 *, 

1004 limit_to: Optional[Sequence[FileSystemExactMatchRule]] = None, 

1005 ) -> "InstallRule": 

1006 return DiscardRule( 

1007 paths, 

1008 condition, 

1009 tuple(limit_to) if limit_to is not None else tuple(), 

1010 definition_source, 

1011 ) 

1012 

1013 

1014class PPFInstallRule(InstallRule): 

1015 __slots__ = ( 

1016 "_ppfs", 

1017 "_substitution", 

1018 "_into", 

1019 ) 

1020 

1021 def __init__( 

1022 self, 

1023 into: BinaryPackage, 

1024 substitution: Substitution, 

1025 ppfs: Sequence["PackagerProvidedFile"], 

1026 ) -> None: 

1027 super().__init__( 

1028 None, 

1029 "<built-in; PPF install rule>", 

1030 ) 

1031 self._substitution = substitution 

1032 self._ppfs = ppfs 

1033 self._into = into 

1034 

1035 def perform_install( 

1036 self, 

1037 path_matcher: SourcePathMatcher, 

1038 install_context: InstallRuleContext, 

1039 condition_context: ConditionContext, 

1040 ) -> None: 

1041 binary_install_context = install_context[self._into.name] 

1042 fs_root = binary_install_context.fs_root 

1043 for ppf in self._ppfs: 

1044 source_path = ppf.path.fs_path 

1045 dest_dir, name = ppf.compute_dest() 

1046 dir_path = fs_root.mkdirs(dest_dir) 

1047 

1048 dir_path.insert_file_from_fs_path( 

1049 name, 

1050 source_path, 

1051 follow_symlinks=True, 

1052 use_fs_path_mode=False, 

1053 mode=ppf.definition.default_mode, 

1054 ) 

1055 

1056 

1057class GenericInstallationRule(InstallRule): 

1058 __slots__ = ( 

1059 "_sources", 

1060 "_into", 

1061 "_dest_paths", 

1062 "_require_single_match", 

1063 ) 

1064 

1065 def __init__( 

1066 self, 

1067 sources: Sequence[FileSystemMatchRule], 

1068 dest_paths: Union[Sequence[Tuple[str, bool]], Callable[[PathMatch], str]], 

1069 into: FrozenSet[BinaryPackage], 

1070 condition: Optional[ManifestCondition], 

1071 definition_source: str, 

1072 *, 

1073 require_single_match: bool = False, 

1074 match_filter: Optional[Callable[["VirtualPath"], bool]] = None, 

1075 ) -> None: 

1076 super().__init__( 

1077 condition, 

1078 definition_source, 

1079 match_filter=match_filter, 

1080 ) 

1081 self._sources = sources 

1082 self._into = into 

1083 self._dest_paths = dest_paths 

1084 self._require_single_match = require_single_match 

1085 if self._require_single_match and len(sources) != 1: 1085 ↛ 1086line 1085 didn't jump to line 1086, because the condition on line 1085 was never true

1086 raise ValueError("require_single_match implies sources must have len 1") 

1087 

1088 def perform_install( 

1089 self, 

1090 path_matcher: SourcePathMatcher, 

1091 install_context: InstallRuleContext, 

1092 condition_context: ConditionContext, 

1093 ) -> None: 

1094 for source in self._sources: 

1095 matches = self._match_pattern( 

1096 path_matcher, 

1097 source, 

1098 condition_context, 

1099 install_context.search_dirs, 

1100 self._into, 

1101 ) 

1102 if self._require_single_match and len(matches) > 1: 

1103 self._check_single_match(source, matches) 

1104 self._install_matches( 

1105 path_matcher, 

1106 matches, 

1107 self._dest_paths, 

1108 install_context, 

1109 self._into, 

1110 condition_context, 

1111 ) 

1112 

1113 

1114class DiscardRule(InstallRule): 

1115 __slots__ = ("_fs_match_rules", "_limit_to") 

1116 

1117 def __init__( 

1118 self, 

1119 fs_match_rules: Sequence[FileSystemMatchRule], 

1120 condition: Optional[ManifestCondition], 

1121 limit_to: Sequence[FileSystemExactMatchRule], 

1122 definition_source: str, 

1123 ) -> None: 

1124 super().__init__(condition, definition_source) 

1125 self._fs_match_rules = fs_match_rules 

1126 self._limit_to = limit_to 

1127 

1128 def perform_install( 

1129 self, 

1130 path_matcher: SourcePathMatcher, 

1131 install_context: InstallRuleContext, 

1132 condition_context: ConditionContext, 

1133 ) -> None: 

1134 into = frozenset() 

1135 limit_to = self._limit_to 

1136 if limit_to: 

1137 matches = {x.match_rule.path for x in limit_to} 

1138 search_dirs = tuple( 

1139 s 

1140 for s in install_context.search_dirs 

1141 if s.search_dir.fs_path in matches 

1142 ) 

1143 if len(limit_to) != len(search_dirs): 

1144 matches.difference(s.search_dir.fs_path for s in search_dirs) 

1145 paths = ":".join(matches) 

1146 _error( 

1147 f"The discard rule defined at {self._definition_source} mentions the following" 

1148 f" search directories that were not known to debputy: {paths}." 

1149 " Either the search dir is missing somewhere else or it should be removed from" 

1150 " the discard rule." 

1151 ) 

1152 else: 

1153 search_dirs = install_context.search_dirs 

1154 

1155 for fs_match_rule in self._fs_match_rules: 

1156 self._match_pattern( 

1157 path_matcher, 

1158 fs_match_rule, 

1159 condition_context, 

1160 search_dirs, 

1161 into, 

1162 )