Coverage for src/debputy/dh_migration/migrators_impl.py: 81%
669 statements
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
1import collections
2import dataclasses
3import functools
4import json
5import os
6import re
7import subprocess
8from typing import (
9 Iterable,
10 Optional,
11 Tuple,
12 List,
13 Set,
14 Mapping,
15 Any,
16 Union,
17 Callable,
18 TypeVar,
19 Dict,
20)
22from debian.deb822 import Deb822
24from debputy import DEBPUTY_DOC_ROOT_DIR
25from debputy.architecture_support import dpkg_architecture_table
26from debputy.deb_packaging_support import dpkg_field_list_pkg_dep
27from debputy.debhelper_emulation import (
28 dhe_filedoublearray,
29 DHConfigFileLine,
30 dhe_pkgfile,
31 parse_drules_for_addons,
32 extract_dh_addons_from_control,
33)
34from debputy.dh_migration.models import (
35 ConflictingChange,
36 FeatureMigration,
37 UnsupportedFeature,
38 AcceptableMigrationIssues,
39 DHMigrationSubstitution,
40)
41from debputy.highlevel_manifest import (
42 MutableYAMLSymlink,
43 HighLevelManifest,
44 MutableYAMLConffileManagementItem,
45 AbstractMutableYAMLInstallRule,
46)
47from debputy.installations import MAN_GUESS_FROM_BASENAME, MAN_GUESS_LANG_FROM_PATH
48from debputy.packages import BinaryPackage
49from debputy.plugin.api import VirtualPath
50from debputy.util import (
51 _error,
52 PKGVERSION_REGEX,
53 PKGNAME_REGEX,
54 _normalize_path,
55 assume_not_none,
56 has_glob_magic,
57)
59MIGRATION_TARGET_DH_DEBPUTY_RRR = "dh-sequence-zz-debputy-rrr"
60MIGRATION_TARGET_DH_DEBPUTY = "dh-sequence-zz-debputy"
63# Align with debputy.py
64DH_COMMANDS_REPLACED = {
65 MIGRATION_TARGET_DH_DEBPUTY_RRR: frozenset(
66 {
67 "dh_fixperms",
68 "dh_shlibdeps",
69 "dh_gencontrol",
70 "dh_md5sums",
71 "dh_builddeb",
72 }
73 ),
74 MIGRATION_TARGET_DH_DEBPUTY: frozenset(
75 {
76 "dh_install",
77 "dh_installdocs",
78 "dh_installchangelogs",
79 "dh_installexamples",
80 "dh_installman",
81 "dh_installcatalogs",
82 "dh_installcron",
83 "dh_installdebconf",
84 "dh_installemacsen",
85 "dh_installifupdown",
86 "dh_installinfo",
87 "dh_installinit",
88 "dh_installsysusers",
89 "dh_installtmpfiles",
90 "dh_installsystemd",
91 "dh_installsystemduser",
92 "dh_installmenu",
93 "dh_installmime",
94 "dh_installmodules",
95 "dh_installlogcheck",
96 "dh_installlogrotate",
97 "dh_installpam",
98 "dh_installppp",
99 "dh_installudev",
100 "dh_installgsettings",
101 "dh_installinitramfs",
102 "dh_installalternatives",
103 "dh_bugfiles",
104 "dh_ucf",
105 "dh_lintian",
106 "dh_icons",
107 "dh_usrlocal",
108 "dh_perl",
109 "dh_link",
110 "dh_installwm",
111 "dh_installxfonts",
112 "dh_strip_nondeterminism",
113 "dh_compress",
114 "dh_fixperms",
115 "dh_dwz",
116 "dh_strip",
117 "dh_makeshlibs",
118 "dh_shlibdeps",
119 "dh_missing",
120 "dh_installdeb",
121 "dh_gencontrol",
122 "dh_md5sums",
123 "dh_builddeb",
124 }
125 ),
126}
128_GS_DOC = f"{DEBPUTY_DOC_ROOT_DIR}/GETTING-STARTED-WITH-dh-debputy.md"
129MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS = {
130 "dh_installinit": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
131 "dh_installsystemd": f"{_GS_DOC}#covert-your-overrides-for-dh_installsystemd-dh_installinit-if-any",
132 "dh_fixperms": f"{_GS_DOC}#convert-your-overrides-or-excludes-for-dh_fixperms-if-any",
133 "dh_gencontrol": f"{_GS_DOC}#convert-your-overrides-for-dh_gencontrol-if-any",
134}
137@dataclasses.dataclass(frozen=True, slots=True)
138class UnsupportedDHConfig:
139 dh_config_basename: str
140 dh_tool: str
141 bug_950723_prefix_matching: bool = False
142 is_missing_migration: bool = False
145@dataclasses.dataclass(frozen=True, slots=True)
146class DHSequenceMigration:
147 debputy_plugin: str
148 remove_dh_sequence: bool = True
149 must_use_zz_debputy: bool = False
152UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY = [
153 UnsupportedDHConfig("config", "dh_installdebconf"),
154 UnsupportedDHConfig("templates", "dh_installdebconf"),
155 UnsupportedDHConfig("emacsen-compat", "dh_installemacsen"),
156 UnsupportedDHConfig("emacsen-install", "dh_installemacsen"),
157 UnsupportedDHConfig("emacsen-remove", "dh_installemacsen"),
158 UnsupportedDHConfig("emacsen-startup", "dh_installemacsen"),
159 # The `upstart` file should be long dead, but we might as well detect it.
160 UnsupportedDHConfig("upstart", "dh_installinit"),
161 # dh_installsystemduser
162 UnsupportedDHConfig(
163 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=False
164 ),
165 UnsupportedDHConfig(
166 "user.path", "dh_installsystemduser", bug_950723_prefix_matching=True
167 ),
168 UnsupportedDHConfig(
169 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=False
170 ),
171 UnsupportedDHConfig(
172 "user.service", "dh_installsystemduser", bug_950723_prefix_matching=True
173 ),
174 UnsupportedDHConfig(
175 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=False
176 ),
177 UnsupportedDHConfig(
178 "user.socket", "dh_installsystemduser", bug_950723_prefix_matching=True
179 ),
180 UnsupportedDHConfig(
181 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=False
182 ),
183 UnsupportedDHConfig(
184 "user.target", "dh_installsystemduser", bug_950723_prefix_matching=True
185 ),
186 UnsupportedDHConfig(
187 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=False
188 ),
189 UnsupportedDHConfig(
190 "user.timer", "dh_installsystemduser", bug_950723_prefix_matching=True
191 ),
192 UnsupportedDHConfig("udev", "dh_installudev"),
193 UnsupportedDHConfig("menu", "dh_installmenu"),
194 UnsupportedDHConfig("menu-method", "dh_installmenu"),
195 UnsupportedDHConfig("ucf", "dh_ucf"),
196 UnsupportedDHConfig("wm", "dh_installwm"),
197 UnsupportedDHConfig("triggers", "dh_installdeb"),
198 UnsupportedDHConfig("postinst", "dh_installdeb"),
199 UnsupportedDHConfig("postrm", "dh_installdeb"),
200 UnsupportedDHConfig("preinst", "dh_installdeb"),
201 UnsupportedDHConfig("prerm", "dh_installdeb"),
202 UnsupportedDHConfig("menutest", "dh_installdeb"),
203 UnsupportedDHConfig("isinstallable", "dh_installdeb"),
204]
205SUPPORTED_DH_ADDONS = frozenset(
206 {
207 # debputy's own
208 "debputy",
209 "zz-debputy",
210 # debhelper provided sequences that should work.
211 "single-binary",
212 }
213)
214DH_ADDONS_TO_REMOVE = frozenset(
215 [
216 # Sequences debputy directly replaces
217 "dwz",
218 "elf-tools",
219 "installinitramfs",
220 "installsysusers",
221 "doxygen",
222 # Sequences that are embedded fully into debputy
223 "bash-completion",
224 "sodeps",
225 ]
226)
227DH_ADDONS_TO_PLUGINS = {
228 "gnome": DHSequenceMigration(
229 "gnome",
230 # The sequence still provides a command for the clean sequence
231 remove_dh_sequence=False,
232 must_use_zz_debputy=True,
233 ),
234 "numpy3": DHSequenceMigration(
235 "numpy3",
236 # The sequence provides (build-time) dependencies that we cannot provide
237 remove_dh_sequence=False,
238 must_use_zz_debputy=True,
239 ),
240 "perl-openssl": DHSequenceMigration(
241 "perl-openssl",
242 # The sequence provides (build-time) dependencies that we cannot provide
243 remove_dh_sequence=False,
244 must_use_zz_debputy=True,
245 ),
246}
249def _dh_config_file(
250 debian_dir: VirtualPath,
251 dctrl_bin: BinaryPackage,
252 basename: str,
253 helper_name: str,
254 acceptable_migration_issues: AcceptableMigrationIssues,
255 feature_migration: FeatureMigration,
256 manifest: HighLevelManifest,
257 support_executable_files: bool = False,
258 allow_dh_exec_rename: bool = False,
259 pkgfile_lookup: bool = True,
260 remove_on_migration: bool = True,
261) -> Union[Tuple[None, None], Tuple[VirtualPath, Iterable[DHConfigFileLine]]]:
262 mutable_manifest = assume_not_none(manifest.mutable_manifest)
263 dh_config_file = (
264 dhe_pkgfile(debian_dir, dctrl_bin, basename)
265 if pkgfile_lookup
266 else debian_dir.get(basename)
267 )
268 if dh_config_file is None or dh_config_file.is_dir:
269 return None, None
270 if dh_config_file.is_executable and not support_executable_files:
271 primary_key = f"executable-{helper_name}-config"
272 if (
273 primary_key in acceptable_migration_issues
274 or "any-executable-dh-configs" in acceptable_migration_issues
275 ):
276 feature_migration.warn(
277 f'TODO: MANUAL MIGRATION of executable dh config "{dh_config_file}" is required.'
278 )
279 return None, None
280 raise UnsupportedFeature(
281 f"Executable configuration files not supported (found: {dh_config_file}).",
282 [primary_key, "any-executable-dh-configs"],
283 )
285 if remove_on_migration:
286 feature_migration.remove_on_success(dh_config_file.fs_path)
287 substitution = DHMigrationSubstitution(
288 dpkg_architecture_table(),
289 acceptable_migration_issues,
290 feature_migration,
291 mutable_manifest,
292 )
293 content = dhe_filedoublearray(
294 dh_config_file,
295 substitution,
296 allow_dh_exec_rename=allow_dh_exec_rename,
297 )
298 return dh_config_file, content
301def _validate_rm_mv_conffile(
302 package: str,
303 config_line: DHConfigFileLine,
304) -> Tuple[str, str, Optional[str], Optional[str], Optional[str]]:
305 cmd, *args = config_line.tokens
306 if "--" in config_line.tokens: 306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true
307 raise ValueError(
308 f'The maintscripts file "{config_line.config_file.path}" for {package} includes a "--" in line'
309 f" {config_line.line_no}. The offending line is: {config_line.original_line}"
310 )
311 if cmd == "rm_conffile":
312 min_args = 1
313 max_args = 3
314 else:
315 min_args = 2
316 max_args = 4
317 if len(args) > max_args or len(args) < min_args: 317 ↛ 318line 317 didn't jump to line 318, because the condition on line 317 was never true
318 raise ValueError(
319 f'The "{cmd}" command takes at least {min_args} and at most {max_args} arguments. However,'
320 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), there'
321 f" are {len(args)} arguments. The offending line is: {config_line.original_line}"
322 )
324 obsolete_conffile = args[0]
325 new_conffile = args[1] if cmd == "mv_conffile" else None
326 prior_version = args[min_args] if len(args) > min_args else None
327 owning_package = args[min_args + 1] if len(args) > min_args + 1 else None
328 if not obsolete_conffile.startswith("/"): 328 ↛ 329line 328 didn't jump to line 329, because the condition on line 328 was never true
329 raise ValueError(
330 f'The (old-)conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
331 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
332 f' as "{obsolete_conffile}". The offending line is: {config_line.original_line}'
333 )
334 if new_conffile is not None and not new_conffile.startswith("/"): 334 ↛ 335line 334 didn't jump to line 335, because the condition on line 334 was never true
335 raise ValueError(
336 f'The new-conffile parameter for {cmd} must be absolute (i.e., start with "/"). However,'
337 f' in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it was specified'
338 f' as "{new_conffile}". The offending line is: {config_line.original_line}'
339 )
340 if prior_version is not None and not PKGVERSION_REGEX.fullmatch(prior_version): 340 ↛ 341line 340 didn't jump to line 341, because the condition on line 340 was never true
341 raise ValueError(
342 f"The prior-version parameter for {cmd} must be a valid package version (i.e., match"
343 f' {PKGVERSION_REGEX}). However, in "{config_line.config_file.path}" line {config_line.line_no}'
344 f' (for {package}), it was specified as "{prior_version}". The offending line is:'
345 f" {config_line.original_line}"
346 )
347 if owning_package is not None and not PKGNAME_REGEX.fullmatch(owning_package): 347 ↛ 348line 347 didn't jump to line 348, because the condition on line 347 was never true
348 raise ValueError(
349 f"The package parameter for {cmd} must be a valid package name (i.e., match {PKGNAME_REGEX})."
350 f' However, in "{config_line.config_file.path}" line {config_line.line_no} (for {package}), it'
351 f' was specified as "{owning_package}". The offending line is: {config_line.original_line}'
352 )
353 return cmd, obsolete_conffile, new_conffile, prior_version, owning_package
356_BASH_COMPLETION_RE = re.compile(
357 r"""
358 (^|[|&;])\s*complete.*-[A-Za-z].*
359 | \$\(.*\)
360 | \s*compgen.*-[A-Za-z].*
361 | \s*if.*;.*then/
362""",
363 re.VERBOSE,
364)
367def migrate_bash_completion(
368 debian_dir: VirtualPath,
369 manifest: HighLevelManifest,
370 acceptable_migration_issues: AcceptableMigrationIssues,
371 feature_migration: FeatureMigration,
372 _migration_target: str,
373) -> None:
374 feature_migration.tagline = "dh_bash-completion files"
375 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
376 mutable_manifest = assume_not_none(manifest.mutable_manifest)
377 installations = mutable_manifest.installations(create_if_absent=False)
379 for dctrl_bin in manifest.all_packages:
380 dh_file = dhe_pkgfile(debian_dir, dctrl_bin, "bash-completion")
381 if dh_file is None:
382 continue
383 is_bash_completion_file = False
384 with dh_file.open() as fd:
385 for line in fd:
386 line = line.strip()
387 if not line or line[0] == "#": 387 ↛ 388line 387 didn't jump to line 388, because the condition on line 387 was never true
388 continue
389 if _BASH_COMPLETION_RE.search(line):
390 is_bash_completion_file = True
391 break
392 if not is_bash_completion_file:
393 _, content = _dh_config_file(
394 debian_dir,
395 dctrl_bin,
396 "bash-completion",
397 "dh_bash-completion",
398 acceptable_migration_issues,
399 feature_migration,
400 manifest,
401 support_executable_files=True,
402 )
403 else:
404 content = None
406 if content:
407 install_dest_sources: List[str] = []
408 install_as_rules: List[Tuple[str, str]] = []
409 for dhe_line in content:
410 if len(dhe_line.tokens) > 2: 410 ↛ 411line 410 didn't jump to line 411, because the condition on line 410 was never true
411 raise UnsupportedFeature(
412 f"The dh_bash-completion file {dh_file.path} more than two words on"
413 f' line {dhe_line.line_no} (line: "{dhe_line.original_line}").'
414 )
415 source = dhe_line.tokens[0]
416 dest_basename = (
417 dhe_line.tokens[1]
418 if len(dhe_line.tokens) > 1
419 else os.path.basename(source)
420 )
421 if source.startswith("debian/") and not has_glob_magic(source):
422 if dctrl_bin.name != dest_basename:
423 dest_path = (
424 f"debian/{dctrl_bin.name}.{dest_basename}.bash-completion"
425 )
426 else:
427 dest_path = f"debian/{dest_basename}.bash-completion"
428 feature_migration.rename_on_success(source, dest_path)
429 elif len(dhe_line.tokens) == 1:
430 install_dest_sources.append(source)
431 else:
432 install_as_rules.append((source, dest_basename))
434 if install_dest_sources: 434 ↛ 448line 434 didn't jump to line 448, because the condition on line 434 was never false
435 sources = (
436 install_dest_sources
437 if len(install_dest_sources) > 1
438 else install_dest_sources[0]
439 )
440 installations.append(
441 AbstractMutableYAMLInstallRule.install_dest(
442 sources=sources,
443 dest_dir="{{path:BASH_COMPLETION_DIR}}",
444 into=dctrl_bin.name if not is_single_binary else None,
445 )
446 )
448 for source, dest_basename in install_as_rules:
449 installations.append(
450 AbstractMutableYAMLInstallRule.install_as(
451 source=source,
452 install_as="{{path:BASH_COMPLETION_DIR}}/" + dest_basename,
453 into=dctrl_bin.name if not is_single_binary else None,
454 )
455 )
458def migrate_dh_installsystemd_files(
459 debian_dir: VirtualPath,
460 manifest: HighLevelManifest,
461 _acceptable_migration_issues: AcceptableMigrationIssues,
462 feature_migration: FeatureMigration,
463 _migration_target: str,
464) -> None:
465 feature_migration.tagline = "dh_installsystemd files"
466 for dctrl_bin in manifest.all_packages:
467 for stem in [
468 "path",
469 "service",
470 "socket",
471 "target",
472 "timer",
473 ]:
474 pkgfile = dhe_pkgfile(
475 debian_dir, dctrl_bin, stem, bug_950723_prefix_matching=True
476 )
477 if not pkgfile:
478 continue
479 if not pkgfile.name.endswith(f".{stem}") or "@." not in pkgfile.name: 479 ↛ 480line 479 didn't jump to line 480, because the condition on line 479 was never true
480 raise UnsupportedFeature(
481 f'Unable to determine the correct name for {pkgfile.fs_path}. It should be a ".@{stem}"'
482 f" file now (foo@.service => foo.@service)"
483 )
484 newname = pkgfile.name.replace("@.", ".")
485 newname = newname[: -len(stem)] + f"@{stem}"
486 feature_migration.rename_on_success(
487 pkgfile.fs_path, os.path.join(debian_dir.fs_path, newname)
488 )
491def migrate_maintscript(
492 debian_dir: VirtualPath,
493 manifest: HighLevelManifest,
494 acceptable_migration_issues: AcceptableMigrationIssues,
495 feature_migration: FeatureMigration,
496 _migration_target: str,
497) -> None:
498 feature_migration.tagline = "dh_installdeb files"
499 mutable_manifest = assume_not_none(manifest.mutable_manifest)
500 for dctrl_bin in manifest.all_packages:
501 mainscript_file, content = _dh_config_file(
502 debian_dir,
503 dctrl_bin,
504 "maintscript",
505 "dh_installdeb",
506 acceptable_migration_issues,
507 feature_migration,
508 manifest,
509 )
511 if mainscript_file is None:
512 continue
513 assert content is not None
515 package_definition = mutable_manifest.package(dctrl_bin.name)
516 conffiles = {
517 it.obsolete_conffile: it
518 for it in package_definition.conffile_management_items()
519 }
520 seen_conffiles = set()
522 for dhe_line in content:
523 cmd = dhe_line.tokens[0]
524 if cmd not in {"rm_conffile", "mv_conffile"}: 524 ↛ 525line 524 didn't jump to line 525, because the condition on line 524 was never true
525 raise UnsupportedFeature(
526 f"The dh_installdeb file {mainscript_file.path} contains the (currently)"
527 f' unsupported command "{cmd}" on line {dhe_line.line_no}'
528 f' (line: "{dhe_line.original_line}")'
529 )
531 try:
532 (
533 _,
534 obsolete_conffile,
535 new_conffile,
536 prior_to_version,
537 owning_package,
538 ) = _validate_rm_mv_conffile(dctrl_bin.name, dhe_line)
539 except ValueError as e:
540 _error(
541 f"Validation error in {mainscript_file} on line {dhe_line.line_no}. The error was: {e.args[0]}."
542 )
544 if obsolete_conffile in seen_conffiles: 544 ↛ 545line 544 didn't jump to line 545, because the condition on line 544 was never true
545 raise ConflictingChange(
546 f'The {mainscript_file} file defines actions for "{obsolete_conffile}" twice!'
547 f" Please ensure that it is defined at most once in that file."
548 )
549 seen_conffiles.add(obsolete_conffile)
551 if cmd == "rm_conffile":
552 item = MutableYAMLConffileManagementItem.rm_conffile(
553 obsolete_conffile,
554 prior_to_version,
555 owning_package,
556 )
557 else:
558 assert cmd == "mv_conffile"
559 item = MutableYAMLConffileManagementItem.mv_conffile(
560 obsolete_conffile,
561 assume_not_none(new_conffile),
562 prior_to_version,
563 owning_package,
564 )
566 existing_def = conffiles.get(item.obsolete_conffile)
567 if existing_def is not None: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true
568 if not (
569 item.command == existing_def.command
570 and item.new_conffile == existing_def.new_conffile
571 and item.prior_to_version == existing_def.prior_to_version
572 and item.owning_package == existing_def.owning_package
573 ):
574 raise ConflictingChange(
575 f"The maintscript defines the action {item.command} for"
576 f' "{obsolete_conffile}" in {mainscript_file}, but there is another'
577 f" conffile management definition for same path defined already (in the"
578 f" existing manifest or an migration e.g., inside {mainscript_file})"
579 )
580 feature_migration.already_present += 1
581 continue
583 package_definition.add_conffile_management(item)
584 feature_migration.successful_manifest_changes += 1
587@dataclasses.dataclass(slots=True)
588class SourcesAndConditional:
589 dest_dir: Optional[str] = None
590 sources: List[str] = dataclasses.field(default_factory=list)
591 conditional: Optional[Union[str, Mapping[str, Any]]] = None
594def _strip_d_tmp(p: str) -> str:
595 if p.startswith("debian/tmp/") and len(p) > 11:
596 return p[11:]
597 return p
600def migrate_install_file(
601 debian_dir: VirtualPath,
602 manifest: HighLevelManifest,
603 acceptable_migration_issues: AcceptableMigrationIssues,
604 feature_migration: FeatureMigration,
605 _migration_target: str,
606) -> None:
607 feature_migration.tagline = "dh_install config files"
608 mutable_manifest = assume_not_none(manifest.mutable_manifest)
609 installations = mutable_manifest.installations(create_if_absent=False)
610 priority_lines = []
611 remaining_install_lines = []
612 warn_about_fixmes_in_dest_dir = False
614 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
616 for dctrl_bin in manifest.all_packages:
617 install_file, content = _dh_config_file(
618 debian_dir,
619 dctrl_bin,
620 "install",
621 "dh_install",
622 acceptable_migration_issues,
623 feature_migration,
624 manifest,
625 support_executable_files=True,
626 allow_dh_exec_rename=True,
627 )
628 if not install_file or not content:
629 continue
630 current_sources = []
631 sources_by_destdir: Dict[Tuple[str, Tuple[str, ...]], SourcesAndConditional] = (
632 {}
633 )
634 install_as_rules = []
635 multi_dest = collections.defaultdict(list)
636 seen_sources = set()
637 multi_dest_sources: Set[str] = set()
639 for dhe_line in content:
640 special_rule = None
641 if "=>" in dhe_line.tokens:
642 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
643 # This rule must be as early as possible to retain the semantics
644 path = _strip_d_tmp(
645 _normalize_path(dhe_line.tokens[1], with_prefix=False)
646 )
647 special_rule = AbstractMutableYAMLInstallRule.install_dest(
648 path,
649 dctrl_bin.name if not is_single_binary else None,
650 dest_dir=None,
651 when=dhe_line.conditional(),
652 )
653 elif len(dhe_line.tokens) != 3: 653 ↛ 654line 653 didn't jump to line 654, because the condition on line 653 was never true
654 _error(
655 f"Validation error in {install_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
656 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
657 )
658 else:
659 install_rule = AbstractMutableYAMLInstallRule.install_as(
660 _strip_d_tmp(
661 _normalize_path(dhe_line.tokens[0], with_prefix=False)
662 ),
663 _normalize_path(dhe_line.tokens[2], with_prefix=False),
664 dctrl_bin.name if not is_single_binary else None,
665 when=dhe_line.conditional(),
666 )
667 install_as_rules.append(install_rule)
668 else:
669 if len(dhe_line.tokens) > 1:
670 sources = list(
671 _strip_d_tmp(_normalize_path(w, with_prefix=False))
672 for w in dhe_line.tokens[:-1]
673 )
674 dest_dir = _normalize_path(dhe_line.tokens[-1], with_prefix=False)
675 else:
676 sources = list(
677 _strip_d_tmp(_normalize_path(w, with_prefix=False))
678 for w in dhe_line.tokens
679 )
680 dest_dir = None
682 multi_dest_sources.update(s for s in sources if s in seen_sources)
683 seen_sources.update(sources)
685 if dest_dir is None and dhe_line.conditional() is None:
686 current_sources.extend(sources)
687 continue
688 key = (dest_dir, dhe_line.conditional_key())
689 ctor = functools.partial(
690 SourcesAndConditional,
691 dest_dir=dest_dir,
692 conditional=dhe_line.conditional(),
693 )
694 md = _fetch_or_create(
695 sources_by_destdir,
696 key,
697 ctor,
698 )
699 md.sources.extend(sources)
701 if special_rule:
702 priority_lines.append(special_rule)
704 remaining_install_lines.extend(install_as_rules)
706 for md in sources_by_destdir.values():
707 if multi_dest_sources:
708 sources = [s for s in md.sources if s not in multi_dest_sources]
709 already_installed = (s for s in md.sources if s in multi_dest_sources)
710 for s in already_installed:
711 # The sources are ignored, so we can reuse the object as-is
712 multi_dest[s].append(md)
713 if not sources:
714 continue
715 else:
716 sources = md.sources
717 install_rule = AbstractMutableYAMLInstallRule.install_dest(
718 sources[0] if len(sources) == 1 else sources,
719 dctrl_bin.name if not is_single_binary else None,
720 dest_dir=md.dest_dir,
721 when=md.conditional,
722 )
723 remaining_install_lines.append(install_rule)
725 if current_sources:
726 if multi_dest_sources:
727 sources = [s for s in current_sources if s not in multi_dest_sources]
728 already_installed = (
729 s for s in current_sources if s in multi_dest_sources
730 )
731 for s in already_installed:
732 # The sources are ignored, so we can reuse the object as-is
733 dest_dir = os.path.dirname(s)
734 if has_glob_magic(dest_dir):
735 warn_about_fixmes_in_dest_dir = True
736 dest_dir = f"FIXME: {dest_dir} (could not reliably compute the dest dir)"
737 multi_dest[s].append(
738 SourcesAndConditional(
739 dest_dir=dest_dir,
740 conditional=None,
741 )
742 )
743 else:
744 sources = current_sources
746 if sources:
747 install_rule = AbstractMutableYAMLInstallRule.install_dest(
748 sources[0] if len(sources) == 1 else sources,
749 dctrl_bin.name if not is_single_binary else None,
750 dest_dir=None,
751 )
752 remaining_install_lines.append(install_rule)
754 if multi_dest:
755 for source, dest_and_conditionals in multi_dest.items():
756 dest_dirs = [dac.dest_dir for dac in dest_and_conditionals]
757 # We assume the conditional is the same.
758 conditional = next(
759 iter(
760 dac.conditional
761 for dac in dest_and_conditionals
762 if dac.conditional is not None
763 ),
764 None,
765 )
766 remaining_install_lines.append(
767 AbstractMutableYAMLInstallRule.multi_dest_install(
768 source,
769 dest_dirs,
770 dctrl_bin.name if not is_single_binary else None,
771 when=conditional,
772 )
773 )
775 if priority_lines:
776 installations.extend(priority_lines)
778 if remaining_install_lines:
779 installations.extend(remaining_install_lines)
781 feature_migration.successful_manifest_changes += len(priority_lines) + len(
782 remaining_install_lines
783 )
784 if warn_about_fixmes_in_dest_dir:
785 feature_migration.warn(
786 "TODO: FIXME left in dest-dir(s) of some installation rules."
787 " Please review these and remove the FIXME (plus correct as necessary)"
788 )
791def migrate_installdocs_file(
792 debian_dir: VirtualPath,
793 manifest: HighLevelManifest,
794 acceptable_migration_issues: AcceptableMigrationIssues,
795 feature_migration: FeatureMigration,
796 _migration_target: str,
797) -> None:
798 feature_migration.tagline = "dh_installdocs config files"
799 mutable_manifest = assume_not_none(manifest.mutable_manifest)
800 installations = mutable_manifest.installations(create_if_absent=False)
802 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
804 for dctrl_bin in manifest.all_packages:
805 install_file, content = _dh_config_file(
806 debian_dir,
807 dctrl_bin,
808 "docs",
809 "dh_installdocs",
810 acceptable_migration_issues,
811 feature_migration,
812 manifest,
813 support_executable_files=True,
814 )
815 if not install_file:
816 continue
817 assert content is not None
818 docs: List[str] = []
819 for dhe_line in content:
820 if dhe_line.arch_filter or dhe_line.build_profile_filter: 820 ↛ 821line 820 didn't jump to line 821, because the condition on line 820 was never true
821 _error(
822 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
823 " Missing support for conditions."
824 )
825 docs.extend(_normalize_path(w, with_prefix=False) for w in dhe_line.tokens)
827 if not docs: 827 ↛ 828line 827 didn't jump to line 828, because the condition on line 827 was never true
828 continue
829 feature_migration.successful_manifest_changes += 1
830 install_rule = AbstractMutableYAMLInstallRule.install_docs(
831 docs if len(docs) > 1 else docs[0],
832 dctrl_bin.name if not is_single_binary else None,
833 )
834 installations.create_definition_if_missing()
835 installations.append(install_rule)
838def migrate_installexamples_file(
839 debian_dir: VirtualPath,
840 manifest: HighLevelManifest,
841 acceptable_migration_issues: AcceptableMigrationIssues,
842 feature_migration: FeatureMigration,
843 _migration_target: str,
844) -> None:
845 feature_migration.tagline = "dh_installexamples config files"
846 mutable_manifest = assume_not_none(manifest.mutable_manifest)
847 installations = mutable_manifest.installations(create_if_absent=False)
848 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
850 for dctrl_bin in manifest.all_packages:
851 install_file, content = _dh_config_file(
852 debian_dir,
853 dctrl_bin,
854 "examples",
855 "dh_installexamples",
856 acceptable_migration_issues,
857 feature_migration,
858 manifest,
859 support_executable_files=True,
860 )
861 if not install_file:
862 continue
863 assert content is not None
864 examples: List[str] = []
865 for dhe_line in content:
866 if dhe_line.arch_filter or dhe_line.build_profile_filter: 866 ↛ 867line 866 didn't jump to line 867, because the condition on line 866 was never true
867 _error(
868 f"Unable to migrate line {dhe_line.line_no} of {install_file.path}."
869 " Missing support for conditions."
870 )
871 examples.extend(
872 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
873 )
875 if not examples: 875 ↛ 876line 875 didn't jump to line 876, because the condition on line 875 was never true
876 continue
877 feature_migration.successful_manifest_changes += 1
878 install_rule = AbstractMutableYAMLInstallRule.install_examples(
879 examples if len(examples) > 1 else examples[0],
880 dctrl_bin.name if not is_single_binary else None,
881 )
882 installations.create_definition_if_missing()
883 installations.append(install_rule)
886@dataclasses.dataclass(slots=True)
887class InfoFilesDefinition:
888 sources: List[str] = dataclasses.field(default_factory=list)
889 conditional: Optional[Union[str, Mapping[str, Any]]] = None
892def migrate_installinfo_file(
893 debian_dir: VirtualPath,
894 manifest: HighLevelManifest,
895 acceptable_migration_issues: AcceptableMigrationIssues,
896 feature_migration: FeatureMigration,
897 _migration_target: str,
898) -> None:
899 feature_migration.tagline = "dh_installinfo config files"
900 mutable_manifest = assume_not_none(manifest.mutable_manifest)
901 installations = mutable_manifest.installations(create_if_absent=False)
902 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
904 for dctrl_bin in manifest.all_packages:
905 info_file, content = _dh_config_file(
906 debian_dir,
907 dctrl_bin,
908 "info",
909 "dh_installinfo",
910 acceptable_migration_issues,
911 feature_migration,
912 manifest,
913 support_executable_files=True,
914 )
915 if not info_file:
916 continue
917 assert content is not None
918 info_files_by_condition: Dict[Tuple[str, ...], InfoFilesDefinition] = {}
919 for dhe_line in content:
920 key = dhe_line.conditional_key()
921 ctr = functools.partial(
922 InfoFilesDefinition, conditional=dhe_line.conditional()
923 )
924 info_def = _fetch_or_create(
925 info_files_by_condition,
926 key,
927 ctr,
928 )
929 info_def.sources.extend(
930 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
931 )
933 if not info_files_by_condition: 933 ↛ 934line 933 didn't jump to line 934, because the condition on line 933 was never true
934 continue
935 feature_migration.successful_manifest_changes += 1
936 installations.create_definition_if_missing()
937 for info_def in info_files_by_condition.values():
938 info_files = info_def.sources
939 install_rule = AbstractMutableYAMLInstallRule.install_docs(
940 info_files if len(info_files) > 1 else info_files[0],
941 dctrl_bin.name if not is_single_binary else None,
942 dest_dir="{{path:GNU_INFO_DIR}}",
943 when=info_def.conditional,
944 )
945 installations.append(install_rule)
948@dataclasses.dataclass(slots=True)
949class ManpageDefinition:
950 sources: List[str] = dataclasses.field(default_factory=list)
951 language: Optional[str] = None
952 conditional: Optional[Union[str, Mapping[str, Any]]] = None
955DK = TypeVar("DK")
956DV = TypeVar("DV")
959def _fetch_or_create(d: Dict[DK, DV], key: DK, factory: Callable[[], DV]) -> DV:
960 v = d.get(key)
961 if v is None:
962 v = factory()
963 d[key] = v
964 return v
967def migrate_installman_file(
968 debian_dir: VirtualPath,
969 manifest: HighLevelManifest,
970 acceptable_migration_issues: AcceptableMigrationIssues,
971 feature_migration: FeatureMigration,
972 _migration_target: str,
973) -> None:
974 feature_migration.tagline = "dh_installman config files"
975 mutable_manifest = assume_not_none(manifest.mutable_manifest)
976 installations = mutable_manifest.installations(create_if_absent=False)
977 is_single_binary = sum(1 for _ in manifest.all_packages) == 1
978 warn_about_basename = False
980 for dctrl_bin in manifest.all_packages:
981 manpages_file, content = _dh_config_file(
982 debian_dir,
983 dctrl_bin,
984 "manpages",
985 "dh_installman",
986 acceptable_migration_issues,
987 feature_migration,
988 manifest,
989 support_executable_files=True,
990 allow_dh_exec_rename=True,
991 )
992 if not manpages_file:
993 continue
994 assert content is not None
996 vanilla_definitions = []
997 install_as_rules = []
998 complex_definitions: Dict[
999 Tuple[Optional[str], Tuple[str, ...]], ManpageDefinition
1000 ] = {}
1001 install_rule: AbstractMutableYAMLInstallRule
1002 for dhe_line in content:
1003 if "=>" in dhe_line.tokens: 1003 ↛ 1006line 1003 didn't jump to line 1006, because the condition on line 1003 was never true
1004 # dh-exec allows renaming features. For `debputy`, we degenerate it into an `install` (w. `as`) feature
1005 # without any of the `install-man` features.
1006 if dhe_line.tokens[0] == "=>" and len(dhe_line.tokens) == 2:
1007 _error(
1008 f'Unsupported "=> DEST" rule for error in {manpages_file.path} on line {dhe_line.line_no}."'
1009 f' Cannot migrate dh-exec renames that is not exactly "SOURCE => TARGET" for d/manpages files.'
1010 )
1011 elif len(dhe_line.tokens) != 3:
1012 _error(
1013 f"Validation error in {manpages_file.path} on line {dhe_line.line_no}. Cannot migrate dh-exec"
1014 ' renames that is not exactly "SOURCE => TARGET" or "=> TARGET".'
1015 )
1016 else:
1017 install_rule = AbstractMutableYAMLInstallRule.install_doc_as(
1018 _normalize_path(dhe_line.tokens[0], with_prefix=False),
1019 _normalize_path(dhe_line.tokens[2], with_prefix=False),
1020 dctrl_bin.name if not is_single_binary else None,
1021 when=dhe_line.conditional(),
1022 )
1023 install_as_rules.append(install_rule)
1024 continue
1026 sources = [_normalize_path(w, with_prefix=False) for w in dhe_line.tokens]
1027 needs_basename = any(
1028 MAN_GUESS_FROM_BASENAME.search(x)
1029 and not MAN_GUESS_LANG_FROM_PATH.search(x)
1030 for x in sources
1031 )
1032 if needs_basename or dhe_line.conditional() is not None:
1033 if needs_basename: 1033 ↛ 1037line 1033 didn't jump to line 1037, because the condition on line 1033 was never false
1034 warn_about_basename = True
1035 language = "derive-from-basename"
1036 else:
1037 language = None
1038 key = (language, dhe_line.conditional_key())
1039 ctor = functools.partial(
1040 ManpageDefinition,
1041 language=language,
1042 conditional=dhe_line.conditional(),
1043 )
1044 manpage_def = _fetch_or_create(
1045 complex_definitions,
1046 key,
1047 ctor,
1048 )
1049 manpage_def.sources.extend(sources)
1050 else:
1051 vanilla_definitions.extend(sources)
1053 if not install_as_rules and not vanilla_definitions and not complex_definitions: 1053 ↛ 1054line 1053 didn't jump to line 1054, because the condition on line 1053 was never true
1054 continue
1055 feature_migration.successful_manifest_changes += 1
1056 installations.create_definition_if_missing()
1057 installations.extend(install_as_rules)
1058 if vanilla_definitions: 1058 ↛ 1070line 1058 didn't jump to line 1070, because the condition on line 1058 was never false
1059 man_source = (
1060 vanilla_definitions
1061 if len(vanilla_definitions) > 1
1062 else vanilla_definitions[0]
1063 )
1064 install_rule = AbstractMutableYAMLInstallRule.install_man(
1065 man_source,
1066 dctrl_bin.name if not is_single_binary else None,
1067 None,
1068 )
1069 installations.append(install_rule)
1070 for manpage_def in complex_definitions.values():
1071 sources = manpage_def.sources
1072 install_rule = AbstractMutableYAMLInstallRule.install_man(
1073 sources if len(sources) > 1 else sources[0],
1074 dctrl_bin.name if not is_single_binary else None,
1075 manpage_def.language,
1076 when=manpage_def.conditional,
1077 )
1078 installations.append(install_rule)
1080 if warn_about_basename:
1081 feature_migration.warn(
1082 'Detected man pages that might rely on "derive-from-basename" logic. Please double check'
1083 " that the generated `install-man` rules are correct"
1084 )
1087def migrate_not_installed_file(
1088 debian_dir: VirtualPath,
1089 manifest: HighLevelManifest,
1090 acceptable_migration_issues: AcceptableMigrationIssues,
1091 feature_migration: FeatureMigration,
1092 _migration_target: str,
1093) -> None:
1094 feature_migration.tagline = "dh_missing's not-installed config file"
1095 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1096 installations = mutable_manifest.installations(create_if_absent=False)
1097 main_binary = [p for p in manifest.all_packages if p.is_main_package][0]
1099 missing_file, content = _dh_config_file(
1100 debian_dir,
1101 main_binary,
1102 "not-installed",
1103 "dh_missing",
1104 acceptable_migration_issues,
1105 feature_migration,
1106 manifest,
1107 support_executable_files=False,
1108 pkgfile_lookup=False,
1109 )
1110 discard_rules: List[str] = []
1111 if missing_file:
1112 assert content is not None
1113 for dhe_line in content:
1114 discard_rules.extend(
1115 _normalize_path(w, with_prefix=False) for w in dhe_line.tokens
1116 )
1118 if discard_rules:
1119 feature_migration.successful_manifest_changes += 1
1120 install_rule = AbstractMutableYAMLInstallRule.discard(
1121 discard_rules if len(discard_rules) > 1 else discard_rules[0],
1122 )
1123 installations.create_definition_if_missing()
1124 installations.append(install_rule)
1127def detect_pam_files(
1128 debian_dir: VirtualPath,
1129 manifest: HighLevelManifest,
1130 _acceptable_migration_issues: AcceptableMigrationIssues,
1131 feature_migration: FeatureMigration,
1132 _migration_target: str,
1133) -> None:
1134 feature_migration.tagline = "detect dh_installpam files (min dh compat)"
1135 for dctrl_bin in manifest.all_packages:
1136 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "pam")
1137 if dh_config_file is not None:
1138 feature_migration.assumed_compat = 14
1139 break
1142def migrate_tmpfile(
1143 debian_dir: VirtualPath,
1144 manifest: HighLevelManifest,
1145 _acceptable_migration_issues: AcceptableMigrationIssues,
1146 feature_migration: FeatureMigration,
1147 _migration_target: str,
1148) -> None:
1149 feature_migration.tagline = "dh_installtmpfiles config files"
1150 for dctrl_bin in manifest.all_packages:
1151 dh_config_file = dhe_pkgfile(debian_dir, dctrl_bin, "tmpfile")
1152 if dh_config_file is not None:
1153 target = (
1154 dh_config_file.name.replace(".tmpfile", ".tmpfiles")
1155 if "." in dh_config_file.name
1156 else "tmpfiles"
1157 )
1158 _rename_file_if_exists(
1159 debian_dir,
1160 dh_config_file.name,
1161 target,
1162 feature_migration,
1163 )
1166def migrate_lintian_overrides_files(
1167 debian_dir: VirtualPath,
1168 manifest: HighLevelManifest,
1169 acceptable_migration_issues: AcceptableMigrationIssues,
1170 feature_migration: FeatureMigration,
1171 _migration_target: str,
1172) -> None:
1173 feature_migration.tagline = "dh_lintian config files"
1174 for dctrl_bin in manifest.all_packages:
1175 # We do not support executable lintian-overrides and `_dh_config_file` handles all of that.
1176 # Therefore, the return value is irrelevant to us.
1177 _dh_config_file(
1178 debian_dir,
1179 dctrl_bin,
1180 "lintian-overrides",
1181 "dh_lintian",
1182 acceptable_migration_issues,
1183 feature_migration,
1184 manifest,
1185 support_executable_files=False,
1186 remove_on_migration=False,
1187 )
1190def migrate_links_files(
1191 debian_dir: VirtualPath,
1192 manifest: HighLevelManifest,
1193 acceptable_migration_issues: AcceptableMigrationIssues,
1194 feature_migration: FeatureMigration,
1195 _migration_target: str,
1196) -> None:
1197 feature_migration.tagline = "dh_link files"
1198 mutable_manifest = assume_not_none(manifest.mutable_manifest)
1199 for dctrl_bin in manifest.all_packages:
1200 links_file, content = _dh_config_file(
1201 debian_dir,
1202 dctrl_bin,
1203 "links",
1204 "dh_link",
1205 acceptable_migration_issues,
1206 feature_migration,
1207 manifest,
1208 support_executable_files=True,
1209 )
1211 if links_file is None:
1212 continue
1213 assert content is not None
1215 package_definition = mutable_manifest.package(dctrl_bin.name)
1216 defined_symlink = {
1217 symlink.symlink_path: symlink.symlink_target
1218 for symlink in package_definition.symlinks()
1219 }
1221 seen_symlinks: Set[str] = set()
1223 for dhe_line in content:
1224 if len(dhe_line.tokens) != 2: 1224 ↛ 1225line 1224 didn't jump to line 1225, because the condition on line 1224 was never true
1225 raise UnsupportedFeature(
1226 f"The dh_link file {links_file.fs_path} did not have exactly two paths on line"
1227 f' {dhe_line.line_no} (line: "{dhe_line.original_line}"'
1228 )
1229 target, source = dhe_line.tokens
1230 if source in seen_symlinks: 1230 ↛ 1232line 1230 didn't jump to line 1232, because the condition on line 1230 was never true
1231 # According to #934499, this has happened in the wild already
1232 raise ConflictingChange(
1233 f"The {links_file.fs_path} file defines the link path {source} twice! Please ensure"
1234 " that it is defined at most once in that file"
1235 )
1236 seen_symlinks.add(source)
1237 # Symlinks in .links are always considered absolute, but you were not required to have a leading slash.
1238 # However, in the debputy manifest, you can have relative links, so we should ensure it is explicitly
1239 # absolute.
1240 if not target.startswith("/"): 1240 ↛ 1242line 1240 didn't jump to line 1242, because the condition on line 1240 was never false
1241 target = "/" + target
1242 existing_target = defined_symlink.get(source)
1243 if existing_target is not None: 1243 ↛ 1244line 1243 didn't jump to line 1244, because the condition on line 1243 was never true
1244 if existing_target != target:
1245 raise ConflictingChange(
1246 f'The symlink "{source}" points to "{target}" in {links_file}, but there is'
1247 f' another symlink with same path pointing to "{existing_target}" defined'
1248 " already (in the existing manifest or an migration e.g., inside"
1249 f" {links_file.fs_path})"
1250 )
1251 feature_migration.already_present += 1
1252 continue
1253 condition = dhe_line.conditional()
1254 package_definition.add_symlink(
1255 MutableYAMLSymlink.new_symlink(
1256 source,
1257 target,
1258 condition,
1259 )
1260 )
1261 feature_migration.successful_manifest_changes += 1
1264def migrate_misspelled_readme_debian_files(
1265 debian_dir: VirtualPath,
1266 manifest: HighLevelManifest,
1267 acceptable_migration_issues: AcceptableMigrationIssues,
1268 feature_migration: FeatureMigration,
1269 _migration_target: str,
1270) -> None:
1271 feature_migration.tagline = "misspelled README.Debian files"
1272 for dctrl_bin in manifest.all_packages:
1273 readme, _ = _dh_config_file(
1274 debian_dir,
1275 dctrl_bin,
1276 "README.debian",
1277 "dh_installdocs",
1278 acceptable_migration_issues,
1279 feature_migration,
1280 manifest,
1281 support_executable_files=False,
1282 remove_on_migration=False,
1283 )
1284 if readme is None:
1285 continue
1286 new_name = readme.name.replace("README.debian", "README.Debian")
1287 assert readme.name != new_name
1288 _rename_file_if_exists(
1289 debian_dir,
1290 readme.name,
1291 new_name,
1292 feature_migration,
1293 )
1296def migrate_doc_base_files(
1297 debian_dir: VirtualPath,
1298 manifest: HighLevelManifest,
1299 _: AcceptableMigrationIssues,
1300 feature_migration: FeatureMigration,
1301 _migration_target: str,
1302) -> None:
1303 feature_migration.tagline = "doc-base files"
1304 # ignore the dh_make ".EX" file if one should still be present. The dh_installdocs tool ignores it too.
1305 possible_effected_doc_base_files = [
1306 f
1307 for f in debian_dir.iterdir
1308 if (
1309 (".doc-base." in f.name or f.name.startswith("doc-base."))
1310 and not f.name.endswith("doc-base.EX")
1311 )
1312 ]
1313 known_packages = {d.name: d for d in manifest.all_packages}
1314 main_package = [d for d in manifest.all_packages if d.is_main_package][0]
1315 for doc_base_file in possible_effected_doc_base_files:
1316 parts = doc_base_file.name.split(".")
1317 owning_package = known_packages.get(parts[0])
1318 if owning_package is None: 1318 ↛ 1319line 1318 didn't jump to line 1319, because the condition on line 1318 was never true
1319 owning_package = main_package
1320 package_part = None
1321 else:
1322 package_part = parts[0]
1323 parts = parts[1:]
1325 if not parts or parts[0] != "doc-base": 1325 ↛ 1327line 1325 didn't jump to line 1327, because the condition on line 1325 was never true
1326 # Not a doc-base file after all
1327 continue
1329 if len(parts) > 1: 1329 ↛ 1336line 1329 didn't jump to line 1336, because the condition on line 1329 was never false
1330 name_part = ".".join(parts[1:])
1331 if package_part is None: 1331 ↛ 1333line 1331 didn't jump to line 1333, because the condition on line 1331 was never true
1332 # Named files must have a package prefix
1333 package_part = owning_package.name
1334 else:
1335 # No rename needed
1336 continue
1338 new_basename = ".".join(filter(None, (package_part, name_part, "doc-base")))
1339 _rename_file_if_exists(
1340 debian_dir,
1341 doc_base_file.name,
1342 new_basename,
1343 feature_migration,
1344 )
1347def migrate_dh_hook_targets(
1348 debian_dir: VirtualPath,
1349 _: HighLevelManifest,
1350 acceptable_migration_issues: AcceptableMigrationIssues,
1351 feature_migration: FeatureMigration,
1352 migration_target: str,
1353) -> None:
1354 feature_migration.tagline = "dh hook targets"
1355 source_root = os.path.dirname(debian_dir.fs_path)
1356 if source_root == "":
1357 source_root = "."
1358 detected_hook_targets = json.loads(
1359 subprocess.check_output(
1360 ["dh_assistant", "detect-hook-targets"],
1361 cwd=source_root,
1362 ).decode("utf-8")
1363 )
1364 sample_hook_target: Optional[str] = None
1365 replaced_commands = DH_COMMANDS_REPLACED[migration_target]
1367 for hook_target_def in detected_hook_targets["hook-targets"]:
1368 if hook_target_def["is-empty"]:
1369 continue
1370 command = hook_target_def["command"]
1371 if command not in replaced_commands:
1372 continue
1373 hook_target = hook_target_def["target-name"]
1374 advice = MIGRATION_AID_FOR_OVERRIDDEN_COMMANDS.get(command)
1375 if advice is None:
1376 if sample_hook_target is None:
1377 sample_hook_target = hook_target
1378 feature_migration.warn(
1379 f"TODO: MANUAL MIGRATION required for hook target {hook_target}"
1380 )
1381 else:
1382 feature_migration.warn(
1383 f"TODO: MANUAL MIGRATION required for hook target {hook_target}. Please see {advice}"
1384 f" for migration advice."
1385 )
1386 if (
1387 feature_migration.warnings
1388 and "dh-hook-targets" not in acceptable_migration_issues
1389 and sample_hook_target is not None
1390 ):
1391 raise UnsupportedFeature(
1392 f"The debian/rules file contains one or more non empty dh hook targets that will not"
1393 f" be run with the requested debputy dh sequence with no known migration advice. One of these would be"
1394 f" {sample_hook_target}.",
1395 ["dh-hook-targets"],
1396 )
1399def detect_unsupported_zz_debputy_features(
1400 debian_dir: VirtualPath,
1401 manifest: HighLevelManifest,
1402 acceptable_migration_issues: AcceptableMigrationIssues,
1403 feature_migration: FeatureMigration,
1404 _migration_target: str,
1405) -> None:
1406 feature_migration.tagline = "Known unsupported features"
1408 for unsupported_config in UNSUPPORTED_DH_CONFIGS_AND_TOOLS_FOR_ZZ_DEBPUTY:
1409 _unsupported_debhelper_config_file(
1410 debian_dir,
1411 manifest,
1412 unsupported_config,
1413 acceptable_migration_issues,
1414 feature_migration,
1415 )
1418def detect_obsolete_substvars(
1419 debian_dir: VirtualPath,
1420 _manifest: HighLevelManifest,
1421 _acceptable_migration_issues: AcceptableMigrationIssues,
1422 feature_migration: FeatureMigration,
1423 _migration_target: str,
1424) -> None:
1425 feature_migration.tagline = (
1426 "Check for obsolete ${foo:var} variables in debian/control"
1427 )
1428 ctrl_file = debian_dir.get("control")
1429 if not ctrl_file: 1429 ↛ 1430line 1429 didn't jump to line 1430, because the condition on line 1429 was never true
1430 feature_migration.warn(
1431 "Cannot find debian/control. Detection of obsolete substvars could not be performed."
1432 )
1433 return
1434 with ctrl_file.open() as fd:
1435 ctrl = list(Deb822.iter_paragraphs(fd))
1437 relationship_fields = dpkg_field_list_pkg_dep()
1438 relationship_fields_lc = frozenset(x.lower() for x in relationship_fields)
1440 for p in ctrl[1:]:
1441 seen_obsolete_relationship_substvars = set()
1442 obsolete_fields = set()
1443 is_essential = p.get("Essential") == "yes"
1444 for df in relationship_fields:
1445 field: Optional[str] = p.get(df)
1446 if field is None:
1447 continue
1448 df_lc = df.lower()
1449 number_of_relations = 0
1450 obsolete_substvars_in_field = set()
1451 for d in (d.strip() for d in field.strip().split(",")):
1452 if not d:
1453 continue
1454 number_of_relations += 1
1455 if not d.startswith("${"):
1456 continue
1457 try:
1458 end_idx = d.index("}")
1459 except ValueError:
1460 continue
1461 substvar_name = d[2:end_idx]
1462 if ":" not in substvar_name: 1462 ↛ 1463line 1462 didn't jump to line 1463, because the condition on line 1462 was never true
1463 continue
1464 _, field = substvar_name.rsplit(":", 1)
1465 field_lc = field.lower()
1466 if field_lc not in relationship_fields_lc: 1466 ↛ 1467line 1466 didn't jump to line 1467, because the condition on line 1466 was never true
1467 continue
1468 is_obsolete = field_lc == df_lc
1469 if (
1470 not is_obsolete
1471 and is_essential
1472 and substvar_name.lower() == "shlibs:depends"
1473 and df_lc == "pre-depends"
1474 ):
1475 is_obsolete = True
1477 if is_obsolete:
1478 obsolete_substvars_in_field.add(d)
1480 if number_of_relations == len(obsolete_substvars_in_field):
1481 obsolete_fields.add(df)
1482 else:
1483 seen_obsolete_relationship_substvars.update(obsolete_substvars_in_field)
1485 package = p.get("Package", "(Missing package name!?)")
1486 if obsolete_fields:
1487 fields = ", ".join(obsolete_fields)
1488 feature_migration.warn(
1489 f"The following relationship fields can be removed from {package}: {fields}."
1490 f" (The content in them would be applied automatically.)"
1491 )
1492 if seen_obsolete_relationship_substvars:
1493 v = ", ".join(sorted(seen_obsolete_relationship_substvars))
1494 feature_migration.warn(
1495 f"The following relationship substitution variables can be removed from {package}: {v}"
1496 )
1499def read_dh_addon_sequences(
1500 debian_dir: VirtualPath,
1501) -> Optional[Tuple[Set[str], Set[str]]]:
1502 ctrl_file = debian_dir.get("control")
1503 if ctrl_file:
1504 dr_sequences: Set[str] = set()
1505 bd_sequences = set()
1507 drules = debian_dir.get("rules")
1508 if drules and drules.is_file: 1508 ↛ 1509line 1508 didn't jump to line 1509, because the condition on line 1508 was never true
1509 with drules.open() as fd:
1510 parse_drules_for_addons(fd, dr_sequences)
1512 with ctrl_file.open() as fd:
1513 ctrl = list(Deb822.iter_paragraphs(fd))
1514 source_paragraph = ctrl[0] if ctrl else {}
1516 extract_dh_addons_from_control(source_paragraph, bd_sequences)
1517 return bd_sequences, dr_sequences
1518 return None
1521def detect_dh_addons_zz_debputy_rrr(
1522 debian_dir: VirtualPath,
1523 _manifest: HighLevelManifest,
1524 _acceptable_migration_issues: AcceptableMigrationIssues,
1525 feature_migration: FeatureMigration,
1526 _migration_target: str,
1527) -> None:
1528 feature_migration.tagline = "Check for dh-sequence-addons"
1529 r = read_dh_addon_sequences(debian_dir)
1530 if r is None:
1531 feature_migration.warn(
1532 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1533 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy-rrr."
1534 )
1535 return
1537 bd_sequences, dr_sequences = r
1539 remaining_sequences = bd_sequences | dr_sequences
1540 saw_dh_debputy = "zz-debputy-rrr" in remaining_sequences
1542 if not saw_dh_debputy:
1543 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy-rrr")
1546def detect_dh_addons(
1547 debian_dir: VirtualPath,
1548 _manifest: HighLevelManifest,
1549 acceptable_migration_issues: AcceptableMigrationIssues,
1550 feature_migration: FeatureMigration,
1551 _migration_target: str,
1552) -> None:
1553 feature_migration.tagline = "Check for dh-sequence-addons"
1554 r = read_dh_addon_sequences(debian_dir)
1555 if r is None:
1556 feature_migration.warn(
1557 "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon"
1558 " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy"
1559 " and not rely on any other debhelper sequence addons except those debputy explicitly supports."
1560 )
1561 return
1563 bd_sequences, dr_sequences = r
1565 remaining_sequences = bd_sequences | dr_sequences
1566 saw_dh_debputy = (
1567 "debputy" in remaining_sequences or "zz-debputy" in remaining_sequences
1568 )
1569 saw_zz_debputy = "zz-debputy" in remaining_sequences
1570 must_use_zz_debputy = False
1571 remaining_sequences -= SUPPORTED_DH_ADDONS
1572 for sequence in remaining_sequences & DH_ADDONS_TO_PLUGINS.keys():
1573 migration = DH_ADDONS_TO_PLUGINS[sequence]
1574 feature_migration.require_plugin(migration.debputy_plugin)
1575 if migration.remove_dh_sequence: 1575 ↛ 1576line 1575 didn't jump to line 1576, because the condition on line 1575 was never true
1576 if migration.must_use_zz_debputy:
1577 must_use_zz_debputy = True
1578 if sequence in bd_sequences:
1579 feature_migration.warn(
1580 f"TODO: MANUAL MIGRATION - Remove build-dependency on dh-sequence-{sequence}"
1581 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1582 )
1583 else:
1584 feature_migration.warn(
1585 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1586 f" (replaced by debputy-plugin-{migration.debputy_plugin})"
1587 )
1589 remaining_sequences -= DH_ADDONS_TO_PLUGINS.keys()
1591 alt_key = "unsupported-dh-sequences"
1592 for sequence in remaining_sequences & DH_ADDONS_TO_REMOVE: 1592 ↛ 1593line 1592 didn't jump to line 1593, because the loop on line 1592 never started
1593 if sequence in bd_sequences:
1594 feature_migration.warn(
1595 f"TODO: MANUAL MIGRATION - Remove build dependency on dh-sequence-{sequence}"
1596 )
1597 else:
1598 feature_migration.warn(
1599 f"TODO: MANUAL MIGRATION - Remove --with {sequence} from dh in d/rules"
1600 )
1602 remaining_sequences -= DH_ADDONS_TO_REMOVE
1604 for sequence in remaining_sequences:
1605 key = f"unsupported-dh-sequence-{sequence}"
1606 msg = f'The dh addon "{sequence}" is not known to work with dh-debputy and might malfunction'
1607 if (
1608 key not in acceptable_migration_issues
1609 and alt_key not in acceptable_migration_issues
1610 ):
1611 raise UnsupportedFeature(msg, [key, alt_key])
1612 feature_migration.warn(msg)
1614 if not saw_dh_debputy:
1615 feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy")
1616 elif must_use_zz_debputy and not saw_zz_debputy: 1616 ↛ 1617line 1616 didn't jump to line 1617, because the condition on line 1616 was never true
1617 feature_migration.warn(
1618 "Please use the zz-debputy sequence rather than the debputy (needed due to dh add-on load order)"
1619 )
1622def _rename_file_if_exists(
1623 debian_dir: VirtualPath,
1624 source: str,
1625 dest: str,
1626 feature_migration: FeatureMigration,
1627) -> None:
1628 source_path = debian_dir.get(source)
1629 dest_path = debian_dir.get(dest)
1630 spath = (
1631 source_path.path
1632 if source_path is not None
1633 else os.path.join(debian_dir.path, source)
1634 )
1635 dpath = (
1636 dest_path.path if dest_path is not None else os.path.join(debian_dir.path, dest)
1637 )
1638 if source_path is not None and source_path.is_file:
1639 if dest_path is not None:
1640 if not dest_path.is_file:
1641 feature_migration.warnings.append(
1642 f'TODO: MANUAL MIGRATION - there is a "{spath}" (file) and "{dpath}" (not a file).'
1643 f' The migration wanted to replace "{spath}" with "{dpath}", but since "{dpath}" is not'
1644 " a file, this step is left as a manual migration."
1645 )
1646 return
1647 if (
1648 subprocess.call(["cmp", "-s", source_path.fs_path, dest_path.fs_path])
1649 != 0
1650 ):
1651 feature_migration.warnings.append(
1652 f'TODO: MANUAL MIGRATION - there is a "{source_path.path}" and "{dest_path.path}"'
1653 f" file. Normally these files are for the same package and there would only be one of"
1654 f" them. In this case, they both exist but their content differs. Be advised that"
1655 f' debputy tool will use the "{dest_path.path}".'
1656 )
1657 else:
1658 feature_migration.remove_on_success(dest_path.fs_path)
1659 else:
1660 feature_migration.rename_on_success(
1661 source_path.fs_path,
1662 os.path.join(debian_dir.fs_path, dest),
1663 )
1664 elif source_path is not None: 1664 ↛ exitline 1664 didn't return from function '_rename_file_if_exists', because the condition on line 1664 was never false
1665 feature_migration.warnings.append(
1666 f'TODO: MANUAL MIGRATION - The migration would normally have renamed "{spath}" to "{dpath}".'
1667 f' However, the migration assumed "{spath}" would be a file and it is not. Therefore, this step'
1668 " as a manual migration."
1669 )
1672def _find_dh_config_file_for_any_pkg(
1673 debian_dir: VirtualPath,
1674 manifest: HighLevelManifest,
1675 unsupported_config: UnsupportedDHConfig,
1676) -> Iterable[VirtualPath]:
1677 for dctrl_bin in manifest.all_packages:
1678 dh_config_file = dhe_pkgfile(
1679 debian_dir,
1680 dctrl_bin,
1681 unsupported_config.dh_config_basename,
1682 bug_950723_prefix_matching=unsupported_config.bug_950723_prefix_matching,
1683 )
1684 if dh_config_file is not None:
1685 yield dh_config_file
1688def _unsupported_debhelper_config_file(
1689 debian_dir: VirtualPath,
1690 manifest: HighLevelManifest,
1691 unsupported_config: UnsupportedDHConfig,
1692 acceptable_migration_issues: AcceptableMigrationIssues,
1693 feature_migration: FeatureMigration,
1694) -> None:
1695 dh_config_files = list(
1696 _find_dh_config_file_for_any_pkg(debian_dir, manifest, unsupported_config)
1697 )
1698 if not dh_config_files:
1699 return
1700 dh_tool = unsupported_config.dh_tool
1701 basename = unsupported_config.dh_config_basename
1702 file_stem = (
1703 f"@{basename}" if unsupported_config.bug_950723_prefix_matching else basename
1704 )
1705 dh_config_file = dh_config_files[0]
1706 if unsupported_config.is_missing_migration:
1707 feature_migration.warn(
1708 f'Missing migration support for the "{dh_config_file.path}" debhelper config file'
1709 f" (used by {dh_tool}). Manual migration may be feasible depending on the exact features"
1710 " required."
1711 )
1712 return
1713 primary_key = f"unsupported-dh-config-file-{file_stem}"
1714 secondary_key = "any-unsupported-dh-config-file"
1715 if (
1716 primary_key not in acceptable_migration_issues
1717 and secondary_key not in acceptable_migration_issues
1718 ):
1719 msg = (
1720 f'The "{dh_config_file.path}" debhelper config file (used by {dh_tool} is currently not'
1721 " supported by debputy."
1722 )
1723 raise UnsupportedFeature(
1724 msg,
1725 [primary_key, secondary_key],
1726 )
1727 for dh_config_file in dh_config_files:
1728 feature_migration.warn(
1729 f'TODO: MANUAL MIGRATION - Use of unsupported "{dh_config_file.path}" file (used by {dh_tool})'
1730 )