Coverage for src/debputy/highlevel_manifest_parser.py: 68%
253 statements
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
1import collections
2import contextlib
3from typing import (
4 Optional,
5 Dict,
6 Callable,
7 List,
8 Any,
9 Union,
10 Mapping,
11 IO,
12 Iterator,
13 cast,
14 Tuple,
15)
17from debian.debian_support import DpkgArchTable
19from debputy.highlevel_manifest import (
20 HighLevelManifest,
21 PackageTransformationDefinition,
22 MutableYAMLManifest,
23)
24from debputy.maintscript_snippet import (
25 MaintscriptSnippet,
26 STD_CONTROL_SCRIPTS,
27 MaintscriptSnippetContainer,
28)
29from debputy.packages import BinaryPackage, SourcePackage
30from debputy.path_matcher import (
31 MatchRuleType,
32 ExactFileSystemPath,
33 MatchRule,
34)
35from debputy.substitution import Substitution
36from debputy.util import (
37 _normalize_path,
38 escape_shell,
39 assume_not_none,
40)
41from debputy.util import _warn, _info
42from ._deb_options_profiles import DebBuildOptionsAndProfiles
43from .architecture_support import DpkgArchitectureBuildProcessValuesTable
44from .filesystem_scan import FSROOverlay
45from .installations import InstallRule, PPFInstallRule
46from .manifest_parser.exceptions import ManifestParseException
47from .manifest_parser.parser_data import ParserContextData
48from .manifest_parser.util import AttributePath
49from .packager_provided_files import detect_all_packager_provided_files
50from .plugin.api import VirtualPath
51from .plugin.api.impl_types import (
52 TP,
53 TTP,
54 DispatchingTableParser,
55 OPARSER_MANIFEST_ROOT,
56 PackageContextData,
57)
58from .plugin.api.feature_set import PluginProvidedFeatureSet
59from .yaml import YAMLError, MANIFEST_YAML
61try:
62 from Levenshtein import distance
63except ImportError:
65 def _detect_possible_typo(
66 _d,
67 _key,
68 _attribute_parent_path: AttributePath,
69 required: bool,
70 ) -> None:
71 if required:
72 _info(
73 "Install python3-levenshtein to have debputy try to detect typos in the manifest."
74 )
76else:
78 def _detect_possible_typo(
79 d,
80 key,
81 _attribute_parent_path: AttributePath,
82 _required: bool,
83 ) -> None:
84 k_len = len(key)
85 for actual_key in d:
86 if abs(k_len - len(actual_key)) > 2:
87 continue
88 d = distance(key, actual_key)
89 if d > 2:
90 continue
91 path = _attribute_parent_path.path
92 ref = f'at "{path}"' if path else "at the manifest root level"
93 _warn(
94 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}'
95 )
98def _per_package_subst_variables(
99 p: BinaryPackage,
100 *,
101 name: Optional[str] = None,
102) -> Dict[str, str]:
103 return {
104 "PACKAGE": name if name is not None else p.name,
105 }
108class HighLevelManifestParser(ParserContextData):
109 def __init__(
110 self,
111 manifest_path: str,
112 source_package: SourcePackage,
113 binary_packages: Mapping[str, BinaryPackage],
114 substitution: Substitution,
115 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable,
116 dpkg_arch_query_table: DpkgArchTable,
117 build_env: DebBuildOptionsAndProfiles,
118 plugin_provided_feature_set: PluginProvidedFeatureSet,
119 *,
120 # Available for testing purposes only
121 debian_dir: Union[str, VirtualPath] = "./debian",
122 ):
123 self.manifest_path = manifest_path
124 self._source_package = source_package
125 self._binary_packages = binary_packages
126 self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None
127 # In source context, some variables are known to be unresolvable. Record this, so
128 # we can give better error messages.
129 self._substitution = substitution
130 self._dpkg_architecture_variables = dpkg_architecture_variables
131 self._dpkg_arch_query_table = dpkg_arch_query_table
132 self._build_env = build_env
133 self._package_state_stack: List[PackageTransformationDefinition] = []
134 self._plugin_provided_feature_set = plugin_provided_feature_set
135 self._declared_variables = {}
137 if isinstance(debian_dir, str): 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true
138 debian_dir = FSROOverlay.create_root_dir("debian", debian_dir)
140 self._debian_dir = debian_dir
142 # Delayed initialized; we rely on this delay to parse the variables.
143 self._all_package_states = None
145 self._install_rules: Optional[List[InstallRule]] = None
146 self._ownership_caches_loaded = False
147 self._used = False
149 def _ensure_package_states_is_initialized(self) -> None:
150 if self._all_package_states is not None:
151 return
152 substitution = self._substitution
153 binary_packages = self._binary_packages
154 assert self._all_package_states is None
156 self._all_package_states = {
157 n: PackageTransformationDefinition(
158 binary_package=p,
159 substitution=substitution.with_extra_substitutions(
160 **_per_package_subst_variables(p)
161 ),
162 is_auto_generated_package=False,
163 maintscript_snippets=collections.defaultdict(
164 MaintscriptSnippetContainer
165 ),
166 )
167 for n, p in binary_packages.items()
168 }
169 for n, p in binary_packages.items():
170 dbgsym_name = f"{n}-dbgsym"
171 if dbgsym_name in self._all_package_states: 171 ↛ 172line 171 didn't jump to line 172, because the condition on line 171 was never true
172 continue
173 self._all_package_states[dbgsym_name] = PackageTransformationDefinition(
174 binary_package=p,
175 substitution=substitution.with_extra_substitutions(
176 **_per_package_subst_variables(p, name=dbgsym_name)
177 ),
178 is_auto_generated_package=True,
179 maintscript_snippets=collections.defaultdict(
180 MaintscriptSnippetContainer
181 ),
182 )
184 @property
185 def binary_packages(self) -> Mapping[str, BinaryPackage]:
186 return self._binary_packages
188 @property
189 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]:
190 assert self._all_package_states is not None
191 return self._all_package_states
193 @property
194 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
195 return self._dpkg_architecture_variables
197 @property
198 def dpkg_arch_query_table(self) -> DpkgArchTable:
199 return self._dpkg_arch_query_table
201 @property
202 def build_env(self) -> DebBuildOptionsAndProfiles:
203 return self._build_env
205 def build_manifest(self) -> HighLevelManifest:
206 if self._used: 206 ↛ 207line 206 didn't jump to line 207, because the condition on line 206 was never true
207 raise TypeError("build_manifest can only be called once!")
208 self._used = True
209 self._ensure_package_states_is_initialized()
210 for var, attribute_path in self._declared_variables.items():
211 if not self.substitution.is_used(var):
212 raise ManifestParseException(
213 f'The variable "{var}" is unused. Either use it or remove it.'
214 f" The variable was declared at {attribute_path.path}."
215 )
216 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None:
217 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest()
218 all_packager_provided_files = detect_all_packager_provided_files(
219 self._plugin_provided_feature_set.packager_provided_files,
220 self._debian_dir,
221 self.binary_packages,
222 )
224 for package in self._package_states:
225 with self.binary_package_context(package) as context:
226 if not context.is_auto_generated_package:
227 ppf_result = all_packager_provided_files[package]
228 if ppf_result.auto_installable: 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true
229 context.install_rules.append(
230 PPFInstallRule(
231 context.binary_package,
232 context.substitution,
233 ppf_result.auto_installable,
234 )
235 )
236 context.reserved_packager_provided_files.update(
237 ppf_result.reserved_only
238 )
239 self._transform_dpkg_maintscript_helpers_to_snippets()
241 return HighLevelManifest(
242 self.manifest_path,
243 self._mutable_yaml_manifest,
244 self._install_rules,
245 self._source_package,
246 self.binary_packages,
247 self.substitution,
248 self._package_states,
249 self._dpkg_architecture_variables,
250 self._dpkg_arch_query_table,
251 self._build_env,
252 self._plugin_provided_feature_set,
253 self._debian_dir,
254 )
256 @contextlib.contextmanager
257 def binary_package_context(
258 self, package_name: str
259 ) -> Iterator[PackageTransformationDefinition]:
260 if package_name not in self._package_states:
261 self._error(
262 f'The package "{package_name}" is not present in the debian/control file (could not find'
263 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one'
264 " for a package in debian/control."
265 )
266 package_state = self._package_states[package_name]
267 self._package_state_stack.append(package_state)
268 ps_len = len(self._package_state_stack)
269 yield package_state
270 if ps_len != len(self._package_state_stack): 270 ↛ 271line 270 didn't jump to line 271, because the condition on line 270 was never true
271 raise RuntimeError("Internal error: Unbalanced stack manipulation detected")
272 self._package_state_stack.pop()
274 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]:
275 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for(
276 rule_type
277 )
278 if t is None:
279 raise AssertionError(
280 f"Internal error: No dispatching parser for {rule_type.__name__}"
281 )
282 return t
284 @property
285 def substitution(self) -> Substitution:
286 if self._package_state_stack:
287 return self._package_state_stack[-1].substitution
288 return self._substitution
290 def add_extra_substitution_variables(
291 self,
292 **extra_substitutions: Tuple[str, AttributePath],
293 ) -> Substitution:
294 if self._package_state_stack or self._all_package_states is not None: 294 ↛ 299line 294 didn't jump to line 299, because the condition on line 294 was never true
295 # For one, it would not "bubble up" correctly when added to the lowest stack.
296 # And if it is not added to the lowest stack, then you get errors about it being
297 # unknown as soon as you leave the stack (which is weird for the user when
298 # the variable is something known, sometimes not)
299 raise RuntimeError("Cannot use add_extra_substitution from this state")
300 for key, (_, path) in extra_substitutions.items():
301 self._declared_variables[key] = path
302 self._substitution = self._substitution.with_extra_substitutions(
303 **{k: v[0] for k, v in extra_substitutions.items()}
304 )
305 return self._substitution
307 @property
308 def current_binary_package_state(self) -> PackageTransformationDefinition:
309 if not self._package_state_stack: 309 ↛ 310line 309 didn't jump to line 310, because the condition on line 309 was never true
310 raise RuntimeError("Invalid state: Not in a binary package context")
311 return self._package_state_stack[-1]
313 @property
314 def is_in_binary_package_state(self) -> bool:
315 return bool(self._package_state_stack)
317 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None:
318 package_state = self.current_binary_package_state
319 for dmh in package_state.dpkg_maintscript_helper_snippets: 319 ↛ 320line 319 didn't jump to line 320, because the loop on line 319 never started
320 snippet = MaintscriptSnippet(
321 definition_source=dmh.definition_source,
322 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n',
323 )
324 for script in STD_CONTROL_SCRIPTS:
325 package_state.maintscript_snippets[script].append(snippet)
327 def normalize_path(
328 self,
329 path: str,
330 definition_source: AttributePath,
331 *,
332 allow_root_dir_match: bool = False,
333 ) -> ExactFileSystemPath:
334 try:
335 normalized = _normalize_path(path)
336 except ValueError:
337 self._error(
338 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the'
339 ' package and not use any ".." or "." segments.'
340 )
341 if normalized == "." and not allow_root_dir_match:
342 self._error(
343 "Manifests must not change the root directory of the deb file. Please correct"
344 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}'
345 )
346 return ExactFileSystemPath(
347 self.substitution.substitute(normalized, definition_source.path)
348 )
350 def parse_path_or_glob(
351 self,
352 path_or_glob: str,
353 definition_source: AttributePath,
354 ) -> MatchRule:
355 match_rule = MatchRule.from_path_or_glob(
356 path_or_glob, definition_source.path, substitution=self.substitution
357 )
358 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob,
359 # so there is no need to check for an exact match on "." like in normalize_path.
360 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING:
361 self._error(
362 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).'
363 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to'
364 f' something that matches "less" than everything.'
365 )
366 return match_rule
368 def parse_manifest(self) -> HighLevelManifest:
369 raise NotImplementedError
372class YAMLManifestParser(HighLevelManifestParser):
373 def _optional_key(
374 self,
375 d: Mapping[str, Any],
376 key: str,
377 attribute_parent_path: AttributePath,
378 expected_type=None,
379 default_value=None,
380 ):
381 v = d.get(key)
382 if v is None:
383 _detect_possible_typo(d, key, attribute_parent_path, False)
384 return default_value
385 if expected_type is not None:
386 return self._ensure_value_is_type(
387 v, expected_type, key, attribute_parent_path
388 )
389 return v
391 def _required_key(
392 self,
393 d: Mapping[str, Any],
394 key: str,
395 attribute_parent_path: AttributePath,
396 expected_type=None,
397 extra: Optional[Union[str, Callable[[], str]]] = None,
398 ):
399 v = d.get(key)
400 if v is None:
401 _detect_possible_typo(d, key, attribute_parent_path, True)
402 if extra is not None:
403 msg = extra if isinstance(extra, str) else extra()
404 extra_info = " " + msg
405 else:
406 extra_info = ""
407 self._error(
408 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.'
409 f"{extra_info}"
410 )
412 if expected_type is not None:
413 return self._ensure_value_is_type(
414 v, expected_type, key, attribute_parent_path
415 )
416 return v
418 def _ensure_value_is_type(
419 self,
420 v,
421 t,
422 key: Union[str, int, AttributePath],
423 attribute_parent_path: Optional[AttributePath],
424 ):
425 if v is None:
426 return None
427 if not isinstance(v, t):
428 if isinstance(t, tuple):
429 t_msg = "one of: " + ", ".join(x.__name__ for x in t)
430 else:
431 t_msg = f"a {t.__name__}"
432 key_path = (
433 key.path
434 if isinstance(key, AttributePath)
435 else assume_not_none(attribute_parent_path)[key].path
436 )
437 self._error(
438 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"'
439 )
440 return v
442 def from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest":
443 attribute_path = AttributePath.root_path()
444 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator
445 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers
446 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT]
447 parsed_data = cast(
448 "ManifestRootRule",
449 manifest_root_parser.parse_input(
450 yaml_data,
451 attribute_path,
452 parser_context=self,
453 ),
454 )
456 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast(
457 "Mapping[str, PackageContextData[Mapping[str, Any]]]",
458 parsed_data.get("packages", {}),
459 )
460 install_rules = parsed_data.get("installations")
461 if install_rules:
462 self._install_rules = install_rules
463 packages_parent_path = attribute_path["packages"]
464 for package_name_raw, pcd in packages_dict.items():
465 definition_source = packages_parent_path[package_name_raw]
466 package_name = pcd.resolved_package_name
467 parsed = pcd.value
469 package_state: PackageTransformationDefinition
470 with self.binary_package_context(package_name) as package_state:
471 if package_state.is_auto_generated_package: 471 ↛ 473line 471 didn't jump to line 473, because the condition on line 471 was never true
472 # Maybe lift (part) of this restriction.
473 self._error(
474 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an'
475 " auto-generated package."
476 )
477 binary_version = parsed.get("binary-version")
478 if binary_version is not None:
479 package_state.binary_version = (
480 package_state.substitution.substitute(
481 binary_version,
482 definition_source["binary-version"].path,
483 )
484 )
485 search_dirs = parsed.get("installation_search_dirs")
486 if search_dirs is not None: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true
487 package_state.search_dirs = search_dirs
488 transformations = parsed.get("transformations")
489 conffile_management = parsed.get("conffile_management")
490 service_rules = parsed.get("services")
491 if transformations:
492 package_state.transformations.extend(transformations)
493 if conffile_management: 493 ↛ 494line 493 didn't jump to line 494, because the condition on line 493 was never true
494 package_state.dpkg_maintscript_helper_snippets.extend(
495 conffile_management
496 )
497 if service_rules: 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 package_state.requested_service_rules.extend(service_rules)
500 return self.build_manifest()
502 def _parse_manifest(self, fd: Union[IO[bytes], str]) -> HighLevelManifest:
503 try:
504 data = MANIFEST_YAML.load(fd)
505 except YAMLError as e:
506 msg = str(e)
507 lines = msg.splitlines(keepends=True)
508 i = -1
509 for i, line in enumerate(lines):
510 # Avoid an irrelevant "how do configure the YAML parser" message, which the
511 # user cannot use.
512 if line.startswith("To suppress this check"):
513 break
514 if i > -1 and len(lines) > i + 1:
515 lines = lines[:i]
516 msg = "".join(lines)
517 msg = msg.rstrip()
518 msg += (
519 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate"
520 " the YAML syntax. The yamllint tool also supports style rules for YAML documents"
521 " (such as indentation rules) in case that is of interest."
522 )
523 raise ManifestParseException(
524 f"Could not parse {self.manifest_path} as a YAML document: {msg}"
525 ) from e
526 self._mutable_yaml_manifest = MutableYAMLManifest(data)
527 return self.from_yaml_dict(data)
529 def parse_manifest(
530 self,
531 *,
532 fd: Optional[Union[IO[bytes], str]] = None,
533 ) -> HighLevelManifest:
534 if fd is None: 534 ↛ 535line 534 didn't jump to line 535, because the condition on line 534 was never true
535 with open(self.manifest_path, "rb") as fd:
536 return self._parse_manifest(fd)
537 else:
538 return self._parse_manifest(fd)