Coverage for src/debputy/highlevel_manifest_parser.py: 68%

253 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2024-04-07 12:14 +0200

1import collections 

2import contextlib 

3from typing import ( 

4 Optional, 

5 Dict, 

6 Callable, 

7 List, 

8 Any, 

9 Union, 

10 Mapping, 

11 IO, 

12 Iterator, 

13 cast, 

14 Tuple, 

15) 

16 

17from debian.debian_support import DpkgArchTable 

18 

19from debputy.highlevel_manifest import ( 

20 HighLevelManifest, 

21 PackageTransformationDefinition, 

22 MutableYAMLManifest, 

23) 

24from debputy.maintscript_snippet import ( 

25 MaintscriptSnippet, 

26 STD_CONTROL_SCRIPTS, 

27 MaintscriptSnippetContainer, 

28) 

29from debputy.packages import BinaryPackage, SourcePackage 

30from debputy.path_matcher import ( 

31 MatchRuleType, 

32 ExactFileSystemPath, 

33 MatchRule, 

34) 

35from debputy.substitution import Substitution 

36from debputy.util import ( 

37 _normalize_path, 

38 escape_shell, 

39 assume_not_none, 

40) 

41from debputy.util import _warn, _info 

42from ._deb_options_profiles import DebBuildOptionsAndProfiles 

43from .architecture_support import DpkgArchitectureBuildProcessValuesTable 

44from .filesystem_scan import FSROOverlay 

45from .installations import InstallRule, PPFInstallRule 

46from .manifest_parser.exceptions import ManifestParseException 

47from .manifest_parser.parser_data import ParserContextData 

48from .manifest_parser.util import AttributePath 

49from .packager_provided_files import detect_all_packager_provided_files 

50from .plugin.api import VirtualPath 

51from .plugin.api.impl_types import ( 

52 TP, 

53 TTP, 

54 DispatchingTableParser, 

55 OPARSER_MANIFEST_ROOT, 

56 PackageContextData, 

57) 

58from .plugin.api.feature_set import PluginProvidedFeatureSet 

59from .yaml import YAMLError, MANIFEST_YAML 

60 

61try: 

62 from Levenshtein import distance 

63except ImportError: 

64 

65 def _detect_possible_typo( 

66 _d, 

67 _key, 

68 _attribute_parent_path: AttributePath, 

69 required: bool, 

70 ) -> None: 

71 if required: 

72 _info( 

73 "Install python3-levenshtein to have debputy try to detect typos in the manifest." 

74 ) 

75 

76else: 

77 

78 def _detect_possible_typo( 

79 d, 

80 key, 

81 _attribute_parent_path: AttributePath, 

82 _required: bool, 

83 ) -> None: 

84 k_len = len(key) 

85 for actual_key in d: 

86 if abs(k_len - len(actual_key)) > 2: 

87 continue 

88 d = distance(key, actual_key) 

89 if d > 2: 

90 continue 

91 path = _attribute_parent_path.path 

92 ref = f'at "{path}"' if path else "at the manifest root level" 

93 _warn( 

94 f'Possible typo: The key "{actual_key}" should probably have been "{key}" {ref}' 

95 ) 

96 

97 

98def _per_package_subst_variables( 

99 p: BinaryPackage, 

100 *, 

101 name: Optional[str] = None, 

102) -> Dict[str, str]: 

103 return { 

104 "PACKAGE": name if name is not None else p.name, 

105 } 

106 

107 

108class HighLevelManifestParser(ParserContextData): 

109 def __init__( 

110 self, 

111 manifest_path: str, 

112 source_package: SourcePackage, 

113 binary_packages: Mapping[str, BinaryPackage], 

114 substitution: Substitution, 

115 dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, 

116 dpkg_arch_query_table: DpkgArchTable, 

117 build_env: DebBuildOptionsAndProfiles, 

118 plugin_provided_feature_set: PluginProvidedFeatureSet, 

119 *, 

120 # Available for testing purposes only 

121 debian_dir: Union[str, VirtualPath] = "./debian", 

122 ): 

123 self.manifest_path = manifest_path 

124 self._source_package = source_package 

125 self._binary_packages = binary_packages 

126 self._mutable_yaml_manifest: Optional[MutableYAMLManifest] = None 

127 # In source context, some variables are known to be unresolvable. Record this, so 

128 # we can give better error messages. 

129 self._substitution = substitution 

130 self._dpkg_architecture_variables = dpkg_architecture_variables 

131 self._dpkg_arch_query_table = dpkg_arch_query_table 

132 self._build_env = build_env 

133 self._package_state_stack: List[PackageTransformationDefinition] = [] 

134 self._plugin_provided_feature_set = plugin_provided_feature_set 

135 self._declared_variables = {} 

136 

137 if isinstance(debian_dir, str): 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true

138 debian_dir = FSROOverlay.create_root_dir("debian", debian_dir) 

139 

140 self._debian_dir = debian_dir 

141 

142 # Delayed initialized; we rely on this delay to parse the variables. 

143 self._all_package_states = None 

144 

145 self._install_rules: Optional[List[InstallRule]] = None 

146 self._ownership_caches_loaded = False 

147 self._used = False 

148 

149 def _ensure_package_states_is_initialized(self) -> None: 

150 if self._all_package_states is not None: 

151 return 

152 substitution = self._substitution 

153 binary_packages = self._binary_packages 

154 assert self._all_package_states is None 

155 

156 self._all_package_states = { 

157 n: PackageTransformationDefinition( 

158 binary_package=p, 

159 substitution=substitution.with_extra_substitutions( 

160 **_per_package_subst_variables(p) 

161 ), 

162 is_auto_generated_package=False, 

163 maintscript_snippets=collections.defaultdict( 

164 MaintscriptSnippetContainer 

165 ), 

166 ) 

167 for n, p in binary_packages.items() 

168 } 

169 for n, p in binary_packages.items(): 

170 dbgsym_name = f"{n}-dbgsym" 

171 if dbgsym_name in self._all_package_states: 171 ↛ 172line 171 didn't jump to line 172, because the condition on line 171 was never true

172 continue 

173 self._all_package_states[dbgsym_name] = PackageTransformationDefinition( 

174 binary_package=p, 

175 substitution=substitution.with_extra_substitutions( 

176 **_per_package_subst_variables(p, name=dbgsym_name) 

177 ), 

178 is_auto_generated_package=True, 

179 maintscript_snippets=collections.defaultdict( 

180 MaintscriptSnippetContainer 

181 ), 

182 ) 

183 

184 @property 

185 def binary_packages(self) -> Mapping[str, BinaryPackage]: 

186 return self._binary_packages 

187 

188 @property 

189 def _package_states(self) -> Mapping[str, PackageTransformationDefinition]: 

190 assert self._all_package_states is not None 

191 return self._all_package_states 

192 

193 @property 

194 def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: 

195 return self._dpkg_architecture_variables 

196 

197 @property 

198 def dpkg_arch_query_table(self) -> DpkgArchTable: 

199 return self._dpkg_arch_query_table 

200 

201 @property 

202 def build_env(self) -> DebBuildOptionsAndProfiles: 

203 return self._build_env 

204 

205 def build_manifest(self) -> HighLevelManifest: 

206 if self._used: 206 ↛ 207line 206 didn't jump to line 207, because the condition on line 206 was never true

207 raise TypeError("build_manifest can only be called once!") 

208 self._used = True 

209 self._ensure_package_states_is_initialized() 

210 for var, attribute_path in self._declared_variables.items(): 

211 if not self.substitution.is_used(var): 

212 raise ManifestParseException( 

213 f'The variable "{var}" is unused. Either use it or remove it.' 

214 f" The variable was declared at {attribute_path.path}." 

215 ) 

216 if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None: 

217 self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest() 

218 all_packager_provided_files = detect_all_packager_provided_files( 

219 self._plugin_provided_feature_set.packager_provided_files, 

220 self._debian_dir, 

221 self.binary_packages, 

222 ) 

223 

224 for package in self._package_states: 

225 with self.binary_package_context(package) as context: 

226 if not context.is_auto_generated_package: 

227 ppf_result = all_packager_provided_files[package] 

228 if ppf_result.auto_installable: 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true

229 context.install_rules.append( 

230 PPFInstallRule( 

231 context.binary_package, 

232 context.substitution, 

233 ppf_result.auto_installable, 

234 ) 

235 ) 

236 context.reserved_packager_provided_files.update( 

237 ppf_result.reserved_only 

238 ) 

239 self._transform_dpkg_maintscript_helpers_to_snippets() 

240 

241 return HighLevelManifest( 

242 self.manifest_path, 

243 self._mutable_yaml_manifest, 

244 self._install_rules, 

245 self._source_package, 

246 self.binary_packages, 

247 self.substitution, 

248 self._package_states, 

249 self._dpkg_architecture_variables, 

250 self._dpkg_arch_query_table, 

251 self._build_env, 

252 self._plugin_provided_feature_set, 

253 self._debian_dir, 

254 ) 

255 

256 @contextlib.contextmanager 

257 def binary_package_context( 

258 self, package_name: str 

259 ) -> Iterator[PackageTransformationDefinition]: 

260 if package_name not in self._package_states: 

261 self._error( 

262 f'The package "{package_name}" is not present in the debian/control file (could not find' 

263 f' "Package: {package_name}" in a binary stanza) nor is it a -dbgsym package for one' 

264 " for a package in debian/control." 

265 ) 

266 package_state = self._package_states[package_name] 

267 self._package_state_stack.append(package_state) 

268 ps_len = len(self._package_state_stack) 

269 yield package_state 

270 if ps_len != len(self._package_state_stack): 270 ↛ 271line 270 didn't jump to line 271, because the condition on line 270 was never true

271 raise RuntimeError("Internal error: Unbalanced stack manipulation detected") 

272 self._package_state_stack.pop() 

273 

274 def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]: 

275 t = self._plugin_provided_feature_set.manifest_parser_generator.dispatch_parser_table_for( 

276 rule_type 

277 ) 

278 if t is None: 

279 raise AssertionError( 

280 f"Internal error: No dispatching parser for {rule_type.__name__}" 

281 ) 

282 return t 

283 

284 @property 

285 def substitution(self) -> Substitution: 

286 if self._package_state_stack: 

287 return self._package_state_stack[-1].substitution 

288 return self._substitution 

289 

290 def add_extra_substitution_variables( 

291 self, 

292 **extra_substitutions: Tuple[str, AttributePath], 

293 ) -> Substitution: 

294 if self._package_state_stack or self._all_package_states is not None: 294 ↛ 299line 294 didn't jump to line 299, because the condition on line 294 was never true

295 # For one, it would not "bubble up" correctly when added to the lowest stack. 

296 # And if it is not added to the lowest stack, then you get errors about it being 

297 # unknown as soon as you leave the stack (which is weird for the user when 

298 # the variable is something known, sometimes not) 

299 raise RuntimeError("Cannot use add_extra_substitution from this state") 

300 for key, (_, path) in extra_substitutions.items(): 

301 self._declared_variables[key] = path 

302 self._substitution = self._substitution.with_extra_substitutions( 

303 **{k: v[0] for k, v in extra_substitutions.items()} 

304 ) 

305 return self._substitution 

306 

307 @property 

308 def current_binary_package_state(self) -> PackageTransformationDefinition: 

309 if not self._package_state_stack: 309 ↛ 310line 309 didn't jump to line 310, because the condition on line 309 was never true

310 raise RuntimeError("Invalid state: Not in a binary package context") 

311 return self._package_state_stack[-1] 

312 

313 @property 

314 def is_in_binary_package_state(self) -> bool: 

315 return bool(self._package_state_stack) 

316 

317 def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None: 

318 package_state = self.current_binary_package_state 

319 for dmh in package_state.dpkg_maintscript_helper_snippets: 319 ↛ 320line 319 didn't jump to line 320, because the loop on line 319 never started

320 snippet = MaintscriptSnippet( 

321 definition_source=dmh.definition_source, 

322 snippet=f'dpkg-maintscript-helper {escape_shell(*dmh.cmdline)} -- "$@"\n', 

323 ) 

324 for script in STD_CONTROL_SCRIPTS: 

325 package_state.maintscript_snippets[script].append(snippet) 

326 

327 def normalize_path( 

328 self, 

329 path: str, 

330 definition_source: AttributePath, 

331 *, 

332 allow_root_dir_match: bool = False, 

333 ) -> ExactFileSystemPath: 

334 try: 

335 normalized = _normalize_path(path) 

336 except ValueError: 

337 self._error( 

338 f'The path "{path}" provided in {definition_source.path} should be relative to the root of the' 

339 ' package and not use any ".." or "." segments.' 

340 ) 

341 if normalized == "." and not allow_root_dir_match: 

342 self._error( 

343 "Manifests must not change the root directory of the deb file. Please correct" 

344 f' "{definition_source.path}" (path: "{path}) in {self.manifest_path}' 

345 ) 

346 return ExactFileSystemPath( 

347 self.substitution.substitute(normalized, definition_source.path) 

348 ) 

349 

350 def parse_path_or_glob( 

351 self, 

352 path_or_glob: str, 

353 definition_source: AttributePath, 

354 ) -> MatchRule: 

355 match_rule = MatchRule.from_path_or_glob( 

356 path_or_glob, definition_source.path, substitution=self.substitution 

357 ) 

358 # NB: "." and "/" will be translated to MATCH_ANYTHING by MatchRule.from_path_or_glob, 

359 # so there is no need to check for an exact match on "." like in normalize_path. 

360 if match_rule.rule_type == MatchRuleType.MATCH_ANYTHING: 

361 self._error( 

362 f'The chosen match rule "{path_or_glob}" matches everything (including the deb root directory).' 

363 f' Please correct "{definition_source.path}" (path: "{path_or_glob}) in {self.manifest_path} to' 

364 f' something that matches "less" than everything.' 

365 ) 

366 return match_rule 

367 

368 def parse_manifest(self) -> HighLevelManifest: 

369 raise NotImplementedError 

370 

371 

372class YAMLManifestParser(HighLevelManifestParser): 

373 def _optional_key( 

374 self, 

375 d: Mapping[str, Any], 

376 key: str, 

377 attribute_parent_path: AttributePath, 

378 expected_type=None, 

379 default_value=None, 

380 ): 

381 v = d.get(key) 

382 if v is None: 

383 _detect_possible_typo(d, key, attribute_parent_path, False) 

384 return default_value 

385 if expected_type is not None: 

386 return self._ensure_value_is_type( 

387 v, expected_type, key, attribute_parent_path 

388 ) 

389 return v 

390 

391 def _required_key( 

392 self, 

393 d: Mapping[str, Any], 

394 key: str, 

395 attribute_parent_path: AttributePath, 

396 expected_type=None, 

397 extra: Optional[Union[str, Callable[[], str]]] = None, 

398 ): 

399 v = d.get(key) 

400 if v is None: 

401 _detect_possible_typo(d, key, attribute_parent_path, True) 

402 if extra is not None: 

403 msg = extra if isinstance(extra, str) else extra() 

404 extra_info = " " + msg 

405 else: 

406 extra_info = "" 

407 self._error( 

408 f'Missing required key {key} at {attribute_parent_path.path} in manifest "{self.manifest_path}.' 

409 f"{extra_info}" 

410 ) 

411 

412 if expected_type is not None: 

413 return self._ensure_value_is_type( 

414 v, expected_type, key, attribute_parent_path 

415 ) 

416 return v 

417 

418 def _ensure_value_is_type( 

419 self, 

420 v, 

421 t, 

422 key: Union[str, int, AttributePath], 

423 attribute_parent_path: Optional[AttributePath], 

424 ): 

425 if v is None: 

426 return None 

427 if not isinstance(v, t): 

428 if isinstance(t, tuple): 

429 t_msg = "one of: " + ", ".join(x.__name__ for x in t) 

430 else: 

431 t_msg = f"a {t.__name__}" 

432 key_path = ( 

433 key.path 

434 if isinstance(key, AttributePath) 

435 else assume_not_none(attribute_parent_path)[key].path 

436 ) 

437 self._error( 

438 f'The key {key_path} must be {t_msg} in manifest "{self.manifest_path}"' 

439 ) 

440 return v 

441 

442 def from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest": 

443 attribute_path = AttributePath.root_path() 

444 parser_generator = self._plugin_provided_feature_set.manifest_parser_generator 

445 dispatchable_object_parsers = parser_generator.dispatchable_object_parsers 

446 manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT] 

447 parsed_data = cast( 

448 "ManifestRootRule", 

449 manifest_root_parser.parse_input( 

450 yaml_data, 

451 attribute_path, 

452 parser_context=self, 

453 ), 

454 ) 

455 

456 packages_dict: Mapping[str, PackageContextData[Mapping[str, Any]]] = cast( 

457 "Mapping[str, PackageContextData[Mapping[str, Any]]]", 

458 parsed_data.get("packages", {}), 

459 ) 

460 install_rules = parsed_data.get("installations") 

461 if install_rules: 

462 self._install_rules = install_rules 

463 packages_parent_path = attribute_path["packages"] 

464 for package_name_raw, pcd in packages_dict.items(): 

465 definition_source = packages_parent_path[package_name_raw] 

466 package_name = pcd.resolved_package_name 

467 parsed = pcd.value 

468 

469 package_state: PackageTransformationDefinition 

470 with self.binary_package_context(package_name) as package_state: 

471 if package_state.is_auto_generated_package: 471 ↛ 473line 471 didn't jump to line 473, because the condition on line 471 was never true

472 # Maybe lift (part) of this restriction. 

473 self._error( 

474 f'Cannot define rules for package "{package_name}" (at {definition_source.path}). It is an' 

475 " auto-generated package." 

476 ) 

477 binary_version = parsed.get("binary-version") 

478 if binary_version is not None: 

479 package_state.binary_version = ( 

480 package_state.substitution.substitute( 

481 binary_version, 

482 definition_source["binary-version"].path, 

483 ) 

484 ) 

485 search_dirs = parsed.get("installation_search_dirs") 

486 if search_dirs is not None: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true

487 package_state.search_dirs = search_dirs 

488 transformations = parsed.get("transformations") 

489 conffile_management = parsed.get("conffile_management") 

490 service_rules = parsed.get("services") 

491 if transformations: 

492 package_state.transformations.extend(transformations) 

493 if conffile_management: 493 ↛ 494line 493 didn't jump to line 494, because the condition on line 493 was never true

494 package_state.dpkg_maintscript_helper_snippets.extend( 

495 conffile_management 

496 ) 

497 if service_rules: 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 package_state.requested_service_rules.extend(service_rules) 

499 

500 return self.build_manifest() 

501 

502 def _parse_manifest(self, fd: Union[IO[bytes], str]) -> HighLevelManifest: 

503 try: 

504 data = MANIFEST_YAML.load(fd) 

505 except YAMLError as e: 

506 msg = str(e) 

507 lines = msg.splitlines(keepends=True) 

508 i = -1 

509 for i, line in enumerate(lines): 

510 # Avoid an irrelevant "how do configure the YAML parser" message, which the 

511 # user cannot use. 

512 if line.startswith("To suppress this check"): 

513 break 

514 if i > -1 and len(lines) > i + 1: 

515 lines = lines[:i] 

516 msg = "".join(lines) 

517 msg = msg.rstrip() 

518 msg += ( 

519 f"\n\nYou can use `yamllint -d relaxed {escape_shell(self.manifest_path)}` to validate" 

520 " the YAML syntax. The yamllint tool also supports style rules for YAML documents" 

521 " (such as indentation rules) in case that is of interest." 

522 ) 

523 raise ManifestParseException( 

524 f"Could not parse {self.manifest_path} as a YAML document: {msg}" 

525 ) from e 

526 self._mutable_yaml_manifest = MutableYAMLManifest(data) 

527 return self.from_yaml_dict(data) 

528 

529 def parse_manifest( 

530 self, 

531 *, 

532 fd: Optional[Union[IO[bytes], str]] = None, 

533 ) -> HighLevelManifest: 

534 if fd is None: 534 ↛ 535line 534 didn't jump to line 535, because the condition on line 534 was never true

535 with open(self.manifest_path, "rb") as fd: 

536 return self._parse_manifest(fd) 

537 else: 

538 return self._parse_manifest(fd)