Coverage for britney2/migration.py: 94%
271 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-06-17 09:00 +0000
1import contextlib
2import copy
3import optparse
4from collections.abc import Iterator
5from typing import TYPE_CHECKING, Optional, cast
7import apt_pkg
9from britney2.transaction import MigrationTransactionState, UndoItem
10from britney2.utils import (
11 MigrationConstraintException,
12 check_installability,
13 clone_nuninst,
14 compute_reverse_tree,
15 find_smooth_updateable_binaries,
16)
18if TYPE_CHECKING: 18 ↛ 19line 18 didn't jump to line 19 because the condition on line 18 was never true
19 from . import BinaryPackage, BinaryPackageId, Suites
20 from .hints import HintCollection
21 from .installability.universe import BinaryPackageUniverse
22 from .migrationitem import MigrationItem, MigrationItemFactory
25def compute_eqv_set(
26 pkg_universe: "BinaryPackageUniverse",
27 updates: set["BinaryPackageId"],
28 rms: set["BinaryPackageId"],
29) -> set[tuple[str, str]]:
30 eqv_set: set[tuple[str, str]] = set()
31 # If we are removing *and* updating packages, then check for eqv. packages
32 if rms and updates:
33 eqv_table = {(x.package_name, x.architecture): x for x in rms}
35 for new_pkg_id in updates:
36 key = (new_pkg_id.package_name, new_pkg_id.architecture)
37 old_pkg_id = eqv_table.get(key)
38 if old_pkg_id is not None:
39 if pkg_universe.are_equivalent(new_pkg_id, old_pkg_id):
40 eqv_set.add(key)
41 return eqv_set
44def is_nuninst_worse(
45 must_be_installable: list[str],
46 nuninst_now_arch: set[str],
47 nuninst_after_arch: set[str],
48 allow_uninst: set[str | None],
49) -> bool:
50 if len(nuninst_after_arch - allow_uninst) > len(nuninst_now_arch - allow_uninst):
51 return True
53 regression = nuninst_after_arch - nuninst_now_arch
54 if not regression.isdisjoint(must_be_installable): 54 ↛ 55line 54 didn't jump to line 55 because the condition on line 54 was never true
55 return True
56 return False
59class MigrationManager:
61 def __init__(
62 self,
63 options: optparse.Values,
64 suite_info: "Suites",
65 all_binaries: dict["BinaryPackageId", "BinaryPackage"],
66 pkg_universe: "BinaryPackageUniverse",
67 constraints: dict[str, list[str]],
68 allow_uninst: dict[str, set[str | None]],
69 migration_item_factory: "MigrationItemFactory",
70 hints: "HintCollection",
71 ) -> None:
72 self.options = options
73 self.suite_info = suite_info
74 self.all_binaries = all_binaries
75 self.pkg_universe = pkg_universe
76 self.constraints = constraints
77 self.allow_uninst = allow_uninst
78 self.hints = hints
79 self._transactions: list[MigrationTransactionState] = []
80 self._all_architectures: frozenset[str] = frozenset(self.options.architectures)
81 self._migration_item_factory = migration_item_factory
83 @property
84 def current_transaction(self) -> MigrationTransactionState | None:
85 return self._transactions[-1] if self._transactions else None
87 def compute_groups(
88 self,
89 item: "MigrationItem",
90 allow_smooth_updates: bool = True,
91 removals: set["BinaryPackageId"] = cast(set["BinaryPackageId"], frozenset()),
92 ) -> tuple[
93 str, set["BinaryPackageId"], set["BinaryPackageId"], set["BinaryPackageId"]
94 ]:
95 """Compute the groups of binaries being migrated by item
97 This method will compute the binaries that will be added to,
98 replaced in or removed from the target suite and which of
99 the removals are smooth updatable.
101 Parameters:
102 * "allow_smooth_updates" is a boolean determining whether smooth-
103 updates are permitted in this migration. When set to False,
104 the "smoothbins" return value will always be the empty set.
105 Any value that would have been there will now be in "rms"
106 instead. (defaults: True)
107 * "removals" is a set of binaries that is assumed to be
108 removed at the same time as this migration (e.g. in the same
109 "easy"-hint). This may affect what if some binaries are
110 smooth updated or not. (defaults: empty-set)
111 - Binaries must be given as ("package-name", "version",
112 "architecture") tuples.
114 Returns a tuple (adds, rms, smoothbins). "adds" is a set of
115 binaries that will updated in or appear after the migration.
116 "rms" is a set of binaries that are not smooth-updatable (or
117 binaries that could be, but there is no reason to let them be
118 smooth updated). "smoothbins" is set of binaries that are to
119 be smooth-updated.
121 Each "binary" in "adds", "rms" and "smoothbins" will be a
122 tuple of ("package-name", "version", "architecture") and are
123 thus tuples suitable for passing on to the
124 InstallabilityTester.
127 Unlike migrate_items_to_target_suite, this will not modify
128 any data structure.
129 """
130 # local copies for better performances
131 item_package = item.package
132 target_suite = self.suite_info.target_suite
133 binaries_t = target_suite.binaries
135 adds = set()
137 # remove all binary packages (if the source already exists)
138 if item.architecture == "source" or not item.is_removal:
139 source_name = item_package
140 if source_name in target_suite.sources:
141 rms, smoothbins = self._compute_removals(
142 item, allow_smooth_updates, removals
143 )
144 else:
145 rms = set()
146 smoothbins = set()
148 # single binary removal; used for clearing up after smooth
149 # updates but not supported as a manual hint
150 else:
151 assert item_package in binaries_t[item.architecture]
152 pkg_id_t = binaries_t[item.architecture][item_package].pkg_id
153 ver = pkg_id_t.version
154 if ver != item.version:
155 raise MigrationConstraintException(
156 "trying cruft removal item %s, while %s has %s/%s on %s"
157 % (
158 item,
159 target_suite.name,
160 pkg_id_t.package_name,
161 ver,
162 pkg_id_t.architecture,
163 )
164 )
165 source_name = binaries_t[item.architecture][item_package].source
166 rms = {pkg_id_t}
167 smoothbins = set()
169 # add the new binary packages (if we are not removing)
170 if not item.is_removal:
171 source_suite = item.suite
172 binaries_s = source_suite.binaries
173 source_data = source_suite.sources[source_name]
174 source_ver_new = source_data.version
175 sources_t = target_suite.sources
176 if source_name in sources_t:
177 source_data_old = sources_t[source_name]
178 source_ver_old = source_data_old.version
179 if apt_pkg.version_compare(source_ver_old, source_ver_new) > 0:
180 raise MigrationConstraintException(
181 "trying src:%s %s, while %s has %s"
182 % (
183 source_name,
184 source_ver_new,
185 target_suite.name,
186 source_ver_old,
187 )
188 )
190 for pkg_id_s in source_data.binaries:
191 ver = pkg_id_s.version
192 parch = pkg_id_s.architecture
193 if item.architecture not in ("source", parch):
194 continue
196 binary = pkg_id_s.package_name
197 if binaries_s[parch][binary].source != source_name:
198 # This binary package has been hijacked by some other source.
199 # So don't add it as part of this update.
200 #
201 # Also, if this isn't a source update, don't remove
202 # the package that's been hijacked if it's present.
203 if item.architecture != "source": 203 ↛ 210line 203 didn't jump to line 210 because the condition on line 203 was always true
204 for rm_item in list(rms):
205 if (
206 rm_item.package_name == binary
207 and rm_item.architecture == parch
208 ):
209 rms.remove(rm_item)
210 continue
212 # Don't add the binary if it is cruft; smooth updates will keep it if possible
213 if (
214 parch not in self.options.outofsync_arches
215 and source_data.version != binaries_s[parch][binary].source_version
216 ):
217 continue
219 if binary in binaries_t[parch]:
220 oldver = binaries_t[parch][binary].version
221 ver = pkg_id_s.version
222 if apt_pkg.version_compare(oldver, ver) > 0:
223 raise MigrationConstraintException(
224 "trying %s %s from src:%s %s, while %s has %s"
225 % (
226 binary,
227 ver,
228 source_name,
229 source_ver_new,
230 target_suite.name,
231 oldver,
232 )
233 )
235 adds.add(pkg_id_s)
237 return (source_name, adds, rms, smoothbins)
239 def _compute_removals(
240 self,
241 item: "MigrationItem",
242 allow_smooth_updates: bool,
243 removals: set["BinaryPackageId"],
244 ) -> tuple[set["BinaryPackageId"], set["BinaryPackageId"]]:
245 pkg_universe = self.pkg_universe
246 source_suite = item.suite
247 target_suite = self.suite_info.target_suite
248 binaries_s = source_suite.binaries
249 binaries_t = target_suite.binaries
250 source_name = item.package
251 source_data = target_suite.sources[source_name]
253 bins: list["BinaryPackageId"] = []
254 # remove all the binaries
256 # first, build a list of eligible binaries
257 for pkg_id in source_data.binaries:
258 parch = pkg_id.architecture
259 if item.architecture != "source" and parch != item.architecture:
260 continue
262 binary = pkg_id.package_name
263 # Work around #815995
264 if ( 264 ↛ 269line 264 didn't jump to line 269
265 item.architecture == "source"
266 and item.is_removal
267 and binary not in binaries_t[parch]
268 ):
269 continue
271 bin_data = binaries_t[parch][binary]
272 # Do not include hijacked binaries nor cruft (cruft is handled separately)
273 if (
274 bin_data.source != source_name
275 or bin_data.source_version != source_data.version
276 ):
277 continue
278 bins.append(pkg_id)
280 if allow_smooth_updates and source_suite.suite_class.is_primary_source:
281 smoothbins = find_smooth_updateable_binaries(
282 bins,
283 source_suite.sources[source_name],
284 pkg_universe,
285 target_suite,
286 binaries_t,
287 binaries_s,
288 removals,
289 self.options.smooth_updates,
290 self.hints,
291 )
292 else:
293 smoothbins = set()
295 # remove all the binaries which aren't being smooth updated
296 if (
297 item.architecture != "source"
298 and source_suite.suite_class.is_additional_source
299 ):
300 # Special-case for pu/tpu:
301 # if this is a binary migration from *pu, only the arch:any
302 # packages will be present. ideally dak would also populate
303 # the arch-indep packages, but as that's not the case we
304 # must keep them around; they will not be re-added by the
305 # migration so will end up missing from testing
306 all_binaries = self.all_binaries
307 rms = {
308 pkg_id
309 for pkg_id in bins
310 if pkg_id not in smoothbins
311 and all_binaries[pkg_id].architecture != "all"
312 }
313 else:
314 rms = {pkg_id for pkg_id in bins if pkg_id not in smoothbins}
316 return rms, smoothbins
318 def _apply_item_to_target_suite(
319 self,
320 item: "MigrationItem",
321 removals: set["BinaryPackageId"] = cast(set["BinaryPackageId"], frozenset()),
322 ) -> tuple[set["BinaryPackageId"], set["BinaryPackageId"]]:
323 """Apply a change to the target suite as requested by `item`
325 An optional set of binaries may be passed in "removals". Binaries listed
326 in this set will be assumed to be removed at the same time as the "item"
327 will migrate. This may change what binaries will be smooth-updated.
328 - Binaries in this set must be instances of BinaryPackageId.
330 This method applies the changes required by the action `item` tracking
331 them so it will be possible to revert them.
333 The method returns a tuple containing a set of packages
334 affected by the change (as (name, arch)-tuples) and the
335 dictionary undo which can be used to rollback the changes.
336 """
337 undo: UndoItem = {"binaries": {}, "sources": {}, "virtual": {}}
339 affected_all = set()
340 updated_binaries = set()
342 # local copies for better performance
343 source_suite = item.suite
344 target_suite = self.suite_info.target_suite
345 packages_t = target_suite.binaries
346 provides_t = target_suite.provides_table
347 pkg_universe = self.pkg_universe
348 transaction = self.current_transaction
350 source_name, updates, rms, smooth_updates = self.compute_groups(
351 item, removals=removals
352 )
353 sources_t = target_suite.sources
354 # Handle the source package
355 old_source = sources_t.get(source_name)
357 # add/update the source package
358 if item.is_removal and item.architecture == "source":
359 del sources_t[source_name]
360 else:
361 # with OUTOFSYNC_ARCHES, the source can be removed before out-of-sync binaries are removed
362 if not item.is_removal or source_name in source_suite.sources: 362 ↛ 371line 362 didn't jump to line 371 because the condition on line 362 was always true
363 # always create a copy of the SourcePackage object
364 sources_t[source_name] = copy.copy(source_suite.sources[source_name])
365 if old_source is not None:
366 # always create a new list of binaries
367 sources_t[source_name].binaries = copy.copy(old_source.binaries)
368 else:
369 sources_t[source_name].binaries = set()
371 undo["sources"][source_name] = old_source
373 eqv_set = compute_eqv_set(pkg_universe, updates, rms)
375 # remove all the binaries which aren't being smooth updated
376 for rm_pkg_id in rms:
377 binary = rm_pkg_id.package_name
378 parch = rm_pkg_id.architecture
379 pkey = (binary, parch)
380 binaries_t_a = packages_t[parch]
381 provides_t_a = provides_t[parch]
383 pkg_data = binaries_t_a[binary]
384 # save the old binary for undo
385 undo["binaries"][pkey] = rm_pkg_id
386 if pkey not in eqv_set:
387 # all the reverse dependencies are affected by
388 # the change
389 affected_all.update(pkg_universe.reverse_dependencies_of(rm_pkg_id))
390 affected_all.update(pkg_universe.negative_dependencies_of(rm_pkg_id))
392 # remove the provided virtual packages
393 for provided_pkg, prov_version, _ in (
394 pkg_data.provides if pkg_data.provides is not None else []
395 ):
396 key = (provided_pkg, parch)
397 if key not in undo["virtual"]:
398 undo["virtual"][key] = provides_t_a[provided_pkg].copy()
399 provides_t_a[provided_pkg].remove((binary, prov_version))
400 if not provides_t_a[provided_pkg]:
401 del provides_t_a[provided_pkg]
402 # for source removal, the source is already gone
403 if source_name in sources_t:
404 sources_t[source_name].binaries.discard(rm_pkg_id)
405 # finally, remove the binary package
406 del binaries_t_a[binary]
407 target_suite.remove_binary(rm_pkg_id)
409 # Add/Update binary packages in testing
410 if updates:
411 packages_s = source_suite.binaries
413 for updated_pkg_id in updates:
414 binary, parch = updated_pkg_id.package_name, updated_pkg_id.architecture
415 key = (binary, parch)
416 binaries_t_a = packages_t[parch]
417 provides_t_a = provides_t[parch]
418 equivalent_replacement = key in eqv_set
420 # obviously, added/modified packages are affected
421 if not equivalent_replacement:
422 affected_all.add(updated_pkg_id)
423 # if the binary already exists in testing, it is currently
424 # built by another source package. we therefore remove the
425 # version built by the other source package, after marking
426 # all of its reverse dependencies as affected
427 if binary in binaries_t_a:
428 old_pkg_data = binaries_t_a[binary]
429 old_pkg_id = old_pkg_data.pkg_id
430 # save the old binary package
431 undo["binaries"][key] = old_pkg_id
432 if not equivalent_replacement: 432 ↛ 437line 432 didn't jump to line 437 because the condition on line 432 was always true
433 # all the reverse conflicts
434 affected_all.update(
435 pkg_universe.reverse_dependencies_of(old_pkg_id)
436 )
437 target_suite.remove_binary(old_pkg_id)
438 elif transaction and transaction.parent_transaction:
439 # the binary isn't in the target suite, but it may have been at
440 # the start of the current hint and have been removed
441 # by an earlier migration. if that's the case then we
442 # will have a record of the older instance of the binary
443 # in the undo information. we can use that to ensure
444 # that the reverse dependencies of the older binary
445 # package are also checked.
446 # reverse dependencies built from this source can be
447 # ignored as their reverse trees are already handled
448 # by this function
449 for tundo, tpkg in transaction.parent_transaction.undo_items:
450 if key in tundo["binaries"]: 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true
451 tpkg_id = tundo["binaries"][key]
452 affected_all.update(
453 pkg_universe.reverse_dependencies_of(tpkg_id)
454 )
456 # add/update the binary package from the source suite
457 new_pkg_data = packages_s[parch][binary]
458 binaries_t_a[binary] = new_pkg_data
459 target_suite.add_binary(updated_pkg_id)
460 updated_binaries.add(updated_pkg_id)
461 # add the binary to the source package
462 sources_t[source_name].binaries.add(updated_pkg_id)
463 # register new provided packages
464 for provided_pkg, prov_version, _ in (
465 new_pkg_data.provides if new_pkg_data.provides is not None else []
466 ):
467 key = (provided_pkg, parch)
468 if key not in undo["virtual"]:
469 restore_as = (
470 provides_t_a[provided_pkg].copy()
471 if provided_pkg in provides_t_a
472 else None
473 )
474 undo["virtual"][key] = restore_as
475 provides_t_a[provided_pkg].add((binary, prov_version))
476 if not equivalent_replacement:
477 # all the reverse dependencies are affected by the change
478 affected_all.add(updated_pkg_id)
479 affected_all.update(
480 pkg_universe.negative_dependencies_of(updated_pkg_id)
481 )
483 # Also include the transitive rdeps of the packages found so far
484 compute_reverse_tree(pkg_universe, affected_all)
485 if transaction:
486 transaction.add_undo_item(undo, updated_binaries)
487 # return the affected packages (direct and than all)
488 return (affected_all, smooth_updates)
490 def _apply_multiple_items_to_target_suite(
491 self, items: list["MigrationItem"]
492 ) -> tuple[
493 bool,
494 frozenset[str] | set[str],
495 set["BinaryPackageId"],
496 set["BinaryPackageId"],
497 ]:
498 is_source_migration = False
499 if len(items) == 1:
500 item = items[0]
501 # apply the changes
502 affected_all, smooth_updates = self._apply_item_to_target_suite(item)
503 if item.architecture == "source":
504 affected_architectures: frozenset[str] | set[str] = (
505 self._all_architectures
506 )
507 is_source_migration = True
508 else:
509 affected_architectures = {item.architecture}
510 else:
511 affected_architectures = set()
512 removals: set[BinaryPackageId] = set()
513 affected_all = set()
514 smooth_updates = set()
515 for item in items:
516 _, _, rms, _ = self.compute_groups(item, allow_smooth_updates=False)
517 removals.update(rms)
518 affected_architectures.add(item.architecture)
520 if "source" in affected_architectures:
521 affected_architectures = self._all_architectures
522 is_source_migration = True
524 for item in items:
525 item_affected_all, item_smooth = self._apply_item_to_target_suite(
526 item, removals=removals
527 )
528 affected_all.update(item_affected_all)
529 smooth_updates.update(item_smooth)
531 return is_source_migration, affected_architectures, affected_all, smooth_updates
533 def migrate_items_to_target_suite(
534 self,
535 items: list["MigrationItem"],
536 nuninst_now: dict[str, set[str]],
537 stop_on_first_regression: bool = True,
538 ) -> tuple[bool, dict[str, set[str]], str | None, set["MigrationItem"]]:
539 is_accepted = True
540 target_suite = self.suite_info.target_suite
541 packages_t = target_suite.binaries
543 nobreakall_arches = self.options.nobreakall_arches
544 new_arches = self.options.new_arches
545 break_arches = self.options.break_arches
546 arch = None
548 is_source_migration, affected_architectures, affected_all, smooth_updates = (
549 self._apply_multiple_items_to_target_suite(items)
550 )
552 # Copy nuninst_comp - we have to deep clone affected
553 # architectures.
555 # NB: We do this *after* updating testing as we have to filter out
556 # removed binaries. Otherwise, uninstallable binaries that were
557 # removed by the item would still be counted.
559 nuninst_after = clone_nuninst(
560 nuninst_now, packages_s=packages_t, architectures=affected_architectures
561 )
562 must_be_installable = self.constraints["keep-installable"]
564 # check the affected packages on all the architectures
565 for arch in sorted(affected_architectures):
566 check_archall = arch in nobreakall_arches
568 check_installability(
569 target_suite,
570 packages_t,
571 arch,
572 affected_all,
573 check_archall,
574 nuninst_after,
575 )
577 # if the uninstallability counter is worse than before, break the loop
578 if stop_on_first_regression:
579 if is_nuninst_worse(
580 must_be_installable,
581 nuninst_now[arch],
582 nuninst_after[arch],
583 self.allow_uninst[arch],
584 ):
585 if arch not in break_arches:
586 is_accepted = False
587 break
588 # ... except for a few special cases:
589 elif is_source_migration or arch in new_arches: 589 ↛ 592line 589 didn't jump to line 592 because the condition on line 589 was always true
590 pass
591 else:
592 is_accepted = False
593 break
595 new_cruft = {
596 self._migration_item_factory.generate_removal_for_cruft_item(x)
597 for x in smooth_updates
598 }
600 return (is_accepted, nuninst_after, arch, new_cruft)
602 @contextlib.contextmanager
603 def start_transaction(self) -> Iterator[MigrationTransactionState]:
604 tmts = MigrationTransactionState(
605 self.suite_info, self.all_binaries, self.current_transaction
606 )
607 self._transactions.append(tmts)
608 try:
609 yield tmts
610 except Exception:
611 if not tmts.is_committed and not tmts.is_rolled_back:
612 tmts.rollback()
613 raise
614 finally:
615 self._transactions.pop() 615 ↛ exitline 615 didn't except from function 'start_transaction' because the raise on line 613 wasn't executed
616 assert tmts.is_rolled_back or tmts.is_committed